def addVolume(dump, volumeId): if volumeId in dump[couch.Volume].values(): return volume = cache[couch.Volume][volumeId] log.debug('gentest', 'adding volume %r' % volume) dump[couch.Volume][volume] = volume.id
def fragmentSetChroma(self, fragment, chroma): """ @param fragment: a fragment in the fragments key @type fragment: L{mapping.AnonymousStruct} @type chroma: L{track.ChromaModel} @returns: whether any field got changed """ # AnonymousStruct does not actually exist as a class assert fragment.__class__.__name__ == 'AnonymousStruct', \ "fragment %r is not a paisley.mapping.AnonymousStruct" % fragment assert isinstance(chroma, track.ChromaPrintModel) changed = False if not 'chroma' in fragment: fragment.chroma = _ChromaPrint() LOOKUP_FIELDS = ['mbid', 'artists', 'title'] KEYS = ['chromaprint', 'duration', 'lookedup'] for key in LOOKUP_FIELDS + KEYS: orig = getattr(fragment.chroma, key, None) value = getattr(chroma, key, None) log.log('mappings', 'key %r, original %r, value %r', key, orig, value) if value and orig != value: setattr(fragment.chroma, key, value) # if all that changed was the lookedup time, we don't # consider it a change if key != 'lookedup': changed = True log.debug('mappings', 'key %r changed to %r', key, value) return changed
def resolveSlices(result): resultList = list(result) log.debug("playlist", "got %r tracks sliced", len(resultList)) d = manydef.DeferredListSpaced() for succeeded, result in resultList: if not succeeded: self.warningFailure(result) if isinstance(result, mappings.Track): # result from resolveIds continue slicesGen = result # FIXME: only keep first audiofile for now slice = list(slicesGen)[0] trackDict[slice.track_id][1] = slice def callable(slice, trackId): d = self.getSliceFile(slice) def cb(audiofile, trackId): # print 'resolveSlices cb', audiofile, trackId trackDict[trackId][2] = audiofile return audiofile, trackId d.addCallback(cb, trackId) return d d.addCallable(callable, slice, slice.track_id) d.start() return d
def processEnded(self, status): log.debug('slavepp', 'processEnded') rc = status.value.exitCode if rc == 0: self.deferred.callback(self) else: self.deferred.errback(rc)
def addScore(dump, scoreId): if scoreId in dump[couch.Score].values(): return score = cache[couch.Score][scoreId] log.debug('gentest', 'adding score %r' % score) dump[couch.Score][score] = score.id
def addTrack(dump, trackId): if trackId in dump[couch.Track].values(): return track = cache[couch.Track][trackId] log.debug('gentest', 'adding track %r' % track) dump[couch.Track][track] = track.id # trackalbums point to tracks d = refs[(couch.TrackAlbum, 'track_id')] if trackId in d: for trackalbumId in d[trackId]: addTrackAlbum(dump, trackalbumId) # slices point to tracks d = refs[(couch.Slice, 'track_id')] if trackId in d: for sliceId in d[trackId]: addSlice(dump, sliceId) # tracks have artists for artist in track.artist_ids: addArtist(dump, artist, back=False) # scores point to tracks d = refs[(couch.Score, 'subject_id')] if trackId in d: for scoreId in d[trackId]: addScore(dump, scoreId)
def load(db, dbName, klazz, viewName): """ @type db: L{dadcouch.extern.paisley.client.CouchDB} @type dbName: str @param klazz: the class to instantiate objects from @param viewName: name of the view to load objects from """ log.debug('load', 'loading %s->%r using view %r', dbName, klazz, viewName) v = views.View(db, dbName, 'dad', viewName, klazz, include_docs=True) d = v.queryView() def eb(failure): log.warning('load', 'Failed to query view: %r', log.getFailureMessage(failure)) return failure d.addErrback(eb) def cb(result): log.debug('load', 'loaded %s->%r using view %r', dbName, klazz, viewName) return result d.addCallback(cb) return d
def connectionMade(self): log.debug('slavepp', 'connectionMade') protocol = self._server.buildProtocol(None) protocol.transport = MyTransport(self.transport, self._childWriteFd) # FIXME: need to get a transport in here, like Broker log.debug('slavepp', 'protocol %r, transport %r', protocol, protocol.transport) self._protocol = protocol protocol.connectionMade()
def spawnSlave(server): log.debug('master', 'spawning slave') # we'll use fd3 for reading from child, and fd4 for writing to it mypp = SlaveProcessProtocol(server, 4) childFDs = {0: 0, 1: 1, 2: 2, 3: 'r', 4: 'w'} transport = reactor.spawnProcess(mypp, sys.argv[0], args=[sys.argv[0], '--slave'], env=os.environ, childFDs=childFDs) mypp.setTransport(transport)
def cacheLoad(cache, dadDB, klazz, view): log.debug('cacheLoad', 'Loading klazz %r using view %r', klazz, view) cache[klazz] = {} d = dadDB.viewDocs(view, klazz, include_docs=True) def cb(result): for item in result: cache[klazz][item.id] = item d.addCallback(cb) return d
def connectStdio(factory): TIMEOUT = 3 c = ProcessClientConnector(factory, TIMEOUT, reactor) log.debug('slave', 'before connecting: %r', c.state) c.connect() log.debug('slave', 'after connecting: state %r, timeoutid %r, transport %r', c.state, c.timeoutID, c.transport) # FIXME: we shouldn't do this manually c.cancelTimeout() return c
def addAudioFile(dump, audiofileId): if audiofileId in dump[couch.AudioFile].values(): return audiofile = cache[couch.AudioFile][audiofileId] log.debug('gentest', 'adding audiofile %r' % audiofile) dump[couch.AudioFile][audiofile] = audiofile.id # audiofiles point to directories addDirectory(dump, audiofile.directory_id)
def addTrackAlbum(dump, trackalbumId): if trackalbumId in dump[couch.TrackAlbum].values(): return trackalbum = cache[couch.TrackAlbum][trackalbumId] log.debug('gentest', 'adding trackalbum %r' % trackalbum) dump[couch.TrackAlbum][trackalbum] = trackalbum.id addAlbum(dump, trackalbum.album_id) addTrack(dump, trackalbum.track_id)
def addOptions(self): # FIXME: is this the right place ? log.init() # from dad.configure import configure log.debug("dad", "This is dad version %s (%s)", "0.0.0", "0") # configure.version, configure.revision) self.parser.add_option('-v', '--version', action="store_true", dest="version", help="show version information")
def showPaths(result): resultList = list(result) log.debug('playlist', 'got %r paths resolved', len(resultList)) for succeeded, result in resultList: if not succeeded: print 'ERROR', result else: (track, slice, path, score, userId) = result print path.encode('utf-8')
def addSlice(dump, sliceId): if sliceId in dump[couch.Slice].values(): return slice = cache[couch.Slice][sliceId] log.debug('gentest', 'adding slice %r' % slice) dump[couch.Slice][slice] = slice.id # slices point to tracks addTrack(dump, slice.track_id) # slices point to audiofiles addAudioFile(dump, slice.audiofile_id)
def mainSlave(): # this is the slave log.debug('slave', 'slave is started') # log in to manager with a pb client over the processprotocol somehow clientFactory = pb.PBClientFactory() c = connectStdio(clientFactory) log.debug('slave', 'clientFactory._broker %r', clientFactory._broker) d = clientFactory.getRootObject() def cb(root): log.debug('slave', 'slave got root: %r' % root) return root.callRemote("echo", "hello network") d.addCallback(cb)
def addDirectory(dump, directoryId): if directoryId in dump[couch.Directory].values(): return directory = cache[couch.Directory][directoryId] log.debug('gentest', 'adding directory %r' % directory) dump[couch.Directory][directory] = directory.id # directories point to directories if directory.parent_id: addDirectory(dump, directory.parent_id) # directories point to volumes if directory.volume_id: addVolume(dump, directory.volume_id)
def count(resultList): log.debug('tracks', 'counting offline tracks out of %r total', len(resultList)) total = len(resultList) online = 0 for i, (succeeded, result) in enumerate(resultList): if result: online += 1 else: print "track %r offline" % tracks[i] print "%d of %d tracks online" % (online, total) log.info('online', "%d cache hits of %d lookups", dadDB.db.hits, dadDB.db.lookups)
def doLater(self, args): # FIXME: view-specific import gtk defer.Deferred.debug = 1 self._doneDeferred = defer.Deferred() # FIXME: allow customizing model and/or view(s) viewTypes = ['GTK', ] db = self.parentCommand.getDatabase() self.debug('Database: %r', db) # get the model amodel = self.parentCommand.getAppModel() self.debug('App model: %r', amodel) acontroller = app.AppController(amodel) for viewType in viewTypes: viewModule = 'dad%s.views.app.%sAppView' % (viewType.lower(), viewType) aview = reflect.namedAny(viewModule)() aview.set_title(self.getFullName()) acontroller.addView(aview) # FIXME: gtk-specific aview.widget.connect('destroy', self.done_cb) vbox = gtk.VBox() hbox = gtk.HBox() vbox.add(hbox) aview.widget.add(vbox) asController, asModel, asViews = acontroller.getTriad('ArtistSelector') hbox.pack_start(asViews[0]) aview.widget.show_all() # start loading artists and albums d = defer.Deferred() d.addCallback(lambda _: log.debug('test', 'asking controller to populate')) d.addCallback(lambda _: asController.populate()) d.callback(None) return self._doneDeferred
def addAlbum(dump, albumId): if albumId in dump[couch.Album].values(): return album = cache[couch.Album][albumId] log.debug('gentest', 'adding album %r' % album) dump[couch.Album][album] = album.id # trackalbums point to albums d = refs[(couch.TrackAlbum, 'album_id')] if albumId in d: for trackalbumId in d[albumId]: addTrackAlbum(dump, trackalbumId) # albums have artists for artist in album.artist_ids: addArtist(dump, artist, back=False)
def resolveTracks(tracks): # returns: a deferred firing a list of # (result, Track/sliceGen alternating # tracks: list of Track, score, userId trackList = list(tracks) log.debug("playlist", "got %r tracks", len(trackList)) d = manydef.DeferredListSpaced() for track, score, userId in trackList: trackDict[track.id] = [track, None, None, score, userId] log.debug("playlist", "track %r has score %f by user %r", track, score, userId) d.addCallable(self.resolveIds, track, "artist_ids", "artists", mappings.Artist) d.addCallable(self.getSlices, track) d.start() return d
def cb(_): log.debug('consistency', 'got objects for %r and %r', sourceKlazz, destKlazz) errors = 0 items = 0 isList = False attr = getattr(sourceKlazz, attribute) if isinstance(attr, mapping.ListField): isList = True values = cache[sourceKlazz].values() log.debug('consistency', 'got %d items for %r', len(values), sourceKlazz) for item in values: log.log('consistency', 'item %r', item) values = getattr(item, attribute) if not isList: values = [getattr(item, attribute), ] log.log('consistency', 'values %r', values) for item_id in values: # allow None/unset id's if item_id is None: continue if item_id not in cache[destKlazz].keys(): print '%r %r has %r %r but does not exist' % ( sourceKlazz, item.id, destKlazz, item_id) import code;code.interact(local=locals()) errors += 1 if cache[destKlazz][item_id].type != sourceKlazz.type: print '%r %r points to %r %r but type is %r' % ( sourceKlazz, item.id, destKlazz, item_id, cache[destKlazz][item_id].type) else: items += 1 if errors == 0: print '%s.%s is consistent (%d items)' % ( sourceKlazz.__name__, attribute, items)
def main(): log.init() server = 'localhost' if len(sys.argv) > 1: server = sys.argv[1] dbName = 'dad' if len(sys.argv) > 2: dbName = sys.argv[2] # this rebinds and makes it break in views # db = couchdb.CouchDB('localhost', dbName='dad') log.debug('main', 'binding to couchdb %r', server) db = couchdb.CouchDB(server) log.debug('main', 'checking database %r', dbName) d = check(db, dbName) d.addCallback(lambda _: reactor.stop()) return d
def doLater(self, args): db = self.parentCommand.getDatabase() sel = selecter.getSelecter(self.options.selecter, self.stdout, database=db) while True: selected = yield sel.select() if not selected: break if not os.path.exists(selected.path): continue text = "# %s - %s\n%s\n" % ( " & ".join(selected.artists).encode('utf-8'), selected.title.encode('utf-8'), selected.path.encode('utf-8')) log.debug('main', 'output: %r', text) self.stdout.write(text) self.stdout.flush()
def main(): log.init() server = 'localhost' if len(sys.argv) > 1: server = sys.argv[1] dbName = 'dad' if len(sys.argv) > 2: dbName = sys.argv[2] # this rebinds and makes it break in views # db = couchdb.CouchDB('localhost', dbName='dad') db = cachedb.CachingCouchDB(server) dadDB = daddb.DADDB(db, dbName) d = defer.Deferred() # get Volume and Directory into the cache so that we don't do a lot # of parallel lookups for them d.addCallback(lambda _: dadDB.viewDocs( 'directory-lookup', couch.Directory, include_docs=True)) d.addCallback(lambda _: log.debug('online', '%d directories', len(list(_)))) d.addCallback(lambda _: dadDB.viewDocs( 'volumes', couch.Volume, include_docs=True)) d.addCallback(lambda _: log.debug('online', '%d volumes', len(list(_)))) # now check stuff # audiofiles first, so we seed the cache before checking tracks d.addCallback(lambda _: audiofiles(dadDB)) d.addCallback(lambda _: tracks(dadDB)) d.addCallback(lambda _: slices(dadDB)) d.addCallback(lambda _: reactor.stop()) def eb(f): print 'ERROR: ', log.getFailureMessage(f) reactor.stop() d.addErrback(eb) d.callback(None) return d
def addArtist(dump, artistId, back=False): if artistId in dump[couch.Artist].values(): return log.debug('gentest', 'adding artist %s', artistId) artist = cache[couch.Artist][artistId] dump[couch.Artist][artist] = artist.id if not back: return # albums point to artists d = refs[(couch.Album, 'artist_ids')] if artistId in d: for albumId in d[artistId]: addAlbum(dump, albumId) d = refs[(couch.Track, 'artist_ids')] if artistId in d: for trackId in d[artistId]: addTrack(dump, trackId)
def _makeTransport(self): log.debug('slave', 'THOMAS: makeTransport: %r', self.state) # protocol = self.factory.buildProtocol(None) # FIXME: addr # FIXME: makeTransport is called before setting timeoutID; # buildProtocol thus resets it to soon for us protocol = self.buildProtocol(None) # FIXME: addr log.debug('slave', 'timeoutID %r', self.timeoutID) log.debug('slave', 'THOMAS: protocol %r', protocol) t = MyStandardIO(protocol, stdin=4, stdout=3) # FIXME: writing works to get it to the server, but how does the # chitchat really start ? # t.write('kakapipi') # t.startReading() log.debug('slave', 'host: %r, peer: %r', t.getHost(), t.getPeer()) return t
def cb(_): # here is where we output all id's artists = cache[couch.Artist].values()[:count] for artist in artists: log.debug('gentest', 'adding root artist %r', artist) addArtist(dump, artist.id, back=True) # now print all ids for klazz, d in dump.items(): for i in d.values(): print i # these are dumped integrally categories = cache[couch.Category].values() for category in categories: print category.id users = cache[couch.User].values() for user in users: print user.id return
def load(res): res = list(res) log.debug('slices', 'loaded %d slices', len(res)) d = manydef.DeferredListSpaced() for slice in res: log.log('check', 'adding online check for slice %r', slice) d.addCallable(isSliceOnline, dadDB, slice) def count(_): total = 0 errors = 0 online = 0 for succeeded, result in d.resultList: total += 1 if not succeeded: errors += 1 else: if result: online += 1 print "%d of %d slices online" % (online, total) if errors: print "%d errors" % errors log.info('online', "%d cache hits of %d lookups", dadDB.db.hits, dadDB.db.lookups) d.addCallback(count) log.debug('online', "counting online slices") d.start() return d