def addVolume(dump, volumeId):
if volumeId in dump[couch.Volume].values():
return
volume = cache[couch.Volume][volumeId]
log.debug('gentest', 'adding volume %r' % volume)
dump[couch.Volume][volume] = volume.id
def fragmentSetChroma(self, fragment, chroma):
"""
@param fragment: a fragment in the fragments key
@type fragment: L{mapping.AnonymousStruct}
@type chroma: L{track.ChromaModel}
@returns: whether any field got changed
"""
# AnonymousStruct does not actually exist as a class
assert fragment.__class__.__name__ == 'AnonymousStruct', \
"fragment %r is not a paisley.mapping.AnonymousStruct" % fragment
assert isinstance(chroma, track.ChromaPrintModel)
changed = False
if not 'chroma' in fragment:
fragment.chroma = _ChromaPrint()
LOOKUP_FIELDS = ['mbid', 'artists', 'title']
KEYS = ['chromaprint', 'duration', 'lookedup']
for key in LOOKUP_FIELDS + KEYS:
orig = getattr(fragment.chroma, key, None)
value = getattr(chroma, key, None)
log.log('mappings', 'key %r, original %r, value %r',
key, orig, value)
if value and orig != value:
setattr(fragment.chroma, key, value)
# if all that changed was the lookedup time, we don't
# consider it a change
if key != 'lookedup':
changed = True
log.debug('mappings', 'key %r changed to %r', key, value)
return changed
def resolveSlices(result):
resultList = list(result)
log.debug("playlist", "got %r tracks sliced", len(resultList))
d = manydef.DeferredListSpaced()
for succeeded, result in resultList:
if not succeeded:
self.warningFailure(result)
if isinstance(result, mappings.Track):
# result from resolveIds
continue
slicesGen = result
# FIXME: only keep first audiofile for now
slice = list(slicesGen)[0]
trackDict[slice.track_id][1] = slice
def callable(slice, trackId):
d = self.getSliceFile(slice)
def cb(audiofile, trackId):
# print 'resolveSlices cb', audiofile, trackId
trackDict[trackId][2] = audiofile
return audiofile, trackId
d.addCallback(cb, trackId)
return d
d.addCallable(callable, slice, slice.track_id)
d.start()
return d
def processEnded(self, status):
log.debug('slavepp', 'processEnded')
rc = status.value.exitCode
if rc == 0:
self.deferred.callback(self)
else:
self.deferred.errback(rc)
def addScore(dump, scoreId):
if scoreId in dump[couch.Score].values():
return
score = cache[couch.Score][scoreId]
log.debug('gentest', 'adding score %r' % score)
dump[couch.Score][score] = score.id
def addTrack(dump, trackId):
if trackId in dump[couch.Track].values():
return
track = cache[couch.Track][trackId]
log.debug('gentest', 'adding track %r' % track)
dump[couch.Track][track] = track.id
# trackalbums point to tracks
d = refs[(couch.TrackAlbum, 'track_id')]
if trackId in d:
for trackalbumId in d[trackId]:
addTrackAlbum(dump, trackalbumId)
# slices point to tracks
d = refs[(couch.Slice, 'track_id')]
if trackId in d:
for sliceId in d[trackId]:
addSlice(dump, sliceId)
# tracks have artists
for artist in track.artist_ids:
addArtist(dump, artist, back=False)
# scores point to tracks
d = refs[(couch.Score, 'subject_id')]
if trackId in d:
for scoreId in d[trackId]:
addScore(dump, scoreId)
def load(db, dbName, klazz, viewName):
"""
@type db: L{dadcouch.extern.paisley.client.CouchDB}
@type dbName: str
@param klazz: the class to instantiate objects from
@param viewName: name of the view to load objects from
"""
log.debug('load', 'loading %s->%r using view %r',
dbName, klazz, viewName)
v = views.View(db, dbName, 'dad', viewName,
klazz, include_docs=True)
d = v.queryView()
def eb(failure):
log.warning('load', 'Failed to query view: %r',
log.getFailureMessage(failure))
return failure
d.addErrback(eb)
def cb(result):
log.debug('load', 'loaded %s->%r using view %r',
dbName, klazz, viewName)
return result
d.addCallback(cb)
return d
def connectionMade(self):
log.debug('slavepp', 'connectionMade')
protocol = self._server.buildProtocol(None)
protocol.transport = MyTransport(self.transport, self._childWriteFd)
# FIXME: need to get a transport in here, like Broker
log.debug('slavepp', 'protocol %r, transport %r',
protocol, protocol.transport)
self._protocol = protocol
protocol.connectionMade()
def spawnSlave(server):
log.debug('master', 'spawning slave')
# we'll use fd3 for reading from child, and fd4 for writing to it
mypp = SlaveProcessProtocol(server, 4)
childFDs = {0: 0, 1: 1, 2: 2, 3: 'r', 4: 'w'}
transport = reactor.spawnProcess(mypp, sys.argv[0],
args=[sys.argv[0], '--slave'],
env=os.environ, childFDs=childFDs)
mypp.setTransport(transport)
def cacheLoad(cache, dadDB, klazz, view):
log.debug('cacheLoad', 'Loading klazz %r using view %r', klazz, view)
cache[klazz] = {}
d = dadDB.viewDocs(view, klazz, include_docs=True)
def cb(result):
for item in result:
cache[klazz][item.id] = item
d.addCallback(cb)
return d
def connectStdio(factory):
TIMEOUT = 3
c = ProcessClientConnector(factory, TIMEOUT, reactor)
log.debug('slave', 'before connecting: %r', c.state)
c.connect()
log.debug('slave', 'after connecting: state %r, timeoutid %r, transport %r',
c.state, c.timeoutID, c.transport)
# FIXME: we shouldn't do this manually
c.cancelTimeout()
return c
def addAudioFile(dump, audiofileId):
if audiofileId in dump[couch.AudioFile].values():
return
audiofile = cache[couch.AudioFile][audiofileId]
log.debug('gentest', 'adding audiofile %r' % audiofile)
dump[couch.AudioFile][audiofile] = audiofile.id
# audiofiles point to directories
addDirectory(dump, audiofile.directory_id)
def addTrackAlbum(dump, trackalbumId):
if trackalbumId in dump[couch.TrackAlbum].values():
return
trackalbum = cache[couch.TrackAlbum][trackalbumId]
log.debug('gentest', 'adding trackalbum %r' % trackalbum)
dump[couch.TrackAlbum][trackalbum] = trackalbum.id
addAlbum(dump, trackalbum.album_id)
addTrack(dump, trackalbum.track_id)
def addOptions(self):
# FIXME: is this the right place ?
log.init()
# from dad.configure import configure
log.debug("dad", "This is dad version %s (%s)", "0.0.0", "0")
# configure.version, configure.revision)
self.parser.add_option('-v', '--version',
action="store_true", dest="version",
help="show version information")
def showPaths(result):
resultList = list(result)
log.debug('playlist', 'got %r paths resolved', len(resultList))
for succeeded, result in resultList:
if not succeeded:
print 'ERROR', result
else:
(track, slice, path, score, userId) = result
print path.encode('utf-8')
def addSlice(dump, sliceId):
if sliceId in dump[couch.Slice].values():
return
slice = cache[couch.Slice][sliceId]
log.debug('gentest', 'adding slice %r' % slice)
dump[couch.Slice][slice] = slice.id
# slices point to tracks
addTrack(dump, slice.track_id)
# slices point to audiofiles
addAudioFile(dump, slice.audiofile_id)
def mainSlave():
# this is the slave
log.debug('slave', 'slave is started')
# log in to manager with a pb client over the processprotocol somehow
clientFactory = pb.PBClientFactory()
c = connectStdio(clientFactory)
log.debug('slave', 'clientFactory._broker %r', clientFactory._broker)
d = clientFactory.getRootObject()
def cb(root):
log.debug('slave', 'slave got root: %r' % root)
return root.callRemote("echo", "hello network")
d.addCallback(cb)
def addDirectory(dump, directoryId):
if directoryId in dump[couch.Directory].values():
return
directory = cache[couch.Directory][directoryId]
log.debug('gentest', 'adding directory %r' % directory)
dump[couch.Directory][directory] = directory.id
# directories point to directories
if directory.parent_id:
addDirectory(dump, directory.parent_id)
# directories point to volumes
if directory.volume_id:
addVolume(dump, directory.volume_id)
def count(resultList):
log.debug('tracks', 'counting offline tracks out of %r total',
len(resultList))
total = len(resultList)
online = 0
for i, (succeeded, result) in enumerate(resultList):
if result:
online += 1
else:
print "track %r offline" % tracks[i]
print "%d of %d tracks online" % (online, total)
log.info('online', "%d cache hits of %d lookups",
dadDB.db.hits, dadDB.db.lookups)
def doLater(self, args):
# FIXME: view-specific
import gtk
defer.Deferred.debug = 1
self._doneDeferred = defer.Deferred()
# FIXME: allow customizing model and/or view(s)
viewTypes = ['GTK', ]
db = self.parentCommand.getDatabase()
self.debug('Database: %r', db)
# get the model
amodel = self.parentCommand.getAppModel()
self.debug('App model: %r', amodel)
acontroller = app.AppController(amodel)
for viewType in viewTypes:
viewModule = 'dad%s.views.app.%sAppView' % (viewType.lower(), viewType)
aview = reflect.namedAny(viewModule)()
aview.set_title(self.getFullName())
acontroller.addView(aview)
# FIXME: gtk-specific
aview.widget.connect('destroy', self.done_cb)
vbox = gtk.VBox()
hbox = gtk.HBox()
vbox.add(hbox)
aview.widget.add(vbox)
asController, asModel, asViews = acontroller.getTriad('ArtistSelector')
hbox.pack_start(asViews[0])
aview.widget.show_all()
# start loading artists and albums
d = defer.Deferred()
d.addCallback(lambda _: log.debug('test', 'asking controller to populate'))
d.addCallback(lambda _: asController.populate())
d.callback(None)
return self._doneDeferred
def addAlbum(dump, albumId):
if albumId in dump[couch.Album].values():
return
album = cache[couch.Album][albumId]
log.debug('gentest', 'adding album %r' % album)
dump[couch.Album][album] = album.id
# trackalbums point to albums
d = refs[(couch.TrackAlbum, 'album_id')]
if albumId in d:
for trackalbumId in d[albumId]:
addTrackAlbum(dump, trackalbumId)
# albums have artists
for artist in album.artist_ids:
addArtist(dump, artist, back=False)
def resolveTracks(tracks):
# returns: a deferred firing a list of
# (result, Track/sliceGen alternating
# tracks: list of Track, score, userId
trackList = list(tracks)
log.debug("playlist", "got %r tracks", len(trackList))
d = manydef.DeferredListSpaced()
for track, score, userId in trackList:
trackDict[track.id] = [track, None, None, score, userId]
log.debug("playlist", "track %r has score %f by user %r", track, score, userId)
d.addCallable(self.resolveIds, track, "artist_ids", "artists", mappings.Artist)
d.addCallable(self.getSlices, track)
d.start()
return d
def cb(_):
log.debug('consistency',
'got objects for %r and %r', sourceKlazz, destKlazz)
errors = 0
items = 0
isList = False
attr = getattr(sourceKlazz, attribute)
if isinstance(attr, mapping.ListField):
isList = True
values = cache[sourceKlazz].values()
log.debug('consistency', 'got %d items for %r',
len(values), sourceKlazz)
for item in values:
log.log('consistency', 'item %r', item)
values = getattr(item, attribute)
if not isList:
values = [getattr(item, attribute), ]
log.log('consistency', 'values %r', values)
for item_id in values:
# allow None/unset id's
if item_id is None:
continue
if item_id not in cache[destKlazz].keys():
print '%r %r has %r %r but does not exist' % (
sourceKlazz, item.id, destKlazz, item_id)
import code;code.interact(local=locals())
errors += 1
if cache[destKlazz][item_id].type != sourceKlazz.type:
print '%r %r points to %r %r but type is %r' % (
sourceKlazz, item.id, destKlazz, item_id,
cache[destKlazz][item_id].type)
else:
items += 1
if errors == 0:
print '%s.%s is consistent (%d items)' % (
sourceKlazz.__name__, attribute, items)
def main():
log.init()
server = 'localhost'
if len(sys.argv) > 1:
server = sys.argv[1]
dbName = 'dad'
if len(sys.argv) > 2:
dbName = sys.argv[2]
# this rebinds and makes it break in views
# db = couchdb.CouchDB('localhost', dbName='dad')
log.debug('main', 'binding to couchdb %r', server)
db = couchdb.CouchDB(server)
log.debug('main', 'checking database %r', dbName)
d = check(db, dbName)
d.addCallback(lambda _: reactor.stop())
return d
def doLater(self, args):
db = self.parentCommand.getDatabase()
sel = selecter.getSelecter(self.options.selecter, self.stdout,
database=db)
while True:
selected = yield sel.select()
if not selected:
break
if not os.path.exists(selected.path):
continue
text = "# %s - %s\n%s\n" % (
" & ".join(selected.artists).encode('utf-8'),
selected.title.encode('utf-8'),
selected.path.encode('utf-8'))
log.debug('main', 'output: %r', text)
self.stdout.write(text)
self.stdout.flush()
def main():
log.init()
server = 'localhost'
if len(sys.argv) > 1:
server = sys.argv[1]
dbName = 'dad'
if len(sys.argv) > 2:
dbName = sys.argv[2]
# this rebinds and makes it break in views
# db = couchdb.CouchDB('localhost', dbName='dad')
db = cachedb.CachingCouchDB(server)
dadDB = daddb.DADDB(db, dbName)
d = defer.Deferred()
# get Volume and Directory into the cache so that we don't do a lot
# of parallel lookups for them
d.addCallback(lambda _: dadDB.viewDocs(
'directory-lookup', couch.Directory, include_docs=True))
d.addCallback(lambda _: log.debug('online', '%d directories', len(list(_))))
d.addCallback(lambda _: dadDB.viewDocs(
'volumes', couch.Volume, include_docs=True))
d.addCallback(lambda _: log.debug('online', '%d volumes', len(list(_))))
# now check stuff
# audiofiles first, so we seed the cache before checking tracks
d.addCallback(lambda _: audiofiles(dadDB))
d.addCallback(lambda _: tracks(dadDB))
d.addCallback(lambda _: slices(dadDB))
d.addCallback(lambda _: reactor.stop())
def eb(f):
print 'ERROR: ', log.getFailureMessage(f)
reactor.stop()
d.addErrback(eb)
d.callback(None)
return d
def addArtist(dump, artistId, back=False):
if artistId in dump[couch.Artist].values():
return
log.debug('gentest', 'adding artist %s', artistId)
artist = cache[couch.Artist][artistId]
dump[couch.Artist][artist] = artist.id
if not back:
return
# albums point to artists
d = refs[(couch.Album, 'artist_ids')]
if artistId in d:
for albumId in d[artistId]:
addAlbum(dump, albumId)
d = refs[(couch.Track, 'artist_ids')]
if artistId in d:
for trackId in d[artistId]:
addTrack(dump, trackId)
def _makeTransport(self):
log.debug('slave', 'THOMAS: makeTransport: %r', self.state)
# protocol = self.factory.buildProtocol(None) # FIXME: addr
# FIXME: makeTransport is called before setting timeoutID;
# buildProtocol thus resets it to soon for us
protocol = self.buildProtocol(None) # FIXME: addr
log.debug('slave', 'timeoutID %r', self.timeoutID)
log.debug('slave', 'THOMAS: protocol %r', protocol)
t = MyStandardIO(protocol, stdin=4, stdout=3)
# FIXME: writing works to get it to the server, but how does the
# chitchat really start ?
# t.write('kakapipi')
# t.startReading()
log.debug('slave', 'host: %r, peer: %r', t.getHost(), t.getPeer())
return t
def cb(_):
# here is where we output all id's
artists = cache[couch.Artist].values()[:count]
for artist in artists:
log.debug('gentest', 'adding root artist %r', artist)
addArtist(dump, artist.id, back=True)
# now print all ids
for klazz, d in dump.items():
for i in d.values():
print i
# these are dumped integrally
categories = cache[couch.Category].values()
for category in categories:
print category.id
users = cache[couch.User].values()
for user in users:
print user.id
return
def load(res):
res = list(res)
log.debug('slices', 'loaded %d slices', len(res))
d = manydef.DeferredListSpaced()
for slice in res:
log.log('check', 'adding online check for slice %r', slice)
d.addCallable(isSliceOnline, dadDB, slice)
def count(_):
total = 0
errors = 0
online = 0
for succeeded, result in d.resultList:
total += 1
if not succeeded:
errors += 1
else:
if result:
online += 1
print "%d of %d slices online" % (online, total)
if errors:
print "%d errors" % errors
log.info('online', "%d cache hits of %d lookups",
dadDB.db.hits, dadDB.db.lookups)
d.addCallback(count)
log.debug('online', "counting online slices")
d.start()
return d