def fragmentSetChroma(self, fragment, chroma):
"""
@param fragment: a fragment in the fragments key
@type fragment: L{mapping.AnonymousStruct}
@type chroma: L{track.ChromaModel}
@returns: whether any field got changed
"""
# AnonymousStruct does not actually exist as a class
assert fragment.__class__.__name__ == 'AnonymousStruct', \
"fragment %r is not a paisley.mapping.AnonymousStruct" % fragment
assert isinstance(chroma, track.ChromaPrintModel)
changed = False
if not 'chroma' in fragment:
fragment.chroma = _ChromaPrint()
LOOKUP_FIELDS = ['mbid', 'artists', 'title']
KEYS = ['chromaprint', 'duration', 'lookedup']
for key in LOOKUP_FIELDS + KEYS:
orig = getattr(fragment.chroma, key, None)
value = getattr(chroma, key, None)
log.log('mappings', 'key %r, original %r, value %r',
key, orig, value)
if value and orig != value:
setattr(fragment.chroma, key, value)
# if all that changed was the lookedup time, we don't
# consider it a change
if key != 'lookedup':
changed = True
log.debug('mappings', 'key %r changed to %r', key, value)
return changed
def load(res):
res = list(res)
d = manydef.DeferredListSpaced()
for audiofile in res:
log.log('check', 'adding online check for audiofile %r', audiofile)
d.addCallable(isFileOnline, dadDB, audiofile)
def count(_):
total = 0
errors = 0
online = 0
for succeeded, result in d.resultList:
total += 1
if not succeeded:
errors += 1
else:
if result:
online += 1
print "%d of %d audiofiles online" % (online, total)
if errors:
print "%d errors" % errors
log.info('online', "%d cache hits of %d lookups",
dadDB.db.hits, dadDB.db.lookups)
d.addCallback(count)
d.start()
return d
def file_getPath(db, file):
# return the full path to the file using multiple db queries
log.log('lookup', 'looking up path for file %r %r', file, file.id)
directory = couch.Directory.load(db, file.directory_id)
path = directory_getPath(db, directory)
return os.path.join(path, file.name)
def cb(path):
log.log('check', 'statting path %r', path)
# nfs can make these calls take long
before = time.time()
res = os.path.exists(path)
if time.time() - before > 2.0:
log.warning('check', 'slow filesystem for %r' % path)
if res:
return True
print "%s offline" % path.encode('utf-8')
def isAnySliceOnline(dadDB, slices):
log.log('check', 'is any of slices %r online ?' % slices)
dl = []
for s in slices:
dl.append(isSliceOnline(dadDB, s))
d = defer.DeferredList(dl)
def consolidate(results):
for succeeded, result in results:
if result is True:
return True
return False
d.addCallback(consolidate)
return d
def findOrAddDirectory(db, path, volumes=None, add=True):
if not volumes:
volumes = getVolumes(db)
volume, rest = findVolume(db, path, volumes)
log.log('lookup', 'findOrAddDirectory: volume %r rest %r', volume, rest)
# split the path and see if we can find it in the db
parts = rest.split(os.path.sep)
# FIXME: apparently os.path.split on a non-slashed dir still
# returns a tuple with an empty string as first member
if parts[0] is '':
parts = parts[1:]
# now find or add each part in turn in the db
directory_id = None
path = volume.path
for part in parts:
volume_id = volume and volume.id or None
if os.path.sep in part:
raise AssertionError, \
"parts should not contain a part with /: %r" % (parts, )
path = os.path.join(path, part)
log.log('lookup', 'looking up directory on '
'volume %r for path %s and part %s', volume, path, part)
result = couch.Directory.view(db, 'dad/directory-lookup',
key=[volume_id, directory_id, part],
include_docs=True)
result = list(result)
if not result:
if not add:
return None
res = os.stat(path)
mtime = time.gmtime(res[stat.ST_MTIME])
log.log('lookup', 'about to add Directory %r %r %r',
part, volume, directory_id)
directory = couch.Directory(name=part, volume_id=volume_id,
parent_id=directory_id,
mtime=mtime, inode=res[stat.ST_INO])
directory.store(db)
else:
log.log('lookup', 'result %r %d', result, len(result))
assert len(result) == 1
directory = result[0]
directory_id = directory.id
volume = None # changes the first time through
volume_id = None
return directory
def cb(_):
log.debug('consistency',
'got objects for %r and %r', sourceKlazz, destKlazz)
errors = 0
items = 0
isList = False
attr = getattr(sourceKlazz, attribute)
if isinstance(attr, mapping.ListField):
isList = True
values = cache[sourceKlazz].values()
log.debug('consistency', 'got %d items for %r',
len(values), sourceKlazz)
for item in values:
log.log('consistency', 'item %r', item)
values = getattr(item, attribute)
if not isList:
values = [getattr(item, attribute), ]
log.log('consistency', 'values %r', values)
for item_id in values:
# allow None/unset id's
if item_id is None:
continue
if item_id not in cache[destKlazz].keys():
print '%r %r has %r %r but does not exist' % (
sourceKlazz, item.id, destKlazz, item_id)
import code;code.interact(local=locals())
errors += 1
if cache[destKlazz][item_id].type != sourceKlazz.type:
print '%r %r points to %r %r but type is %r' % (
sourceKlazz, item.id, destKlazz, item_id,
cache[destKlazz][item_id].type)
else:
items += 1
if errors == 0:
print '%s.%s is consistent (%d items)' % (
sourceKlazz.__name__, attribute, items)
def isFileOnline(dadDB, file):
log.log('check', 'is file %r online ?' % file)
d = dadDB.getFilePath(file)
def cb(path):
log.log('check', 'statting path %r', path)
# nfs can make these calls take long
before = time.time()
res = os.path.exists(path)
if time.time() - before > 2.0:
log.warning('check', 'slow filesystem for %r' % path)
if res:
return True
print "%s offline" % path.encode('utf-8')
d.addCallback(cb)
return d
def trackLoadedCb(tracks):
tracks = list(tracks)
log.debug('tracks', 'loaded %d tracks', len(tracks))
d = manydef.DeferredListSpaced()
for track in tracks:
log.log('tracks', 'adding online check for track %r', track)
if track.id not in track_ids:
# no slices referencing this track, so offline
log.log('tracks', 'Track %r has 0 slices', track)
d.addCallable(lambda: False)
else:
if len(track_ids[track.id]) > 1:
log.debug('tracks',
"Track %r has %d slices" % (
track, len(track_ids[track.id])))
d.addCallable(isAnySliceOnline, dadDB, track_ids[track.id])
log.debug('tracks', 'checking offline tracks')
def count(resultList):
log.debug('tracks', 'counting offline tracks out of %r total',
len(resultList))
total = len(resultList)
online = 0
for i, (succeeded, result) in enumerate(resultList):
if result:
online += 1
else:
print "track %r offline" % tracks[i]
print "%d of %d tracks online" % (online, total)
log.info('online', "%d cache hits of %d lookups",
dadDB.db.hits, dadDB.db.lookups)
d.addCallback(count)
d.start()
return d
def directory_getPath(db, directory):
# return the full path to the directory using multiple db queries
parts = []
log.log('lookup', 'looking up path for directory %r %r',
directory, directory.id)
while True:
parts.append(directory.name)
if directory.volume_id is not None:
# gotten to the root, get volume path and stop
volume = couch.Volume.load(db, directory.volume_id)
parts.append(volume.path)
break
else:
directory = couch.Directory.load(db, directory.parent_id)
print parts
parts.reverse()
return "/".join(parts)
def findVolume(db, path, volumes=None):
"""
Given the path, find the Volume for that path.
@returns: tuple of volume and rest of path
@rtype: tuple of (L{models.Volume}, str), or (None, None)
"""
if not volumes:
volumes = getVolumes(db)
for vp in volumes.keys():
if path.startswith(vp):
volume = volumes[vp]
# split the path and see if we can find it in the db
rest = path[len(vp) + 1:]
log.log('lookup', 'findVolume: returning %r', (volume, rest))
return (volume, rest)
print "Did not find volume for %s" % path
raise KeyError, "Did not find volume for %s" % path
def isSliceOnline(dadDB, slice):
log.log('check', 'is slice %r online ?' % slice)
d = dadDB.db.map(dadDB.dbName, str(slice.audiofile_id), couch.AudioFile)
d.addCallback(lambda f: isFileOnline(dadDB, f))
return d
