def fragmentSetChroma(self, fragment, chroma): """ @param fragment: a fragment in the fragments key @type fragment: L{mapping.AnonymousStruct} @type chroma: L{track.ChromaModel} @returns: whether any field got changed """ # AnonymousStruct does not actually exist as a class assert fragment.__class__.__name__ == 'AnonymousStruct', \ "fragment %r is not a paisley.mapping.AnonymousStruct" % fragment assert isinstance(chroma, track.ChromaPrintModel) changed = False if not 'chroma' in fragment: fragment.chroma = _ChromaPrint() LOOKUP_FIELDS = ['mbid', 'artists', 'title'] KEYS = ['chromaprint', 'duration', 'lookedup'] for key in LOOKUP_FIELDS + KEYS: orig = getattr(fragment.chroma, key, None) value = getattr(chroma, key, None) log.log('mappings', 'key %r, original %r, value %r', key, orig, value) if value and orig != value: setattr(fragment.chroma, key, value) # if all that changed was the lookedup time, we don't # consider it a change if key != 'lookedup': changed = True log.debug('mappings', 'key %r changed to %r', key, value) return changed
def load(res): res = list(res) d = manydef.DeferredListSpaced() for audiofile in res: log.log('check', 'adding online check for audiofile %r', audiofile) d.addCallable(isFileOnline, dadDB, audiofile) def count(_): total = 0 errors = 0 online = 0 for succeeded, result in d.resultList: total += 1 if not succeeded: errors += 1 else: if result: online += 1 print "%d of %d audiofiles online" % (online, total) if errors: print "%d errors" % errors log.info('online', "%d cache hits of %d lookups", dadDB.db.hits, dadDB.db.lookups) d.addCallback(count) d.start() return d
def file_getPath(db, file): # return the full path to the file using multiple db queries log.log('lookup', 'looking up path for file %r %r', file, file.id) directory = couch.Directory.load(db, file.directory_id) path = directory_getPath(db, directory) return os.path.join(path, file.name)
def cb(path): log.log('check', 'statting path %r', path) # nfs can make these calls take long before = time.time() res = os.path.exists(path) if time.time() - before > 2.0: log.warning('check', 'slow filesystem for %r' % path) if res: return True print "%s offline" % path.encode('utf-8')
def isAnySliceOnline(dadDB, slices): log.log('check', 'is any of slices %r online ?' % slices) dl = [] for s in slices: dl.append(isSliceOnline(dadDB, s)) d = defer.DeferredList(dl) def consolidate(results): for succeeded, result in results: if result is True: return True return False d.addCallback(consolidate) return d
def findOrAddDirectory(db, path, volumes=None, add=True): if not volumes: volumes = getVolumes(db) volume, rest = findVolume(db, path, volumes) log.log('lookup', 'findOrAddDirectory: volume %r rest %r', volume, rest) # split the path and see if we can find it in the db parts = rest.split(os.path.sep) # FIXME: apparently os.path.split on a non-slashed dir still # returns a tuple with an empty string as first member if parts[0] is '': parts = parts[1:] # now find or add each part in turn in the db directory_id = None path = volume.path for part in parts: volume_id = volume and volume.id or None if os.path.sep in part: raise AssertionError, \ "parts should not contain a part with /: %r" % (parts, ) path = os.path.join(path, part) log.log('lookup', 'looking up directory on ' 'volume %r for path %s and part %s', volume, path, part) result = couch.Directory.view(db, 'dad/directory-lookup', key=[volume_id, directory_id, part], include_docs=True) result = list(result) if not result: if not add: return None res = os.stat(path) mtime = time.gmtime(res[stat.ST_MTIME]) log.log('lookup', 'about to add Directory %r %r %r', part, volume, directory_id) directory = couch.Directory(name=part, volume_id=volume_id, parent_id=directory_id, mtime=mtime, inode=res[stat.ST_INO]) directory.store(db) else: log.log('lookup', 'result %r %d', result, len(result)) assert len(result) == 1 directory = result[0] directory_id = directory.id volume = None # changes the first time through volume_id = None return directory
def cb(_): log.debug('consistency', 'got objects for %r and %r', sourceKlazz, destKlazz) errors = 0 items = 0 isList = False attr = getattr(sourceKlazz, attribute) if isinstance(attr, mapping.ListField): isList = True values = cache[sourceKlazz].values() log.debug('consistency', 'got %d items for %r', len(values), sourceKlazz) for item in values: log.log('consistency', 'item %r', item) values = getattr(item, attribute) if not isList: values = [getattr(item, attribute), ] log.log('consistency', 'values %r', values) for item_id in values: # allow None/unset id's if item_id is None: continue if item_id not in cache[destKlazz].keys(): print '%r %r has %r %r but does not exist' % ( sourceKlazz, item.id, destKlazz, item_id) import code;code.interact(local=locals()) errors += 1 if cache[destKlazz][item_id].type != sourceKlazz.type: print '%r %r points to %r %r but type is %r' % ( sourceKlazz, item.id, destKlazz, item_id, cache[destKlazz][item_id].type) else: items += 1 if errors == 0: print '%s.%s is consistent (%d items)' % ( sourceKlazz.__name__, attribute, items)
def isFileOnline(dadDB, file): log.log('check', 'is file %r online ?' % file) d = dadDB.getFilePath(file) def cb(path): log.log('check', 'statting path %r', path) # nfs can make these calls take long before = time.time() res = os.path.exists(path) if time.time() - before > 2.0: log.warning('check', 'slow filesystem for %r' % path) if res: return True print "%s offline" % path.encode('utf-8') d.addCallback(cb) return d
def trackLoadedCb(tracks): tracks = list(tracks) log.debug('tracks', 'loaded %d tracks', len(tracks)) d = manydef.DeferredListSpaced() for track in tracks: log.log('tracks', 'adding online check for track %r', track) if track.id not in track_ids: # no slices referencing this track, so offline log.log('tracks', 'Track %r has 0 slices', track) d.addCallable(lambda: False) else: if len(track_ids[track.id]) > 1: log.debug('tracks', "Track %r has %d slices" % ( track, len(track_ids[track.id]))) d.addCallable(isAnySliceOnline, dadDB, track_ids[track.id]) log.debug('tracks', 'checking offline tracks') def count(resultList): log.debug('tracks', 'counting offline tracks out of %r total', len(resultList)) total = len(resultList) online = 0 for i, (succeeded, result) in enumerate(resultList): if result: online += 1 else: print "track %r offline" % tracks[i] print "%d of %d tracks online" % (online, total) log.info('online', "%d cache hits of %d lookups", dadDB.db.hits, dadDB.db.lookups) d.addCallback(count) d.start() return d
def directory_getPath(db, directory): # return the full path to the directory using multiple db queries parts = [] log.log('lookup', 'looking up path for directory %r %r', directory, directory.id) while True: parts.append(directory.name) if directory.volume_id is not None: # gotten to the root, get volume path and stop volume = couch.Volume.load(db, directory.volume_id) parts.append(volume.path) break else: directory = couch.Directory.load(db, directory.parent_id) print parts parts.reverse() return "/".join(parts)
def findVolume(db, path, volumes=None): """ Given the path, find the Volume for that path. @returns: tuple of volume and rest of path @rtype: tuple of (L{models.Volume}, str), or (None, None) """ if not volumes: volumes = getVolumes(db) for vp in volumes.keys(): if path.startswith(vp): volume = volumes[vp] # split the path and see if we can find it in the db rest = path[len(vp) + 1:] log.log('lookup', 'findVolume: returning %r', (volume, rest)) return (volume, rest) print "Did not find volume for %s" % path raise KeyError, "Did not find volume for %s" % path
def isSliceOnline(dadDB, slice): log.log('check', 'is slice %r online ?' % slice) d = dadDB.db.map(dadDB.dbName, str(slice.audiofile_id), couch.AudioFile) d.addCallback(lambda f: isFileOnline(dadDB, f)) return d