def lookup(self, uri): if urlparse.urlsplit(uri).scheme not in self.backend.uri_schemes: return [] try: data = self._scanner.scan(uri) track = scan.audio_data_to_track(data) except exceptions.ScannerError as e: logger.warning('Problem looking up %s: %s', uri, e) track = Track(uri=uri, name=uri) return [track]
def lookup(self, uri): if urlparse.urlsplit(uri).scheme not in self.backend.uri_schemes: return [] if self._blacklist_re.match(uri): logger.debug('URI matched metadata lookup blacklist: %s', uri) return [Track(uri=uri)] try: data = self._scanner.scan(uri) track = scan.audio_data_to_track(data) except exceptions.ScannerError as e: logger.warning('Problem looking up %s: %s', uri, e) track = Track(uri=uri) return [track]
def run(self, args, config): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] flush_threshold = config['local']['scan_flush_threshold'] excluded_file_extensions = config['local']['excluded_file_extensions'] excluded_file_extensions = tuple( bytes(file_ext.lower()) for file_ext in excluded_file_extensions) library = _get_library(args, config) uris_in_library = set() uris_to_update = set() uris_to_remove = set() file_mtimes = path.find_mtimes(media_dir) logger.info('Found %d files in media_dir.', len(file_mtimes)) num_tracks = library.load() logger.info('Checking %d tracks from library.', num_tracks) for track in library.begin(): abspath = translator.local_track_uri_to_path(track.uri, media_dir) mtime = file_mtimes.pop(abspath, None) if mtime is None: logger.debug('Missing file %s', track.uri) uris_to_remove.add(track.uri) elif mtime > track.last_modified: uris_in_library.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_to_remove)) for uri in uris_to_remove: library.remove(uri) for abspath in file_mtimes: relpath = os.path.relpath(abspath, media_dir) uri = translator.path_to_local_track_uri(relpath) if relpath.lower().endswith(excluded_file_extensions): logger.debug('Skipped %s: File extension excluded.', uri) continue uris_to_update.add(uri) logger.info( 'Found %d tracks which need to be updated.', len(uris_to_update)) logger.info('Scanning...') uris_to_update = sorted(uris_to_update, key=lambda v: v.lower()) uris_to_update = uris_to_update[:args.limit] scanner = scan.Scanner(scan_timeout) progress = _Progress(flush_threshold, len(uris_to_update)) for uri in uris_to_update: try: relpath = translator.local_track_uri_to_path(uri, media_dir) file_uri = path.path_to_uri(os.path.join(media_dir, relpath)) data = scanner.scan(file_uri) track = scan.audio_data_to_track(data).copy(uri=uri) library.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) if progress.increment(): progress.log() if library.flush(): logger.debug('Progress flushed.') progress.log() library.close() logger.info('Done scanning.') return 0
def check(self, expected): actual = scan.audio_data_to_track(self.data) self.assertEqual(expected, actual)
def check(self): expected = self.build_track() actual = scan.audio_data_to_track(self.data) self.assertEqual(expected, actual)
def run(self, args, config, extensions): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] excluded_file_extensions = set( ext.lower() for ext in config['local']['excluded_file_extensions']) updaters = {} for e in extensions: for updater_class in e.get_library_updaters(): if updater_class and 'local' in updater_class.uri_schemes: updaters[e.ext_name] = updater_class if not updaters: logger.error('No usable library updaters found.') return 1 elif len(updaters) > 1: logger.error( 'More than one library updater found. ' 'Provided by: %s', ', '.join(updaters.keys())) return 1 local_updater = updaters.values()[0](config) # TODO: cleanup to consistently use local urls, not a random mix of # local and file uris depending on how the data was loaded. uris_library = set() uris_update = set() uris_remove = set() tracks = local_updater.load() logger.info('Checking %d tracks from library.', len(tracks)) for track in tracks: try: uri = translator.local_to_file_uri(track.uri, media_dir) stat = os.stat(path.uri_to_path(uri)) if int(stat.st_mtime) > track.last_modified: uris_update.add(uri) uris_library.add(uri) except OSError: logger.debug('Missing file %s', track.uri) uris_remove.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_remove)) for uri in uris_remove: local_updater.remove(uri) logger.info('Checking %s for unknown tracks.', media_dir) for uri in path.find_uris(media_dir): file_extension = os.path.splitext(path.uri_to_path(uri))[1] if file_extension.lower() in excluded_file_extensions: logger.debug('Skipped %s: File extension excluded.', uri) continue if uri not in uris_library: uris_update.add(uri) logger.info('Found %d unknown tracks.', len(uris_update)) logger.info('Scanning...') scanner = scan.Scanner(scan_timeout) progress = Progress(len(uris_update)) for uri in sorted(uris_update): try: data = scanner.scan(uri) track = scan.audio_data_to_track(data) local_updater.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) progress.increment() logger.info('Commiting changes.') local_updater.commit() return 0
def run(self, args, config): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] flush_threshold = config['local']['scan_flush_threshold'] excluded_file_extensions = config['local']['excluded_file_extensions'] excluded_file_extensions = tuple( bytes(file_ext.lower()) for file_ext in excluded_file_extensions) library = _get_library(args, config) uris_to_update = set() uris_to_remove = set() file_mtimes = path.find_mtimes(media_dir) logger.info('Found %d files in media_dir.', len(file_mtimes)) num_tracks = library.load() logger.info('Checking %d tracks from library.', num_tracks) for track in library.begin(): abspath = translator.local_track_uri_to_path(track.uri, media_dir) mtime = file_mtimes.pop(abspath, None) if mtime is None: logger.debug('Missing file %s', track.uri) uris_to_remove.add(track.uri) elif mtime > track.last_modified: uris_to_update.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_to_remove)) for uri in uris_to_remove: library.remove(uri) for abspath in file_mtimes: relpath = os.path.relpath(abspath, media_dir) uri = translator.path_to_local_track_uri(relpath) if relpath.lower().endswith(excluded_file_extensions): logger.debug('Skipped %s: File extension excluded.', uri) continue uris_to_update.add(uri) logger.info('Found %d tracks which need to be updated.', len(uris_to_update)) logger.info('Scanning...') uris_to_update = sorted(uris_to_update, key=lambda v: v.lower()) uris_to_update = uris_to_update[:args.limit] scanner = scan.Scanner(scan_timeout) progress = _Progress(flush_threshold, len(uris_to_update)) for uri in uris_to_update: try: relpath = translator.local_track_uri_to_path(uri, media_dir) file_uri = path.path_to_uri(os.path.join(media_dir, relpath)) data = scanner.scan(file_uri) track = scan.audio_data_to_track(data).copy(uri=uri) library.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) if progress.increment(): progress.log() if library.flush(): logger.debug('Progress flushed.') progress.log() library.close() logger.info('Done scanning.') return 0
def run(self, args, config): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] flush_threshold = config['local']['scan_flush_threshold'] excluded_file_extensions = config['local']['excluded_file_extensions'] excluded_file_extensions = set( file_ext.lower() for file_ext in excluded_file_extensions) library = _get_library(args, config) uri_path_mapping = {} uris_in_library = set() uris_to_update = set() uris_to_remove = set() num_tracks = library.load() logger.info('Checking %d tracks from library.', num_tracks) for track in library.begin(): uri_path_mapping[track.uri] = translator.local_track_uri_to_path( track.uri, media_dir) try: stat = os.stat(uri_path_mapping[track.uri]) if int(stat.st_mtime) > track.last_modified: uris_to_update.add(track.uri) uris_in_library.add(track.uri) except OSError: logger.debug('Missing file %s', track.uri) uris_to_remove.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_to_remove)) for uri in uris_to_remove: library.remove(uri) logger.info('Checking %s for unknown tracks.', media_dir) for relpath in path.find_files(media_dir): uri = translator.path_to_local_track_uri(relpath) file_extension = os.path.splitext(relpath)[1] if file_extension.lower() in excluded_file_extensions: logger.debug('Skipped %s: File extension excluded.', uri) continue if uri not in uris_in_library: uris_to_update.add(uri) uri_path_mapping[uri] = os.path.join(media_dir, relpath) logger.info('Found %d unknown tracks.', len(uris_to_update)) logger.info('Scanning...') uris_to_update = sorted(uris_to_update)[:args.limit] scanner = scan.Scanner(scan_timeout) progress = _Progress(flush_threshold, len(uris_to_update)) for uri in uris_to_update: try: data = scanner.scan(path.path_to_uri(uri_path_mapping[uri])) track = scan.audio_data_to_track(data).copy(uri=uri) library.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) if progress.increment(): progress.log() if library.flush(): logger.debug('Progress flushed.') progress.log() library.close() logger.info('Done scanning.') return 0
def run(self, args, config, extensions): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] excluded_file_extensions = set( ext.lower() for ext in config['local']['excluded_file_extensions']) updaters = {} for e in extensions: for updater_class in e.get_library_updaters(): if updater_class and 'local' in updater_class.uri_schemes: updaters[e.ext_name] = updater_class if not updaters: logger.error('No usable library updaters found.') return 1 elif len(updaters) > 1: logger.error('More than one library updater found. ' 'Provided by: %s', ', '.join(updaters.keys())) return 1 local_updater = updaters.values()[0](config) # TODO: cleanup to consistently use local urls, not a random mix of # local and file uris depending on how the data was loaded. uris_library = set() uris_update = set() uris_remove = set() tracks = local_updater.load() logger.info('Checking %d tracks from library.', len(tracks)) for track in tracks: try: uri = translator.local_to_file_uri(track.uri, media_dir) stat = os.stat(path.uri_to_path(uri)) if int(stat.st_mtime) > track.last_modified: uris_update.add(uri) uris_library.add(uri) except OSError: logger.debug('Missing file %s', track.uri) uris_remove.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_remove)) for uri in uris_remove: local_updater.remove(uri) logger.info('Checking %s for unknown tracks.', media_dir) for uri in path.find_uris(media_dir): file_extension = os.path.splitext(path.uri_to_path(uri))[1] if file_extension.lower() in excluded_file_extensions: logger.debug('Skipped %s: File extension excluded.', uri) continue if uri not in uris_library: uris_update.add(uri) logger.info('Found %d unknown tracks.', len(uris_update)) logger.info('Scanning...') scanner = scan.Scanner(scan_timeout) progress = Progress(len(uris_update)) for uri in sorted(uris_update): try: data = scanner.scan(uri) track = scan.audio_data_to_track(data) local_updater.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) progress.increment() logger.info('Commiting changes.') local_updater.commit() return 0