def lookup(self, uri):
if urlparse.urlsplit(uri).scheme not in self.backend.uri_schemes:
return []
try:
data = self._scanner.scan(uri)
track = scan.audio_data_to_track(data)
except exceptions.ScannerError as e:
logger.warning('Problem looking up %s: %s', uri, e)
track = Track(uri=uri, name=uri)
return [track]
def lookup(self, uri):
if urlparse.urlsplit(uri).scheme not in self.backend.uri_schemes:
return []
if self._blacklist_re.match(uri):
logger.debug('URI matched metadata lookup blacklist: %s', uri)
return [Track(uri=uri)]
try:
data = self._scanner.scan(uri)
track = scan.audio_data_to_track(data)
except exceptions.ScannerError as e:
logger.warning('Problem looking up %s: %s', uri, e)
track = Track(uri=uri)
return [track]
def lookup(self, uri):
if urlparse.urlsplit(uri).scheme not in self.backend.uri_schemes:
return []
if self._blacklist_re.match(uri):
logger.debug('URI matched metadata lookup blacklist: %s', uri)
return [Track(uri=uri)]
try:
data = self._scanner.scan(uri)
track = scan.audio_data_to_track(data)
except exceptions.ScannerError as e:
logger.warning('Problem looking up %s: %s', uri, e)
track = Track(uri=uri)
return [track]
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def check(self, expected):
actual = scan.audio_data_to_track(self.data)
self.assertEqual(expected, actual)
def check(self):
expected = self.build_track()
actual = scan.audio_data_to_track(self.data)
self.assertEqual(expected, actual)
def run(self, args, config, extensions):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
excluded_file_extensions = set(
ext.lower() for ext in config['local']['excluded_file_extensions'])
updaters = {}
for e in extensions:
for updater_class in e.get_library_updaters():
if updater_class and 'local' in updater_class.uri_schemes:
updaters[e.ext_name] = updater_class
if not updaters:
logger.error('No usable library updaters found.')
return 1
elif len(updaters) > 1:
logger.error(
'More than one library updater found. '
'Provided by: %s', ', '.join(updaters.keys()))
return 1
local_updater = updaters.values()[0](config)
# TODO: cleanup to consistently use local urls, not a random mix of
# local and file uris depending on how the data was loaded.
uris_library = set()
uris_update = set()
uris_remove = set()
tracks = local_updater.load()
logger.info('Checking %d tracks from library.', len(tracks))
for track in tracks:
try:
uri = translator.local_to_file_uri(track.uri, media_dir)
stat = os.stat(path.uri_to_path(uri))
if int(stat.st_mtime) > track.last_modified:
uris_update.add(uri)
uris_library.add(uri)
except OSError:
logger.debug('Missing file %s', track.uri)
uris_remove.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_remove))
for uri in uris_remove:
local_updater.remove(uri)
logger.info('Checking %s for unknown tracks.', media_dir)
for uri in path.find_uris(media_dir):
file_extension = os.path.splitext(path.uri_to_path(uri))[1]
if file_extension.lower() in excluded_file_extensions:
logger.debug('Skipped %s: File extension excluded.', uri)
continue
if uri not in uris_library:
uris_update.add(uri)
logger.info('Found %d unknown tracks.', len(uris_update))
logger.info('Scanning...')
scanner = scan.Scanner(scan_timeout)
progress = Progress(len(uris_update))
for uri in sorted(uris_update):
try:
data = scanner.scan(uri)
track = scan.audio_data_to_track(data)
local_updater.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
progress.increment()
logger.info('Commiting changes.')
local_updater.commit()
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_to_update.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info('Found %d tracks which need to be updated.',
len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = set(
file_ext.lower() for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uri_path_mapping = {}
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
uri_path_mapping[track.uri] = translator.local_track_uri_to_path(
track.uri, media_dir)
try:
stat = os.stat(uri_path_mapping[track.uri])
if int(stat.st_mtime) > track.last_modified:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
except OSError:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
logger.info('Checking %s for unknown tracks.', media_dir)
for relpath in path.find_files(media_dir):
uri = translator.path_to_local_track_uri(relpath)
file_extension = os.path.splitext(relpath)[1]
if file_extension.lower() in excluded_file_extensions:
logger.debug('Skipped %s: File extension excluded.', uri)
continue
if uri not in uris_in_library:
uris_to_update.add(uri)
uri_path_mapping[uri] = os.path.join(media_dir, relpath)
logger.info('Found %d unknown tracks.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update)[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
data = scanner.scan(path.path_to_uri(uri_path_mapping[uri]))
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config, extensions):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
excluded_file_extensions = set(
ext.lower() for ext in config['local']['excluded_file_extensions'])
updaters = {}
for e in extensions:
for updater_class in e.get_library_updaters():
if updater_class and 'local' in updater_class.uri_schemes:
updaters[e.ext_name] = updater_class
if not updaters:
logger.error('No usable library updaters found.')
return 1
elif len(updaters) > 1:
logger.error('More than one library updater found. '
'Provided by: %s', ', '.join(updaters.keys()))
return 1
local_updater = updaters.values()[0](config)
# TODO: cleanup to consistently use local urls, not a random mix of
# local and file uris depending on how the data was loaded.
uris_library = set()
uris_update = set()
uris_remove = set()
tracks = local_updater.load()
logger.info('Checking %d tracks from library.', len(tracks))
for track in tracks:
try:
uri = translator.local_to_file_uri(track.uri, media_dir)
stat = os.stat(path.uri_to_path(uri))
if int(stat.st_mtime) > track.last_modified:
uris_update.add(uri)
uris_library.add(uri)
except OSError:
logger.debug('Missing file %s', track.uri)
uris_remove.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_remove))
for uri in uris_remove:
local_updater.remove(uri)
logger.info('Checking %s for unknown tracks.', media_dir)
for uri in path.find_uris(media_dir):
file_extension = os.path.splitext(path.uri_to_path(uri))[1]
if file_extension.lower() in excluded_file_extensions:
logger.debug('Skipped %s: File extension excluded.', uri)
continue
if uri not in uris_library:
uris_update.add(uri)
logger.info('Found %d unknown tracks.', len(uris_update))
logger.info('Scanning...')
scanner = scan.Scanner(scan_timeout)
progress = Progress(len(uris_update))
for uri in sorted(uris_update):
try:
data = scanner.scan(uri)
track = scan.audio_data_to_track(data)
local_updater.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
progress.increment()
logger.info('Commiting changes.')
local_updater.commit()
return 0
def check(self, expected):
actual = scan.audio_data_to_track(self.data)
self.assertEqual(expected, actual)
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = set(
file_ext.lower() for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uri_path_mapping = {}
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
uri_path_mapping[track.uri] = translator.local_track_uri_to_path(
track.uri, media_dir)
try:
stat = os.stat(uri_path_mapping[track.uri])
if int(stat.st_mtime) > track.last_modified:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
except OSError:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
logger.info('Checking %s for unknown tracks.', media_dir)
for relpath in path.find_files(media_dir):
uri = translator.path_to_local_track_uri(relpath)
file_extension = os.path.splitext(relpath)[1]
if file_extension.lower() in excluded_file_extensions:
logger.debug('Skipped %s: File extension excluded.', uri)
continue
if uri not in uris_in_library:
uris_to_update.add(uri)
uri_path_mapping[uri] = os.path.join(media_dir, relpath)
logger.info('Found %d unknown tracks.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update)[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
data = scanner.scan(path.path_to_uri(uri_path_mapping[uri]))
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
