def test_local_uri_to_path(uri, path):
media_dir = b'/home/alice/Music'
assert translator.local_uri_to_path(uri, media_dir) == path
# Legacy version to keep old versions of Mopidy-Local-Sqlite working
assert translator.local_track_uri_to_path(uri, media_dir) == path
def test_local_uri_to_path(uri, path):
media_dir = b'/home/alice/Music'
assert translator.local_uri_to_path(uri, media_dir) == path
# Legacy version to keep old versions of Mopidy-Local-Sqlite working
assert translator.local_track_uri_to_path(uri, media_dir) == path
def _track_to_refs(track):
track_path = translator.local_track_uri_to_path(track.uri, b'/')
track_path = track_path.decode(sys.getfilesystemencoding(), 'replace')
parts = re.findall(r'([^/]+)', track_path)
track_ref = Ref.track(uri=track.uri, name=parts.pop())
refs = [Ref.directory(uri='local:directory')]
for i in range(len(parts)):
directory = '/'.join(parts[:i+1])
uri = translator.path_to_local_directory_uri(directory)
refs.append(Ref.directory(uri=unicode(uri), name=parts[i]))
return refs + [track_ref]
def __init__(self, uris):
self._cache = {
local.Library.ROOT_DIRECTORY_URI: collections.OrderedDict()
}
for track_uri in uris:
path = translator.local_track_uri_to_path(track_uri, b'/')
parts = self.splitpath_re.findall(
path.decode(self.encoding, 'replace'))
track_ref = models.Ref.track(uri=track_uri, name=parts.pop())
# Look for our parents backwards as this is faster than having to
# do a complete search for each add.
parent_uri = None
child = None
for i in reversed(range(len(parts))):
directory = '/'.join(parts[:i + 1])
uri = translator.path_to_local_directory_uri(directory)
# First dir we process is our parent
if not parent_uri:
parent_uri = uri
# We found ourselves and we exist, done.
if uri in self._cache:
if child:
self._cache[uri][child.uri] = child
break
# Initialize ourselves, store child if present, and add
# ourselves as child for next loop.
self._cache[uri] = collections.OrderedDict()
if child:
self._cache[uri][child.uri] = child
child = models.Ref.directory(uri=uri, name=parts[i])
else:
# Loop completed, so final child needs to be added to root.
if child:
self._cache[local.Library.ROOT_DIRECTORY_URI][
child.uri] = child
# If no parent was set we belong in the root.
if not parent_uri:
parent_uri = local.Library.ROOT_DIRECTORY_URI
self._cache[parent_uri][track_uri] = track_ref
def __init__(self, uris):
self._cache = {
local.Library.ROOT_DIRECTORY_URI: collections.OrderedDict()}
for track_uri in uris:
path = translator.local_track_uri_to_path(track_uri, b'/')
parts = self.splitpath_re.findall(
path.decode(self.encoding, 'replace'))
track_ref = models.Ref.track(uri=track_uri, name=parts.pop())
# Look for our parents backwards as this is faster than having to
# do a complete search for each add.
parent_uri = None
child = None
for i in reversed(range(len(parts))):
directory = '/'.join(parts[:i + 1])
uri = translator.path_to_local_directory_uri(directory)
# First dir we process is our parent
if not parent_uri:
parent_uri = uri
# We found ourselves and we exist, done.
if uri in self._cache:
if child:
self._cache[uri][child.uri] = child
break
# Initialize ourselves, store child if present, and add
# ourselves as child for next loop.
self._cache[uri] = collections.OrderedDict()
if child:
self._cache[uri][child.uri] = child
child = models.Ref.directory(uri=uri, name=parts[i])
else:
# Loop completed, so final child needs to be added to root.
if child:
self._cache[
local.Library.ROOT_DIRECTORY_URI][child.uri] = child
# If no parent was set we belong in the root.
if not parent_uri:
parent_uri = local.Library.ROOT_DIRECTORY_URI
self._cache[parent_uri][track_uri] = track_ref
def _validate_track(self, track, encoding=sys.getfilesystemencoding()):
if not track.uri:
raise ValueError('Empty track URI')
if track.name:
name = track.name
else:
path = translator.local_track_uri_to_path(track.uri, b'')
name = os.path.basename(path).decode(encoding, errors='replace')
if track.album and track.album.name:
album = self._validate_album(track.album)
else:
album = None
return track.copy(
name=name,
album=album,
artists=map(self._validate_artist, track.artists),
composers=map(self._validate_artist, track.composers),
performers=map(self._validate_artist, track.performers)
)
def _validate_track(self, track, encoding=sys.getfilesystemencoding()):
if not track.uri:
raise ValueError('Empty track URI')
if track.name:
name = track.name
else:
path = translator.local_track_uri_to_path(track.uri, b'')
name = os.path.basename(path).decode(encoding, errors='replace')
if track.album and track.album.name:
album = self._validate_album(track.album)
else:
album = None
return track.copy(
name=name,
album=album,
artists=map(self._validate_artist, track.artists),
composers=map(self._validate_artist, track.composers),
performers=map(self._validate_artist, track.performers)
)
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
file_mtimes, file_errors = path.find_mtimes(
media_dir, follow=config['local']['scan_follow_symlinks'])
logger.info('Found %d files in media_dir.', len(file_mtimes))
if file_errors:
logger.warning('Encountered %d errors while scanning media_dir.',
len(file_errors))
for name in file_errors:
logger.debug('Scan error %r for %r', file_errors[name], name)
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
uris_to_update = set()
uris_to_remove = set()
uris_in_library = set()
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.get(abspath)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified or args.force:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if b'/.' in relpath:
logger.debug('Skipped %s: Hidden directory/file.', uri)
elif relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
elif uri not in uris_in_library:
uris_to_update.add(uri)
logger.info('Found %d tracks which need to be updated.',
len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
result = scanner.scan(file_uri)
tags, duration = result.tags, result.duration
if not result.playable:
logger.warning('Failed %s: No audio found in file.', uri)
elif duration < MIN_DURATION_MS:
logger.warning('Failed %s: Track shorter than %dms', uri,
MIN_DURATION_MS)
else:
mtime = file_mtimes.get(os.path.join(media_dir, relpath))
track = utils.convert_tags_to_track(tags).copy(
uri=uri, length=duration, last_modified=mtime)
if library.add_supports_tags_and_duration:
library.add(track, tags=tags, duration=duration)
else:
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
if library is None:
return 1
file_mtimes, file_errors = path.find_mtimes(
media_dir, follow=config['local']['scan_follow_symlinks'])
logger.info('Found %d files in media_dir.', len(file_mtimes))
if file_errors:
logger.warning('Encountered %d errors while scanning media_dir.',
len(file_errors))
for name in file_errors:
logger.debug('Scan error %r for %r', file_errors[name], name)
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
uris_to_update = set()
uris_to_remove = set()
uris_in_library = set()
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.get(abspath)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified or args.force:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if b'/.' in relpath or relpath.startswith(b'.'):
logger.debug('Skipped %s: Hidden directory/file.', uri)
elif relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
elif uri not in uris_in_library:
uris_to_update.add(uri)
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
result = scanner.scan(file_uri)
if not result.playable:
logger.warning('Failed %s: No audio found in file.', uri)
elif result.duration < MIN_DURATION_MS:
logger.warning('Failed %s: Track shorter than %dms',
uri, MIN_DURATION_MS)
else:
mtime = file_mtimes.get(os.path.join(media_dir, relpath))
track = tags.convert_tags_to_track(result.tags).replace(
uri=uri, length=result.duration, last_modified=mtime)
if library.add_supports_tags_and_duration:
library.add(
track, tags=result.tags, duration=result.duration)
else:
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_to_update.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info('Found %d tracks which need to be updated.',
len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
