def test_empty_dir(self):
"""Empty directories should not show up in results"""
self.mkdir('empty')
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(result, {})
self.assertEqual(errors, {})
def test_file_as_the_root(self):
"""Specifying a file as the root should just return the file"""
single = self.touch('single')
result, errors = path.find_mtimes(single)
self.assertEqual(result, {single: tests.any_int})
self.assertEqual(errors, {})
def test_empty_dir(self):
"""Empty directories should not show up in results"""
self.mkdir('empty')
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(result, {})
self.assertEqual(errors, {})
def test_file_as_the_root(self):
"""Specifying a file as the root should just return the file"""
single = self.touch('single')
result, errors = path.find_mtimes(single)
self.assertEqual(result, {single: tests.any_int})
self.assertEqual(errors, {})
def test_symlink_pointing_at_itself_fails(self):
"""Symlink pointing at itself should give as an OS error"""
link = os.path.join(self.tmpdir, 'link')
os.symlink(link, link)
result, errors = path.find_mtimes(link, follow=True)
self.assertEqual({}, result)
self.assertEqual({link: tests.IsA(exceptions.FindError)}, errors)
def test_missing_permission_to_directory(self):
"""Missing permissions to a directory is an error"""
directory = self.mkdir('no-permission')
os.chmod(directory, 0)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual({}, result)
self.assertEqual({directory: tests.IsA(exceptions.FindError)}, errors)
def test_missing_permission_to_file(self):
"""Missing permissions to a file is not a search error"""
target = self.touch('no-permission')
os.chmod(target, 0)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual({target: tests.any_int}, result)
self.assertEqual({}, errors)
def test_symlink_pointing_at_itself_fails(self):
"""Symlink pointing at itself should give as an OS error"""
link = os.path.join(self.tmpdir, 'link')
os.symlink(link, link)
result, errors = path.find_mtimes(link, follow=True)
self.assertEqual({}, result)
self.assertEqual({link: tests.IsA(exceptions.FindError)}, errors)
def test_symlink_pointing_at_parent_fails(self):
"""We should detect a loop via the parent and give up on the branch"""
os.symlink(self.tmpdir, os.path.join(self.tmpdir, 'link'))
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual({}, result)
self.assertEqual(1, len(errors))
self.assertEqual(tests.IsA(Exception), errors.values()[0])
def test_symlink_pointing_at_parent_fails(self):
"""We should detect a loop via the parent and give up on the branch"""
os.symlink(self.tmpdir, os.path.join(self.tmpdir, 'link'))
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual({}, result)
self.assertEqual(1, len(errors))
self.assertEqual(tests.IsA(Exception), errors.values()[0])
def test_missing_permission_to_directory(self):
"""Missing permissions to a directory is an error"""
directory = self.mkdir('no-permission')
os.chmod(directory, 0)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual({}, result)
self.assertEqual({directory: tests.IsA(exceptions.FindError)}, errors)
def test_missing_permission_to_file(self):
"""Missing permissions to a file is not a search error"""
target = self.touch('no-permission')
os.chmod(target, 0)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual({target: tests.any_int}, result)
self.assertEqual({}, errors)
def test_symlinks_are_ignored(self):
"""By default symlinks should be treated as an error"""
target = self.touch('target')
link = os.path.join(self.tmpdir, 'link')
os.symlink(target, link)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(result, {target: tests.any_int})
self.assertEqual(errors, {link: tests.IsA(exceptions.FindError)})
def test_symlink_to_file_as_root_is_followed(self):
"""Passing a symlink as the root should be followed when follow=True"""
target = self.touch('target')
link = os.path.join(self.tmpdir, 'link')
os.symlink(target, link)
result, errors = path.find_mtimes(link, follow=True)
self.assertEqual({link: tests.any_int}, result)
self.assertEqual({}, errors)
def test_symlink_to_file_as_root_is_followed(self):
"""Passing a symlink as the root should be followed when follow=True"""
target = self.touch('target')
link = os.path.join(self.tmpdir, 'link')
os.symlink(target, link)
result, errors = path.find_mtimes(link, follow=True)
self.assertEqual({link: tests.any_int}, result)
self.assertEqual({}, errors)
def test_symlinks_are_ignored(self):
"""By default symlinks should be treated as an error"""
target = self.touch('target')
link = os.path.join(self.tmpdir, 'link')
os.symlink(target, link)
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(result, {target: tests.any_int})
self.assertEqual(errors, {link: tests.IsA(exceptions.FindError)})
def test_indirect_symlink_loop(self):
"""More indirect loops should also be detected"""
# Setup tmpdir/directory/loop where loop points to tmpdir
directory = os.path.join(self.tmpdir, b'directory')
loop = os.path.join(directory, b'loop')
os.mkdir(directory)
os.symlink(self.tmpdir, loop)
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual({}, result)
self.assertEqual({loop: tests.IsA(Exception)}, errors)
def test_indirect_symlink_loop(self):
"""More indirect loops should also be detected"""
# Setup tmpdir/directory/loop where loop points to tmpdir
directory = os.path.join(self.tmpdir, b'directory')
loop = os.path.join(directory, b'loop')
os.mkdir(directory)
os.symlink(self.tmpdir, loop)
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual({}, result)
self.assertEqual({loop: tests.IsA(Exception)}, errors)
def test_symlink_branches_are_not_excluded(self):
"""Using symlinks to make a file show up multiple times should work"""
self.mkdir('directory')
target = self.touch('directory', 'target')
link1 = os.path.join(self.tmpdir, b'link1')
link2 = os.path.join(self.tmpdir, b'link2')
os.symlink(target, link1)
os.symlink(target, link2)
expected = {target: tests.any_int,
link1: tests.any_int,
link2: tests.any_int}
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual(expected, result)
self.assertEqual({}, errors)
def test_nested_directories(self):
"""Searching nested directories should find all files"""
# Setup foo/bar and baz directories
self.mkdir('foo')
self.mkdir('foo', 'bar')
self.mkdir('baz')
# Touch foo/file foo/bar/file and baz/file
foo_file = self.touch('foo', 'file')
foo_bar_file = self.touch('foo', 'bar', 'file')
baz_file = self.touch('baz', 'file')
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(result, {foo_file: tests.any_int,
foo_bar_file: tests.any_int,
baz_file: tests.any_int})
self.assertEqual(errors, {})
def test_symlink_branches_are_not_excluded(self):
"""Using symlinks to make a file show up multiple times should work"""
self.mkdir('directory')
target = self.touch('directory', 'target')
link1 = os.path.join(self.tmpdir, b'link1')
link2 = os.path.join(self.tmpdir, b'link2')
os.symlink(target, link1)
os.symlink(target, link2)
expected = {
target: tests.any_int,
link1: tests.any_int,
link2: tests.any_int
}
result, errors = path.find_mtimes(self.tmpdir, follow=True)
self.assertEqual(expected, result)
self.assertEqual({}, errors)
def test_nested_directories(self):
"""Searching nested directories should find all files"""
# Setup foo/bar and baz directories
self.mkdir('foo')
self.mkdir('foo', 'bar')
self.mkdir('baz')
# Touch foo/file foo/bar/file and baz/file
foo_file = self.touch('foo', 'file')
foo_bar_file = self.touch('foo', 'bar', 'file')
baz_file = self.touch('baz', 'file')
result, errors = path.find_mtimes(self.tmpdir)
self.assertEqual(
result, {
foo_file: tests.any_int,
foo_bar_file: tests.any_int,
baz_file: tests.any_int
})
self.assertEqual(errors, {})
def test_nonexistent_dir(self):
"""Non existent search roots are an error"""
missing = os.path.join(self.tmpdir, 'does-not-exist')
result, errors = path.find_mtimes(missing)
self.assertEqual(result, {})
self.assertEqual(errors, {missing: tests.IsA(exceptions.FindError)})
def find(self, value):
return path.find_mtimes(path_to_data_dir(value))
def find(self, path):
media_dir = path_to_data_dir(path)
for path in path_lib.find_mtimes(media_dir):
yield os.path.join(media_dir, path)
def test_names_are_bytestrings(self):
"""We shouldn't be mixing in unicode for paths."""
result, errors = path.find_mtimes(tests.path_to_data_dir(''))
for name in result.keys() + errors.keys():
self.assertEqual(name, tests.IsA(bytes))
def find(self, path):
media_dir = path_to_data_dir(path)
result, errors = path_lib.find_mtimes(media_dir)
for path in result:
yield os.path.join(media_dir, path)
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_to_update.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info('Found %d tracks which need to be updated.',
len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
file_mtimes, file_errors = path.find_mtimes(
media_dir, follow=config['local']['scan_follow_symlinks'])
logger.info('Found %d files in media_dir.', len(file_mtimes))
if file_errors:
logger.warning('Encountered %d errors while scanning media_dir.',
len(file_errors))
for name in file_errors:
logger.debug('Scan error %r for %r', file_errors[name], name)
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
uris_to_update = set()
uris_to_remove = set()
uris_in_library = set()
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.get(abspath)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified or args.force:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if b'/.' in relpath:
logger.debug('Skipped %s: Hidden directory/file.', uri)
elif relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
elif uri not in uris_in_library:
uris_to_update.add(uri)
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
result = scanner.scan(file_uri)
tags, duration = result.tags, result.duration
if duration < MIN_DURATION_MS:
logger.warning('Failed %s: Track shorter than %dms',
uri, MIN_DURATION_MS)
else:
mtime = file_mtimes.get(os.path.join(media_dir, relpath))
track = utils.convert_tags_to_track(tags).copy(
uri=uri, length=duration, last_modified=mtime)
if library.add_supports_tags_and_duration:
library.add(track, tags=tags, duration=duration)
else:
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def test_names_are_bytestrings(self):
"""We shouldn't be mixing in unicode for paths."""
result, errors = path.find_mtimes(tests.path_to_data_dir(''))
for name in result.keys() + errors.keys():
self.assertEqual(name, tests.IsA(bytes))
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
file_mtimes, file_errors = path.find_mtimes(
media_dir, follow=config['local']['scan_follow_symlinks'])
logger.info('Found %d files in media_dir.', len(file_mtimes))
if file_errors:
logger.warning('Encountered %d errors while scanning media_dir.',
len(file_errors))
for name in file_errors:
logger.debug('Scan error %r for %r', file_errors[name], name)
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
uris_to_update = set()
uris_to_remove = set()
uris_in_library = set()
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.get(abspath)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified or args.force:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if b'/.' in relpath:
logger.debug('Skipped %s: Hidden directory/file.', uri)
elif relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
elif uri not in uris_in_library:
uris_to_update.add(uri)
logger.info('Found %d tracks which need to be updated.',
len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
result = scanner.scan(file_uri)
tags, duration = result.tags, result.duration
if not result.playable:
logger.warning('Failed %s: No audio found in file.', uri)
elif duration < MIN_DURATION_MS:
logger.warning('Failed %s: Track shorter than %dms', uri,
MIN_DURATION_MS)
else:
mtime = file_mtimes.get(os.path.join(media_dir, relpath))
track = utils.convert_tags_to_track(tags).copy(
uri=uri, length=duration, last_modified=mtime)
if library.add_supports_tags_and_duration:
library.add(track, tags=tags, duration=duration)
else:
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def test_nonexistent_dir(self):
"""Non existent search roots are an error"""
missing = os.path.join(self.tmpdir, 'does-not-exist')
result, errors = path.find_mtimes(missing)
self.assertEqual(result, {})
self.assertEqual(errors, {missing: tests.IsA(exceptions.FindError)})
def run(self, args, config):
media_dir = config['local']['media_dir']
scan_timeout = config['local']['scan_timeout']
flush_threshold = config['local']['scan_flush_threshold']
excluded_file_extensions = config['local']['excluded_file_extensions']
excluded_file_extensions = tuple(
bytes(file_ext.lower()) for file_ext in excluded_file_extensions)
library = _get_library(args, config)
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
if relpath.lower().endswith(excluded_file_extensions):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
uris_to_update.add(uri)
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)
except exceptions.ScannerError as error:
logger.warning('Failed %s: %s', uri, error)
if progress.increment():
progress.log()
if library.flush():
logger.debug('Progress flushed.')
progress.log()
library.close()
logger.info('Done scanning.')
return 0
def find(self, path):
media_dir = path_to_data_dir(path)
for path in path_lib.find_mtimes(media_dir):
yield os.path.join(media_dir, path)