示例#1
0
    def test_empty_dir(self):
        """Empty directories should not show up in results"""
        self.mkdir('empty')

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(result, {})
        self.assertEqual(errors, {})
示例#2
0
    def test_file_as_the_root(self):
        """Specifying a file as the root should just return the file"""
        single = self.touch('single')

        result, errors = path.find_mtimes(single)
        self.assertEqual(result, {single: tests.any_int})
        self.assertEqual(errors, {})
示例#3
0
    def test_empty_dir(self):
        """Empty directories should not show up in results"""
        self.mkdir('empty')

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(result, {})
        self.assertEqual(errors, {})
示例#4
0
    def test_file_as_the_root(self):
        """Specifying a file as the root should just return the file"""
        single = self.touch('single')

        result, errors = path.find_mtimes(single)
        self.assertEqual(result, {single: tests.any_int})
        self.assertEqual(errors, {})
示例#5
0
    def test_symlink_pointing_at_itself_fails(self):
        """Symlink pointing at itself should give as an OS error"""
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(link, link)

        result, errors = path.find_mtimes(link, follow=True)
        self.assertEqual({}, result)
        self.assertEqual({link: tests.IsA(exceptions.FindError)}, errors)
示例#6
0
    def test_missing_permission_to_directory(self):
        """Missing permissions to a directory is an error"""
        directory = self.mkdir('no-permission')
        os.chmod(directory, 0)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual({}, result)
        self.assertEqual({directory: tests.IsA(exceptions.FindError)}, errors)
示例#7
0
    def test_missing_permission_to_file(self):
        """Missing permissions to a file is not a search error"""
        target = self.touch('no-permission')
        os.chmod(target, 0)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual({target: tests.any_int}, result)
        self.assertEqual({}, errors)
示例#8
0
    def test_symlink_pointing_at_itself_fails(self):
        """Symlink pointing at itself should give as an OS error"""
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(link, link)

        result, errors = path.find_mtimes(link, follow=True)
        self.assertEqual({}, result)
        self.assertEqual({link: tests.IsA(exceptions.FindError)}, errors)
示例#9
0
    def test_symlink_pointing_at_parent_fails(self):
        """We should detect a loop via the parent and give up on the branch"""
        os.symlink(self.tmpdir, os.path.join(self.tmpdir, 'link'))

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual({}, result)
        self.assertEqual(1, len(errors))
        self.assertEqual(tests.IsA(Exception), errors.values()[0])
示例#10
0
    def test_symlink_pointing_at_parent_fails(self):
        """We should detect a loop via the parent and give up on the branch"""
        os.symlink(self.tmpdir, os.path.join(self.tmpdir, 'link'))

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual({}, result)
        self.assertEqual(1, len(errors))
        self.assertEqual(tests.IsA(Exception), errors.values()[0])
示例#11
0
    def test_missing_permission_to_directory(self):
        """Missing permissions to a directory is an error"""
        directory = self.mkdir('no-permission')
        os.chmod(directory, 0)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual({}, result)
        self.assertEqual({directory: tests.IsA(exceptions.FindError)}, errors)
示例#12
0
    def test_missing_permission_to_file(self):
        """Missing permissions to a file is not a search error"""
        target = self.touch('no-permission')
        os.chmod(target, 0)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual({target: tests.any_int}, result)
        self.assertEqual({}, errors)
示例#13
0
    def test_symlinks_are_ignored(self):
        """By default symlinks should be treated as an error"""
        target = self.touch('target')
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(target, link)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(result, {target: tests.any_int})
        self.assertEqual(errors, {link: tests.IsA(exceptions.FindError)})
示例#14
0
    def test_symlink_to_file_as_root_is_followed(self):
        """Passing a symlink as the root should be followed when follow=True"""
        target = self.touch('target')
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(target, link)

        result, errors = path.find_mtimes(link, follow=True)
        self.assertEqual({link: tests.any_int}, result)
        self.assertEqual({}, errors)
示例#15
0
    def test_symlink_to_file_as_root_is_followed(self):
        """Passing a symlink as the root should be followed when follow=True"""
        target = self.touch('target')
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(target, link)

        result, errors = path.find_mtimes(link, follow=True)
        self.assertEqual({link: tests.any_int}, result)
        self.assertEqual({}, errors)
示例#16
0
    def test_symlinks_are_ignored(self):
        """By default symlinks should be treated as an error"""
        target = self.touch('target')
        link = os.path.join(self.tmpdir, 'link')
        os.symlink(target, link)

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(result, {target: tests.any_int})
        self.assertEqual(errors, {link: tests.IsA(exceptions.FindError)})
示例#17
0
    def test_indirect_symlink_loop(self):
        """More indirect loops should also be detected"""
        # Setup tmpdir/directory/loop where loop points to tmpdir
        directory = os.path.join(self.tmpdir, b'directory')
        loop = os.path.join(directory, b'loop')

        os.mkdir(directory)
        os.symlink(self.tmpdir, loop)

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual({}, result)
        self.assertEqual({loop: tests.IsA(Exception)}, errors)
示例#18
0
    def test_indirect_symlink_loop(self):
        """More indirect loops should also be detected"""
        # Setup tmpdir/directory/loop where loop points to tmpdir
        directory = os.path.join(self.tmpdir, b'directory')
        loop = os.path.join(directory, b'loop')

        os.mkdir(directory)
        os.symlink(self.tmpdir, loop)

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual({}, result)
        self.assertEqual({loop: tests.IsA(Exception)}, errors)
示例#19
0
    def test_symlink_branches_are_not_excluded(self):
        """Using symlinks to make a file show up multiple times should work"""
        self.mkdir('directory')
        target = self.touch('directory', 'target')
        link1 = os.path.join(self.tmpdir, b'link1')
        link2 = os.path.join(self.tmpdir, b'link2')

        os.symlink(target, link1)
        os.symlink(target, link2)

        expected = {target: tests.any_int,
                    link1: tests.any_int,
                    link2: tests.any_int}

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual(expected, result)
        self.assertEqual({}, errors)
示例#20
0
    def test_nested_directories(self):
        """Searching nested directories should find all files"""

        # Setup foo/bar and baz directories
        self.mkdir('foo')
        self.mkdir('foo', 'bar')
        self.mkdir('baz')

        # Touch foo/file foo/bar/file and baz/file
        foo_file = self.touch('foo', 'file')
        foo_bar_file = self.touch('foo', 'bar', 'file')
        baz_file = self.touch('baz', 'file')

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(result, {foo_file: tests.any_int,
                                  foo_bar_file: tests.any_int,
                                  baz_file: tests.any_int})
        self.assertEqual(errors, {})
示例#21
0
    def test_symlink_branches_are_not_excluded(self):
        """Using symlinks to make a file show up multiple times should work"""
        self.mkdir('directory')
        target = self.touch('directory', 'target')
        link1 = os.path.join(self.tmpdir, b'link1')
        link2 = os.path.join(self.tmpdir, b'link2')

        os.symlink(target, link1)
        os.symlink(target, link2)

        expected = {
            target: tests.any_int,
            link1: tests.any_int,
            link2: tests.any_int
        }

        result, errors = path.find_mtimes(self.tmpdir, follow=True)
        self.assertEqual(expected, result)
        self.assertEqual({}, errors)
示例#22
0
    def test_nested_directories(self):
        """Searching nested directories should find all files"""

        # Setup foo/bar and baz directories
        self.mkdir('foo')
        self.mkdir('foo', 'bar')
        self.mkdir('baz')

        # Touch foo/file foo/bar/file and baz/file
        foo_file = self.touch('foo', 'file')
        foo_bar_file = self.touch('foo', 'bar', 'file')
        baz_file = self.touch('baz', 'file')

        result, errors = path.find_mtimes(self.tmpdir)
        self.assertEqual(
            result, {
                foo_file: tests.any_int,
                foo_bar_file: tests.any_int,
                baz_file: tests.any_int
            })
        self.assertEqual(errors, {})
示例#23
0
 def test_nonexistent_dir(self):
     """Non existent search roots are an error"""
     missing = os.path.join(self.tmpdir, 'does-not-exist')
     result, errors = path.find_mtimes(missing)
     self.assertEqual(result, {})
     self.assertEqual(errors, {missing: tests.IsA(exceptions.FindError)})
示例#24
0
 def find(self, value):
     return path.find_mtimes(path_to_data_dir(value))
示例#25
0
 def find(self, path):
     media_dir = path_to_data_dir(path)
     for path in path_lib.find_mtimes(media_dir):
         yield os.path.join(media_dir, path)
示例#26
0
 def test_names_are_bytestrings(self):
     """We shouldn't be mixing in unicode for paths."""
     result, errors = path.find_mtimes(tests.path_to_data_dir(''))
     for name in result.keys() + errors.keys():
         self.assertEqual(name, tests.IsA(bytes))
示例#27
0
 def find(self, path):
     media_dir = path_to_data_dir(path)
     result, errors = path_lib.find_mtimes(media_dir)
     for path in result:
         yield os.path.join(media_dir, path)
示例#28
0
    def run(self, args, config):
        media_dir = config['local']['media_dir']
        scan_timeout = config['local']['scan_timeout']
        flush_threshold = config['local']['scan_flush_threshold']
        excluded_file_extensions = config['local']['excluded_file_extensions']
        excluded_file_extensions = tuple(
            bytes(file_ext.lower()) for file_ext in excluded_file_extensions)

        library = _get_library(args, config)

        uris_to_update = set()
        uris_to_remove = set()

        file_mtimes = path.find_mtimes(media_dir)
        logger.info('Found %d files in media_dir.', len(file_mtimes))

        num_tracks = library.load()
        logger.info('Checking %d tracks from library.', num_tracks)

        for track in library.begin():
            abspath = translator.local_track_uri_to_path(track.uri, media_dir)
            mtime = file_mtimes.pop(abspath, None)
            if mtime is None:
                logger.debug('Missing file %s', track.uri)
                uris_to_remove.add(track.uri)
            elif mtime > track.last_modified:
                uris_to_update.add(track.uri)

        logger.info('Removing %d missing tracks.', len(uris_to_remove))
        for uri in uris_to_remove:
            library.remove(uri)

        for abspath in file_mtimes:
            relpath = os.path.relpath(abspath, media_dir)
            uri = translator.path_to_local_track_uri(relpath)

            if relpath.lower().endswith(excluded_file_extensions):
                logger.debug('Skipped %s: File extension excluded.', uri)
                continue

            uris_to_update.add(uri)

        logger.info('Found %d tracks which need to be updated.',
                    len(uris_to_update))
        logger.info('Scanning...')

        uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
        uris_to_update = uris_to_update[:args.limit]

        scanner = scan.Scanner(scan_timeout)
        progress = _Progress(flush_threshold, len(uris_to_update))

        for uri in uris_to_update:
            try:
                relpath = translator.local_track_uri_to_path(uri, media_dir)
                file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
                data = scanner.scan(file_uri)
                track = scan.audio_data_to_track(data).copy(uri=uri)
                library.add(track)
                logger.debug('Added %s', track.uri)
            except exceptions.ScannerError as error:
                logger.warning('Failed %s: %s', uri, error)

            if progress.increment():
                progress.log()
                if library.flush():
                    logger.debug('Progress flushed.')

        progress.log()
        library.close()
        logger.info('Done scanning.')
        return 0
示例#29
0
    def run(self, args, config):
        media_dir = config['local']['media_dir']
        scan_timeout = config['local']['scan_timeout']
        flush_threshold = config['local']['scan_flush_threshold']
        excluded_file_extensions = config['local']['excluded_file_extensions']
        excluded_file_extensions = tuple(
            bytes(file_ext.lower()) for file_ext in excluded_file_extensions)

        library = _get_library(args, config)

        file_mtimes, file_errors = path.find_mtimes(
            media_dir, follow=config['local']['scan_follow_symlinks'])

        logger.info('Found %d files in media_dir.', len(file_mtimes))

        if file_errors:
            logger.warning('Encountered %d errors while scanning media_dir.',
                           len(file_errors))
        for name in file_errors:
            logger.debug('Scan error %r for %r', file_errors[name], name)

        num_tracks = library.load()
        logger.info('Checking %d tracks from library.', num_tracks)

        uris_to_update = set()
        uris_to_remove = set()
        uris_in_library = set()

        for track in library.begin():
            abspath = translator.local_track_uri_to_path(track.uri, media_dir)
            mtime = file_mtimes.get(abspath)
            if mtime is None:
                logger.debug('Missing file %s', track.uri)
                uris_to_remove.add(track.uri)
            elif mtime > track.last_modified or args.force:
                uris_to_update.add(track.uri)
            uris_in_library.add(track.uri)

        logger.info('Removing %d missing tracks.', len(uris_to_remove))
        for uri in uris_to_remove:
            library.remove(uri)

        for abspath in file_mtimes:
            relpath = os.path.relpath(abspath, media_dir)
            uri = translator.path_to_local_track_uri(relpath)

            if b'/.' in relpath:
                logger.debug('Skipped %s: Hidden directory/file.', uri)
            elif relpath.lower().endswith(excluded_file_extensions):
                logger.debug('Skipped %s: File extension excluded.', uri)
            elif uri not in uris_in_library:
                uris_to_update.add(uri)

        logger.info(
            'Found %d tracks which need to be updated.', len(uris_to_update))
        logger.info('Scanning...')

        uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
        uris_to_update = uris_to_update[:args.limit]

        scanner = scan.Scanner(scan_timeout)
        progress = _Progress(flush_threshold, len(uris_to_update))

        for uri in uris_to_update:
            try:
                relpath = translator.local_track_uri_to_path(uri, media_dir)
                file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
                result = scanner.scan(file_uri)
                tags, duration = result.tags, result.duration
                if duration < MIN_DURATION_MS:
                    logger.warning('Failed %s: Track shorter than %dms',
                                   uri, MIN_DURATION_MS)
                else:
                    mtime = file_mtimes.get(os.path.join(media_dir, relpath))
                    track = utils.convert_tags_to_track(tags).copy(
                        uri=uri, length=duration, last_modified=mtime)
                    if library.add_supports_tags_and_duration:
                        library.add(track, tags=tags, duration=duration)
                    else:
                        library.add(track)
                    logger.debug('Added %s', track.uri)
            except exceptions.ScannerError as error:
                logger.warning('Failed %s: %s', uri, error)

            if progress.increment():
                progress.log()
                if library.flush():
                    logger.debug('Progress flushed.')

        progress.log()
        library.close()
        logger.info('Done scanning.')
        return 0
示例#30
0
 def test_names_are_bytestrings(self):
     """We shouldn't be mixing in unicode for paths."""
     result, errors = path.find_mtimes(tests.path_to_data_dir(''))
     for name in result.keys() + errors.keys():
         self.assertEqual(name, tests.IsA(bytes))
示例#31
0
    def run(self, args, config):
        media_dir = config['local']['media_dir']
        scan_timeout = config['local']['scan_timeout']
        flush_threshold = config['local']['scan_flush_threshold']
        excluded_file_extensions = config['local']['excluded_file_extensions']
        excluded_file_extensions = tuple(
            bytes(file_ext.lower()) for file_ext in excluded_file_extensions)

        library = _get_library(args, config)

        file_mtimes, file_errors = path.find_mtimes(
            media_dir, follow=config['local']['scan_follow_symlinks'])

        logger.info('Found %d files in media_dir.', len(file_mtimes))

        if file_errors:
            logger.warning('Encountered %d errors while scanning media_dir.',
                           len(file_errors))
        for name in file_errors:
            logger.debug('Scan error %r for %r', file_errors[name], name)

        num_tracks = library.load()
        logger.info('Checking %d tracks from library.', num_tracks)

        uris_to_update = set()
        uris_to_remove = set()
        uris_in_library = set()

        for track in library.begin():
            abspath = translator.local_track_uri_to_path(track.uri, media_dir)
            mtime = file_mtimes.get(abspath)
            if mtime is None:
                logger.debug('Missing file %s', track.uri)
                uris_to_remove.add(track.uri)
            elif mtime > track.last_modified or args.force:
                uris_to_update.add(track.uri)
            uris_in_library.add(track.uri)

        logger.info('Removing %d missing tracks.', len(uris_to_remove))
        for uri in uris_to_remove:
            library.remove(uri)

        for abspath in file_mtimes:
            relpath = os.path.relpath(abspath, media_dir)
            uri = translator.path_to_local_track_uri(relpath)

            if b'/.' in relpath:
                logger.debug('Skipped %s: Hidden directory/file.', uri)
            elif relpath.lower().endswith(excluded_file_extensions):
                logger.debug('Skipped %s: File extension excluded.', uri)
            elif uri not in uris_in_library:
                uris_to_update.add(uri)

        logger.info('Found %d tracks which need to be updated.',
                    len(uris_to_update))
        logger.info('Scanning...')

        uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
        uris_to_update = uris_to_update[:args.limit]

        scanner = scan.Scanner(scan_timeout)
        progress = _Progress(flush_threshold, len(uris_to_update))

        for uri in uris_to_update:
            try:
                relpath = translator.local_track_uri_to_path(uri, media_dir)
                file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
                result = scanner.scan(file_uri)
                tags, duration = result.tags, result.duration
                if not result.playable:
                    logger.warning('Failed %s: No audio found in file.', uri)
                elif duration < MIN_DURATION_MS:
                    logger.warning('Failed %s: Track shorter than %dms', uri,
                                   MIN_DURATION_MS)
                else:
                    mtime = file_mtimes.get(os.path.join(media_dir, relpath))
                    track = utils.convert_tags_to_track(tags).copy(
                        uri=uri, length=duration, last_modified=mtime)
                    if library.add_supports_tags_and_duration:
                        library.add(track, tags=tags, duration=duration)
                    else:
                        library.add(track)
                    logger.debug('Added %s', track.uri)
            except exceptions.ScannerError as error:
                logger.warning('Failed %s: %s', uri, error)

            if progress.increment():
                progress.log()
                if library.flush():
                    logger.debug('Progress flushed.')

        progress.log()
        library.close()
        logger.info('Done scanning.')
        return 0
示例#32
0
 def test_nonexistent_dir(self):
     """Non existent search roots are an error"""
     missing = os.path.join(self.tmpdir, 'does-not-exist')
     result, errors = path.find_mtimes(missing)
     self.assertEqual(result, {})
     self.assertEqual(errors, {missing: tests.IsA(exceptions.FindError)})
示例#33
0
    def run(self, args, config):
        media_dir = config['local']['media_dir']
        scan_timeout = config['local']['scan_timeout']
        flush_threshold = config['local']['scan_flush_threshold']
        excluded_file_extensions = config['local']['excluded_file_extensions']
        excluded_file_extensions = tuple(
            bytes(file_ext.lower()) for file_ext in excluded_file_extensions)

        library = _get_library(args, config)

        uris_in_library = set()
        uris_to_update = set()
        uris_to_remove = set()

        file_mtimes = path.find_mtimes(media_dir)
        logger.info('Found %d files in media_dir.', len(file_mtimes))

        num_tracks = library.load()
        logger.info('Checking %d tracks from library.', num_tracks)

        for track in library.begin():
            abspath = translator.local_track_uri_to_path(track.uri, media_dir)
            mtime = file_mtimes.pop(abspath, None)
            if mtime is None:
                logger.debug('Missing file %s', track.uri)
                uris_to_remove.add(track.uri)
            elif mtime > track.last_modified:
                uris_in_library.add(track.uri)

        logger.info('Removing %d missing tracks.', len(uris_to_remove))
        for uri in uris_to_remove:
            library.remove(uri)

        for abspath in file_mtimes:
            relpath = os.path.relpath(abspath, media_dir)
            uri = translator.path_to_local_track_uri(relpath)

            if relpath.lower().endswith(excluded_file_extensions):
                logger.debug('Skipped %s: File extension excluded.', uri)
                continue

            uris_to_update.add(uri)

        logger.info(
            'Found %d tracks which need to be updated.', len(uris_to_update))
        logger.info('Scanning...')

        uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
        uris_to_update = uris_to_update[:args.limit]

        scanner = scan.Scanner(scan_timeout)
        progress = _Progress(flush_threshold, len(uris_to_update))

        for uri in uris_to_update:
            try:
                relpath = translator.local_track_uri_to_path(uri, media_dir)
                file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
                data = scanner.scan(file_uri)
                track = scan.audio_data_to_track(data).copy(uri=uri)
                library.add(track)
                logger.debug('Added %s', track.uri)
            except exceptions.ScannerError as error:
                logger.warning('Failed %s: %s', uri, error)

            if progress.increment():
                progress.log()
                if library.flush():
                    logger.debug('Progress flushed.')

        progress.log()
        library.close()
        logger.info('Done scanning.')
        return 0
示例#34
0
 def find(self, path):
     media_dir = path_to_data_dir(path)
     for path in path_lib.find_mtimes(media_dir):
         yield os.path.join(media_dir, path)