def test_constrained_parallel(self): l = [] pl = pipeline.Pipeline(( _produce(1000), (_work(), _work()), _consume(l) )) pl.run_parallel(1) self.assertEqual(set(l), set(i * 2 for i in range(1000)))
def test_constrained(self): l = [] # Do a "significant" amount of work... pl = pipeline.Pipeline((_produce(1000), _work(), _consume(l))) # ... with only a single queue slot. pl.run_parallel(1) self.assertEqual(l, [i * 2 for i in range(1000)])
def test_stage_decorator(self): @pipeline.stage def add(n, i): return i + n pl = pipeline.Pipeline([iter([1, 2, 3]), add(2)]) self.assertEqual(list(pl.pull()), [3, 4, 5])
def run_import(**kwargs): """Run an import. The keyword arguments are the same as those to ImportConfig. """ config = ImportConfig(**kwargs) # Set up the pipeline. if config.singletons: # Singleton importer. stages = [read_items(config)] if config.autot: stages += [item_lookup(config), item_query(config)] else: stages += [item_progress(config)] else: # Whole-album importer. stages = [read_albums(config)] if config.autot: # Only look up and query the user when autotagging. stages += [initial_lookup(config), user_query(config)] else: # When not autotagging, just display progress. stages += [show_progress(config)] stages += [apply_choices(config)] pl = pipeline.Pipeline(stages) # Run the pipeline. try: if config.threaded: pl.run_parallel(QUEUE_SIZE) else: pl.run_sequential() except ImportAbort: # User aborted operation. Silently stop. pass
def user_query(session, task): """A coroutine for interfacing with the user about the tagging process. The coroutine accepts an ImportTask objects. It uses the session's `choose_match` method to determine the `action` for this task. Depending on the action additional stages are exectuted and the processed task is yielded. It emits the ``import_task_choice`` event for plugins. Plugins have acces to the choice via the ``taks.choice_flag`` property and may choose to change it. """ if task.skip: return task # Ask the user for a choice. task.choose_match(session) plugins.send('import_task_choice', session=session, task=task) # As-tracks: transition to singleton workflow. if task.choice_flag is action.TRACKS: # Set up a little pipeline for dealing with the singletons. def emitter(task): for item in task.items: yield SingletonImportTask(task.toppath, item) yield SentinelImportTask(task.toppath, task.paths) ipl = pipeline.Pipeline([ emitter(task), lookup_candidates(session), user_query(session), ]) return pipeline.multiple(ipl.pull()) # As albums: group items by albums and create task for each album if task.choice_flag is action.ALBUMS: ipl = pipeline.Pipeline([ iter([task]), group_albums(session), lookup_candidates(session), user_query(session) ]) return pipeline.multiple(ipl.pull()) resolve_duplicates(session, task) return task
def run(self): stages = [ read_tasks(self), lookup_candidates(self), identify_release(self) ] pl = pipeline.Pipeline(stages) pl.run_parallel(QUEUE_SIZE)
def test_pull(self): pl = pipeline.Pipeline((_produce(), _exc_work())) pull = pl.pull() for i in range(3): next(pull) if six.PY2: self.assertRaises(TestException, pull.next) else: self.assertRaises(TestException, pull.__next__)
def user_query(session): """A coroutine for interfacing with the user about the tagging process. lib is the Library to import into and logfile may be a file-like object for logging the import process. The coroutine accepts and yields ImportTask objects. """ recent = set() task = None while True: task = yield task if task.sentinel: continue # Ask the user for a choice. choice = session.choose_match(task) task.set_choice(choice) session.log_choice(task) plugins.send('import_task_choice', session=session, task=task) # As-tracks: transition to singleton workflow. if choice is action.TRACKS: # Set up a little pipeline for dealing with the singletons. item_tasks = [] def emitter(): for item in task.items: yield ImportTask.item_task(item) yield ImportTask.progress_sentinel(task.toppath, task.paths) def collector(): while True: item_task = yield item_tasks.append(item_task) ipl = pipeline.Pipeline((emitter(), item_lookup(session), item_query(session), collector())) ipl.run_sequential() task = pipeline.multiple(item_tasks) continue # Check for duplicates if we have a match (or ASIS). if task.choice_flag in (action.ASIS, action.APPLY): ident = task.chosen_ident() # The "recent" set keeps track of identifiers for recently # imported albums -- those that haven't reached the database # yet. task.duplicates = _duplicate_check(session.lib, task) if ident in recent: # TODO: Somehow manage duplicate hooks for recents pass plugins.send('import_task_duplicate', session=session, task=task) if task.duplicates: session.resolve_duplicate(task) session.log_choice(task, True) recent.add(ident)
def test_mutator_stage_decorator(self): @pipeline.mutator_stage def setkey(key, item): item[key] = True pl = pipeline.Pipeline([ iter([{'x': False}, {'a': False}]), setkey('x'), ]) self.assertEqual(list(pl.pull()), [{'x': True}, {'a': False, 'x': True}])
def new_pipeline(self, tasks, stages): if type(tasks) == list: task_iter = iter(tasks) else: task_iter = tasks pl = pipeline.Pipeline([task_iter] + stages) plugins.send('import_begin', session=self) if config['threaded']: pl.run_parallel(QUEUE_SIZE) else: pl.run_sequential()
def user_query(config): """A coroutine for interfacing with the user about the tagging process. lib is the Library to import into and logfile may be a file-like object for logging the import process. The coroutine accepts and yields ImportTask objects. """ lib = _reopen_lib(config.lib) task = None while True: task = yield task if task.sentinel: continue # Ask the user for a choice. choice = config.choose_match_func(task, config) task.set_choice(choice) # As-tracks: transition to singleton workflow. if choice is action.TRACKS: # Set up a little pipeline for dealing with the singletons. item_tasks = [] def emitter(): for item in task.items: yield ImportTask.item_task(item) def collector(): while True: item_task = yield item_tasks.append(item_task) ipl = pipeline.Pipeline((emitter(), item_lookup(config), item_query(config), collector())) ipl.run_sequential() task = pipeline.multiple(item_tasks) # Log certain choices. if choice is action.ASIS: tag_log(config.logfile, 'asis', task.path) elif choice is action.SKIP: tag_log(config.logfile, 'skip', task.path) # Check for duplicates if we have a match (or ASIS). if choice is action.ASIS or isinstance(choice, tuple): if choice is action.ASIS: artist = task.cur_artist album = task.cur_album else: artist = task.info['artist'] album = task.info['album'] if _duplicate_check(lib, artist, album): tag_log(config.logfile, 'duplicate', task.path) log.warn("This album is already in the library!") task.set_choice(action.SKIP)
def run(self): """Run the import task. """ self._amend_config() # Set up the pipeline. if self.query is None: stages = [read_tasks(self)] else: stages = [query_tasks(self)] if config['import']['singletons']: # Singleton importer. if config['import']['autotag']: stages += [item_lookup(self), item_query(self)] else: stages += [item_progress(self)] else: # Whole-album importer. if config['import']['group_albums']: # Split directory tasks into one task for each album stages += [group_albums(self)] if config['import']['autotag']: # Only look up and query the user when autotagging. stages += [initial_lookup(self), user_query(self)] else: # When not autotagging, just display progress. stages += [show_progress(self)] stages += [apply_choices(self)] for stage_func in plugins.import_stages(): stages.append(plugin_stage(self, stage_func)) stages += [manipulate_files(self)] stages += [finalize(self)] pl = pipeline.Pipeline(stages) # Run the pipeline. try: if config['threaded']: pl.run_parallel(QUEUE_SIZE) else: pl.run_sequential() except ImportAbort: # User aborted operation. Silently stop. pass
def run_import(**kwargs): """Run an import. The keyword arguments are the same as those to ImportConfig. """ config = ImportConfig(**kwargs) # Set up the pipeline. if config.query is None: stages = [read_tasks(config)] else: stages = [query_tasks(config)] if config.singletons: # Singleton importer. if config.autot: stages += [item_lookup(config), item_query(config)] else: stages += [item_progress(config)] else: # Whole-album importer. if config.autot: # Only look up and query the user when autotagging. stages += [initial_lookup(config), user_query(config)] else: # When not autotagging, just display progress. stages += [show_progress(config)] stages += [apply_choices(config)] for stage_func in plugins.import_stages(): stages.append(plugin_stage(config, stage_func)) stages += [manipulate_files(config)] if config.art: stages += [fetch_art(config)] stages += [finalize(config)] pl = pipeline.Pipeline(stages) # Run the pipeline. try: if config.threaded: pl.run_parallel(QUEUE_SIZE) else: pl.run_sequential() except ImportAbort: # User aborted operation. Silently stop. pass
def run(self): """Run the import task. """ self.set_config(config['import']) # Set up the pipeline. if self.query is None: stages = [read_tasks(self)] else: stages = [query_tasks(self)] if self.config['pretend']: # Only log the imported files and end the pipeline stages += [log_files(self)] else: if self.config['group_albums'] and \ not self.config['singletons']: # Split directory tasks into one task for each album stages += [group_albums(self)] if self.config['autotag']: # FIXME We should also resolve duplicates when not # autotagging. This is currently handled in `user_query` stages += [lookup_candidates(self), user_query(self)] else: stages += [import_asis(self)] stages += [apply_choices(self)] for stage_func in plugins.import_stages(): stages.append(plugin_stage(self, stage_func)) stages += [manipulate_files(self)] pl = pipeline.Pipeline(stages) # Run the pipeline. plugins.send('import_begin', session=self) try: if config['threaded']: pl.run_parallel(QUEUE_SIZE) else: pl.run_sequential() except ImportAbort: # User aborted operation. Silently stop. pass
def setUp(self): self.l = [] self.pl = pipeline.Pipeline((_produce(), _work(), _consume(self.l)))
def test_pull(self): pl = pipeline.Pipeline((_produce(), _multi_work())) self.assertEqual(list(pl.pull()), [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])
def test_constrained_exception(self): # Raise an exception in a constrained pipeline. l = [] pl = pipeline.Pipeline((_produce(1000), _exc_work(), _consume(l))) self.assertRaises(TestException, pl.run_parallel, 1)
def user_query(session): """A coroutine for interfacing with the user about the tagging process. The coroutine accepts an ImportTask objects. It uses the session's ``choose_match`` method to determine the ``action`` for this task. Depending on the action additional stages are exectuted and the processed task is yielded. It emits the ``import_task_choice`` event for plugins. Plugins have acces to the choice via the ``taks.choice_flag`` property and may choose to change it. """ recent = set() task = None while True: task = yield task if task.should_skip(): continue # Ask the user for a choice. choice = session.choose_match(task) task.set_choice(choice) session.log_choice(task) plugins.send('import_task_choice', session=session, task=task) # As-tracks: transition to singleton workflow. if task.choice_flag is action.TRACKS: # Set up a little pipeline for dealing with the singletons. def emitter(task): for item in task.items: yield ImportTask.item_task(item) yield ImportTask.progress_sentinel(task.toppath, task.paths) ipl = pipeline.Pipeline([ emitter(task), item_lookup(session), item_query(session), ]) task = pipeline.multiple(ipl.pull()) continue # As albums: group items by albums and create task for each album if task.choice_flag is action.ALBUMS: def emitter(task): yield task ipl = pipeline.Pipeline([ emitter(task), group_albums(session), initial_lookup(session), user_query(session) ]) task = pipeline.multiple(ipl.pull()) continue # Check for duplicates if we have a match (or ASIS). if task.choice_flag in (action.ASIS, action.APPLY): ident = task.chosen_ident() # The "recent" set keeps track of identifiers for recently # imported albums -- those that haven't reached the database # yet. if ident in recent or _duplicate_check(session.lib, task): session.resolve_duplicate(task) session.log_choice(task, True) recent.add(ident)
def test_pull(self): pl = pipeline.Pipeline((_produce(), _exc_work())) pull = pl.pull() for i in range(3): next(pull) self.assertRaises(ExceptionFixture, pull.__next__)
def test_pull(self): pl = pipeline.Pipeline((_produce(), _work())) self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])
def test_pull_chain(self): pl = pipeline.Pipeline((_produce(), _work())) pl2 = pipeline.Pipeline((pl.pull(), _work())) self.assertEqual(list(pl2.pull()), [0, 4, 8, 12, 16])
def test_pull(self): pl = pipeline.Pipeline((_produce(), _exc_work())) pull = pl.pull() for i in range(3): pull.next() self.assertRaises(TestException, pull.next)