def test_constrained_parallel(self):
l = []
pl = pipeline.Pipeline((
_produce(1000), (_work(), _work()), _consume(l)
))
pl.run_parallel(1)
self.assertEqual(set(l), set(i * 2 for i in range(1000)))
def test_constrained(self):
l = []
# Do a "significant" amount of work...
pl = pipeline.Pipeline((_produce(1000), _work(), _consume(l)))
# ... with only a single queue slot.
pl.run_parallel(1)
self.assertEqual(l, [i * 2 for i in range(1000)])
def test_stage_decorator(self):
@pipeline.stage
def add(n, i):
return i + n
pl = pipeline.Pipeline([iter([1, 2, 3]), add(2)])
self.assertEqual(list(pl.pull()), [3, 4, 5])
def run_import(**kwargs):
"""Run an import. The keyword arguments are the same as those to
ImportConfig.
"""
config = ImportConfig(**kwargs)
# Set up the pipeline.
if config.singletons:
# Singleton importer.
stages = [read_items(config)]
if config.autot:
stages += [item_lookup(config), item_query(config)]
else:
stages += [item_progress(config)]
else:
# Whole-album importer.
stages = [read_albums(config)]
if config.autot:
# Only look up and query the user when autotagging.
stages += [initial_lookup(config), user_query(config)]
else:
# When not autotagging, just display progress.
stages += [show_progress(config)]
stages += [apply_choices(config)]
pl = pipeline.Pipeline(stages)
# Run the pipeline.
try:
if config.threaded:
pl.run_parallel(QUEUE_SIZE)
else:
pl.run_sequential()
except ImportAbort:
# User aborted operation. Silently stop.
pass
def user_query(session, task):
"""A coroutine for interfacing with the user about the tagging
process.
The coroutine accepts an ImportTask objects. It uses the
session's `choose_match` method to determine the `action` for
this task. Depending on the action additional stages are exectuted
and the processed task is yielded.
It emits the ``import_task_choice`` event for plugins. Plugins have
acces to the choice via the ``taks.choice_flag`` property and may
choose to change it.
"""
if task.skip:
return task
# Ask the user for a choice.
task.choose_match(session)
plugins.send('import_task_choice', session=session, task=task)
# As-tracks: transition to singleton workflow.
if task.choice_flag is action.TRACKS:
# Set up a little pipeline for dealing with the singletons.
def emitter(task):
for item in task.items:
yield SingletonImportTask(task.toppath, item)
yield SentinelImportTask(task.toppath, task.paths)
ipl = pipeline.Pipeline([
emitter(task),
lookup_candidates(session),
user_query(session),
])
return pipeline.multiple(ipl.pull())
# As albums: group items by albums and create task for each album
if task.choice_flag is action.ALBUMS:
ipl = pipeline.Pipeline([
iter([task]),
group_albums(session),
lookup_candidates(session),
user_query(session)
])
return pipeline.multiple(ipl.pull())
resolve_duplicates(session, task)
return task
def run(self):
stages = [
read_tasks(self),
lookup_candidates(self),
identify_release(self)
]
pl = pipeline.Pipeline(stages)
pl.run_parallel(QUEUE_SIZE)
def test_pull(self):
pl = pipeline.Pipeline((_produce(), _exc_work()))
pull = pl.pull()
for i in range(3):
next(pull)
if six.PY2:
self.assertRaises(TestException, pull.next)
else:
self.assertRaises(TestException, pull.__next__)
def user_query(session):
"""A coroutine for interfacing with the user about the tagging
process. lib is the Library to import into and logfile may be
a file-like object for logging the import process. The coroutine
accepts and yields ImportTask objects.
"""
recent = set()
task = None
while True:
task = yield task
if task.sentinel:
continue
# Ask the user for a choice.
choice = session.choose_match(task)
task.set_choice(choice)
session.log_choice(task)
plugins.send('import_task_choice', session=session, task=task)
# As-tracks: transition to singleton workflow.
if choice is action.TRACKS:
# Set up a little pipeline for dealing with the singletons.
item_tasks = []
def emitter():
for item in task.items:
yield ImportTask.item_task(item)
yield ImportTask.progress_sentinel(task.toppath, task.paths)
def collector():
while True:
item_task = yield
item_tasks.append(item_task)
ipl = pipeline.Pipeline((emitter(), item_lookup(session),
item_query(session), collector()))
ipl.run_sequential()
task = pipeline.multiple(item_tasks)
continue
# Check for duplicates if we have a match (or ASIS).
if task.choice_flag in (action.ASIS, action.APPLY):
ident = task.chosen_ident()
# The "recent" set keeps track of identifiers for recently
# imported albums -- those that haven't reached the database
# yet.
task.duplicates = _duplicate_check(session.lib, task)
if ident in recent:
# TODO: Somehow manage duplicate hooks for recents
pass
plugins.send('import_task_duplicate', session=session, task=task)
if task.duplicates:
session.resolve_duplicate(task)
session.log_choice(task, True)
recent.add(ident)
def test_mutator_stage_decorator(self):
@pipeline.mutator_stage
def setkey(key, item):
item[key] = True
pl = pipeline.Pipeline([
iter([{'x': False}, {'a': False}]),
setkey('x'),
])
self.assertEqual(list(pl.pull()),
[{'x': True}, {'a': False, 'x': True}])
def new_pipeline(self, tasks, stages):
if type(tasks) == list:
task_iter = iter(tasks)
else:
task_iter = tasks
pl = pipeline.Pipeline([task_iter] + stages)
plugins.send('import_begin', session=self)
if config['threaded']:
pl.run_parallel(QUEUE_SIZE)
else:
pl.run_sequential()
def user_query(config):
"""A coroutine for interfacing with the user about the tagging
process. lib is the Library to import into and logfile may be
a file-like object for logging the import process. The coroutine
accepts and yields ImportTask objects.
"""
lib = _reopen_lib(config.lib)
task = None
while True:
task = yield task
if task.sentinel:
continue
# Ask the user for a choice.
choice = config.choose_match_func(task, config)
task.set_choice(choice)
# As-tracks: transition to singleton workflow.
if choice is action.TRACKS:
# Set up a little pipeline for dealing with the singletons.
item_tasks = []
def emitter():
for item in task.items:
yield ImportTask.item_task(item)
def collector():
while True:
item_task = yield
item_tasks.append(item_task)
ipl = pipeline.Pipeline((emitter(), item_lookup(config),
item_query(config), collector()))
ipl.run_sequential()
task = pipeline.multiple(item_tasks)
# Log certain choices.
if choice is action.ASIS:
tag_log(config.logfile, 'asis', task.path)
elif choice is action.SKIP:
tag_log(config.logfile, 'skip', task.path)
# Check for duplicates if we have a match (or ASIS).
if choice is action.ASIS or isinstance(choice, tuple):
if choice is action.ASIS:
artist = task.cur_artist
album = task.cur_album
else:
artist = task.info['artist']
album = task.info['album']
if _duplicate_check(lib, artist, album):
tag_log(config.logfile, 'duplicate', task.path)
log.warn("This album is already in the library!")
task.set_choice(action.SKIP)
def run(self):
"""Run the import task.
"""
self._amend_config()
# Set up the pipeline.
if self.query is None:
stages = [read_tasks(self)]
else:
stages = [query_tasks(self)]
if config['import']['singletons']:
# Singleton importer.
if config['import']['autotag']:
stages += [item_lookup(self), item_query(self)]
else:
stages += [item_progress(self)]
else:
# Whole-album importer.
if config['import']['group_albums']:
# Split directory tasks into one task for each album
stages += [group_albums(self)]
if config['import']['autotag']:
# Only look up and query the user when autotagging.
stages += [initial_lookup(self), user_query(self)]
else:
# When not autotagging, just display progress.
stages += [show_progress(self)]
stages += [apply_choices(self)]
for stage_func in plugins.import_stages():
stages.append(plugin_stage(self, stage_func))
stages += [manipulate_files(self)]
stages += [finalize(self)]
pl = pipeline.Pipeline(stages)
# Run the pipeline.
try:
if config['threaded']:
pl.run_parallel(QUEUE_SIZE)
else:
pl.run_sequential()
except ImportAbort:
# User aborted operation. Silently stop.
pass
def run_import(**kwargs):
"""Run an import. The keyword arguments are the same as those to
ImportConfig.
"""
config = ImportConfig(**kwargs)
# Set up the pipeline.
if config.query is None:
stages = [read_tasks(config)]
else:
stages = [query_tasks(config)]
if config.singletons:
# Singleton importer.
if config.autot:
stages += [item_lookup(config), item_query(config)]
else:
stages += [item_progress(config)]
else:
# Whole-album importer.
if config.autot:
# Only look up and query the user when autotagging.
stages += [initial_lookup(config), user_query(config)]
else:
# When not autotagging, just display progress.
stages += [show_progress(config)]
stages += [apply_choices(config)]
for stage_func in plugins.import_stages():
stages.append(plugin_stage(config, stage_func))
stages += [manipulate_files(config)]
if config.art:
stages += [fetch_art(config)]
stages += [finalize(config)]
pl = pipeline.Pipeline(stages)
# Run the pipeline.
try:
if config.threaded:
pl.run_parallel(QUEUE_SIZE)
else:
pl.run_sequential()
except ImportAbort:
# User aborted operation. Silently stop.
pass
def run(self):
"""Run the import task.
"""
self.set_config(config['import'])
# Set up the pipeline.
if self.query is None:
stages = [read_tasks(self)]
else:
stages = [query_tasks(self)]
if self.config['pretend']:
# Only log the imported files and end the pipeline
stages += [log_files(self)]
else:
if self.config['group_albums'] and \
not self.config['singletons']:
# Split directory tasks into one task for each album
stages += [group_albums(self)]
if self.config['autotag']:
# FIXME We should also resolve duplicates when not
# autotagging. This is currently handled in `user_query`
stages += [lookup_candidates(self), user_query(self)]
else:
stages += [import_asis(self)]
stages += [apply_choices(self)]
for stage_func in plugins.import_stages():
stages.append(plugin_stage(self, stage_func))
stages += [manipulate_files(self)]
pl = pipeline.Pipeline(stages)
# Run the pipeline.
plugins.send('import_begin', session=self)
try:
if config['threaded']:
pl.run_parallel(QUEUE_SIZE)
else:
pl.run_sequential()
except ImportAbort:
# User aborted operation. Silently stop.
pass
def setUp(self):
self.l = []
self.pl = pipeline.Pipeline((_produce(), _work(), _consume(self.l)))
def test_pull(self):
pl = pipeline.Pipeline((_produce(), _multi_work()))
self.assertEqual(list(pl.pull()), [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])
def test_constrained_exception(self):
# Raise an exception in a constrained pipeline.
l = []
pl = pipeline.Pipeline((_produce(1000), _exc_work(), _consume(l)))
self.assertRaises(TestException, pl.run_parallel, 1)
def user_query(session):
"""A coroutine for interfacing with the user about the tagging
process.
The coroutine accepts an ImportTask objects. It uses the
session's ``choose_match`` method to determine the ``action`` for
this task. Depending on the action additional stages are exectuted
and the processed task is yielded.
It emits the ``import_task_choice`` event for plugins. Plugins have
acces to the choice via the ``taks.choice_flag`` property and may
choose to change it.
"""
recent = set()
task = None
while True:
task = yield task
if task.should_skip():
continue
# Ask the user for a choice.
choice = session.choose_match(task)
task.set_choice(choice)
session.log_choice(task)
plugins.send('import_task_choice', session=session, task=task)
# As-tracks: transition to singleton workflow.
if task.choice_flag is action.TRACKS:
# Set up a little pipeline for dealing with the singletons.
def emitter(task):
for item in task.items:
yield ImportTask.item_task(item)
yield ImportTask.progress_sentinel(task.toppath, task.paths)
ipl = pipeline.Pipeline([
emitter(task),
item_lookup(session),
item_query(session),
])
task = pipeline.multiple(ipl.pull())
continue
# As albums: group items by albums and create task for each album
if task.choice_flag is action.ALBUMS:
def emitter(task):
yield task
ipl = pipeline.Pipeline([
emitter(task),
group_albums(session),
initial_lookup(session),
user_query(session)
])
task = pipeline.multiple(ipl.pull())
continue
# Check for duplicates if we have a match (or ASIS).
if task.choice_flag in (action.ASIS, action.APPLY):
ident = task.chosen_ident()
# The "recent" set keeps track of identifiers for recently
# imported albums -- those that haven't reached the database
# yet.
if ident in recent or _duplicate_check(session.lib, task):
session.resolve_duplicate(task)
session.log_choice(task, True)
recent.add(ident)
def test_pull(self):
pl = pipeline.Pipeline((_produce(), _exc_work()))
pull = pl.pull()
for i in range(3):
next(pull)
self.assertRaises(ExceptionFixture, pull.__next__)
def test_pull(self):
pl = pipeline.Pipeline((_produce(), _work()))
self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])
def test_pull_chain(self):
pl = pipeline.Pipeline((_produce(), _work()))
pl2 = pipeline.Pipeline((pl.pull(), _work()))
self.assertEqual(list(pl2.pull()), [0, 4, 8, 12, 16])
def test_pull(self):
pl = pipeline.Pipeline((_produce(), _exc_work()))
pull = pl.pull()
for i in range(3):
pull.next()
self.assertRaises(TestException, pull.next)