def test_delayed_creation(self):
def creator():
yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']}
yield {'name': 'foo2', 'actions': None, 'targets': ['bar']}
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = [Task('t0', None, task_dep=['t1']),
Task('t1', None, loader=delayed_loader),
Task('t2', None)]
control = TaskControl(tasks)
control.process(['t0'])
disp = control.task_dispatcher()
gen = disp.generator
nt2 = next(gen)
assert nt2.task.name == "t2"
# wait for t2 to be executed
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
# delayed creation of tasks for t1 does not mess existing info
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
nf2 = gen.send(nt2)
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
assert nf2.task.name == "t1:foo2"
nf1 = gen.send(nf2)
assert nf1.task.name == "t1:foo1"
assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added
nt1 = gen.send(nf1)
assert nt1.task.name == "t1"
nt0 = gen.send(nt1)
assert nt0.task.name == "t0"
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def testPosParam(self):
tasks = list(TASKS_SAMPLE)
tasks.append(Task("tP", [""],[],[], pos_arg='myp'))
tc = TaskControl(tasks)
args = ["tP", "hello option!", "t1"]
assert ['tP',] == tc._filter_tasks(args)
assert ["hello option!", "t1"] == tc.tasks['tP'].pos_arg_val
def test_wild(self):
tasks = [Task('t1', None, task_dep=['foo*']),
Task(
'foo4',
None,
)]
TaskControl(tasks)
assert 'foo4' in tasks[0].task_dep
def test_filter_delayed_subtask(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None, loader=DelayedLoader(lambda: None))
control = TaskControl([t1, t2])
control._filter_tasks(['taskY:foo'])
assert isinstance(t2.loader, DelayedLoader)
# sub-task will use same loader, and keep parent basename
assert control.tasks['taskY:foo'].loader.basename == 'taskY'
assert control.tasks['taskY:foo'].loader is t2.loader
def _execute(self, subtasks, reverse, horizontal, outfile, pos_args=None):
# init
control = TaskControl(self.task_list)
self.tasks = control.tasks
self.subtasks = subtasks
self._edges = set() # used to avoid adding same edge twice
# create graph
self.graph = pygraphviz.AGraph(strict=False, directed=True)
self.graph.node_attr['color'] = 'lightblue2'
self.graph.node_attr['style'] = 'filled'
if (horizontal):
self.graph.graph_attr.update(rankdir='LR')
# populate graph
processed = set() # str - task name
if pos_args:
to_process = deque(pos_args)
else:
to_process = deque(control.tasks.keys())
while to_process:
task = control.tasks[to_process.popleft()]
if task.name in processed:
continue
processed.add(task.name)
# add nodes
node_attrs = {}
if task.has_subtask:
node_attrs['peripheries'] = '2'
if (not task.subtask_of) or subtasks:
self.graph.add_node(task.name, **node_attrs)
# add edges
for sink_name in task.setup_tasks:
self.add_edge(task.name, sink_name, arrowhead='empty')
if sink_name not in processed:
to_process.append(sink_name)
for sink_name in task.task_dep:
self.add_edge(task.name, sink_name, arrowhead='')
if sink_name not in processed:
to_process.append(sink_name)
if not outfile:
name = pos_args[0] if len(pos_args) == 1 else 'tasks'
outfile = '{}.dot'.format(name)
print('Generated file: {}'.format(outfile))
if (reverse):
self.graph.reverse().write(outfile)
else:
self.graph.write(outfile)
def test_normal(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task("t2", None,)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher().generator
n2 = next(gen)
assert tasks[1] == n2.task
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
assert tasks[0] == gen.send(n2).task
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def test_delayed_creation_target_regex(self):
def creator():
yield Task('foo', None, targets=['tgt1'])
delayed_loader = DelayedLoader(creator,
executed='t2',
target_regex='tgt1')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
tc = TaskControl(list(tasks.values()))
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
n2.run_status = 'done'
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen2 = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n5 = next(gen2)
assert n5.task.name == 'foo'
# get internal created task
n5.run_status = 'done'
td._update_waiting(n5)
n6 = next(gen2)
assert n6.name == '_regex_target_tgt1:t1'
# file_dep is removed because foo might not be task
# that creates this task (support for multi regex matches)
assert n6.file_dep == {}
def test_include_setup(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task(
"t2",
None,
)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher(include_setup=True).generator
# dont wait for tasks
assert tasks[0] == gen.send(None).task
assert tasks[1] == gen.send(None).task
pytest.raises(StopIteration, gen.send, None)
def test_filter_delayed_multi_select(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None,
loader=DelayedLoader(lambda: None, target_regex='a.*'))
t3 = Task("taskZ", None,
loader=DelayedLoader(lambda: None, target_regex='b.*'))
t4 = Task("taskW", None,
loader=DelayedLoader(lambda: None))
control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
selected = control._filter_tasks(['abc', 'att'])
assert isinstance(t2.loader, DelayedLoader)
assert len(selected) == 2
assert selected[0] == '_regex_target_abc:taskY'
assert selected[1] == '_regex_target_att:taskY'
def test_regex_not_found(self):
def creator1():
yield Task('foo1', None, targets=['tgt1'])
delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')
t1 = Task('t1', None, loader=delayed_loader1)
tc = TaskControl([t1])
selection = tc._filter_tasks(['tgt666'])
assert ['_regex_target_tgt666:t1'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt666:t1')
gen = td._add_task(n1)
# target not found after generating all tasks from regex group
pytest.raises(InvalidCommand, next, gen)
def test_regex_group_already_created(self):
# this is required to avoid loading more tasks than required, GH-#60
def creator1():
yield Task('foo1', None, targets=['tgt1'])
delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')
def creator2(): # pragma: no cover
yield Task('foo2', None, targets=['tgt2'])
delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*')
t1 = Task('t1', None, loader=delayed_loader1)
t2 = Task('t2', None, loader=delayed_loader2)
tc = TaskControl([t1, t2])
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# delayed loader executed, so generator is reset
n1b = next(gen)
assert n1b == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen1b = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n1c = next(gen1b)
assert n1c.task.name == 'foo1'
# get internal created task
n1c.run_status = 'done'
td._update_waiting(n1c)
n1d = next(gen1b)
assert n1d.name == '_regex_target_tgt1:t1'
## go for second selected task
n2 = td._gen_node(None, '_regex_target_tgt1:t2')
gen2 = td._add_task(n2)
# loader is not executed because target t1 was already found
pytest.raises(StopIteration, next, gen2)
def test_filter_delayed_regex_auto(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None,
loader=DelayedLoader(lambda: None, target_regex='a.*'))
t3 = Task("taskZ", None,
loader=DelayedLoader(lambda: None))
control = TaskControl([t1, t2, t3], auto_delayed_regex=True)
selected = control._filter_tasks(['abc'])
assert len(selected) == 2
assert (sorted(selected) ==
['_regex_target_abc:taskY', '_regex_target_abc:taskZ'])
assert control.tasks['_regex_target_abc:taskY'].file_dep == {'abc'}
assert control.tasks['_regex_target_abc:taskZ'].file_dep == {'abc'}
assert (control.tasks['_regex_target_abc:taskY'].loader.basename ==
t2.name)
assert (control.tasks['_regex_target_abc:taskZ'].loader.basename ==
t3.name)
def test_filter_delayed_regex_single(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None,
loader=DelayedLoader(lambda: None, target_regex='a.*'))
t3 = Task("taskZ", None,
loader=DelayedLoader(lambda: None, target_regex='b.*'))
t4 = Task("taskW", None,
loader=DelayedLoader(lambda: None))
control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
selected = control._filter_tasks(['abc'])
assert isinstance(t2.loader, DelayedLoader)
assert len(selected) == 1
assert selected[0] == '_regex_target_abc:taskY'
sel_task = control.tasks['_regex_target_abc:taskY']
assert sel_task.file_dep == {'abc'}
assert sel_task.loader.basename == 'taskY'
assert sel_task.loader is t2.loader
def _execute(self,
outfile=sys.stdout,
verbosity=None,
always=False,
continue_=False,
reporter='default',
num_process=0,
par_type='process',
single=False,
pipeline_name="Custom Pipeline"):
"""
@param reporter: (str) one of provided reporters or ...
(class) user defined reporter class
(can only be specified
from DOIT_CONFIG - never from command line)
(reporter instance) - only used in unittests
"""
# get tasks to be executed
# self.control is saved on instance to be used by 'auto' command
self.control = TaskControl(self.task_list)
self.control.process(self.sel_tasks)
if single:
for task_name in self.sel_tasks:
task = self.control.tasks[task_name]
if task.has_subtask:
for task_name in task.task_dep:
sub_task = self.control.tasks[task_name]
sub_task.task_dep = []
else:
task.task_dep = []
# reporter
if isinstance(reporter, six.string_types):
if reporter not in REPORTERS:
msg = ("No reporter named '%s'."
" Type 'doit help run' to see a list "
"of available reporters.")
raise InvalidCommand(msg % reporter)
reporter_cls = REPORTERS[reporter]
else:
# user defined class
reporter_cls = reporter
# verbosity
if verbosity is None:
use_verbosity = Task.DEFAULT_VERBOSITY
else:
use_verbosity = verbosity
show_out = use_verbosity < 2 # show on error report
# outstream
if isinstance(outfile, six.string_types):
outstream = codecs.open(outfile, 'w', encoding='utf-8')
else: # outfile is a file-like object (like StringIO or sys.stdout)
outstream = outfile
# run
try:
# FIXME stderr will be shown twice in case of task error/failure
if isinstance(reporter_cls, type):
reporter_obj = reporter_cls(
outstream, {
'show_out': show_out,
'show_err': True,
'reporter_url': self.opt_values['reporter_url'],
'auth_info': self.opt_values['auth_info']
})
else: # also accepts reporter instances
reporter_obj = reporter_cls
run_args = [
self.dep_class, self.dep_file, reporter_obj, continue_, always,
verbosity
]
run_kwargs = {}
RunnerClass = RUNNER_MAP.get(self.opt_values["runner"])
if not RunnerClass:
RunnerClass = self._discover_runner_class(
num_process, par_type)
elif self.opt_values['runner'] in GRID_RUNNER_MAP:
if not self.opt_values.get('partition', None):
raise InvalidCommand("--partition option is required "
"when using a grid runner")
run_args = [
self.opt_values['partition'], self.opt_values['perf_url'],
self.opt_values['tmpfiledir'], self.opt_values['grid_args']
] + run_args
run_kwargs['num_process'] = num_process if num_process else 1
runner = RunnerClass(*run_args, **run_kwargs)
runner.pipeline_name = pipeline_name
return runner.run_all(self.control.task_dispatcher())
finally:
if isinstance(outfile, str):
outstream.close()
def testFilterPattern(self, tasks_sample):
tc = TaskControl(tasks_sample)
assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
def testOptions(self):
options = ["t3", "--message", "hello option!", "t1"]
tc = TaskControl(TASKS_SAMPLE)
assert ['t3', 't1'] == tc._filter_tasks(options)
assert "hello option!" == tc.tasks['t3'].options['opt1']
def testFilterEmptyList(self):
filter_ = []
tc = TaskControl(TASKS_SAMPLE)
assert filter_ == tc._filter_tasks(filter_)
def testFilterWrongName(self):
tc = TaskControl(TASKS_SAMPLE)
pytest.raises(InvalidCommand, tc._filter_tasks, ['no'])
def test_targetDependency(self):
t1 = Task("taskX", None, [], ['intermediate'])
t2 = Task("taskY", None, ['intermediate'], [])
TaskControl([t1, t2])
assert ['taskX'] == t2.task_dep
def test_addTask(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None)
tc = TaskControl([t1, t2])
assert 2 == len(tc.tasks)
def testProcessAll(self, tasks_sample):
tc = TaskControl(tasks_sample)
tc.process(None)
assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testProcessAll(self):
tc = TaskControl(TASKS_SAMPLE)
tc.process(None)
assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testFilter(self):
filter_ = ['t2', 't3']
tc = TaskControl(TASKS_SAMPLE)
assert filter_ == tc._filter_tasks(filter_)
def test_bug770150_task_dependency_from_target(self):
t1 = Task("taskX", None, file_dep=[], targets=['intermediate'])
t2 = Task("taskY", None, file_dep=['intermediate'], task_dep=['taskZ'])
t3 = Task("taskZ", None)
TaskControl([t1, t2, t3])
assert ['taskZ', 'taskX'] == t2.task_dep
def testFilterWrongSubtaskName(self):
t1 = Task("taskX", None)
t2 = Task("taskY", None)
tc = TaskControl([t1, t2])
pytest.raises(InvalidCommand, tc._filter_tasks, ['taskX:no'])
def testFilterSubtask(self):
filter_ = ["t1", "g1.b"]
tc = TaskControl(TASKS_SAMPLE)
assert filter_ == tc._filter_tasks(filter_)
def testProcessSelection(self):
filter_ = ['t2', 't3']
tc = TaskControl(TASKS_SAMPLE)
tc.process(filter_)
assert filter_ == tc.selected_tasks
def testFilterTarget(self):
tasks = list(TASKS_SAMPLE)
tasks.append(Task("tX", [""], [], ["targetX"]))
tc = TaskControl(tasks)
assert ['tX'] == tc._filter_tasks(["targetX"])
def testFilterPattern(self):
tc = TaskControl(TASKS_SAMPLE)
assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
def testProcessSelection(self, tasks_sample):
filter_ = ['t2', 't3']
tc = TaskControl(tasks_sample)
tc.process(filter_)
assert filter_ == tc.selected_tasks
