def test_calc_dep(self):
def calc_intermediate():
return {'file_dep': ['intermediate']}
tasks = {'t1': Task('t1', None, calc_dep=['t2']),
't2': Task('t2', [calc_intermediate]),
't3': Task('t3', None, targets=['intermediate']),
}
td = TaskDispatcher(tasks, {'intermediate': 't3'}, None)
n1 = td._gen_node(None, 't1')
n2 = n1.step()
assert tasks['t2'] == n2.task
assert 'wait' == n1.step()
# execute t2 to process calc_dep
tasks['t2'].execute()
td.nodes['t2'].run_status = 'done'
td._update_waiting(n2)
n3 = n1.step()
assert tasks['t3'] == n3.task
assert 'intermediate' in tasks['t1'].file_dep
assert 't3' in tasks['t1'].task_dep
# t3 was added by calc dep
assert 'wait' == n1.step()
n3.run_status = 'done'
td._update_waiting(n3)
assert tasks['t1'] == n1.step()
def test_delayed_creation(self):
def creator():
yield Task('foo', None, loader=DelayedLoader(lambda: None))
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# recursive loader is preserved
assert isinstance(td.tasks['foo'].loader, DelayedLoader)
pytest.raises(AssertionError, next, gen)
def test_delayed_creation(self):
def creator():
yield Task('foo', None, loader=DelayedLoader(lambda : None))
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = {'t1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# recursive loader is preserved
assert isinstance(td.tasks['foo'].loader, DelayedLoader)
pytest.raises(AssertionError, next, gen)
def test_calc_dep(self):
def calc_intermediate():
return {'file_dep': ['intermediate']}
tasks = {
't1': Task('t1', None, calc_dep=['t2']),
't2': Task('t2', [calc_intermediate]),
't3': Task('t3', None, targets=['intermediate']),
}
td = TaskDispatcher(tasks, {'intermediate': 't3'}, None)
n1 = td._gen_node(None, 't1')
n2 = n1.step()
assert tasks['t2'] == n2.task
assert 'wait' == n1.step()
# execute t2 to process calc_dep
tasks['t2'].execute()
td.nodes['t2'].run_status = 'done'
td._update_waiting(n2)
n3 = n1.step()
assert tasks['t3'] == n3.task
assert 'intermediate' in tasks['t1'].file_dep
assert 't3' in tasks['t1'].task_dep
# t3 was added by calc dep
assert 'wait' == n1.step()
n3.run_status = 'done'
td._update_waiting(n3)
assert tasks['t1'] == n1.step()
def test_wait_select(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n2 = td._gen_node(None, 't2')
n2.wait_select = True
n2.run_status = 'run'
td.waiting.add(n2)
td._update_waiting(n2)
assert False == n2.wait_select
assert deque([n2]) == td.ready
def test_task_deps_already_created(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
assert 'wait' == n1.step()
assert 'wait' == n1.step()
#tasks['t2'].run_status = 'done'
td._update_waiting(n2)
assert tasks['t1'] == n1.step()
def test_wait_select(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n2 = td._gen_node(None, 't2')
n2.wait_select = True
n2.run_status = 'run'
td.waiting.add(n2)
td._update_waiting(n2)
assert False == n2.wait_select
assert deque([n2]) == td.ready
def test_task_deps_already_created(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
assert 'wait' == n1.step()
assert 'wait' == n1.step()
#tasks['t2'].run_status = 'done'
td._update_waiting(n2)
assert tasks['t1'] == n1.step()
def test_delayed_creation_target_regex(self):
def creator():
yield Task('foo', None, targets=['tgt1'])
delayed_loader = DelayedLoader(creator,
executed='t2',
target_regex='tgt1')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
tc = TaskControl(list(tasks.values()))
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
n2.run_status = 'done'
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen2 = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n5 = next(gen2)
assert n5.task.name == 'foo'
# get internal created task
n5.run_status = 'done'
td._update_waiting(n5)
n6 = next(gen2)
assert n6.name == '_regex_target_tgt1:t1'
# file_dep is removed because foo might not be task
# that creates this task (support for multi regex matches)
assert n6.file_dep == {}
def test_delayed_creation_sub_task(self):
# usually a repeated loader is replaced by the real task
# when it is first executed, the problem arises when the
# the expected task is not actually created
def creator():
yield Task('t1:foo', None)
yield Task('t1:bar', None)
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
# simulate a sub-task from delayed created added to task_list
tasks['t1:foo'] = Task('t1:foo', None, loader=delayed_loader)
tasks['t1:xxx'] = Task('t1:xxx', None, loader=delayed_loader)
delayed_loader.basename = 't1'
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1:foo')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n1b = next(gen)
assert n1b.task.name == 't2'
# wait until t2 is finished
n1c = next(gen)
assert n1c == 'wait'
# after t2 is done, generator is reseted
n1b.run_status = 'successful'
td._update_waiting(n1b)
n1d = next(gen)
assert n1d == "reset generator"
assert 't1:foo' in td.tasks
assert 't1:bar' in td.tasks
# finish with t1:foo
gen2 = td._add_task(n1)
n1.reset_task(td.tasks[n1.task.name], gen2)
n2 = next(gen2)
assert n2.name == 't1:foo'
pytest.raises(StopIteration, next, gen2)
# try non-existent t1:xxx
n3 = td._gen_node(None, 't1:xxx')
gen3 = td._add_task(n3)
# ? should raise a runtime error?
assert next(gen3) == 'reset generator'
def test_wait_run_deps_not_ok(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
td._node_add_wait_run(n1, ['t2'])
n2.run_status = 'failure'
td.waiting.add(n1)
td._update_waiting(n2)
assert n1.bad_deps
assert deque([n1]) == td.ready
assert 0 == len(td.waiting)
def test_delayed_creation_target_regex(self):
def creator():
yield Task('foo', None, targets=['tgt1'])
delayed_loader = DelayedLoader(creator,
executed='t2', target_regex='tgt1')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
tc = TaskControl(list(tasks.values()))
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
n2.run_status = 'done'
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen2 = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n5 = next(gen2)
assert n5.task.name == 'foo'
# get internal created task
n5.run_status = 'done'
td._update_waiting(n5)
n6 = next(gen2)
assert n6.name == '_regex_target_tgt1:t1'
# file_dep is removed because foo might not be task
# that creates this task (support for multi regex matches)
assert n6.file_dep == {}
def test_wait_run_deps_not_ok(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
td._node_add_wait_run(n1, ['t2'])
n2.run_status = 'failure'
td.waiting.add(n1)
td._update_waiting(n2)
assert n1.bad_deps
assert deque([n1]) == td.ready
assert 0 == len(td.waiting)
def test_delayed_creation_sub_task(self):
# usually a repeated loader is replaced by the real task
# when it is first executed, the problem arises when the
# the expected task is not actually created
def creator():
yield Task('t1:foo', None)
yield Task('t1:bar', None)
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),}
# simulate a sub-task from delayed created added to task_list
tasks['t1:foo'] = Task('t1:foo', None, loader=delayed_loader)
tasks['t1:xxx'] = Task('t1:xxx', None, loader=delayed_loader)
delayed_loader.basename = 't1'
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1:foo')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n1b = next(gen)
assert n1b.task.name == 't2'
# wait until t2 is finished
n1c = next(gen)
assert n1c == 'wait'
# after t2 is done, generator is reseted
n1b.run_status = 'successful'
td._update_waiting(n1b)
n1d = next(gen)
assert n1d == "reset generator"
assert 't1:foo' in td.tasks
assert 't1:bar' in td.tasks
# finish with t1:foo
gen2 = td._add_task(n1)
n1.reset_task(td.tasks[n1.task.name], gen2)
n2 = next(gen2)
assert n2.name == 't1:foo'
pytest.raises(StopIteration, next, gen2)
# try non-existent t1:xxx
n3 = td._gen_node(None, 't1:xxx')
gen3 = td._add_task(n3)
# ? should raise a runtime error?
assert next(gen3) == 'reset generator'
def test_regex_group_already_created(self):
# this is required to avoid loading more tasks than required, GH-#60
def creator1():
yield Task('foo1', None, targets=['tgt1'])
delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')
def creator2(): # pragma: no cover
yield Task('foo2', None, targets=['tgt2'])
delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*')
t1 = Task('t1', None, loader=delayed_loader1)
t2 = Task('t2', None, loader=delayed_loader2)
tc = TaskControl([t1, t2])
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# delayed loader executed, so generator is reset
n1b = next(gen)
assert n1b == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen1b = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n1c = next(gen1b)
assert n1c.task.name == 'foo1'
# get internal created task
n1c.run_status = 'done'
td._update_waiting(n1c)
n1d = next(gen1b)
assert n1d.name == '_regex_target_tgt1:t1'
## go for second selected task
n2 = td._gen_node(None, '_regex_target_tgt1:t2')
gen2 = td._add_task(n2)
# loader is not executed because target t1 was already found
pytest.raises(StopIteration, next, gen2)
def test_regex_group_already_created(self):
# this is required to avoid loading more tasks than required, GH-#60
def creator1():
yield Task('foo1', None, targets=['tgt1'])
delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')
def creator2(): # pragma: no cover
yield Task('foo2', None, targets=['tgt2'])
delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*')
t1 = Task('t1', None, loader=delayed_loader1)
t2 = Task('t2', None, loader=delayed_loader2)
tc = TaskControl([t1, t2])
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# delayed loader executed, so generator is reset
n1b = next(gen)
assert n1b == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen1b = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n1c = next(gen1b)
assert n1c.task.name == 'foo1'
# get internal created task
n1c.run_status = 'done'
td._update_waiting(n1c)
n1d = next(gen1b)
assert n1d.name == '_regex_target_tgt1:t1'
## go for second selected task
n2 = td._gen_node(None, '_regex_target_tgt1:t2')
gen2 = td._add_task(n2)
# loader is not executed because target t1 was already found
pytest.raises(StopIteration, next, gen2)
def test_task_deps(self):
tasks = {'t1': Task('t1', None, task_dep=['t2', 't3']),
't2': Task('t2', None),
't3': Task('t3', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
gen = td._add_task(n1)
n2 = next(gen)
assert tasks['t2'] == n2.task
n3 = next(gen)
assert tasks['t3'] == n3.task
assert 'wait' == next(gen)
tasks['t2'].run_status = 'done'
td._update_waiting(n2)
tasks['t3'].run_status = 'done'
td._update_waiting(n3)
assert tasks['t1'] == next(gen)
def test_task_deps(self):
tasks = {'t1': Task('t1', None, task_dep=['t2', 't3']),
't2': Task('t2', None),
't3': Task('t3', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
gen = td._add_task(n1)
n2 = next(gen)
assert tasks['t2'] == n2.task
n3 = next(gen)
assert tasks['t3'] == n3.task
assert 'wait' == next(gen)
tasks['t2'].run_status = 'done'
td._update_waiting(n2)
tasks['t3'].run_status = 'done'
td._update_waiting(n3)
assert tasks['t1'] == next(gen)
def test_waiting_node_updated(self):
tasks = {'t1': Task('t1', None, calc_dep=['t2'], task_dep=['t4']),
't2': Task('t2', None),
't3': Task('t3', None),
't4': Task('t4', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n1_gen = td._add_task(n1)
n2 = next(n1_gen)
assert 't2' == n2.task.name
assert 't4' == next(n1_gen).task.name
assert 'wait' == next(n1_gen)
assert set() == n1.calc_dep
assert td.waiting == set()
n2.run_status = 'done'
n2.task.values = {'calc_dep': ['t2', 't3'], 'task_dep':['t5']}
assert n1.calc_dep == set()
assert n1.task_dep == []
td._update_waiting(n2)
assert n1.calc_dep == set(['t3'])
assert n1.task_dep == ['t5']
def test_waiting_node_updated(self):
tasks = {'t1': Task('t1', None, calc_dep=['t2'], task_dep=['t4']),
't2': Task('t2', None),
't3': Task('t3', None),
't4': Task('t4', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n1_gen = td._add_task(n1)
n2 = next(n1_gen)
assert 't2' == n2.task.name
assert 't4' == next(n1_gen).task.name
assert 'wait' == next(n1_gen)
assert set() == n1.calc_dep
assert td.waiting == set()
n2.run_status = 'done'
n2.task.values = {'calc_dep': ['t2', 't3'], 'task_dep':['t5']}
assert n1.calc_dep == set()
assert n1.task_dep == []
td._update_waiting(n2)
assert n1.calc_dep == set(['t3'])
assert n1.task_dep == ['t5']
