def test_calc_dep(self): def calc_intermediate(): return {'file_dep': ['intermediate']} tasks = {'t1': Task('t1', None, calc_dep=['t2']), 't2': Task('t2', [calc_intermediate]), 't3': Task('t3', None, targets=['intermediate']), } td = TaskDispatcher(tasks, {'intermediate': 't3'}, None) n1 = td._gen_node(None, 't1') n2 = n1.step() assert tasks['t2'] == n2.task assert 'wait' == n1.step() # execute t2 to process calc_dep tasks['t2'].execute() td.nodes['t2'].run_status = 'done' td._update_waiting(n2) n3 = n1.step() assert tasks['t3'] == n3.task assert 'intermediate' in tasks['t1'].file_dep assert 't3' in tasks['t1'].task_dep # t3 was added by calc dep assert 'wait' == n1.step() n3.run_status = 'done' td._update_waiting(n3) assert tasks['t1'] == n1.step()
def test_delayed_creation(self): def creator(): yield Task('foo', None, loader=DelayedLoader(lambda: None)) delayed_loader = DelayedLoader(creator, executed='t2') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # recursive loader is preserved assert isinstance(td.tasks['foo'].loader, DelayedLoader) pytest.raises(AssertionError, next, gen)
def test_delayed_creation(self): def creator(): yield Task('foo', None, loader=DelayedLoader(lambda : None)) delayed_loader = DelayedLoader(creator, executed='t2') tasks = {'t1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # recursive loader is preserved assert isinstance(td.tasks['foo'].loader, DelayedLoader) pytest.raises(AssertionError, next, gen)
def test_calc_dep(self): def calc_intermediate(): return {'file_dep': ['intermediate']} tasks = { 't1': Task('t1', None, calc_dep=['t2']), 't2': Task('t2', [calc_intermediate]), 't3': Task('t3', None, targets=['intermediate']), } td = TaskDispatcher(tasks, {'intermediate': 't3'}, None) n1 = td._gen_node(None, 't1') n2 = n1.step() assert tasks['t2'] == n2.task assert 'wait' == n1.step() # execute t2 to process calc_dep tasks['t2'].execute() td.nodes['t2'].run_status = 'done' td._update_waiting(n2) n3 = n1.step() assert tasks['t3'] == n3.task assert 'intermediate' in tasks['t1'].file_dep assert 't3' in tasks['t1'].task_dep # t3 was added by calc dep assert 'wait' == n1.step() n3.run_status = 'done' td._update_waiting(n3) assert tasks['t1'] == n1.step()
def test_wait_select(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n2 = td._gen_node(None, 't2') n2.wait_select = True n2.run_status = 'run' td.waiting.add(n2) td._update_waiting(n2) assert False == n2.wait_select assert deque([n2]) == td.ready
def test_task_deps_already_created(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') assert 'wait' == n1.step() assert 'wait' == n1.step() #tasks['t2'].run_status = 'done' td._update_waiting(n2) assert tasks['t1'] == n1.step()
def test_wait_select(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n2 = td._gen_node(None, 't2') n2.wait_select = True n2.run_status = 'run' td.waiting.add(n2) td._update_waiting(n2) assert False == n2.wait_select assert deque([n2]) == td.ready
def test_task_deps_already_created(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') assert 'wait' == n1.step() assert 'wait' == n1.step() #tasks['t2'].run_status = 'done' td._update_waiting(n2) assert tasks['t1'] == n1.step()
def test_delayed_creation_target_regex(self): def creator(): yield Task('foo', None, targets=['tgt1']) delayed_loader = DelayedLoader(creator, executed='t2', target_regex='tgt1') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } tc = TaskControl(list(tasks.values())) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted n2.run_status = 'done' td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen2 = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n5 = next(gen2) assert n5.task.name == 'foo' # get internal created task n5.run_status = 'done' td._update_waiting(n5) n6 = next(gen2) assert n6.name == '_regex_target_tgt1:t1' # file_dep is removed because foo might not be task # that creates this task (support for multi regex matches) assert n6.file_dep == {}
def test_delayed_creation_sub_task(self): # usually a repeated loader is replaced by the real task # when it is first executed, the problem arises when the # the expected task is not actually created def creator(): yield Task('t1:foo', None) yield Task('t1:bar', None) delayed_loader = DelayedLoader(creator, executed='t2') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } # simulate a sub-task from delayed created added to task_list tasks['t1:foo'] = Task('t1:foo', None, loader=delayed_loader) tasks['t1:xxx'] = Task('t1:xxx', None, loader=delayed_loader) delayed_loader.basename = 't1' td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1:foo') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n1b = next(gen) assert n1b.task.name == 't2' # wait until t2 is finished n1c = next(gen) assert n1c == 'wait' # after t2 is done, generator is reseted n1b.run_status = 'successful' td._update_waiting(n1b) n1d = next(gen) assert n1d == "reset generator" assert 't1:foo' in td.tasks assert 't1:bar' in td.tasks # finish with t1:foo gen2 = td._add_task(n1) n1.reset_task(td.tasks[n1.task.name], gen2) n2 = next(gen2) assert n2.name == 't1:foo' pytest.raises(StopIteration, next, gen2) # try non-existent t1:xxx n3 = td._gen_node(None, 't1:xxx') gen3 = td._add_task(n3) # ? should raise a runtime error? assert next(gen3) == 'reset generator'
def test_wait_run_deps_not_ok(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') td._node_add_wait_run(n1, ['t2']) n2.run_status = 'failure' td.waiting.add(n1) td._update_waiting(n2) assert n1.bad_deps assert deque([n1]) == td.ready assert 0 == len(td.waiting)
def test_delayed_creation_target_regex(self): def creator(): yield Task('foo', None, targets=['tgt1']) delayed_loader = DelayedLoader(creator, executed='t2', target_regex='tgt1') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } tc = TaskControl(list(tasks.values())) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted n2.run_status = 'done' td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen2 = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n5 = next(gen2) assert n5.task.name == 'foo' # get internal created task n5.run_status = 'done' td._update_waiting(n5) n6 = next(gen2) assert n6.name == '_regex_target_tgt1:t1' # file_dep is removed because foo might not be task # that creates this task (support for multi regex matches) assert n6.file_dep == {}
def test_wait_run_deps_not_ok(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') td._node_add_wait_run(n1, ['t2']) n2.run_status = 'failure' td.waiting.add(n1) td._update_waiting(n2) assert n1.bad_deps assert deque([n1]) == td.ready assert 0 == len(td.waiting)
def test_delayed_creation_sub_task(self): # usually a repeated loader is replaced by the real task # when it is first executed, the problem arises when the # the expected task is not actually created def creator(): yield Task('t1:foo', None) yield Task('t1:bar', None) delayed_loader = DelayedLoader(creator, executed='t2') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None),} # simulate a sub-task from delayed created added to task_list tasks['t1:foo'] = Task('t1:foo', None, loader=delayed_loader) tasks['t1:xxx'] = Task('t1:xxx', None, loader=delayed_loader) delayed_loader.basename = 't1' td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1:foo') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n1b = next(gen) assert n1b.task.name == 't2' # wait until t2 is finished n1c = next(gen) assert n1c == 'wait' # after t2 is done, generator is reseted n1b.run_status = 'successful' td._update_waiting(n1b) n1d = next(gen) assert n1d == "reset generator" assert 't1:foo' in td.tasks assert 't1:bar' in td.tasks # finish with t1:foo gen2 = td._add_task(n1) n1.reset_task(td.tasks[n1.task.name], gen2) n2 = next(gen2) assert n2.name == 't1:foo' pytest.raises(StopIteration, next, gen2) # try non-existent t1:xxx n3 = td._gen_node(None, 't1:xxx') gen3 = td._add_task(n3) # ? should raise a runtime error? assert next(gen3) == 'reset generator'
def test_regex_group_already_created(self): # this is required to avoid loading more tasks than required, GH-#60 def creator1(): yield Task('foo1', None, targets=['tgt1']) delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*') def creator2(): # pragma: no cover yield Task('foo2', None, targets=['tgt2']) delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*') t1 = Task('t1', None, loader=delayed_loader1) t2 = Task('t2', None, loader=delayed_loader2) tc = TaskControl([t1, t2]) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # delayed loader executed, so generator is reset n1b = next(gen) assert n1b == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen1b = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n1c = next(gen1b) assert n1c.task.name == 'foo1' # get internal created task n1c.run_status = 'done' td._update_waiting(n1c) n1d = next(gen1b) assert n1d.name == '_regex_target_tgt1:t1' ## go for second selected task n2 = td._gen_node(None, '_regex_target_tgt1:t2') gen2 = td._add_task(n2) # loader is not executed because target t1 was already found pytest.raises(StopIteration, next, gen2)
def test_regex_group_already_created(self): # this is required to avoid loading more tasks than required, GH-#60 def creator1(): yield Task('foo1', None, targets=['tgt1']) delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*') def creator2(): # pragma: no cover yield Task('foo2', None, targets=['tgt2']) delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*') t1 = Task('t1', None, loader=delayed_loader1) t2 = Task('t2', None, loader=delayed_loader2) tc = TaskControl([t1, t2]) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # delayed loader executed, so generator is reset n1b = next(gen) assert n1b == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen1b = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n1c = next(gen1b) assert n1c.task.name == 'foo1' # get internal created task n1c.run_status = 'done' td._update_waiting(n1c) n1d = next(gen1b) assert n1d.name == '_regex_target_tgt1:t1' ## go for second selected task n2 = td._gen_node(None, '_regex_target_tgt1:t2') gen2 = td._add_task(n2) # loader is not executed because target t1 was already found pytest.raises(StopIteration, next, gen2)
def test_task_deps(self): tasks = {'t1': Task('t1', None, task_dep=['t2', 't3']), 't2': Task('t2', None), 't3': Task('t3', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') gen = td._add_task(n1) n2 = next(gen) assert tasks['t2'] == n2.task n3 = next(gen) assert tasks['t3'] == n3.task assert 'wait' == next(gen) tasks['t2'].run_status = 'done' td._update_waiting(n2) tasks['t3'].run_status = 'done' td._update_waiting(n3) assert tasks['t1'] == next(gen)
def test_task_deps(self): tasks = {'t1': Task('t1', None, task_dep=['t2', 't3']), 't2': Task('t2', None), 't3': Task('t3', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') gen = td._add_task(n1) n2 = next(gen) assert tasks['t2'] == n2.task n3 = next(gen) assert tasks['t3'] == n3.task assert 'wait' == next(gen) tasks['t2'].run_status = 'done' td._update_waiting(n2) tasks['t3'].run_status = 'done' td._update_waiting(n3) assert tasks['t1'] == next(gen)
def test_waiting_node_updated(self): tasks = {'t1': Task('t1', None, calc_dep=['t2'], task_dep=['t4']), 't2': Task('t2', None), 't3': Task('t3', None), 't4': Task('t4', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n1_gen = td._add_task(n1) n2 = next(n1_gen) assert 't2' == n2.task.name assert 't4' == next(n1_gen).task.name assert 'wait' == next(n1_gen) assert set() == n1.calc_dep assert td.waiting == set() n2.run_status = 'done' n2.task.values = {'calc_dep': ['t2', 't3'], 'task_dep':['t5']} assert n1.calc_dep == set() assert n1.task_dep == [] td._update_waiting(n2) assert n1.calc_dep == set(['t3']) assert n1.task_dep == ['t5']
def test_waiting_node_updated(self): tasks = {'t1': Task('t1', None, calc_dep=['t2'], task_dep=['t4']), 't2': Task('t2', None), 't3': Task('t3', None), 't4': Task('t4', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n1_gen = td._add_task(n1) n2 = next(n1_gen) assert 't2' == n2.task.name assert 't4' == next(n1_gen).task.name assert 'wait' == next(n1_gen) assert set() == n1.calc_dep assert td.waiting == set() n2.run_status = 'done' n2.task.values = {'calc_dep': ['t2', 't3'], 'task_dep':['t5']} assert n1.calc_dep == set() assert n1.task_dep == [] td._update_waiting(n2) assert n1.calc_dep == set(['t3']) assert n1.task_dep == ['t5']