def test_continue_dont_execute_parent_of_failed_task( self, reporter, RunnerClass, depfile_name): t1 = Task("t1", [(_error, )]) t2 = Task("t2", [(ok, )], task_dep=['t1']) t3 = Task("t3", [(ok, )]) my_runner = RunnerClass(Dependency, depfile_name, reporter, continue_=True) disp = TaskDispatcher({ 't1': t1, 't2': t2, 't3': t3 }, [], ['t1', 't2', 't3']) my_runner.run_tasks(disp) assert runner.ERROR == my_runner.finish() assert ('start', t1) == reporter.log.pop(0) assert ('execute', t1) == reporter.log.pop(0) assert ('fail', t1) == reporter.log.pop(0) assert ('start', t2) == reporter.log.pop(0) assert ('fail', t2) == reporter.log.pop(0) assert ('start', t3) == reporter.log.pop(0) assert ('execute', t3) == reporter.log.pop(0) assert ('success', t3) == reporter.log.pop(0) assert 0 == len(reporter.log)
def test_none(self): tasks = { 't1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) assert None == td._get_next_node([], [])
def test_delayed_creation(self): def creator(): yield Task('foo', None, loader=DelayedLoader(lambda: None)) delayed_loader = DelayedLoader(creator, executed='t2') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # recursive loader is preserved assert isinstance(td.tasks['foo'].loader, DelayedLoader) pytest.raises(AssertionError, next, gen)
def test_no_deps(self): tasks = { 't1': Task('t1', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') assert [tasks['t1']] == list(td._add_task(n1))
def test_calc_dep(self): def calc_intermediate(): return {'file_dep': ['intermediate']} tasks = { 't1': Task('t1', None, calc_dep=['t2']), 't2': Task('t2', [calc_intermediate]), 't3': Task('t3', None, targets=['intermediate']), } td = TaskDispatcher(tasks, {'intermediate': 't3'}, None) n1 = td._gen_node(None, 't1') n2 = n1.step() assert tasks['t2'] == n2.task assert 'wait' == n1.step() # execute t2 to process calc_dep tasks['t2'].execute() td.nodes['t2'].run_status = 'done' td._update_waiting(n2) n3 = n1.step() assert tasks['t3'] == n3.task assert 'intermediate' in tasks['t1'].file_dep assert 't3' in tasks['t1'].task_dep # t3 was added by calc dep assert 'wait' == n1.step() n3.run_status = 'done' td._update_waiting(n3) assert tasks['t1'] == n1.step()
def test_to_run_none(self): tasks = {'t1': Task('t1', None), } td = TaskDispatcher(tasks, [], None) td._gen_node(None, 't1') # t1 was already created to_run = ['t1'] assert None == td._get_next_node([], to_run) assert [] == to_run
def test_stop_running(self, reporter, dep_manager): t1 = Task('t1', []) t2 = Task('t2', []) run = runner.MRunner(dep_manager, reporter) run._run_tasks_init(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) assert t1.name == run.get_next_job(None).name run._stop_running = True assert None == run.get_next_job(None)
def test_cyclic(self): tasks = {'t1': Task('t1', None), 't2': Task('t2', None) } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(n1, 't2') pytest.raises(InvalidDodoFile, td._gen_node, n2, 't1')
def test_already_created(self): tasks = {'t1': Task('t1', None), 't2': Task('t2', None) } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') td._gen_node(n1, 't2') assert None == td._gen_node(None, 't1')
def test_MRunner_pickable(dep_manager): t1 = Task('t1', []) import sys reporter = ConsoleReporter(sys.stdout, {}) run = runner.MRunner(dep_manager, reporter) run._run_tasks_init(TaskDispatcher({'t1': t1}, [], ['t1'])) # assert nothing is raised pickle.dumps(run)
def test_teardown(self, reporter, RunnerClass, dep_manager): t1 = Task('t1', [], teardown=[ok]) t2 = Task('t2', []) my_runner = RunnerClass(dep_manager, reporter) assert [] == my_runner.teardown_list my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) my_runner.finish() assert ('teardown', t1) == reporter.log[-1]
def test_task_cloudpicklabe_multiprocess(self, reporter, dep_manager): t1 = Task("t1", [(my_print, ["out a"] )] ) t2 = Task("t2", None, loader=DelayedLoader( non_pickable_creator, executed='t1')) my_runner = runner.MRunner(dep_manager, reporter) dispatcher = TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']) my_runner.run_tasks(dispatcher) assert runner.SUCCESS == my_runner.finish()
def test_task_not_picklabe_multiprocess(self, reporter, dep_manager): def creator(): return {'basename': 't2', 'actions': [lambda: 5]} t1 = Task("t1", [(my_print, ["out a"])]) t2 = Task("t2", None, loader=DelayedLoader(creator, executed='t1')) my_runner = runner.MRunner(dep_manager, reporter) dispatcher = TaskDispatcher({'t1': t1, 't2': t2}, [], ['t1', 't2']) pytest.raises(InvalidTask, my_runner.run_tasks, dispatcher)
def test_ready(self): tasks = {'t1': Task('t1', None), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') ready = deque([n1]) assert n1 == td._get_next_node(ready, ['t2']) assert 0 == len(ready)
def test_deps_not_ok(self): tasks = {'t1': Task('t1', None), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') n2.run_status = 'failure' td._node_add_wait_run(n1, ['t2']) assert n1.bad_deps
def test_none(self): tasks = {'t1': Task('t1', None), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') n2.run_status = 'done' td._node_add_wait_run(n1, ['t2']) assert not n1.wait_run
def test_task_deps_no_wait(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') n2.run_status = 'done' gen = td._add_task(n1) assert tasks['t1'] == next(gen)
def test_reporter_runtime_error(self, reporter, dep_manager): t1 = Task('t1', [], calc_dep=['t2']) t2 = Task('t2', [lambda: {'file_dep':[1]}]) my_runner = runner.Runner(dep_manager, reporter) my_runner.run_all(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) assert runner.ERROR == my_runner.final_result assert ('start', t2) == reporter.log.pop(0) assert ('execute', t2) == reporter.log.pop(0) assert ('success', t2) == reporter.log.pop(0) assert ('runtime_error',) == reporter.log.pop(0) assert not reporter.log
def test_to_run(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) to_run = ['t2', 't1'] td._gen_node(None, 't1') # t1 was already created got = td._get_next_node([], to_run) assert isinstance(got, ExecNode) assert 't2' == got.task.name assert [] == to_run
def test_error(self, reporter, RunnerClass, dep_manager): t1 = Task("t1", [_error]) t2 = Task("t2", [_error]) my_runner = RunnerClass(dep_manager, reporter) my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) assert runner.ERROR == my_runner.finish() assert ('start', t1) == reporter.log.pop(0) assert ('execute', t1) == reporter.log.pop(0) assert ('fail', t1) == reporter.log.pop(0) # second task is not executed assert 0 == len(reporter.log)
def test_run_task(self, reporter, depfile): t1 = Task('t1', []) t2 = Task('t2', []) run = runner.MRunner(depfile.name, reporter) run._run_tasks_init( TaskDispatcher({ 't1': t1, 't2': t2 }, [], ['t1', 't2'])) assert t1 == run.get_next_task(None).task assert t2 == run.get_next_task(None).task assert None == run.get_next_task(None)
def test_task_deps_already_created(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n1 = td._gen_node(None, 't1') n2 = td._gen_node(None, 't2') assert 'wait' == n1.step() assert 'wait' == n1.step() #tasks['t2'].run_status = 'done' td._update_waiting(n2) assert tasks['t1'] == n1.step()
def test_success(self, reporter, RunnerClass, dep_manager): t1 = Task("t1", [(my_print, ["out a"] )] ) t2 = Task("t2", [(my_print, ["out a"] )] ) my_runner = RunnerClass(dep_manager, reporter) my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) assert runner.SUCCESS == my_runner.finish() assert ('start', t1) == reporter.log.pop(0), reporter.log assert ('execute', t1) == reporter.log.pop(0) assert ('success', t1) == reporter.log.pop(0) assert ('start', t2) == reporter.log.pop(0) assert ('execute', t2) == reporter.log.pop(0) assert ('success', t2) == reporter.log.pop(0)
def test_wait_select(self): tasks = {'t1': Task('t1', None, task_dep=['t2']), 't2': Task('t2', None), } td = TaskDispatcher(tasks, [], None) n2 = td._gen_node(None, 't2') n2.wait_select = True n2.run_status = 'run' td.waiting.add(n2) td._update_waiting(n2) assert False == n2.wait_select assert deque([n2]) == td.ready
def testActionModifiesFiledep(self, reporter, RunnerClass, dep_manager): extra_dep = os.path.join(os.path.dirname(__file__), 'sample_md5.txt') t1 = Task("t1", [(my_print, ["out a"] ), (action_add_filedep, (), {'extra_dep': extra_dep}) ] ) my_runner = RunnerClass(dep_manager, reporter) my_runner.run_tasks(TaskDispatcher({'t1':t1}, [], ['t1'])) assert runner.SUCCESS == my_runner.finish() assert ('start', t1) == reporter.log.pop(0), reporter.log assert ('execute', t1) == reporter.log.pop(0) assert ('success', t1) == reporter.log.pop(0) assert t1.file_dep == set([extra_dep])
def test_continue_dep_error(self, reporter, RunnerClass, dep_manager): t1 = Task("t1", [(ok, )], file_dep=['i_dont_exist']) t2 = Task("t2", [(ok, )], task_dep=['t1']) my_runner = RunnerClass(dep_manager, reporter, continue_=True) disp = TaskDispatcher({'t1': t1, 't2': t2}, [], ['t1', 't2']) my_runner.run_tasks(disp) assert runner.ERROR == my_runner.finish() assert ('start', t1) == reporter.log.pop(0) assert ('fail', t1) == reporter.log.pop(0) assert ('start', t2) == reporter.log.pop(0) assert ('fail', t2) == reporter.log.pop(0) assert 0 == len(reporter.log)
def test_stop_running(self, reporter, depfile_name): t1 = Task('t1', []) t2 = Task('t2', []) run = runner.MRunner(Dependency, depfile_name, reporter) run._run_tasks_init( TaskDispatcher({ 't1': t1, 't2': t2 }, [], ['t1', 't2'])) assert t1 == run.get_next_task(None).task run._stop_running = True assert None == run.get_next_task(None)
def test_delayed_creation_target_regex(self): def creator(): yield Task('foo', None, targets=['tgt1']) delayed_loader = DelayedLoader(creator, executed='t2', target_regex='tgt1') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } tc = TaskControl(list(tasks.values())) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted n2.run_status = 'done' td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen2 = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n5 = next(gen2) assert n5.task.name == 'foo' # get internal created task n5.run_status = 'done' td._update_waiting(n5) n6 = next(gen2) assert n6.name == '_regex_target_tgt1:t1' # file_dep is removed because foo might not be task # that creates this task (support for multi regex matches) assert n6.file_dep == {}
def test_result(self, reporter, RunnerClass, dep_manager): task = Task("taskY", [my_action]) my_runner = RunnerClass(dep_manager, reporter) assert None == task.result assert {} == task.values assert [None] == [a.out for a in task.actions] assert [None] == [a.err for a in task.actions] my_runner.run_tasks(TaskDispatcher({'taskY': task}, [], ['taskY'])) assert runner.SUCCESS == my_runner.finish() assert {'bb': 5} == task.result assert {'bb': 5} == task.values assert ['out here'] == [a.out for a in task.actions] assert ['err here'] == [a.err for a in task.actions]
def test_waiting_controller(self, reporter, dep_manager): t1 = Task('t1', []) t2 = Task('t2', [], calc_dep=('t1',)) run = runner.MRunner(dep_manager, reporter) run._run_tasks_init(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])) # first task ok assert t1.name == run.get_next_job(None).name # hold until t1 finishes assert 0 == run.free_proc assert isinstance(run.get_next_job(None), runner.JobHold) assert 1 == run.free_proc