def test_continue_dont_execute_parent_of_failed_task(
self, reporter, RunnerClass, depfile_name):
t1 = Task("t1", [(_error, )])
t2 = Task("t2", [(ok, )], task_dep=['t1'])
t3 = Task("t3", [(ok, )])
my_runner = RunnerClass(Dependency,
depfile_name,
reporter,
continue_=True)
disp = TaskDispatcher({
't1': t1,
't2': t2,
't3': t3
}, [], ['t1', 't2', 't3'])
my_runner.run_tasks(disp)
assert runner.ERROR == my_runner.finish()
assert ('start', t1) == reporter.log.pop(0)
assert ('execute', t1) == reporter.log.pop(0)
assert ('fail', t1) == reporter.log.pop(0)
assert ('start', t2) == reporter.log.pop(0)
assert ('fail', t2) == reporter.log.pop(0)
assert ('start', t3) == reporter.log.pop(0)
assert ('execute', t3) == reporter.log.pop(0)
assert ('success', t3) == reporter.log.pop(0)
assert 0 == len(reporter.log)
def test_none(self):
tasks = {
't1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
assert None == td._get_next_node([], [])
def test_delayed_creation(self):
def creator():
yield Task('foo', None, loader=DelayedLoader(lambda: None))
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# recursive loader is preserved
assert isinstance(td.tasks['foo'].loader, DelayedLoader)
pytest.raises(AssertionError, next, gen)
def test_no_deps(self):
tasks = {
't1': Task('t1', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
assert [tasks['t1']] == list(td._add_task(n1))
def test_calc_dep(self):
def calc_intermediate():
return {'file_dep': ['intermediate']}
tasks = {
't1': Task('t1', None, calc_dep=['t2']),
't2': Task('t2', [calc_intermediate]),
't3': Task('t3', None, targets=['intermediate']),
}
td = TaskDispatcher(tasks, {'intermediate': 't3'}, None)
n1 = td._gen_node(None, 't1')
n2 = n1.step()
assert tasks['t2'] == n2.task
assert 'wait' == n1.step()
# execute t2 to process calc_dep
tasks['t2'].execute()
td.nodes['t2'].run_status = 'done'
td._update_waiting(n2)
n3 = n1.step()
assert tasks['t3'] == n3.task
assert 'intermediate' in tasks['t1'].file_dep
assert 't3' in tasks['t1'].task_dep
# t3 was added by calc dep
assert 'wait' == n1.step()
n3.run_status = 'done'
td._update_waiting(n3)
assert tasks['t1'] == n1.step()
def test_to_run_none(self):
tasks = {'t1': Task('t1', None),
}
td = TaskDispatcher(tasks, [], None)
td._gen_node(None, 't1') # t1 was already created
to_run = ['t1']
assert None == td._get_next_node([], to_run)
assert [] == to_run
def test_stop_running(self, reporter, dep_manager):
t1 = Task('t1', [])
t2 = Task('t2', [])
run = runner.MRunner(dep_manager, reporter)
run._run_tasks_init(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
assert t1.name == run.get_next_job(None).name
run._stop_running = True
assert None == run.get_next_job(None)
def test_cyclic(self):
tasks = {'t1': Task('t1', None),
't2': Task('t2', None)
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(n1, 't2')
pytest.raises(InvalidDodoFile, td._gen_node, n2, 't1')
def test_already_created(self):
tasks = {'t1': Task('t1', None),
't2': Task('t2', None)
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
td._gen_node(n1, 't2')
assert None == td._gen_node(None, 't1')
def test_MRunner_pickable(dep_manager):
t1 = Task('t1', [])
import sys
reporter = ConsoleReporter(sys.stdout, {})
run = runner.MRunner(dep_manager, reporter)
run._run_tasks_init(TaskDispatcher({'t1': t1}, [], ['t1']))
# assert nothing is raised
pickle.dumps(run)
def test_teardown(self, reporter, RunnerClass, dep_manager):
t1 = Task('t1', [], teardown=[ok])
t2 = Task('t2', [])
my_runner = RunnerClass(dep_manager, reporter)
assert [] == my_runner.teardown_list
my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
my_runner.finish()
assert ('teardown', t1) == reporter.log[-1]
def test_task_cloudpicklabe_multiprocess(self, reporter, dep_manager):
t1 = Task("t1", [(my_print, ["out a"] )] )
t2 = Task("t2", None, loader=DelayedLoader(
non_pickable_creator, executed='t1'))
my_runner = runner.MRunner(dep_manager, reporter)
dispatcher = TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2'])
my_runner.run_tasks(dispatcher)
assert runner.SUCCESS == my_runner.finish()
def test_task_not_picklabe_multiprocess(self, reporter, dep_manager):
def creator():
return {'basename': 't2', 'actions': [lambda: 5]}
t1 = Task("t1", [(my_print, ["out a"])])
t2 = Task("t2", None, loader=DelayedLoader(creator, executed='t1'))
my_runner = runner.MRunner(dep_manager, reporter)
dispatcher = TaskDispatcher({'t1': t1, 't2': t2}, [], ['t1', 't2'])
pytest.raises(InvalidTask, my_runner.run_tasks, dispatcher)
def test_ready(self):
tasks = {'t1': Task('t1', None),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
ready = deque([n1])
assert n1 == td._get_next_node(ready, ['t2'])
assert 0 == len(ready)
def test_deps_not_ok(self):
tasks = {'t1': Task('t1', None),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
n2.run_status = 'failure'
td._node_add_wait_run(n1, ['t2'])
assert n1.bad_deps
def test_none(self):
tasks = {'t1': Task('t1', None),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
n2.run_status = 'done'
td._node_add_wait_run(n1, ['t2'])
assert not n1.wait_run
def test_task_deps_no_wait(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
n2.run_status = 'done'
gen = td._add_task(n1)
assert tasks['t1'] == next(gen)
def test_reporter_runtime_error(self, reporter, dep_manager):
t1 = Task('t1', [], calc_dep=['t2'])
t2 = Task('t2', [lambda: {'file_dep':[1]}])
my_runner = runner.Runner(dep_manager, reporter)
my_runner.run_all(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
assert runner.ERROR == my_runner.final_result
assert ('start', t2) == reporter.log.pop(0)
assert ('execute', t2) == reporter.log.pop(0)
assert ('success', t2) == reporter.log.pop(0)
assert ('runtime_error',) == reporter.log.pop(0)
assert not reporter.log
def test_to_run(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
to_run = ['t2', 't1']
td._gen_node(None, 't1') # t1 was already created
got = td._get_next_node([], to_run)
assert isinstance(got, ExecNode)
assert 't2' == got.task.name
assert [] == to_run
def test_error(self, reporter, RunnerClass, dep_manager):
t1 = Task("t1", [_error])
t2 = Task("t2", [_error])
my_runner = RunnerClass(dep_manager, reporter)
my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
assert runner.ERROR == my_runner.finish()
assert ('start', t1) == reporter.log.pop(0)
assert ('execute', t1) == reporter.log.pop(0)
assert ('fail', t1) == reporter.log.pop(0)
# second task is not executed
assert 0 == len(reporter.log)
def test_run_task(self, reporter, depfile):
t1 = Task('t1', [])
t2 = Task('t2', [])
run = runner.MRunner(depfile.name, reporter)
run._run_tasks_init(
TaskDispatcher({
't1': t1,
't2': t2
}, [], ['t1', 't2']))
assert t1 == run.get_next_task(None).task
assert t2 == run.get_next_task(None).task
assert None == run.get_next_task(None)
def test_task_deps_already_created(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n1 = td._gen_node(None, 't1')
n2 = td._gen_node(None, 't2')
assert 'wait' == n1.step()
assert 'wait' == n1.step()
#tasks['t2'].run_status = 'done'
td._update_waiting(n2)
assert tasks['t1'] == n1.step()
def test_success(self, reporter, RunnerClass, dep_manager):
t1 = Task("t1", [(my_print, ["out a"] )] )
t2 = Task("t2", [(my_print, ["out a"] )] )
my_runner = RunnerClass(dep_manager, reporter)
my_runner.run_tasks(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
assert runner.SUCCESS == my_runner.finish()
assert ('start', t1) == reporter.log.pop(0), reporter.log
assert ('execute', t1) == reporter.log.pop(0)
assert ('success', t1) == reporter.log.pop(0)
assert ('start', t2) == reporter.log.pop(0)
assert ('execute', t2) == reporter.log.pop(0)
assert ('success', t2) == reporter.log.pop(0)
def test_wait_select(self):
tasks = {'t1': Task('t1', None, task_dep=['t2']),
't2': Task('t2', None),
}
td = TaskDispatcher(tasks, [], None)
n2 = td._gen_node(None, 't2')
n2.wait_select = True
n2.run_status = 'run'
td.waiting.add(n2)
td._update_waiting(n2)
assert False == n2.wait_select
assert deque([n2]) == td.ready
def testActionModifiesFiledep(self, reporter, RunnerClass, dep_manager):
extra_dep = os.path.join(os.path.dirname(__file__), 'sample_md5.txt')
t1 = Task("t1", [(my_print, ["out a"] ),
(action_add_filedep, (), {'extra_dep': extra_dep})
] )
my_runner = RunnerClass(dep_manager, reporter)
my_runner.run_tasks(TaskDispatcher({'t1':t1}, [], ['t1']))
assert runner.SUCCESS == my_runner.finish()
assert ('start', t1) == reporter.log.pop(0), reporter.log
assert ('execute', t1) == reporter.log.pop(0)
assert ('success', t1) == reporter.log.pop(0)
assert t1.file_dep == set([extra_dep])
def test_continue_dep_error(self, reporter, RunnerClass, dep_manager):
t1 = Task("t1", [(ok, )], file_dep=['i_dont_exist'])
t2 = Task("t2", [(ok, )], task_dep=['t1'])
my_runner = RunnerClass(dep_manager, reporter, continue_=True)
disp = TaskDispatcher({'t1': t1, 't2': t2}, [], ['t1', 't2'])
my_runner.run_tasks(disp)
assert runner.ERROR == my_runner.finish()
assert ('start', t1) == reporter.log.pop(0)
assert ('fail', t1) == reporter.log.pop(0)
assert ('start', t2) == reporter.log.pop(0)
assert ('fail', t2) == reporter.log.pop(0)
assert 0 == len(reporter.log)
def test_stop_running(self, reporter, depfile_name):
t1 = Task('t1', [])
t2 = Task('t2', [])
run = runner.MRunner(Dependency, depfile_name, reporter)
run._run_tasks_init(
TaskDispatcher({
't1': t1,
't2': t2
}, [], ['t1', 't2']))
assert t1 == run.get_next_task(None).task
run._stop_running = True
assert None == run.get_next_task(None)
def test_delayed_creation_target_regex(self):
def creator():
yield Task('foo', None, targets=['tgt1'])
delayed_loader = DelayedLoader(creator,
executed='t2',
target_regex='tgt1')
tasks = {
't1': Task('t1', None, loader=delayed_loader),
't2': Task('t2', None),
}
tc = TaskControl(list(tasks.values()))
selection = tc._filter_tasks(['tgt1'])
assert ['_regex_target_tgt1:t1'] == selection
td = TaskDispatcher(tc.tasks, tc.targets, selection)
n1 = td._gen_node(None, '_regex_target_tgt1:t1')
gen = td._add_task(n1)
# first returned node is `t2` because it is an implicit task_dep
n2 = next(gen)
assert n2.task.name == 't2'
# wait until t2 is finished
n3 = next(gen)
assert n3 == 'wait'
# after t2 is done, generator is reseted
n2.run_status = 'done'
td._update_waiting(n2)
n4 = next(gen)
assert n4 == "reset generator"
# manually reset generator
n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))
# get the delayed created task
gen2 = n1.generator # n1 generator was reset / replaced
# get t1 because of its target was a file_dep of _regex_target_tgt1
n5 = next(gen2)
assert n5.task.name == 'foo'
# get internal created task
n5.run_status = 'done'
td._update_waiting(n5)
n6 = next(gen2)
assert n6.name == '_regex_target_tgt1:t1'
# file_dep is removed because foo might not be task
# that creates this task (support for multi regex matches)
assert n6.file_dep == {}
def test_result(self, reporter, RunnerClass, dep_manager):
task = Task("taskY", [my_action])
my_runner = RunnerClass(dep_manager, reporter)
assert None == task.result
assert {} == task.values
assert [None] == [a.out for a in task.actions]
assert [None] == [a.err for a in task.actions]
my_runner.run_tasks(TaskDispatcher({'taskY': task}, [], ['taskY']))
assert runner.SUCCESS == my_runner.finish()
assert {'bb': 5} == task.result
assert {'bb': 5} == task.values
assert ['out here'] == [a.out for a in task.actions]
assert ['err here'] == [a.err for a in task.actions]
def test_waiting_controller(self, reporter, dep_manager):
t1 = Task('t1', [])
t2 = Task('t2', [], calc_dep=('t1',))
run = runner.MRunner(dep_manager, reporter)
run._run_tasks_init(TaskDispatcher({'t1':t1, 't2':t2}, [], ['t1', 't2']))
# first task ok
assert t1.name == run.get_next_job(None).name
# hold until t1 finishes
assert 0 == run.free_proc
assert isinstance(run.get_next_job(None), runner.JobHold)
assert 1 == run.free_proc
