def doit_auto(dependency_file, task_list, filter_tasks, loop_callback=None):
"""Re-execute tasks automatically a depedency changes
@param filter_tasks (list -str): print only tasks from this list
@loop_callback: used to stop loop on unittests
"""
task_control = TaskControl(task_list)
task_control.process(filter_tasks)
tasks_to_run = list(set([t for t in task_control.task_dispatcher(True)]))
watch_tasks = [t.name for t in tasks_to_run]
watch_files = list(itertools.chain(*[s.file_dep for s in tasks_to_run]))
watch_files = list(set(watch_files))
class DoitAutoRun(FileModifyWatcher):
"""Execute doit on event handler of file changes """
def handle_event(self, event):
doit_run(dependency_file, task_list, sys.stdout,
watch_tasks, reporter='executed-only')
# reset run_status
for task in task_list:
task.run_status = None
file_watcher = DoitAutoRun(watch_files)
# always run once when started
file_watcher.handle_event(None)
file_watcher.loop(loop_callback)
def test_delayed_creation(self):
def creator():
yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']}
yield {'name': 'foo2', 'actions': None, 'targets': ['bar']}
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = [Task('t0', None, task_dep=['t1']),
Task('t1', None, loader=delayed_loader),
Task('t2', None)]
control = TaskControl(tasks)
control.process(['t0'])
disp = control.task_dispatcher()
gen = disp.generator
nt2 = next(gen)
assert nt2.task.name == "t2"
# wait for t2 to be executed
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
# delayed creation of tasks for t1 does not mess existing info
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
nf2 = gen.send(nt2)
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
assert nf2.task.name == "t1:foo2"
nf1 = gen.send(nf2)
assert nf1.task.name == "t1:foo1"
assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added
nt1 = gen.send(nf1)
assert nt1.task.name == "t1"
nt0 = gen.send(nt1)
assert nt0.task.name == "t0"
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def testSystemExitRaises(self, reporter, RunnerClass, depfile):
t1 = Task("x", [_exit])
my_runner = RunnerClass(depfile.name, reporter)
tc = TaskControl([t1])
tc.process(None)
pytest.raises(SystemExit, my_runner.run_tasks, tc)
my_runner.finish()
def testDetectCyclicReference(self):
tasks = [Task("taskX",None,task_dep=["taskY"]),
Task("taskY",None,task_dep=["taskX"])]
tc = TaskControl(tasks)
tc.process(None)
gen = tc._add_task(0, "taskX", False)
py.test.raises(InvalidDodoFile, gen.next)
def test_delayed_creation(self):
def creator():
yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']}
yield {'name': 'foo2', 'actions': None, 'targets': ['bar']}
delayed_loader = DelayedLoader(creator, executed='t2')
tasks = [Task('t0', None, task_dep=['t1']),
Task('t1', None, loader=delayed_loader),
Task('t2', None)]
control = TaskControl(tasks)
control.process(['t0'])
disp = control.task_dispatcher()
gen = disp.generator
nt2 = next(gen)
assert nt2.task.name == "t2"
# wait for t2 to be executed
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
# delayed creation of tasks for t1 does not mess existing info
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
nf2 = gen.send(nt2)
assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
assert nf2.task.name == "t1:foo2"
nf1 = gen.send(nf2)
assert nf1.task.name == "t1:foo1"
assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added
nt1 = gen.send(nf1)
assert nt1.task.name == "t1"
nt0 = gen.send(nt1)
assert nt0.task.name == "t0"
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def testChangeOrder_AddJustOnce(self):
tasks = [Task("taskX",None,task_dep=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(None)
assert [tasks[1], tasks[0]] == [x for x in tc._add_task(0, 'taskX', False)]
# both tasks were already added. so no tasks left..
assert [] == [x for x in tc._add_task(0, 'taskY', False)]
def test_successRunOnce(self, reporter, RunnerClass):
tasks = [Task("taskX", [my_print], run_once=True)]
my_runner = RunnerClass(TESTDB, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.SUCCESS == my_runner.finish()
d = Dependency(TESTDB)
assert '1' == d._get('taskX', 'run-once:')
def test_reporter_runtime_error(self, reporter):
t1 = Task('t1', [], setup=['make_invalid'])
my_runner = runner.Runner(TESTDB, reporter)
tc = TaskControl([t1])
tc.process(None)
my_runner.run_all(tc)
assert ('start', t1) == reporter.log.pop(0)
assert ('runtime_error',) == reporter.log.pop(0)
assert not reporter.log
def testSetupInvalid(self):
tasks = [Task("taskX",None,setup=["taskZZZZZZZZ"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', False)
assert tasks[0] == gen.next() # tasks with setup are yield twice
tasks[0].run_status = 'run' # should be executed
py.test.raises(InvalidTask, gen.next) # execute setup before
def test_stop_running(self, reporter):
t1 = Task('t1', [])
t2 = Task('t2', [])
tc = TaskControl([t1, t2])
tc.process(None)
run = runner.MRunner(TESTDB, reporter)
run._run_tasks_init(tc)
assert t1 == run.get_next_task()
run._stop_running = True
assert None == run.get_next_task()
def testSetupTasksDontRun(self):
tasks = [Task("taskX",None,setup=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', False)
assert tasks[0] == gen.next()
# X is up-to-date
tasks[0].run_status = 'up-to-date'
py.test.raises(StopIteration, gen.next)
def testParallel(self):
tasks = [Task("taskX",None,task_dep=["taskY"]),
Task("taskY",None)]
tc = TaskControl(tasks)
tc.process(None)
gen1 = tc._add_task(0, "taskX", False)
assert tasks[1] == gen1.next()
# gen2 wont get any task, because it was already being processed
gen2 = tc._add_task(1, "taskY", False)
py.test.raises(StopIteration, gen2.next)
def test_include_setup(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task("t2", None,)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher(include_setup=True).generator
# dont wait for tasks
assert tasks[0] == gen.send(None).task
assert tasks[1] == gen.send(None).task
pytest.raises(StopIteration, gen.send, None)
def test_teardown(self, reporter, RunnerClass):
t1 = Task('t1', [], teardown=[ok])
t2 = Task('t2', [])
my_runner = RunnerClass(TESTDB, reporter)
tc = TaskControl([t1, t2])
tc.process(None)
assert [] == my_runner.teardown_list
my_runner.run_tasks(tc)
my_runner.finish()
assert ('teardown', t1) == reporter.log[-1]
def test_run_task(self, reporter, depfile):
t1 = Task('t1', [])
t2 = Task('t2', [])
tc = TaskControl([t1, t2])
tc.process(None)
run = runner.MRunner(depfile.name, reporter)
run._run_tasks_init(tc)
assert t1 == run.get_next_task()
assert t2 == run.get_next_task()
assert None == run.get_next_task()
def doit_run(dependency_file, task_list, output, options=None,
verbosity=None, always_execute=False, continue_=False,
reporter='default', num_process=0):
"""
@param reporter: (str) one of provided reporters or ...
(class) user defined reporter class (can only be specified
from DOIT_CONFIG - never from command line)
"""
# get tasks to be executed
task_control = TaskControl(task_list)
task_control.process(options)
# reporter
if isinstance(reporter, str):
if reporter not in REPORTERS:
msg = ("No reporter named '%s'."
" Type 'doit help run' to see a list "
"of available reporters.")
raise InvalidCommand(msg % reporter)
reporter_cls = REPORTERS[reporter]
else:
# user defined class
reporter_cls = reporter
# verbosity
if verbosity is None:
use_verbosity = Task.DEFAULT_VERBOSITY
else:
use_verbosity = verbosity
show_out = use_verbosity < 2 # show on error report
# outstream
if isinstance(output, str):
outstream = open(output, 'w')
else: # outfile is a file-like object (like StringIO or sys.stdout)
outstream = output
# run
try:
# FIXME stderr will be shown twice in case of task error/failure
reporter_obj = reporter_cls(outstream, {'show_out':show_out,
'show_err': True})
if num_process == 0:
runner = Runner(dependency_file, reporter_obj, continue_,
always_execute, verbosity)
else:
runner = MRunner(dependency_file, reporter_obj, continue_,
always_execute, verbosity, num_process)
return runner.run_all(task_control)
finally:
if isinstance(output, str):
outstream.close()
def testSetupTasksRun(self):
tasks = [Task("taskX",None,setup=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', False)
assert tasks[0] == gen.next() # tasks with setup are yield twice
tasks[0].run_status = 'run' # should be executed
assert tasks[1] == gen.next() # execute setup before
assert tasks[0] == gen.next() # second time, ok
py.test.raises(StopIteration, gen.next) # nothing left
def testIncludeSetup(self):
# with include_setup yield all tasks without waiting for setup tasks to
# be ready
tasks = [Task("taskX",None,setup=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', True) # <== include_setup
assert tasks[0] == gen.next() # tasks with setup are yield twice
assert tasks[1] == gen.next() # execute setup before
assert tasks[0] == gen.next() # second time, ok
py.test.raises(StopIteration, gen.next) # nothing left
def test_normal(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task("t2", None,)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher().generator
n2 = next(gen)
assert tasks[1] == n2.task
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
assert tasks[0] == gen.send(n2).task
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def testAllTasksWaiting(self):
tasks = [Task("taskX",None,setup=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc.task_dispatcher()
assert tasks[0] == gen.next() # tasks with setup are yield twice
assert "hold on" == gen.next() # nothing else really available
tasks[0].run_status = 'run' # should be executed
assert tasks[1] == gen.next() # execute setup before
assert tasks[0] == gen.next() # second time, ok
py.test.raises(StopIteration, gen.next) # nothing left
def test_normal(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task("t2", None,)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher().generator
n2 = next(gen)
assert tasks[1] == n2.task
assert "hold on" == next(gen)
assert "hold on" == next(gen) # hold until t2 is done
assert tasks[0] == gen.send(n2).task
pytest.raises(StopIteration, lambda gen: next(gen), gen)
def test_error(self, reporter, RunnerClass):
tasks = [Task("taskX", [_error]),
Task("taskY", [_error])]
my_runner = RunnerClass(TESTDB, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.ERROR == my_runner.finish()
assert ('start', tasks[0]) == reporter.log.pop(0)
assert ('execute', tasks[0]) == reporter.log.pop(0)
assert ('fail', tasks[0]) == reporter.log.pop(0)
# second task is not executed
assert 0 == len(reporter.log)
def test_include_setup(self):
tasks = [Task("t1", None, task_dep=["t2"]),
Task(
"t2",
None,
)]
control = TaskControl(tasks)
control.process(['t1'])
gen = control.task_dispatcher(include_setup=True).generator
# dont wait for tasks
assert tasks[0] == gen.send(None).task
assert tasks[1] == gen.send(None).task
pytest.raises(StopIteration, gen.send, None)
def test_failureOutput(self, reporter, RunnerClass, depfile):
tasks = [Task("taskX", [_fail]),
Task("taskY", [_fail])]
my_runner = RunnerClass(depfile.name, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.FAILURE == my_runner.finish()
assert ('start', tasks[0]) == reporter.log.pop(0)
assert ('execute', tasks[0]) == reporter.log.pop(0)
assert ('fail', tasks[0]) == reporter.log.pop(0)
# second task is not executed
assert 0 == len(reporter.log)
def test_less_processes(self, reporter, monkeypatch):
mock_process = Mock()
monkeypatch.setattr(runner, 'Process', mock_process)
t1 = Task('t1', [])
tc = TaskControl([t1])
tc.process(None)
run = runner.MRunner(TESTDB, reporter, num_process=2)
run._run_tasks_init(tc)
result_q = Queue()
task_q = Queue()
proc_list = run._run_start_processes(task_q, result_q)
assert 1 == len(proc_list)
assert t1.name == task_q.get()[0]
def test_success(self, reporter, RunnerClass):
tasks = [Task("taskX", [(my_print, ["out a"] )] ),
Task("taskY", [(my_print, ["out a"] )] )]
my_runner = RunnerClass(TESTDB, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.SUCCESS == my_runner.finish()
assert ('start', tasks[0]) == reporter.log.pop(0), reporter.log
assert ('execute', tasks[0]) == reporter.log.pop(0)
assert ('success', tasks[0]) == reporter.log.pop(0)
assert ('start', tasks[1]) == reporter.log.pop(0)
assert ('execute', tasks[1]) == reporter.log.pop(0)
assert ('success', tasks[1]) == reporter.log.pop(0)
def test_waiting_controller(self, reporter):
t1 = Task('t1', [])
t2a = Task('t2A', [], calc_dep=('t1',))
tc = TaskControl([t1, t2a])
tc.process(None)
run = runner.MRunner(TESTDB, reporter)
run._run_tasks_init(tc)
# first task ok
assert t1 == run.get_next_task()
# hold until t1 finishes
assert 0 == run.free_proc
assert isinstance(run.get_next_task(), runner.Hold)
assert 1 == run.free_proc
def testWaitSetup(self):
tasks = [Task("taskX",None,setup=["taskY"]),
Task("taskY",None,)]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', False)
assert tasks[0] == gen.next() # tasks with setup are yield twice
# wait for taskX run_status
wait = gen.next()
assert wait.task_name == 'taskX'
assert isinstance(wait, WaitSelectTask)
tasks[0].run_status = 'run' # should be executed
assert tasks[1] == gen.next() # execute setup before
assert tasks[0] == gen.next() # second time, ok
py.test.raises(StopIteration, gen.next) # nothing left
def testCalcDep(self):
def get_deps():
print "gget"
return {'file_dep': ('a', 'b')}
tasks = [Task("taskX", None, calc_dep=['task_dep']),
Task("task_dep", [(get_deps,)]),
]
tc = TaskControl(tasks)
tc.process(['taskX'])
gen = tc._add_task(0, 'taskX', False)
assert tasks[1] == gen.next()
assert isinstance(gen.next(), WaitRunTask)
tasks[1].execute()
assert tasks[0] == gen.next()
assert set(['a', 'b']) == tasks[0].file_dep
def test_waiting_process(self, reporter, monkeypatch, depfile):
mock_process = Mock()
monkeypatch.setattr(runner, 'Process', mock_process)
t1 = Task('t1', [])
t2 = Task('t2', [], task_dep=['t1'])
tc = TaskControl([t1, t2])
tc.process(None)
run = runner.MRunner(depfile.name, reporter, num_process=2)
run._run_tasks_init(tc)
result_q = Queue()
task_q = Queue()
proc_list = run._run_start_processes(task_q, result_q)
run.finish()
assert 2 == len(proc_list)
assert t1.name == task_q.get().name
assert isinstance(task_q.get(), runner.Hold)
def test_getargs(self, reporter, RunnerClass, depfile):
def use_args(arg1):
print arg1
def make_args(): return {'myarg':1}
tasks = [Task("task1", [(use_args,)], getargs=dict(arg1="task2.myarg") ),
Task("task2", [(make_args,)])]
my_runner = RunnerClass(depfile.name, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.SUCCESS == my_runner.finish()
assert ('start', tasks[0]) == reporter.log.pop(0)
assert ('start', tasks[1]) == reporter.log.pop(0)
assert ('execute', tasks[1]) == reporter.log.pop(0)
assert ('success', tasks[1]) == reporter.log.pop(0)
assert ('execute', tasks[0]) == reporter.log.pop(0)
assert ('success', tasks[0]) == reporter.log.pop(0)
assert 0 == len(reporter.log)
def test_continue(self, reporter, RunnerClass):
tasks = [Task("task1", [(_fail,)] ),
Task("task2", [(_error,)] ),
Task("task3", [(ok,)])]
my_runner = RunnerClass(TESTDB, reporter, continue_=True)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.ERROR == my_runner.finish()
assert ('start', tasks[0]) == reporter.log.pop(0)
assert ('execute', tasks[0]) == reporter.log.pop(0)
assert ('fail', tasks[0]) == reporter.log.pop(0)
assert ('start', tasks[1]) == reporter.log.pop(0)
assert ('execute', tasks[1]) == reporter.log.pop(0)
assert ('fail', tasks[1]) == reporter.log.pop(0)
assert ('start', tasks[2]) == reporter.log.pop(0)
assert ('execute', tasks[2]) == reporter.log.pop(0)
assert ('success', tasks[2]) == reporter.log.pop(0)
assert 0 == len(reporter.log)
def test_updateDependencies(self, reporter, RunnerClass):
depPath = os.path.join(os.path.dirname(__file__),"data/dependency1")
ff = open(depPath,"a")
ff.write("xxx")
ff.close()
dependencies = [depPath]
filePath = os.path.join(os.path.dirname(__file__),"data/target")
ff = open(filePath,"a")
ff.write("xxx")
ff.close()
targets = [filePath]
tasks = [Task("taskX", [my_print], dependencies, targets)]
my_runner = RunnerClass(TESTDB, reporter)
tc = TaskControl(tasks)
tc.process(None)
my_runner.run_tasks(tc)
assert runner.SUCCESS == my_runner.finish()
d = Dependency(TESTDB)
assert d._get("taskX", os.path.abspath(depPath))
class Run(AnadamaCmdBase, DoitRun):
my_opts = (opt_runner, opt_pipeline_name, opt_grid_part, opt_perf_url,
opt_tmpfiles, opt_grid_args, opt_reporter_url, opt_auth_info)
def _execute(self,
outfile=sys.stdout,
verbosity=None,
always=False,
continue_=False,
reporter='default',
num_process=0,
par_type='process',
single=False,
pipeline_name="Custom Pipeline"):
"""
@param reporter: (str) one of provided reporters or ...
(class) user defined reporter class
(can only be specified
from DOIT_CONFIG - never from command line)
(reporter instance) - only used in unittests
"""
# get tasks to be executed
# self.control is saved on instance to be used by 'auto' command
self.control = TaskControl(self.task_list)
self.control.process(self.sel_tasks)
if single:
for task_name in self.sel_tasks:
task = self.control.tasks[task_name]
if task.has_subtask:
for task_name in task.task_dep:
sub_task = self.control.tasks[task_name]
sub_task.task_dep = []
else:
task.task_dep = []
# reporter
if isinstance(reporter, six.string_types):
if reporter not in REPORTERS:
msg = ("No reporter named '%s'."
" Type 'doit help run' to see a list "
"of available reporters.")
raise InvalidCommand(msg % reporter)
reporter_cls = REPORTERS[reporter]
else:
# user defined class
reporter_cls = reporter
# verbosity
if verbosity is None:
use_verbosity = Task.DEFAULT_VERBOSITY
else:
use_verbosity = verbosity
show_out = use_verbosity < 2 # show on error report
# outstream
if isinstance(outfile, six.string_types):
outstream = codecs.open(outfile, 'w', encoding='utf-8')
else: # outfile is a file-like object (like StringIO or sys.stdout)
outstream = outfile
# run
try:
# FIXME stderr will be shown twice in case of task error/failure
if isinstance(reporter_cls, type):
reporter_obj = reporter_cls(
outstream, {
'show_out': show_out,
'show_err': True,
'reporter_url': self.opt_values['reporter_url'],
'auth_info': self.opt_values['auth_info']
})
else: # also accepts reporter instances
reporter_obj = reporter_cls
run_args = [
self.dep_class, self.dep_file, reporter_obj, continue_, always,
verbosity
]
run_kwargs = {}
RunnerClass = RUNNER_MAP.get(self.opt_values["runner"])
if not RunnerClass:
RunnerClass = self._discover_runner_class(
num_process, par_type)
elif self.opt_values['runner'] in GRID_RUNNER_MAP:
if not self.opt_values.get('partition', None):
raise InvalidCommand("--partition option is required "
"when using a grid runner")
run_args = [
self.opt_values['partition'], self.opt_values['perf_url'],
self.opt_values['tmpfiledir'], self.opt_values['grid_args']
] + run_args
run_kwargs['num_process'] = num_process if num_process else 1
runner = RunnerClass(*run_args, **run_kwargs)
runner.pipeline_name = pipeline_name
return runner.run_all(self.control.task_dispatcher())
finally:
if isinstance(outfile, str):
outstream.close()
def _discover_runner_class(self, num_process, par_type):
if num_process == 0:
return Runner
else:
if par_type == 'process':
if MRunner.available():
return partial(MRunner, num_process=num_process)
else:
sys.stderr.write(
"WARNING: multiprocessing module not available, " +
"running in parallel using threads.")
elif par_type == 'thread':
return partial(MThreadRunner, num_process=num_process)
else:
msg = "Invalid parallel type %s"
raise InvalidCommand(msg % par_type)
def testProcessSelection(self):
filter_ = ['t2', 't3']
tc = TaskControl(TASKS_SAMPLE)
tc.process(filter_)
assert filter_ == tc.selected_tasks
def testProcessAll(self, tasks_sample):
tc = TaskControl(tasks_sample)
tc.process(None)
assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testProcessAll(self):
tc = TaskControl(TASKS_SAMPLE)
tc.process(None)
assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testProcessSelection(self, tasks_sample):
filter_ = ['t2', 't3']
tc = TaskControl(tasks_sample)
tc.process(filter_)
assert filter_ == tc.selected_tasks
