def testSystemExitRaises(self, reporter, RunnerClass, depfile): t1 = Task("x", [_exit]) my_runner = RunnerClass(depfile.name, reporter) tc = TaskControl([t1]) tc.process(None) pytest.raises(SystemExit, my_runner.run_tasks, tc) my_runner.finish()
def test_delayed_creation(self): def creator(): yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']} yield {'name': 'foo2', 'actions': None, 'targets': ['bar']} delayed_loader = DelayedLoader(creator, executed='t2') tasks = [Task('t0', None, task_dep=['t1']), Task('t1', None, loader=delayed_loader), Task('t2', None)] control = TaskControl(tasks) control.process(['t0']) disp = control.task_dispatcher() gen = disp.generator nt2 = next(gen) assert nt2.task.name == "t2" # wait for t2 to be executed assert "hold on" == next(gen) assert "hold on" == next(gen) # hold until t2 is done # delayed creation of tasks for t1 does not mess existing info assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']]) nf2 = gen.send(nt2) assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']]) assert nf2.task.name == "t1:foo2" nf1 = gen.send(nf2) assert nf1.task.name == "t1:foo1" assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added nt1 = gen.send(nf1) assert nt1.task.name == "t1" nt0 = gen.send(nt1) assert nt0.task.name == "t0" pytest.raises(StopIteration, lambda gen: next(gen), gen)
def doit_auto(dependency_file, task_list, filter_tasks, loop_callback=None): """Re-execute tasks automatically a depedency changes @param filter_tasks (list -str): print only tasks from this list @loop_callback: used to stop loop on unittests """ task_control = TaskControl(task_list) task_control.process(filter_tasks) tasks_to_run = list(set([t for t in task_control.task_dispatcher(True)])) watch_tasks = [t.name for t in tasks_to_run] watch_files = list(itertools.chain(*[s.file_dep for s in tasks_to_run])) watch_files = list(set(watch_files)) class DoitAutoRun(FileModifyWatcher): """Execute doit on event handler of file changes """ def handle_event(self, event): doit_run(dependency_file, task_list, sys.stdout, watch_tasks, reporter='executed-only') # reset run_status for task in task_list: task.run_status = None file_watcher = DoitAutoRun(watch_files) # always run once when started file_watcher.handle_event(None) file_watcher.loop(loop_callback)
def testPosParam(self): tasks = list(TASKS_SAMPLE) tasks.append(Task("tP", [""],[],[], pos_arg='myp')) tc = TaskControl(tasks) args = ["tP", "hello option!", "t1"] assert ['tP',] == tc._filter_tasks(args) assert ["hello option!", "t1"] == tc.tasks['tP'].pos_arg_val
def testDetectCyclicReference(self): tasks = [Task("taskX",None,task_dep=["taskY"]), Task("taskY",None,task_dep=["taskX"])] tc = TaskControl(tasks) tc.process(None) gen = tc._add_task(0, "taskX", False) py.test.raises(InvalidDodoFile, gen.next)
def testChangeOrder_AddJustOnce(self): tasks = [Task("taskX",None,task_dep=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(None) assert [tasks[1], tasks[0]] == [x for x in tc._add_task(0, 'taskX', False)] # both tasks were already added. so no tasks left.. assert [] == [x for x in tc._add_task(0, 'taskY', False)]
def test_successRunOnce(self, reporter, RunnerClass): tasks = [Task("taskX", [my_print], run_once=True)] my_runner = RunnerClass(TESTDB, reporter) tc = TaskControl(tasks) tc.process(None) my_runner.run_tasks(tc) assert runner.SUCCESS == my_runner.finish() d = Dependency(TESTDB) assert '1' == d._get('taskX', 'run-once:')
def test_filter_delayed_subtask(self): t1 = Task("taskX", None) t2 = Task("taskY", None, loader=DelayedLoader(lambda: None)) control = TaskControl([t1, t2]) control._filter_tasks(['taskY:foo']) assert isinstance(t2.loader, DelayedLoader) # sub-task will use same loader, and keep parent basename assert control.tasks['taskY:foo'].loader.basename == 'taskY' assert control.tasks['taskY:foo'].loader is t2.loader
def testSetupInvalid(self): tasks = [Task("taskX",None,setup=["taskZZZZZZZZ"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', False) assert tasks[0] == gen.next() # tasks with setup are yield twice tasks[0].run_status = 'run' # should be executed py.test.raises(InvalidTask, gen.next) # execute setup before
def test_reporter_runtime_error(self, reporter): t1 = Task('t1', [], setup=['make_invalid']) my_runner = runner.Runner(TESTDB, reporter) tc = TaskControl([t1]) tc.process(None) my_runner.run_all(tc) assert ('start', t1) == reporter.log.pop(0) assert ('runtime_error',) == reporter.log.pop(0) assert not reporter.log
def test_teardown(self, reporter, RunnerClass): t1 = Task('t1', [], teardown=[ok]) t2 = Task('t2', []) my_runner = RunnerClass(TESTDB, reporter) tc = TaskControl([t1, t2]) tc.process(None) assert [] == my_runner.teardown_list my_runner.run_tasks(tc) my_runner.finish() assert ('teardown', t1) == reporter.log[-1]
def test_stop_running(self, reporter): t1 = Task('t1', []) t2 = Task('t2', []) tc = TaskControl([t1, t2]) tc.process(None) run = runner.MRunner(TESTDB, reporter) run._run_tasks_init(tc) assert t1 == run.get_next_task() run._stop_running = True assert None == run.get_next_task()
def testSetupTasksDontRun(self): tasks = [Task("taskX",None,setup=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', False) assert tasks[0] == gen.next() # X is up-to-date tasks[0].run_status = 'up-to-date' py.test.raises(StopIteration, gen.next)
def testParallel(self): tasks = [Task("taskX",None,task_dep=["taskY"]), Task("taskY",None)] tc = TaskControl(tasks) tc.process(None) gen1 = tc._add_task(0, "taskX", False) assert tasks[1] == gen1.next() # gen2 wont get any task, because it was already being processed gen2 = tc._add_task(1, "taskY", False) py.test.raises(StopIteration, gen2.next)
def test_run_task(self, reporter, depfile): t1 = Task('t1', []) t2 = Task('t2', []) tc = TaskControl([t1, t2]) tc.process(None) run = runner.MRunner(depfile.name, reporter) run._run_tasks_init(tc) assert t1 == run.get_next_task() assert t2 == run.get_next_task() assert None == run.get_next_task()
def test_include_setup(self): tasks = [Task("t1", None, task_dep=["t2"]), Task("t2", None,)] control = TaskControl(tasks) control.process(['t1']) gen = control.task_dispatcher(include_setup=True).generator # dont wait for tasks assert tasks[0] == gen.send(None).task assert tasks[1] == gen.send(None).task pytest.raises(StopIteration, gen.send, None)
def doit_run(dependency_file, task_list, output, options=None, verbosity=None, always_execute=False, continue_=False, reporter='default', num_process=0): """ @param reporter: (str) one of provided reporters or ... (class) user defined reporter class (can only be specified from DOIT_CONFIG - never from command line) """ # get tasks to be executed task_control = TaskControl(task_list) task_control.process(options) # reporter if isinstance(reporter, str): if reporter not in REPORTERS: msg = ("No reporter named '%s'." " Type 'doit help run' to see a list " "of available reporters.") raise InvalidCommand(msg % reporter) reporter_cls = REPORTERS[reporter] else: # user defined class reporter_cls = reporter # verbosity if verbosity is None: use_verbosity = Task.DEFAULT_VERBOSITY else: use_verbosity = verbosity show_out = use_verbosity < 2 # show on error report # outstream if isinstance(output, str): outstream = open(output, 'w') else: # outfile is a file-like object (like StringIO or sys.stdout) outstream = output # run try: # FIXME stderr will be shown twice in case of task error/failure reporter_obj = reporter_cls(outstream, {'show_out':show_out, 'show_err': True}) if num_process == 0: runner = Runner(dependency_file, reporter_obj, continue_, always_execute, verbosity) else: runner = MRunner(dependency_file, reporter_obj, continue_, always_execute, verbosity, num_process) return runner.run_all(task_control) finally: if isinstance(output, str): outstream.close()
def testSetupTasksRun(self): tasks = [Task("taskX",None,setup=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', False) assert tasks[0] == gen.next() # tasks with setup are yield twice tasks[0].run_status = 'run' # should be executed assert tasks[1] == gen.next() # execute setup before assert tasks[0] == gen.next() # second time, ok py.test.raises(StopIteration, gen.next) # nothing left
def test_normal(self): tasks = [Task("t1", None, task_dep=["t2"]), Task("t2", None,)] control = TaskControl(tasks) control.process(['t1']) gen = control.task_dispatcher().generator n2 = next(gen) assert tasks[1] == n2.task assert "hold on" == next(gen) assert "hold on" == next(gen) # hold until t2 is done assert tasks[0] == gen.send(n2).task pytest.raises(StopIteration, lambda gen: next(gen), gen)
def testIncludeSetup(self): # with include_setup yield all tasks without waiting for setup tasks to # be ready tasks = [Task("taskX",None,setup=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', True) # <== include_setup assert tasks[0] == gen.next() # tasks with setup are yield twice assert tasks[1] == gen.next() # execute setup before assert tasks[0] == gen.next() # second time, ok py.test.raises(StopIteration, gen.next) # nothing left
def testAllTasksWaiting(self): tasks = [Task("taskX",None,setup=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc.task_dispatcher() assert tasks[0] == gen.next() # tasks with setup are yield twice assert "hold on" == gen.next() # nothing else really available tasks[0].run_status = 'run' # should be executed assert tasks[1] == gen.next() # execute setup before assert tasks[0] == gen.next() # second time, ok py.test.raises(StopIteration, gen.next) # nothing left
def test_delayed_creation_target_regex(self): def creator(): yield Task('foo', None, targets=['tgt1']) delayed_loader = DelayedLoader(creator, executed='t2', target_regex='tgt1') tasks = { 't1': Task('t1', None, loader=delayed_loader), 't2': Task('t2', None), } tc = TaskControl(list(tasks.values())) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # first returned node is `t2` because it is an implicit task_dep n2 = next(gen) assert n2.task.name == 't2' # wait until t2 is finished n3 = next(gen) assert n3 == 'wait' # after t2 is done, generator is reseted n2.run_status = 'done' td._update_waiting(n2) n4 = next(gen) assert n4 == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen2 = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n5 = next(gen2) assert n5.task.name == 'foo' # get internal created task n5.run_status = 'done' td._update_waiting(n5) n6 = next(gen2) assert n6.name == '_regex_target_tgt1:t1' # file_dep is removed because foo might not be task # that creates this task (support for multi regex matches) assert n6.file_dep == {}
def test_include_setup(self): tasks = [Task("t1", None, task_dep=["t2"]), Task( "t2", None, )] control = TaskControl(tasks) control.process(['t1']) gen = control.task_dispatcher(include_setup=True).generator # dont wait for tasks assert tasks[0] == gen.send(None).task assert tasks[1] == gen.send(None).task pytest.raises(StopIteration, gen.send, None)
def test_failureOutput(self, reporter, RunnerClass, depfile): tasks = [Task("taskX", [_fail]), Task("taskY", [_fail])] my_runner = RunnerClass(depfile.name, reporter) tc = TaskControl(tasks) tc.process(None) my_runner.run_tasks(tc) assert runner.FAILURE == my_runner.finish() assert ('start', tasks[0]) == reporter.log.pop(0) assert ('execute', tasks[0]) == reporter.log.pop(0) assert ('fail', tasks[0]) == reporter.log.pop(0) # second task is not executed assert 0 == len(reporter.log)
def test_error(self, reporter, RunnerClass): tasks = [Task("taskX", [_error]), Task("taskY", [_error])] my_runner = RunnerClass(TESTDB, reporter) tc = TaskControl(tasks) tc.process(None) my_runner.run_tasks(tc) assert runner.ERROR == my_runner.finish() assert ('start', tasks[0]) == reporter.log.pop(0) assert ('execute', tasks[0]) == reporter.log.pop(0) assert ('fail', tasks[0]) == reporter.log.pop(0) # second task is not executed assert 0 == len(reporter.log)
def test_filter_delayed_multi_select(self): t1 = Task("taskX", None) t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*')) t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None, target_regex='b.*')) t4 = Task("taskW", None, loader=DelayedLoader(lambda: None)) control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False) selected = control._filter_tasks(['abc', 'att']) assert isinstance(t2.loader, DelayedLoader) assert len(selected) == 2 assert selected[0] == '_regex_target_abc:taskY' assert selected[1] == '_regex_target_att:taskY'
def test_less_processes(self, reporter, monkeypatch): mock_process = Mock() monkeypatch.setattr(runner, 'Process', mock_process) t1 = Task('t1', []) tc = TaskControl([t1]) tc.process(None) run = runner.MRunner(TESTDB, reporter, num_process=2) run._run_tasks_init(tc) result_q = Queue() task_q = Queue() proc_list = run._run_start_processes(task_q, result_q) assert 1 == len(proc_list) assert t1.name == task_q.get()[0]
def test_success(self, reporter, RunnerClass): tasks = [Task("taskX", [(my_print, ["out a"] )] ), Task("taskY", [(my_print, ["out a"] )] )] my_runner = RunnerClass(TESTDB, reporter) tc = TaskControl(tasks) tc.process(None) my_runner.run_tasks(tc) assert runner.SUCCESS == my_runner.finish() assert ('start', tasks[0]) == reporter.log.pop(0), reporter.log assert ('execute', tasks[0]) == reporter.log.pop(0) assert ('success', tasks[0]) == reporter.log.pop(0) assert ('start', tasks[1]) == reporter.log.pop(0) assert ('execute', tasks[1]) == reporter.log.pop(0) assert ('success', tasks[1]) == reporter.log.pop(0)
def testCalcDep(self): def get_deps(): print "gget" return {'file_dep': ('a', 'b')} tasks = [Task("taskX", None, calc_dep=['task_dep']), Task("task_dep", [(get_deps,)]), ] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', False) assert tasks[1] == gen.next() assert isinstance(gen.next(), WaitRunTask) tasks[1].execute() assert tasks[0] == gen.next() assert set(['a', 'b']) == tasks[0].file_dep
def testWaitSetup(self): tasks = [Task("taskX",None,setup=["taskY"]), Task("taskY",None,)] tc = TaskControl(tasks) tc.process(['taskX']) gen = tc._add_task(0, 'taskX', False) assert tasks[0] == gen.next() # tasks with setup are yield twice # wait for taskX run_status wait = gen.next() assert wait.task_name == 'taskX' assert isinstance(wait, WaitSelectTask) tasks[0].run_status = 'run' # should be executed assert tasks[1] == gen.next() # execute setup before assert tasks[0] == gen.next() # second time, ok py.test.raises(StopIteration, gen.next) # nothing left
def test_waiting_controller(self, reporter): t1 = Task('t1', []) t2a = Task('t2A', [], calc_dep=('t1',)) tc = TaskControl([t1, t2a]) tc.process(None) run = runner.MRunner(TESTDB, reporter) run._run_tasks_init(tc) # first task ok assert t1 == run.get_next_task() # hold until t1 finishes assert 0 == run.free_proc assert isinstance(run.get_next_task(), runner.Hold) assert 1 == run.free_proc
def test_regex_not_found(self): def creator1(): yield Task('foo1', None, targets=['tgt1']) delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*') t1 = Task('t1', None, loader=delayed_loader1) tc = TaskControl([t1]) selection = tc._filter_tasks(['tgt666']) assert ['_regex_target_tgt666:t1'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt666:t1') gen = td._add_task(n1) # target not found after generating all tasks from regex group pytest.raises(InvalidCommand, next, gen)
def test_regex_group_already_created(self): # this is required to avoid loading more tasks than required, GH-#60 def creator1(): yield Task('foo1', None, targets=['tgt1']) delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*') def creator2(): # pragma: no cover yield Task('foo2', None, targets=['tgt2']) delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*') t1 = Task('t1', None, loader=delayed_loader1) t2 = Task('t2', None, loader=delayed_loader2) tc = TaskControl([t1, t2]) selection = tc._filter_tasks(['tgt1']) assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection td = TaskDispatcher(tc.tasks, tc.targets, selection) n1 = td._gen_node(None, '_regex_target_tgt1:t1') gen = td._add_task(n1) # delayed loader executed, so generator is reset n1b = next(gen) assert n1b == "reset generator" # manually reset generator n1.reset_task(td.tasks[n1.task.name], td._add_task(n1)) # get the delayed created task gen1b = n1.generator # n1 generator was reset / replaced # get t1 because of its target was a file_dep of _regex_target_tgt1 n1c = next(gen1b) assert n1c.task.name == 'foo1' # get internal created task n1c.run_status = 'done' td._update_waiting(n1c) n1d = next(gen1b) assert n1d.name == '_regex_target_tgt1:t1' ## go for second selected task n2 = td._gen_node(None, '_regex_target_tgt1:t2') gen2 = td._add_task(n2) # loader is not executed because target t1 was already found pytest.raises(StopIteration, next, gen2)
def test_wild(self): tasks = [Task('t1', None, task_dep=['foo*']), Task( 'foo4', None, )] TaskControl(tasks) assert 'foo4' in tasks[0].task_dep
def test_filter_delayed_regex_auto(self): t1 = Task("taskX", None) t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*')) t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None)) control = TaskControl([t1, t2, t3], auto_delayed_regex=True) selected = control._filter_tasks(['abc']) assert len(selected) == 2 assert (sorted(selected) == ['_regex_target_abc:taskY', '_regex_target_abc:taskZ']) assert control.tasks['_regex_target_abc:taskY'].file_dep == {'abc'} assert control.tasks['_regex_target_abc:taskZ'].file_dep == {'abc'} assert (control.tasks['_regex_target_abc:taskY'].loader.basename == t2.name) assert (control.tasks['_regex_target_abc:taskZ'].loader.basename == t3.name)
def test_filter_delayed_regex_single(self): t1 = Task("taskX", None) t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*')) t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None, target_regex='b.*')) t4 = Task("taskW", None, loader=DelayedLoader(lambda: None)) control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False) selected = control._filter_tasks(['abc']) assert isinstance(t2.loader, DelayedLoader) assert len(selected) == 1 assert selected[0] == '_regex_target_abc:taskY' sel_task = control.tasks['_regex_target_abc:taskY'] assert sel_task.file_dep == {'abc'} assert sel_task.loader.basename == 'taskY' assert sel_task.loader is t2.loader
def _execute(self, subtasks, reverse, horizontal, outfile, pos_args=None): # init control = TaskControl(self.task_list) self.tasks = control.tasks self.subtasks = subtasks self._edges = set() # used to avoid adding same edge twice # create graph self.graph = pygraphviz.AGraph(strict=False, directed=True) self.graph.node_attr['color'] = 'lightblue2' self.graph.node_attr['style'] = 'filled' if (horizontal): self.graph.graph_attr.update(rankdir='LR') # populate graph processed = set() # str - task name if pos_args: to_process = deque(pos_args) else: to_process = deque(control.tasks.keys()) while to_process: task = control.tasks[to_process.popleft()] if task.name in processed: continue processed.add(task.name) # add nodes node_attrs = {} if task.has_subtask: node_attrs['peripheries'] = '2' if (not task.subtask_of) or subtasks: self.graph.add_node(task.name, **node_attrs) # add edges for sink_name in task.setup_tasks: self.add_edge(task.name, sink_name, arrowhead='empty') if sink_name not in processed: to_process.append(sink_name) for sink_name in task.task_dep: self.add_edge(task.name, sink_name, arrowhead='') if sink_name not in processed: to_process.append(sink_name) if not outfile: name = pos_args[0] if len(pos_args) == 1 else 'tasks' outfile = '{}.dot'.format(name) print('Generated file: {}'.format(outfile)) if (reverse): self.graph.reverse().write(outfile) else: self.graph.write(outfile)
def testOptions(self): options = ["t3", "--message", "hello option!", "t1"] tc = TaskControl(TASKS_SAMPLE) assert ['t3', 't1'] == tc._filter_tasks(options) assert "hello option!" == tc.tasks['t3'].options['opt1']
def testFilterEmptyList(self): filter_ = [] tc = TaskControl(TASKS_SAMPLE) assert filter_ == tc._filter_tasks(filter_)
def testFilterWrongName(self): tc = TaskControl(TASKS_SAMPLE) pytest.raises(InvalidCommand, tc._filter_tasks, ['no'])
def test_targetDependency(self): t1 = Task("taskX", None, [], ['intermediate']) t2 = Task("taskY", None, ['intermediate'], []) TaskControl([t1, t2]) assert ['taskX'] == t2.task_dep
def test_addTask(self): t1 = Task("taskX", None) t2 = Task("taskY", None) tc = TaskControl([t1, t2]) assert 2 == len(tc.tasks)
def _execute(self, outfile=sys.stdout, verbosity=None, always=False, continue_=False, reporter='default', num_process=0, par_type='process', single=False, pipeline_name="Custom Pipeline"): """ @param reporter: (str) one of provided reporters or ... (class) user defined reporter class (can only be specified from DOIT_CONFIG - never from command line) (reporter instance) - only used in unittests """ # get tasks to be executed # self.control is saved on instance to be used by 'auto' command self.control = TaskControl(self.task_list) self.control.process(self.sel_tasks) if single: for task_name in self.sel_tasks: task = self.control.tasks[task_name] if task.has_subtask: for task_name in task.task_dep: sub_task = self.control.tasks[task_name] sub_task.task_dep = [] else: task.task_dep = [] # reporter if isinstance(reporter, six.string_types): if reporter not in REPORTERS: msg = ("No reporter named '%s'." " Type 'doit help run' to see a list " "of available reporters.") raise InvalidCommand(msg % reporter) reporter_cls = REPORTERS[reporter] else: # user defined class reporter_cls = reporter # verbosity if verbosity is None: use_verbosity = Task.DEFAULT_VERBOSITY else: use_verbosity = verbosity show_out = use_verbosity < 2 # show on error report # outstream if isinstance(outfile, six.string_types): outstream = codecs.open(outfile, 'w', encoding='utf-8') else: # outfile is a file-like object (like StringIO or sys.stdout) outstream = outfile # run try: # FIXME stderr will be shown twice in case of task error/failure if isinstance(reporter_cls, type): reporter_obj = reporter_cls( outstream, { 'show_out': show_out, 'show_err': True, 'reporter_url': self.opt_values['reporter_url'], 'auth_info': self.opt_values['auth_info'] }) else: # also accepts reporter instances reporter_obj = reporter_cls run_args = [ self.dep_class, self.dep_file, reporter_obj, continue_, always, verbosity ] run_kwargs = {} RunnerClass = RUNNER_MAP.get(self.opt_values["runner"]) if not RunnerClass: RunnerClass = self._discover_runner_class( num_process, par_type) elif self.opt_values['runner'] in GRID_RUNNER_MAP: if not self.opt_values.get('partition', None): raise InvalidCommand("--partition option is required " "when using a grid runner") run_args = [ self.opt_values['partition'], self.opt_values['perf_url'], self.opt_values['tmpfiledir'], self.opt_values['grid_args'] ] + run_args run_kwargs['num_process'] = num_process if num_process else 1 runner = RunnerClass(*run_args, **run_kwargs) runner.pipeline_name = pipeline_name return runner.run_all(self.control.task_dispatcher()) finally: if isinstance(outfile, str): outstream.close()
def testFilter(self): filter_ = ['t2', 't3'] tc = TaskControl(TASKS_SAMPLE) assert filter_ == tc._filter_tasks(filter_)
def test_bug770150_task_dependency_from_target(self): t1 = Task("taskX", None, file_dep=[], targets=['intermediate']) t2 = Task("taskY", None, file_dep=['intermediate'], task_dep=['taskZ']) t3 = Task("taskZ", None) TaskControl([t1, t2, t3]) assert ['taskZ', 'taskX'] == t2.task_dep
def testProcessAll(self, tasks_sample): tc = TaskControl(tasks_sample) tc.process(None) assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testFilterPattern(self, tasks_sample): tc = TaskControl(tasks_sample) assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
def testProcessAll(self): tc = TaskControl(TASKS_SAMPLE) tc.process(None) assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
def testFilterSubtask(self): filter_ = ["t1", "g1.b"] tc = TaskControl(TASKS_SAMPLE) assert filter_ == tc._filter_tasks(filter_)
def testFilterTarget(self): tasks = list(TASKS_SAMPLE) tasks.append(Task("tX", [""], [], ["targetX"])) tc = TaskControl(tasks) assert ['tX'] == tc._filter_tasks(["targetX"])
def testProcessSelection(self): filter_ = ['t2', 't3'] tc = TaskControl(TASKS_SAMPLE) tc.process(filter_) assert filter_ == tc.selected_tasks
def testFilterWrongSubtaskName(self): t1 = Task("taskX", None) t2 = Task("taskY", None) tc = TaskControl([t1, t2]) pytest.raises(InvalidCommand, tc._filter_tasks, ['taskX:no'])
def testFilterPattern(self): tc = TaskControl(TASKS_SAMPLE) assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
def testProcessSelection(self, tasks_sample): filter_ = ['t2', 't3'] tc = TaskControl(tasks_sample) tc.process(filter_) assert filter_ == tc.selected_tasks