def testSystemExitRaises(self, reporter, RunnerClass, depfile):
     t1 = Task("x", [_exit])
     my_runner = RunnerClass(depfile.name, reporter)
     tc = TaskControl([t1])
     tc.process(None)
     pytest.raises(SystemExit, my_runner.run_tasks, tc)
     my_runner.finish()
示例#2
0
    def test_delayed_creation(self):
        def creator():
            yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']}
            yield {'name': 'foo2', 'actions': None, 'targets': ['bar']}

        delayed_loader = DelayedLoader(creator, executed='t2')
        tasks = [Task('t0', None, task_dep=['t1']),
                 Task('t1', None, loader=delayed_loader),
                 Task('t2', None)]

        control = TaskControl(tasks)
        control.process(['t0'])
        disp = control.task_dispatcher()
        gen = disp.generator
        nt2 = next(gen)
        assert nt2.task.name == "t2"

        # wait for t2 to be executed
        assert "hold on" == next(gen)
        assert "hold on" == next(gen) # hold until t2 is done

        # delayed creation of tasks for t1 does not mess existing info
        assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
        nf2 = gen.send(nt2)
        assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])

        assert nf2.task.name == "t1:foo2"
        nf1 = gen.send(nf2)
        assert nf1.task.name == "t1:foo1"
        assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added
        nt1 = gen.send(nf1)
        assert nt1.task.name == "t1"
        nt0 = gen.send(nt1)
        assert nt0.task.name == "t0"
        pytest.raises(StopIteration, lambda gen: next(gen), gen)
示例#3
0
    def test_delayed_creation(self):
        def creator():
            yield {'name': 'foo1', 'actions': None, 'file_dep': ['bar']}
            yield {'name': 'foo2', 'actions': None, 'targets': ['bar']}

        delayed_loader = DelayedLoader(creator, executed='t2')
        tasks = [Task('t0', None, task_dep=['t1']),
                 Task('t1', None, loader=delayed_loader),
                 Task('t2', None)]

        control = TaskControl(tasks)
        control.process(['t0'])
        disp = control.task_dispatcher()
        gen = disp.generator
        nt2 = next(gen)
        assert nt2.task.name == "t2"

        # wait for t2 to be executed
        assert "hold on" == next(gen)
        assert "hold on" == next(gen) # hold until t2 is done

        # delayed creation of tasks for t1 does not mess existing info
        assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])
        nf2 = gen.send(nt2)
        assert disp.nodes['t1'].waiting_me == set([disp.nodes['t0']])

        assert nf2.task.name == "t1:foo2"
        nf1 = gen.send(nf2)
        assert nf1.task.name == "t1:foo1"
        assert nf1.task.task_dep == ['t1:foo2'] # implicit dep added
        nt1 = gen.send(nf1)
        assert nt1.task.name == "t1"
        nt0 = gen.send(nt1)
        assert nt0.task.name == "t0"
        pytest.raises(StopIteration, lambda gen: next(gen), gen)
示例#4
0
def doit_auto(dependency_file, task_list, filter_tasks, loop_callback=None):
    """Re-execute tasks automatically a depedency changes

    @param filter_tasks (list -str): print only tasks from this list
    @loop_callback: used to stop loop on unittests
    """
    task_control = TaskControl(task_list)
    task_control.process(filter_tasks)
    tasks_to_run = list(set([t for t in task_control.task_dispatcher(True)]))
    watch_tasks = [t.name for t in tasks_to_run]
    watch_files = list(itertools.chain(*[s.file_dep for s in tasks_to_run]))
    watch_files = list(set(watch_files))

    class DoitAutoRun(FileModifyWatcher):
        """Execute doit on event handler of file changes """
        def handle_event(self, event):
            doit_run(dependency_file, task_list, sys.stdout,
                     watch_tasks, reporter='executed-only')
            # reset run_status
            for task in task_list:
                task.run_status = None

    file_watcher = DoitAutoRun(watch_files)
    # always run once when started
    file_watcher.handle_event(None)
    file_watcher.loop(loop_callback)
示例#5
0
 def testPosParam(self):
     tasks = list(TASKS_SAMPLE)
     tasks.append(Task("tP", [""],[],[], pos_arg='myp'))
     tc = TaskControl(tasks)
     args = ["tP", "hello option!", "t1"]
     assert ['tP',] == tc._filter_tasks(args)
     assert ["hello option!", "t1"] == tc.tasks['tP'].pos_arg_val
示例#6
0
 def testDetectCyclicReference(self):
     tasks = [Task("taskX",None,task_dep=["taskY"]),
              Task("taskY",None,task_dep=["taskX"])]
     tc = TaskControl(tasks)
     tc.process(None)
     gen = tc._add_task(0, "taskX", False)
     py.test.raises(InvalidDodoFile, gen.next)
示例#7
0
 def testPosParam(self):
     tasks = list(TASKS_SAMPLE)
     tasks.append(Task("tP", [""],[],[], pos_arg='myp'))
     tc = TaskControl(tasks)
     args = ["tP", "hello option!", "t1"]
     assert ['tP',] == tc._filter_tasks(args)
     assert ["hello option!", "t1"] == tc.tasks['tP'].pos_arg_val
示例#8
0
 def testChangeOrder_AddJustOnce(self):
     tasks = [Task("taskX",None,task_dep=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(None)
     assert [tasks[1], tasks[0]] == [x for x in tc._add_task(0, 'taskX', False)]
     # both tasks were already added. so no tasks left..
     assert [] == [x for x in tc._add_task(0, 'taskY', False)]
示例#9
0
 def test_successRunOnce(self, reporter, RunnerClass):
     tasks = [Task("taskX", [my_print], run_once=True)]
     my_runner = RunnerClass(TESTDB, reporter)
     tc = TaskControl(tasks)
     tc.process(None)
     my_runner.run_tasks(tc)
     assert runner.SUCCESS == my_runner.finish()
     d = Dependency(TESTDB)
     assert '1' == d._get('taskX', 'run-once:')
示例#10
0
 def test_filter_delayed_subtask(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None, loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2])
     control._filter_tasks(['taskY:foo'])
     assert isinstance(t2.loader, DelayedLoader)
     # sub-task will use same loader, and keep parent basename
     assert control.tasks['taskY:foo'].loader.basename == 'taskY'
     assert control.tasks['taskY:foo'].loader is t2.loader
示例#11
0
 def testSetupInvalid(self):
     tasks = [Task("taskX",None,setup=["taskZZZZZZZZ"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', False)
     assert tasks[0] == gen.next() # tasks with setup are yield twice
     tasks[0].run_status = 'run' # should be executed
     py.test.raises(InvalidTask, gen.next) # execute setup before
示例#12
0
 def test_filter_delayed_subtask(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None, loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2])
     control._filter_tasks(['taskY:foo'])
     assert isinstance(t2.loader, DelayedLoader)
     # sub-task will use same loader, and keep parent basename
     assert control.tasks['taskY:foo'].loader.basename == 'taskY'
     assert control.tasks['taskY:foo'].loader is t2.loader
示例#13
0
 def test_reporter_runtime_error(self, reporter):
     t1 = Task('t1', [], setup=['make_invalid'])
     my_runner = runner.Runner(TESTDB, reporter)
     tc = TaskControl([t1])
     tc.process(None)
     my_runner.run_all(tc)
     assert ('start', t1) == reporter.log.pop(0)
     assert ('runtime_error',) == reporter.log.pop(0)
     assert not reporter.log
示例#14
0
 def test_teardown(self, reporter, RunnerClass):
     t1 = Task('t1', [], teardown=[ok])
     t2 = Task('t2', [])
     my_runner = RunnerClass(TESTDB, reporter)
     tc = TaskControl([t1, t2])
     tc.process(None)
     assert [] == my_runner.teardown_list
     my_runner.run_tasks(tc)
     my_runner.finish()
     assert ('teardown', t1) == reporter.log[-1]
示例#15
0
 def test_stop_running(self, reporter):
     t1 = Task('t1', [])
     t2 = Task('t2', [])
     tc = TaskControl([t1, t2])
     tc.process(None)
     run = runner.MRunner(TESTDB, reporter)
     run._run_tasks_init(tc)
     assert t1 == run.get_next_task()
     run._stop_running = True
     assert None == run.get_next_task()
示例#16
0
 def testSetupTasksDontRun(self):
     tasks = [Task("taskX",None,setup=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', False)
     assert tasks[0] == gen.next()
     # X is up-to-date
     tasks[0].run_status = 'up-to-date'
     py.test.raises(StopIteration, gen.next)
示例#17
0
 def testParallel(self):
     tasks = [Task("taskX",None,task_dep=["taskY"]),
              Task("taskY",None)]
     tc = TaskControl(tasks)
     tc.process(None)
     gen1 = tc._add_task(0, "taskX", False)
     assert tasks[1] == gen1.next()
     # gen2 wont get any task, because it was already being processed
     gen2 = tc._add_task(1, "taskY", False)
     py.test.raises(StopIteration, gen2.next)
示例#18
0
 def test_run_task(self, reporter, depfile):
     t1 = Task('t1', [])
     t2 = Task('t2', [])
     tc = TaskControl([t1, t2])
     tc.process(None)
     run = runner.MRunner(depfile.name, reporter)
     run._run_tasks_init(tc)
     assert t1 == run.get_next_task()
     assert t2 == run.get_next_task()
     assert None == run.get_next_task()
示例#19
0
 def test_include_setup(self):
     tasks = [Task("t1", None, task_dep=["t2"]),
              Task("t2", None,)]
     control = TaskControl(tasks)
     control.process(['t1'])
     gen = control.task_dispatcher(include_setup=True).generator
     # dont wait for tasks
     assert tasks[0] == gen.send(None).task
     assert tasks[1] == gen.send(None).task
     pytest.raises(StopIteration, gen.send, None)
示例#20
0
def doit_run(dependency_file, task_list, output, options=None,
             verbosity=None, always_execute=False, continue_=False,
             reporter='default', num_process=0):
    """
    @param reporter: (str) one of provided reporters or ...
                     (class) user defined reporter class (can only be specified
           from DOIT_CONFIG - never from command line)
    """
    # get tasks to be executed
    task_control = TaskControl(task_list)
    task_control.process(options)

    # reporter
    if isinstance(reporter, str):
        if reporter not in REPORTERS:
            msg = ("No reporter named '%s'."
                   " Type 'doit help run' to see a list "
                   "of available reporters.")
            raise InvalidCommand(msg % reporter)
        reporter_cls = REPORTERS[reporter]
    else:
        # user defined class
        reporter_cls = reporter

    # verbosity
    if verbosity is None:
        use_verbosity = Task.DEFAULT_VERBOSITY
    else:
        use_verbosity = verbosity
    show_out = use_verbosity < 2 # show on error report

    # outstream
    if isinstance(output, str):
        outstream = open(output, 'w')
    else: # outfile is a file-like object (like StringIO or sys.stdout)
        outstream = output

    # run
    try:
        # FIXME stderr will be shown twice in case of task error/failure
        reporter_obj = reporter_cls(outstream, {'show_out':show_out,
                                                'show_err': True})

        if num_process == 0:
            runner = Runner(dependency_file, reporter_obj, continue_,
                            always_execute, verbosity)
        else:
            runner = MRunner(dependency_file, reporter_obj, continue_,
                               always_execute, verbosity, num_process)

        return runner.run_all(task_control)
    finally:
        if isinstance(output, str):
            outstream.close()
示例#21
0
 def testSetupTasksRun(self):
     tasks = [Task("taskX",None,setup=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', False)
     assert tasks[0] == gen.next() # tasks with setup are yield twice
     tasks[0].run_status = 'run' # should be executed
     assert tasks[1] == gen.next() # execute setup before
     assert tasks[0] == gen.next() # second time, ok
     py.test.raises(StopIteration, gen.next) # nothing left
示例#22
0
 def test_normal(self):
     tasks = [Task("t1", None, task_dep=["t2"]),
              Task("t2", None,)]
     control = TaskControl(tasks)
     control.process(['t1'])
     gen = control.task_dispatcher().generator
     n2 = next(gen)
     assert tasks[1] == n2.task
     assert "hold on" == next(gen)
     assert "hold on" == next(gen) # hold until t2 is done
     assert tasks[0] == gen.send(n2).task
     pytest.raises(StopIteration, lambda gen: next(gen), gen)
示例#23
0
 def testIncludeSetup(self):
     # with include_setup yield all tasks without waiting for setup tasks to
     # be ready
     tasks = [Task("taskX",None,setup=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', True) # <== include_setup
     assert tasks[0] == gen.next() # tasks with setup are yield twice
     assert tasks[1] == gen.next() # execute setup before
     assert tasks[0] == gen.next() # second time, ok
     py.test.raises(StopIteration, gen.next) # nothing left
示例#24
0
 def test_normal(self):
     tasks = [Task("t1", None, task_dep=["t2"]),
              Task("t2", None,)]
     control = TaskControl(tasks)
     control.process(['t1'])
     gen = control.task_dispatcher().generator
     n2 = next(gen)
     assert tasks[1] == n2.task
     assert "hold on" == next(gen)
     assert "hold on" == next(gen) # hold until t2 is done
     assert tasks[0] == gen.send(n2).task
     pytest.raises(StopIteration, lambda gen: next(gen), gen)
示例#25
0
 def testAllTasksWaiting(self):
     tasks = [Task("taskX",None,setup=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc.task_dispatcher()
     assert tasks[0] == gen.next() # tasks with setup are yield twice
     assert "hold on" == gen.next() # nothing else really available
     tasks[0].run_status = 'run' # should be executed
     assert tasks[1] == gen.next() # execute setup before
     assert tasks[0] == gen.next() # second time, ok
     py.test.raises(StopIteration, gen.next) # nothing left
示例#26
0
    def test_delayed_creation_target_regex(self):
        def creator():
            yield Task('foo', None, targets=['tgt1'])

        delayed_loader = DelayedLoader(creator,
                                       executed='t2',
                                       target_regex='tgt1')
        tasks = {
            't1': Task('t1', None, loader=delayed_loader),
            't2': Task('t2', None),
        }

        tc = TaskControl(list(tasks.values()))
        selection = tc._filter_tasks(['tgt1'])
        assert ['_regex_target_tgt1:t1'] == selection
        td = TaskDispatcher(tc.tasks, tc.targets, selection)

        n1 = td._gen_node(None, '_regex_target_tgt1:t1')
        gen = td._add_task(n1)

        # first returned node is `t2` because it is an implicit task_dep
        n2 = next(gen)
        assert n2.task.name == 't2'

        # wait until t2 is finished
        n3 = next(gen)
        assert n3 == 'wait'

        # after t2 is done, generator is reseted
        n2.run_status = 'done'
        td._update_waiting(n2)
        n4 = next(gen)
        assert n4 == "reset generator"

        # manually reset generator
        n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))

        # get the delayed created task
        gen2 = n1.generator  # n1 generator was reset / replaced
        # get t1 because of its target was a file_dep of _regex_target_tgt1
        n5 = next(gen2)
        assert n5.task.name == 'foo'

        # get internal created task
        n5.run_status = 'done'
        td._update_waiting(n5)
        n6 = next(gen2)
        assert n6.name == '_regex_target_tgt1:t1'

        # file_dep is removed because foo might not be task
        # that creates this task (support for multi regex matches)
        assert n6.file_dep == {}
示例#27
0
 def test_include_setup(self):
     tasks = [Task("t1", None, task_dep=["t2"]),
              Task(
                  "t2",
                  None,
              )]
     control = TaskControl(tasks)
     control.process(['t1'])
     gen = control.task_dispatcher(include_setup=True).generator
     # dont wait for tasks
     assert tasks[0] == gen.send(None).task
     assert tasks[1] == gen.send(None).task
     pytest.raises(StopIteration, gen.send, None)
示例#28
0
 def test_failureOutput(self, reporter, RunnerClass, depfile):
     tasks = [Task("taskX", [_fail]),
              Task("taskY", [_fail])]
     my_runner = RunnerClass(depfile.name, reporter)
     tc = TaskControl(tasks)
     tc.process(None)
     my_runner.run_tasks(tc)
     assert runner.FAILURE == my_runner.finish()
     assert ('start', tasks[0]) == reporter.log.pop(0)
     assert ('execute', tasks[0]) == reporter.log.pop(0)
     assert ('fail', tasks[0]) == reporter.log.pop(0)
     # second task is not executed
     assert 0 == len(reporter.log)
示例#29
0
 def test_error(self, reporter, RunnerClass):
     tasks = [Task("taskX", [_error]),
              Task("taskY", [_error])]
     my_runner = RunnerClass(TESTDB, reporter)
     tc = TaskControl(tasks)
     tc.process(None)
     my_runner.run_tasks(tc)
     assert runner.ERROR == my_runner.finish()
     assert ('start', tasks[0]) == reporter.log.pop(0)
     assert ('execute', tasks[0]) == reporter.log.pop(0)
     assert ('fail', tasks[0]) == reporter.log.pop(0)
     # second task is not executed
     assert 0 == len(reporter.log)
示例#30
0
 def test_filter_delayed_multi_select(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None,
               loader=DelayedLoader(lambda: None, target_regex='a.*'))
     t3 = Task("taskZ", None,
               loader=DelayedLoader(lambda: None, target_regex='b.*'))
     t4 = Task("taskW", None,
               loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
     selected = control._filter_tasks(['abc', 'att'])
     assert isinstance(t2.loader, DelayedLoader)
     assert len(selected) == 2
     assert selected[0] == '_regex_target_abc:taskY'
     assert selected[1] == '_regex_target_att:taskY'
示例#31
0
    def test_less_processes(self, reporter, monkeypatch):
        mock_process = Mock()
        monkeypatch.setattr(runner, 'Process', mock_process)
        t1 = Task('t1', [])
        tc = TaskControl([t1])
        tc.process(None)
        run = runner.MRunner(TESTDB, reporter, num_process=2)
        run._run_tasks_init(tc)
        result_q = Queue()
        task_q = Queue()

        proc_list = run._run_start_processes(task_q, result_q)
        assert 1 == len(proc_list)
        assert t1.name == task_q.get()[0]
示例#32
0
    def test_delayed_creation_target_regex(self):
        def creator():
            yield Task('foo', None, targets=['tgt1'])
        delayed_loader = DelayedLoader(creator,
                                       executed='t2', target_regex='tgt1')
        tasks = {
            't1': Task('t1', None, loader=delayed_loader),
            't2': Task('t2', None),
        }

        tc = TaskControl(list(tasks.values()))
        selection = tc._filter_tasks(['tgt1'])
        assert ['_regex_target_tgt1:t1'] == selection
        td = TaskDispatcher(tc.tasks, tc.targets, selection)

        n1 = td._gen_node(None, '_regex_target_tgt1:t1')
        gen = td._add_task(n1)

        # first returned node is `t2` because it is an implicit task_dep
        n2 = next(gen)
        assert n2.task.name == 't2'

        # wait until t2 is finished
        n3 = next(gen)
        assert n3 == 'wait'

        # after t2 is done, generator is reseted
        n2.run_status = 'done'
        td._update_waiting(n2)
        n4 = next(gen)
        assert n4 == "reset generator"

        # manually reset generator
        n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))

        # get the delayed created task
        gen2 = n1.generator  # n1 generator was reset / replaced
        # get t1 because of its target was a file_dep of _regex_target_tgt1
        n5 = next(gen2)
        assert n5.task.name == 'foo'

        # get internal created task
        n5.run_status = 'done'
        td._update_waiting(n5)
        n6 = next(gen2)
        assert n6.name == '_regex_target_tgt1:t1'

        # file_dep is removed because foo might not be task
        # that creates this task (support for multi regex matches)
        assert n6.file_dep == {}
示例#33
0
 def test_filter_delayed_multi_select(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None,
               loader=DelayedLoader(lambda: None, target_regex='a.*'))
     t3 = Task("taskZ", None,
               loader=DelayedLoader(lambda: None, target_regex='b.*'))
     t4 = Task("taskW", None,
               loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
     selected = control._filter_tasks(['abc', 'att'])
     assert isinstance(t2.loader, DelayedLoader)
     assert len(selected) == 2
     assert selected[0] == '_regex_target_abc:taskY'
     assert selected[1] == '_regex_target_att:taskY'
示例#34
0
 def test_success(self, reporter, RunnerClass):
     tasks = [Task("taskX", [(my_print, ["out a"] )] ),
              Task("taskY", [(my_print, ["out a"] )] )]
     my_runner = RunnerClass(TESTDB, reporter)
     tc = TaskControl(tasks)
     tc.process(None)
     my_runner.run_tasks(tc)
     assert runner.SUCCESS == my_runner.finish()
     assert ('start', tasks[0]) == reporter.log.pop(0), reporter.log
     assert ('execute', tasks[0]) == reporter.log.pop(0)
     assert ('success', tasks[0]) == reporter.log.pop(0)
     assert ('start', tasks[1]) == reporter.log.pop(0)
     assert ('execute', tasks[1]) == reporter.log.pop(0)
     assert ('success', tasks[1]) == reporter.log.pop(0)
示例#35
0
 def testCalcDep(self):
     def get_deps():
         print "gget"
         return {'file_dep': ('a', 'b')}
     tasks = [Task("taskX", None, calc_dep=['task_dep']),
              Task("task_dep", [(get_deps,)]),
              ]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', False)
     assert tasks[1] == gen.next()
     assert isinstance(gen.next(), WaitRunTask)
     tasks[1].execute()
     assert tasks[0] == gen.next()
     assert set(['a', 'b']) == tasks[0].file_dep
示例#36
0
 def testWaitSetup(self):
     tasks = [Task("taskX",None,setup=["taskY"]),
              Task("taskY",None,)]
     tc = TaskControl(tasks)
     tc.process(['taskX'])
     gen = tc._add_task(0, 'taskX', False)
     assert tasks[0] == gen.next() # tasks with setup are yield twice
     # wait for taskX run_status
     wait = gen.next()
     assert wait.task_name == 'taskX'
     assert isinstance(wait, WaitSelectTask)
     tasks[0].run_status = 'run' # should be executed
     assert tasks[1] == gen.next() # execute setup before
     assert tasks[0] == gen.next() # second time, ok
     py.test.raises(StopIteration, gen.next) # nothing left
示例#37
0
    def test_waiting_controller(self, reporter):
        t1 = Task('t1', [])
        t2a = Task('t2A', [], calc_dep=('t1',))
        tc = TaskControl([t1, t2a])
        tc.process(None)
        run = runner.MRunner(TESTDB, reporter)
        run._run_tasks_init(tc)

        # first task ok
        assert t1 == run.get_next_task()

        # hold until t1 finishes
        assert 0 == run.free_proc
        assert isinstance(run.get_next_task(), runner.Hold)
        assert 1 == run.free_proc
示例#38
0
    def test_regex_not_found(self):
        def creator1():
            yield Task('foo1', None, targets=['tgt1'])
        delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')

        t1 = Task('t1', None, loader=delayed_loader1)

        tc = TaskControl([t1])
        selection = tc._filter_tasks(['tgt666'])
        assert ['_regex_target_tgt666:t1'] == selection
        td = TaskDispatcher(tc.tasks, tc.targets, selection)

        n1 = td._gen_node(None, '_regex_target_tgt666:t1')
        gen = td._add_task(n1)
        # target not found after generating all tasks from regex group
        pytest.raises(InvalidCommand, next, gen)
示例#39
0
    def test_regex_group_already_created(self):
        # this is required to avoid loading more tasks than required, GH-#60
        def creator1():
            yield Task('foo1', None, targets=['tgt1'])

        delayed_loader1 = DelayedLoader(creator1, target_regex='tgt.*')

        def creator2():  # pragma: no cover
            yield Task('foo2', None, targets=['tgt2'])

        delayed_loader2 = DelayedLoader(creator2, target_regex='tgt.*')

        t1 = Task('t1', None, loader=delayed_loader1)
        t2 = Task('t2', None, loader=delayed_loader2)

        tc = TaskControl([t1, t2])
        selection = tc._filter_tasks(['tgt1'])
        assert ['_regex_target_tgt1:t1', '_regex_target_tgt1:t2'] == selection
        td = TaskDispatcher(tc.tasks, tc.targets, selection)

        n1 = td._gen_node(None, '_regex_target_tgt1:t1')
        gen = td._add_task(n1)

        # delayed loader executed, so generator is reset
        n1b = next(gen)
        assert n1b == "reset generator"

        # manually reset generator
        n1.reset_task(td.tasks[n1.task.name], td._add_task(n1))

        # get the delayed created task
        gen1b = n1.generator  # n1 generator was reset / replaced
        # get t1 because of its target was a file_dep of _regex_target_tgt1
        n1c = next(gen1b)
        assert n1c.task.name == 'foo1'

        # get internal created task
        n1c.run_status = 'done'
        td._update_waiting(n1c)
        n1d = next(gen1b)
        assert n1d.name == '_regex_target_tgt1:t1'

        ## go for second selected task
        n2 = td._gen_node(None, '_regex_target_tgt1:t2')
        gen2 = td._add_task(n2)
        # loader is not executed because target t1 was already found
        pytest.raises(StopIteration, next, gen2)
示例#40
0
 def test_wild(self):
     tasks = [Task('t1', None, task_dep=['foo*']),
              Task(
                  'foo4',
                  None,
              )]
     TaskControl(tasks)
     assert 'foo4' in tasks[0].task_dep
示例#41
0
 def test_filter_delayed_regex_auto(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None,
               loader=DelayedLoader(lambda: None, target_regex='a.*'))
     t3 = Task("taskZ", None,
               loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2, t3], auto_delayed_regex=True)
     selected = control._filter_tasks(['abc'])
     assert len(selected) == 2
     assert (sorted(selected) ==
             ['_regex_target_abc:taskY', '_regex_target_abc:taskZ'])
     assert control.tasks['_regex_target_abc:taskY'].file_dep == {'abc'}
     assert control.tasks['_regex_target_abc:taskZ'].file_dep == {'abc'}
     assert (control.tasks['_regex_target_abc:taskY'].loader.basename ==
             t2.name)
     assert (control.tasks['_regex_target_abc:taskZ'].loader.basename ==
             t3.name)
示例#42
0
 def test_filter_delayed_regex_single(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None,
               loader=DelayedLoader(lambda: None, target_regex='a.*'))
     t3 = Task("taskZ", None,
               loader=DelayedLoader(lambda: None, target_regex='b.*'))
     t4 = Task("taskW", None,
               loader=DelayedLoader(lambda: None))
     control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
     selected = control._filter_tasks(['abc'])
     assert isinstance(t2.loader, DelayedLoader)
     assert len(selected) == 1
     assert selected[0] == '_regex_target_abc:taskY'
     sel_task = control.tasks['_regex_target_abc:taskY']
     assert sel_task.file_dep == {'abc'}
     assert sel_task.loader.basename == 'taskY'
     assert sel_task.loader is t2.loader
示例#43
0
    def _execute(self, subtasks, reverse, horizontal, outfile, pos_args=None):
        # init
        control = TaskControl(self.task_list)
        self.tasks = control.tasks
        self.subtasks = subtasks
        self._edges = set()  # used to avoid adding same edge twice

        # create graph
        self.graph = pygraphviz.AGraph(strict=False, directed=True)
        self.graph.node_attr['color'] = 'lightblue2'
        self.graph.node_attr['style'] = 'filled'

        if (horizontal):
            self.graph.graph_attr.update(rankdir='LR')

        # populate graph
        processed = set()  # str - task name
        if pos_args:
            to_process = deque(pos_args)
        else:
            to_process = deque(control.tasks.keys())

        while to_process:
            task = control.tasks[to_process.popleft()]
            if task.name in processed:
                continue
            processed.add(task.name)

            # add nodes
            node_attrs = {}
            if task.has_subtask:
                node_attrs['peripheries'] = '2'
            if (not task.subtask_of) or subtasks:
                self.graph.add_node(task.name, **node_attrs)

            # add edges
            for sink_name in task.setup_tasks:
                self.add_edge(task.name, sink_name, arrowhead='empty')
                if sink_name not in processed:
                    to_process.append(sink_name)
            for sink_name in task.task_dep:
                self.add_edge(task.name, sink_name, arrowhead='')
                if sink_name not in processed:
                    to_process.append(sink_name)

        if not outfile:
            name = pos_args[0] if len(pos_args) == 1 else 'tasks'
            outfile = '{}.dot'.format(name)
        print('Generated file: {}'.format(outfile))
        if (reverse):
            self.graph.reverse().write(outfile)
        else:
            self.graph.write(outfile)
示例#44
0
 def testOptions(self):
     options = ["t3", "--message", "hello option!", "t1"]
     tc = TaskControl(TASKS_SAMPLE)
     assert ['t3', 't1'] == tc._filter_tasks(options)
     assert "hello option!" == tc.tasks['t3'].options['opt1']
示例#45
0
 def testFilterEmptyList(self):
     filter_ = []
     tc = TaskControl(TASKS_SAMPLE)
     assert filter_ == tc._filter_tasks(filter_)
示例#46
0
 def testFilterWrongName(self):
     tc = TaskControl(TASKS_SAMPLE)
     pytest.raises(InvalidCommand, tc._filter_tasks, ['no'])
示例#47
0
 def test_targetDependency(self):
     t1 = Task("taskX", None, [], ['intermediate'])
     t2 = Task("taskY", None, ['intermediate'], [])
     TaskControl([t1, t2])
     assert ['taskX'] == t2.task_dep
示例#48
0
 def test_addTask(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None)
     tc = TaskControl([t1, t2])
     assert 2 == len(tc.tasks)
示例#49
0
文件: run.py 项目: biobakery/anadama
    def _execute(self,
                 outfile=sys.stdout,
                 verbosity=None,
                 always=False,
                 continue_=False,
                 reporter='default',
                 num_process=0,
                 par_type='process',
                 single=False,
                 pipeline_name="Custom Pipeline"):
        """
        @param reporter: (str) one of provided reporters or ...
                         (class) user defined reporter class 
                                 (can only be specified
                                 from DOIT_CONFIG - never from command line)
                         (reporter instance) - only used in unittests
        """
        # get tasks to be executed
        # self.control is saved on instance to be used by 'auto' command
        self.control = TaskControl(self.task_list)
        self.control.process(self.sel_tasks)

        if single:
            for task_name in self.sel_tasks:
                task = self.control.tasks[task_name]
                if task.has_subtask:
                    for task_name in task.task_dep:
                        sub_task = self.control.tasks[task_name]
                        sub_task.task_dep = []
                else:
                    task.task_dep = []

        # reporter
        if isinstance(reporter, six.string_types):
            if reporter not in REPORTERS:
                msg = ("No reporter named '%s'."
                       " Type 'doit help run' to see a list "
                       "of available reporters.")
                raise InvalidCommand(msg % reporter)
            reporter_cls = REPORTERS[reporter]
        else:
            # user defined class
            reporter_cls = reporter

        # verbosity
        if verbosity is None:
            use_verbosity = Task.DEFAULT_VERBOSITY
        else:
            use_verbosity = verbosity
        show_out = use_verbosity < 2  # show on error report

        # outstream
        if isinstance(outfile, six.string_types):
            outstream = codecs.open(outfile, 'w', encoding='utf-8')
        else:  # outfile is a file-like object (like StringIO or sys.stdout)
            outstream = outfile

        # run
        try:
            # FIXME stderr will be shown twice in case of task error/failure
            if isinstance(reporter_cls, type):
                reporter_obj = reporter_cls(
                    outstream, {
                        'show_out': show_out,
                        'show_err': True,
                        'reporter_url': self.opt_values['reporter_url'],
                        'auth_info': self.opt_values['auth_info']
                    })
            else:  # also accepts reporter instances
                reporter_obj = reporter_cls

            run_args = [
                self.dep_class, self.dep_file, reporter_obj, continue_, always,
                verbosity
            ]
            run_kwargs = {}
            RunnerClass = RUNNER_MAP.get(self.opt_values["runner"])
            if not RunnerClass:
                RunnerClass = self._discover_runner_class(
                    num_process, par_type)
            elif self.opt_values['runner'] in GRID_RUNNER_MAP:
                if not self.opt_values.get('partition', None):
                    raise InvalidCommand("--partition option is required "
                                         "when using a grid runner")
                run_args = [
                    self.opt_values['partition'], self.opt_values['perf_url'],
                    self.opt_values['tmpfiledir'], self.opt_values['grid_args']
                ] + run_args
                run_kwargs['num_process'] = num_process if num_process else 1

            runner = RunnerClass(*run_args, **run_kwargs)
            runner.pipeline_name = pipeline_name
            return runner.run_all(self.control.task_dispatcher())
        finally:
            if isinstance(outfile, str):
                outstream.close()
示例#50
0
 def testFilter(self):
     filter_ = ['t2', 't3']
     tc = TaskControl(TASKS_SAMPLE)
     assert filter_ == tc._filter_tasks(filter_)
示例#51
0
 def test_bug770150_task_dependency_from_target(self):
     t1 = Task("taskX", None, file_dep=[], targets=['intermediate'])
     t2 = Task("taskY", None, file_dep=['intermediate'], task_dep=['taskZ'])
     t3 = Task("taskZ", None)
     TaskControl([t1, t2, t3])
     assert ['taskZ', 'taskX'] == t2.task_dep
示例#52
0
 def testProcessAll(self, tasks_sample):
     tc = TaskControl(tasks_sample)
     tc.process(None)
     assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
示例#53
0
 def testFilterPattern(self, tasks_sample):
     tc = TaskControl(tasks_sample)
     assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
示例#54
0
 def testProcessAll(self):
     tc = TaskControl(TASKS_SAMPLE)
     tc.process(None)
     assert ['t1', 't2', 'g1', 'g1.a', 'g1.b', 't3'] == tc.selected_tasks
示例#55
0
 def testFilterSubtask(self):
     filter_ = ["t1", "g1.b"]
     tc = TaskControl(TASKS_SAMPLE)
     assert filter_ == tc._filter_tasks(filter_)
示例#56
0
 def testFilterTarget(self):
     tasks = list(TASKS_SAMPLE)
     tasks.append(Task("tX", [""], [], ["targetX"]))
     tc = TaskControl(tasks)
     assert ['tX'] == tc._filter_tasks(["targetX"])
示例#57
0
 def testProcessSelection(self):
     filter_ = ['t2', 't3']
     tc = TaskControl(TASKS_SAMPLE)
     tc.process(filter_)
     assert filter_ == tc.selected_tasks
示例#58
0
 def testFilterWrongSubtaskName(self):
     t1 = Task("taskX", None)
     t2 = Task("taskY", None)
     tc = TaskControl([t1, t2])
     pytest.raises(InvalidCommand, tc._filter_tasks, ['taskX:no'])
示例#59
0
 def testFilterPattern(self):
     tc = TaskControl(TASKS_SAMPLE)
     assert ['t1', 'g1', 'g1.a', 'g1.b'] == tc._filter_tasks(['*1*'])
示例#60
0
 def testProcessSelection(self, tasks_sample):
     filter_ = ['t2', 't3']
     tc = TaskControl(tasks_sample)
     tc.process(filter_)
     assert filter_ == tc.selected_tasks