Example #1
0
 def testProcessRunEmptyFilter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
                          task_list=tasks_sample(), sel_tasks=[])
     cmd_run._execute(output)
     got = output.getvalue().split("\n")[:-1]
     assert [] == got
Example #2
0
 def test_no_children(self):
     my_task = Task("t2", [""], file_dep=['d2.txt'])
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
     cmd._execute(graph_type='json', no_children=True)
     got = output.getvalue()
     self.assertNotIn("d2.txt", got)
Example #3
0
 def testProcessRunMThread(self, dependency1, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample())
     result = cmd_run._execute(output, num_process=1, par_type="thread")
     assert 0 == result
     got = output.getvalue().split("\n")[:-1]
     assert [".  t1", ".  t2", ".  g1.a", ".  g1.b", ".  t3"] == got
Example #4
0
 def testFilterAll(self):
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=tasks_sample())
     cmd_list._execute(subtasks=True, pos_args=['g1'])
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['g1', 'g1.a', 'g1.b']
     assert expected == got
Example #5
0
 def testDependencies(self):
     my_task = Task("t2", [""], file_dep=['d2.txt'])
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=[my_task])
     cmd_list._execute(list_deps=True)
     got = output.getvalue()
     assert "d2.txt" in got
Example #6
0
    def test_missing_dep_and_target(self, depfile, dependency1, dependency2):

        task_a = Task("task_a", [""],
                      file_dep=['tests/data/dependency1'],
                      targets=['tests/data/dependency2'])
        task_b = Task("task_b", [""],
                      file_dep=['tests/data/dependency2'],
                      targets=['tests/data/dependency3'])
        task_c = Task("task_c", [""],
                      file_dep=['tests/data/dependency3'],
                      targets=['tests/data/dependency4'])

        output = StringIO()
        tasks = [task_a, task_b, task_c]
        cmd_reset = CmdFactory(ResetDep,
                               outstream=output,
                               task_list=tasks,
                               dep_manager=depfile)
        cmd_reset._execute()

        got = output.getvalue()
        assert ("processed task_a\n"
                "processed task_b\n"
                "failed task_c (Dependent file 'tests/data/dependency3'"
                " does not exist.)\n") == got
Example #7
0
 def testCustomReporter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
                          task_list=[tasks_sample()[0]])
     cmd_run._execute(output, reporter=MyReporter)
     got = output.getvalue().split("\n")[:-1]
     assert 'MyReporter.start t1' == got[0]
Example #8
0
 def test_no_children(self):
     my_task = Task("t2", [""], file_dep=['d2.txt'])
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
     cmd._execute(graph_type='json', no_children=True)
     got = output.getvalue()
     self.assertNotIn("d2.txt", got)
Example #9
0
 def testProcessRunSingle(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample(), sel_tasks=["t3"])
     cmd_run._execute(output, single=True)
     got = output.getvalue().split("\n")[:-1]
     # t1 is a depenendency of t3 but not included
     assert [".  t3"] == got
Example #10
0
 def test_unicode_name(self, depfile):
     task_list = [Task(six.u("t做"), [""], doc=six.u("t1 doc string 做")),]
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
                           task_list=task_list)
     cmd_list._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     assert six.u('t做') == got[0]
Example #11
0
 def test_filter(self, depfile, dependency1):
     output = StringIO()
     tasks = tasks_sample()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
                            dep_manager=depfile)
     cmd_reset._execute(pos_args=['t2'])
     got = output.getvalue()
     assert "processed t2\n" == got
Example #12
0
 def test_store_json_stdout(self):
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=_sample_tasks())
     cmd._execute(graph_type='json')
     got = output.getvalue()
     self.assertIn("read", got)
     self.assertIn("t3", got)
     self.assertIn("join_files", got)
Example #13
0
 def test_children(self):
     # TODO: Implement no-child option.
     my_task = Task("t2", [""], file_dep=['d2.txt'])
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
     cmd._execute(graph_type='json', no_children=False)
     got = output.getvalue()
     self.assertIn("d2.txt", got)
Example #14
0
 def test_children(self):
     # TODO: Implement no-child option.
     my_task = Task("t2", [""], file_dep=['d2.txt'])
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
     cmd._execute(graph_type='json', no_children=False)
     got = output.getvalue()
     self.assertIn("d2.txt", got)
Example #15
0
 def test_store_json_stdout(self):
     output = StringIO()
     cmd = CmdFactory(Graphx, outstream=output, task_list=_sample_tasks())
     cmd._execute(graph_type='json')
     got = output.getvalue()
     self.assertIn("read", got)
     self.assertIn("t3", got)
     self.assertIn("join_files", got)
Example #16
0
 def test_filter(self, depfile, dependency1):
     output = StringIO()
     tasks = tasks_sample()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
                            dep_manager=depfile)
     cmd_reset._execute(pos_args=['t2'])
     got = output.getvalue()
     assert "processed t2\n" == got
Example #17
0
 def testSubTask(self):
     output = StringIO()
     tasks = tasks_sample()
     cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
     cmd_list._execute(subtasks=True)
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = [t.name for t in sorted(tasks)]
     assert expected == got
Example #18
0
 def testQuiet(self):
     output = StringIO()
     tasks = tasks_sample()
     cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
     cmd_list._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = [t.name for t in tasks if not t.is_subtask]
     assert sorted(expected) == got
Example #19
0
 def testCustomTemplate(self):
     output = StringIO()
     tasks = tasks_sample()
     cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
     cmd_list._execute(template='xxx {name} xxx {doc}')
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     assert 'xxx g1 xxx g1 doc string' == got[0]
     assert 'xxx t3 xxx t3 doc string' == got[3]
Example #20
0
 def test_execute(self, depfile, dependency1):
     output = StringIO()
     tasks = tasks_sample()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
                            dep_manager=depfile)
     cmd_reset._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ["processed %s" % t.name for t in tasks]
     assert sorted(expected) == sorted(got)
Example #21
0
 def test_missing_file_dep(self, depfile):
     my_task = Task("t2", [""], file_dep=['tests/data/missing'])
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert ("failed t2 (Dependent file 'tests/data/missing' does not "
             "exist.)\n") == got
Example #22
0
 def testSortByDefinition(self):
     # test sorting task list by order of definition
     task_list = list(tasks_sample())
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
     cmd_list._execute(sort='definition')
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['t1', 't2', 'g1', 't3']
     assert expected == got
Example #23
0
 def testSortByName(self):
     # by default, the task list should be ordered by name
     task_list = list(tasks_sample())
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
     cmd_list._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['g1', 't1', 't2', 't3']
     assert expected == got
Example #24
0
 def testCustomReporter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=[tasks_sample()[0]])
     cmd_run._execute(output, reporter=MyReporter)
     got = output.getvalue().split("\n")[:-1]
     assert 'MyReporter.start t1' == got[0]
Example #25
0
 def testSortByDefinition(self):
     # test sorting task list by order of definition
     task_list = list(tasks_sample())
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
     cmd_list._execute(sort='definition')
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['t1', 't2', 'g1', 't3']
     assert expected == got
Example #26
0
 def testSortByName(self):
     # by default, the task list should be ordered by name
     task_list = list(tasks_sample())
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
     cmd_list._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['g1', 't1', 't2', 't3']
     assert expected == got
Example #27
0
 def test_file_dep_up_to_date(self, depfile, dependency1):
     my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
     depfile.save_success(my_task)
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert "skip t2\n" == got
Example #28
0
 def test_execute(self, depfile, dependency1):
     output = StringIO()
     tasks = tasks_sample()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
                            dep_manager=depfile)
     cmd_reset._execute()
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ["processed %s" % t.name for t in tasks]
     assert sorted(expected) == sorted(got)
Example #29
0
 def test_file_dep_up_to_date(self, depfile, dependency1):
     my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
     depfile.save_success(my_task)
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert "skip t2\n" == got
Example #30
0
 def test_missing_file_dep(self, depfile):
     my_task = Task("t2", [""], file_dep=['tests/data/missing'])
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert ("failed t2 (Dependent file 'tests/data/missing' does not "
             "exist.)\n") == got
Example #31
0
 def testWithPrivate(self):
     task_list = list(tasks_sample())
     task_list.append(Task("_s3", [""]))
     output = StringIO()
     cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
     cmd_list._execute(private=True, pos_args=['_s3'])
     got = [line.strip() for line in output.getvalue().split('\n') if line]
     expected = ['_s3']
     assert expected == got
Example #32
0
 def testPluginReporter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(
         Run, backend='dbm',
         dep_file=depfile_name,
         task_list=[tasks_sample()[0]],
         config={'REPORTER':{'my': 'tests.test_cmd_run:MyReporter'}})
     cmd_run._execute(output, reporter='my')
     got = output.getvalue().split("\n")[:-1]
     assert 'MyReporter.start t1' == got[0]
Example #33
0
 def testProcessRunFilter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=tasks_sample(),
                          sel_tasks=["g1.a"])
     cmd_run._execute(output)
     got = output.getvalue().split("\n")[:-1]
     assert [".  g1.a"] == got
Example #34
0
 def test_file_dep_change_checker(self, depfile, dependency1):
     my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
     depfile.save_success(my_task)
     depfile.checker = TimestampChecker()
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert "processed t2\n" == got
Example #35
0
 def testProcessRunMThread(self, dependency1, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=tasks_sample())
     result = cmd_run._execute(output, num_process=1, par_type='thread')
     assert 0 == result
     got = output.getvalue().split("\n")[:-1]
     assert [".  t1", ".  t2", ".  g1.a", ".  g1.b", ".  t3"] == got
Example #36
0
 def testProcessRunSingleSubtasks(self, depfile_name):
     output = StringIO()
     task_list = tasks_sample()
     assert task_list[4].name == "g1.b"
     task_list[4].task_dep = ["t3"]
     cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=task_list, sel_tasks=["g1"])
     cmd_run._execute(output, single=True)
     got = output.getvalue().split("\n")[:-1]
     # t3 is a depenendency of g1.b but not included
     assert [".  g1.a", ".  g1.b"] == got
Example #37
0
 def test_file_dep_change_checker(self, depfile, dependency1):
     my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
     depfile.save_success(my_task)
     depfile.checker = TimestampChecker()
     output = StringIO()
     cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                            dep_manager=depfile)
     cmd_reset._execute()
     got = output.getvalue()
     assert "processed t2\n" == got
Example #38
0
 def testSetVerbosity(self, depfile_name):
     output = StringIO()
     t = Task('x', None)
     used_verbosity = []
     def my_execute(out, err, verbosity):
         used_verbosity.append(verbosity)
     t.execute = my_execute
     cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
                          task_list=[t])
     cmd_run._execute(output, verbosity=2)
     assert 2 == used_verbosity[0], used_verbosity
Example #39
0
 def testProcessRunSingle(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=tasks_sample(),
                          sel_tasks=["t3"])
     cmd_run._execute(output, single=True)
     got = output.getvalue().split("\n")[:-1]
     # t1 is a depenendency of t3 but not included
     assert [".  t3"] == got
Example #40
0
 def test_outfile(self, depfile_name):
     cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample(), sel_tasks=["g1.a"])
     cmd_run._execute("test.out")
     try:
         outfile = open("test.out", "r")
         got = outfile.read()
         outfile.close()
         assert ".  g1.a\n" == got
     finally:
         if os.path.exists("test.out"):
             os.remove("test.out")
Example #41
0
    def testErrorStatus(self, dependency1, dep_manager):
        """Check that problematic tasks show an 'E' as status."""
        task_list = tasks_bad_sample()

        output = StringIO()
        cmd_list = CmdFactory(List, outstream=output, dep_manager=dep_manager,
                              task_list=task_list)
        cmd_list._execute(status=True)
        for line in output.getvalue().split('\n'):
            if line:
                assert line.strip().startswith('E ')
Example #42
0
    def test_file_dep(self, depfile, dependency1):
        my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
        output = StringIO()
        cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                               dep_manager=depfile)
        cmd_reset._execute()
        got = output.getvalue()
        assert "processed t2\n" == got

        dep = list(my_task.file_dep)[0]
        timestamp, size, md5 = depfile._get(my_task.name, dep)
        assert get_file_md5(get_abspath("data/dependency1")) == md5
Example #43
0
 def testMP_not_available(self, dependency1, depfile_name, capsys, monkeypatch):
     # make sure MRunner wont be used
     monkeypatch.setattr(runner.MRunner, "available", Mock(return_value=False))
     output = StringIO()
     cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample())
     result = cmd_run._execute(output, num_process=1)
     assert 0 == result
     got = output.getvalue().split("\n")[:-1]
     assert [".  t1", ".  t2", ".  g1.a", ".  g1.b", ".  t3"] == got
     err = capsys.readouterr()[1]
     assert "WARNING:" in err
     assert "parallel using threads" in err
Example #44
0
 def testPluginReporter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(
         Run,
         backend="dbm",
         dep_file=depfile_name,
         task_list=[tasks_sample()[0]],
         config={"REPORTER": {"my": "tests.test_cmd_run:MyReporter"}},
     )
     cmd_run._execute(output, reporter="my")
     got = output.getvalue().split("\n")[:-1]
     assert "MyReporter.start t1" == got[0]
Example #45
0
    def test_file_dep(self, depfile, dependency1):
        my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
        output = StringIO()
        cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
                               dep_manager=depfile)
        cmd_reset._execute()
        got = output.getvalue()
        assert "processed t2\n" == got

        dep = list(my_task.file_dep)[0]
        timestamp, size, md5 = depfile._get(my_task.name, dep)
        assert get_file_md5(get_abspath("data/dependency1")) == md5
Example #46
0
    def testErrorStatus(self, dependency1, dep_manager):
        """Check that problematic tasks show an 'E' as status."""
        task_list = tasks_bad_sample()

        output = StringIO()
        cmd_list = CmdFactory(List,
                              outstream=output,
                              dep_manager=dep_manager,
                              task_list=task_list)
        cmd_list._execute(status=True)
        for line in output.getvalue().split('\n'):
            if line:
                assert line.strip().startswith('E ')
Example #47
0
 def testDoc(self):
     output = StringIO()
     tasks = tasks_sample()
     cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
     cmd_list._execute(quiet=False)
     got = [line for line in output.getvalue().split('\n') if line]
     expected = []
     for t in sorted(tasks):
         if not t.is_subtask:
             expected.append([t.name, t.doc])
     assert len(expected) == len(got)
     for exp1, got1 in zip(expected, got):
         assert exp1 == got1.split(None, 1)
Example #48
0
 def testPluginReporter(self, depfile_name):
     output = StringIO()
     cmd_run = CmdFactory(
         Run,
         backend='dbm',
         dep_file=depfile_name,
         task_list=[tasks_sample()[0]],
         config={'REPORTER': {
             'my': 'tests.test_cmd_run:MyReporter'
         }})
     cmd_run._execute(output, reporter='my')
     got = output.getvalue().split("\n")[:-1]
     assert 'MyReporter.start t1' == got[0]
Example #49
0
 def testProcessRunSingleSubtasks(self, depfile_name):
     output = StringIO()
     task_list = tasks_sample()
     assert task_list[4].name == 'g1.b'
     task_list[4].task_dep = ['t3']
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=task_list,
                          sel_tasks=["g1"])
     cmd_run._execute(output, single=True)
     got = output.getvalue().split("\n")[:-1]
     # t3 is a depenendency of g1.b but not included
     assert [".  g1.a", ".  g1.b"] == got
Example #50
0
    def testStatus(self, dependency1, depfile):
        task_list = tasks_sample()
        depfile.ignore(task_list[0]) # t1
        depfile.save_success(task_list[1]) # t2
        depfile.close()

        output = StringIO()
        cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
                        backend='dbm', task_list=task_list)
        cmd_list._execute(status=True)
        got = [line.strip() for line in output.getvalue().split('\n') if line]
        assert 'R g1' in got
        assert 'I t1' in got
        assert 'U t2' in got
Example #51
0
    def testStatus_result_dep_bug_gh44(self, dependency1, depfile):
        # make sure task dict is passed when checking up-to-date
        task_list = [Task("t1", [""], doc="t1 doc string"),
                     Task("t2", [""], uptodate=[result_dep('t1')]),]

        depfile.save_success(task_list[0]) # t1
        depfile.close()

        output = StringIO()
        cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
                              backend='dbm', task_list=task_list)
        cmd_list._execute(status=True)
        got = [line.strip() for line in output.getvalue().split('\n') if line]
        assert 'R t1' in got
        assert 'R t2' in got
Example #52
0
 def test_outfile(self, depfile_name):
     cmd_run = CmdFactory(Run,
                          backend='dbm',
                          dep_file=depfile_name,
                          task_list=tasks_sample(),
                          sel_tasks=["g1.a"])
     cmd_run._execute('test.out')
     try:
         outfile = open('test.out', 'r')
         got = outfile.read()
         outfile.close()
         assert ".  g1.a\n" == got
     finally:
         if os.path.exists('test.out'):
             os.remove('test.out')