def testProcessRunEmptyFilter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
task_list=tasks_sample(), sel_tasks=[])
cmd_run._execute(output)
got = output.getvalue().split("\n")[:-1]
assert [] == got
def test_no_children(self):
my_task = Task("t2", [""], file_dep=['d2.txt'])
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
cmd._execute(graph_type='json', no_children=True)
got = output.getvalue()
self.assertNotIn("d2.txt", got)
def testProcessRunMThread(self, dependency1, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample())
result = cmd_run._execute(output, num_process=1, par_type="thread")
assert 0 == result
got = output.getvalue().split("\n")[:-1]
assert [". t1", ". t2", ". g1.a", ". g1.b", ". t3"] == got
def testFilterAll(self):
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=tasks_sample())
cmd_list._execute(subtasks=True, pos_args=['g1'])
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['g1', 'g1.a', 'g1.b']
assert expected == got
def testDependencies(self):
my_task = Task("t2", [""], file_dep=['d2.txt'])
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=[my_task])
cmd_list._execute(list_deps=True)
got = output.getvalue()
assert "d2.txt" in got
def test_missing_dep_and_target(self, depfile, dependency1, dependency2):
task_a = Task("task_a", [""],
file_dep=['tests/data/dependency1'],
targets=['tests/data/dependency2'])
task_b = Task("task_b", [""],
file_dep=['tests/data/dependency2'],
targets=['tests/data/dependency3'])
task_c = Task("task_c", [""],
file_dep=['tests/data/dependency3'],
targets=['tests/data/dependency4'])
output = StringIO()
tasks = [task_a, task_b, task_c]
cmd_reset = CmdFactory(ResetDep,
outstream=output,
task_list=tasks,
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert ("processed task_a\n"
"processed task_b\n"
"failed task_c (Dependent file 'tests/data/dependency3'"
" does not exist.)\n") == got
def testCustomReporter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
task_list=[tasks_sample()[0]])
cmd_run._execute(output, reporter=MyReporter)
got = output.getvalue().split("\n")[:-1]
assert 'MyReporter.start t1' == got[0]
def test_no_children(self):
my_task = Task("t2", [""], file_dep=['d2.txt'])
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
cmd._execute(graph_type='json', no_children=True)
got = output.getvalue()
self.assertNotIn("d2.txt", got)
def testProcessRunSingle(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample(), sel_tasks=["t3"])
cmd_run._execute(output, single=True)
got = output.getvalue().split("\n")[:-1]
# t1 is a depenendency of t3 but not included
assert [". t3"] == got
def test_unicode_name(self, depfile):
task_list = [Task(six.u("t做"), [""], doc=six.u("t1 doc string 做")),]
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
task_list=task_list)
cmd_list._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
assert six.u('t做') == got[0]
def test_filter(self, depfile, dependency1):
output = StringIO()
tasks = tasks_sample()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
dep_manager=depfile)
cmd_reset._execute(pos_args=['t2'])
got = output.getvalue()
assert "processed t2\n" == got
def test_store_json_stdout(self):
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=_sample_tasks())
cmd._execute(graph_type='json')
got = output.getvalue()
self.assertIn("read", got)
self.assertIn("t3", got)
self.assertIn("join_files", got)
def test_children(self):
# TODO: Implement no-child option.
my_task = Task("t2", [""], file_dep=['d2.txt'])
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
cmd._execute(graph_type='json', no_children=False)
got = output.getvalue()
self.assertIn("d2.txt", got)
def test_children(self):
# TODO: Implement no-child option.
my_task = Task("t2", [""], file_dep=['d2.txt'])
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=[my_task])
cmd._execute(graph_type='json', no_children=False)
got = output.getvalue()
self.assertIn("d2.txt", got)
def test_store_json_stdout(self):
output = StringIO()
cmd = CmdFactory(Graphx, outstream=output, task_list=_sample_tasks())
cmd._execute(graph_type='json')
got = output.getvalue()
self.assertIn("read", got)
self.assertIn("t3", got)
self.assertIn("join_files", got)
def test_filter(self, depfile, dependency1):
output = StringIO()
tasks = tasks_sample()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
dep_manager=depfile)
cmd_reset._execute(pos_args=['t2'])
got = output.getvalue()
assert "processed t2\n" == got
def testSubTask(self):
output = StringIO()
tasks = tasks_sample()
cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
cmd_list._execute(subtasks=True)
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = [t.name for t in sorted(tasks)]
assert expected == got
def testQuiet(self):
output = StringIO()
tasks = tasks_sample()
cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
cmd_list._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = [t.name for t in tasks if not t.is_subtask]
assert sorted(expected) == got
def testCustomTemplate(self):
output = StringIO()
tasks = tasks_sample()
cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
cmd_list._execute(template='xxx {name} xxx {doc}')
got = [line.strip() for line in output.getvalue().split('\n') if line]
assert 'xxx g1 xxx g1 doc string' == got[0]
assert 'xxx t3 xxx t3 doc string' == got[3]
def test_execute(self, depfile, dependency1):
output = StringIO()
tasks = tasks_sample()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
dep_manager=depfile)
cmd_reset._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ["processed %s" % t.name for t in tasks]
assert sorted(expected) == sorted(got)
def test_missing_file_dep(self, depfile):
my_task = Task("t2", [""], file_dep=['tests/data/missing'])
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert ("failed t2 (Dependent file 'tests/data/missing' does not "
"exist.)\n") == got
def testSortByDefinition(self):
# test sorting task list by order of definition
task_list = list(tasks_sample())
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute(sort='definition')
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['t1', 't2', 'g1', 't3']
assert expected == got
def testSortByName(self):
# by default, the task list should be ordered by name
task_list = list(tasks_sample())
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['g1', 't1', 't2', 't3']
assert expected == got
def testCustomReporter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=[tasks_sample()[0]])
cmd_run._execute(output, reporter=MyReporter)
got = output.getvalue().split("\n")[:-1]
assert 'MyReporter.start t1' == got[0]
def testSortByDefinition(self):
# test sorting task list by order of definition
task_list = list(tasks_sample())
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute(sort='definition')
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['t1', 't2', 'g1', 't3']
assert expected == got
def testSortByName(self):
# by default, the task list should be ordered by name
task_list = list(tasks_sample())
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['g1', 't1', 't2', 't3']
assert expected == got
def test_file_dep_up_to_date(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
depfile.save_success(my_task)
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "skip t2\n" == got
def test_execute(self, depfile, dependency1):
output = StringIO()
tasks = tasks_sample()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=tasks,
dep_manager=depfile)
cmd_reset._execute()
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ["processed %s" % t.name for t in tasks]
assert sorted(expected) == sorted(got)
def test_file_dep_up_to_date(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
depfile.save_success(my_task)
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "skip t2\n" == got
def test_missing_file_dep(self, depfile):
my_task = Task("t2", [""], file_dep=['tests/data/missing'])
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert ("failed t2 (Dependent file 'tests/data/missing' does not "
"exist.)\n") == got
def testWithPrivate(self):
task_list = list(tasks_sample())
task_list.append(Task("_s3", [""]))
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute(private=True, pos_args=['_s3'])
got = [line.strip() for line in output.getvalue().split('\n') if line]
expected = ['_s3']
assert expected == got
def testPluginReporter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(
Run, backend='dbm',
dep_file=depfile_name,
task_list=[tasks_sample()[0]],
config={'REPORTER':{'my': 'tests.test_cmd_run:MyReporter'}})
cmd_run._execute(output, reporter='my')
got = output.getvalue().split("\n")[:-1]
assert 'MyReporter.start t1' == got[0]
def testProcessRunFilter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=tasks_sample(),
sel_tasks=["g1.a"])
cmd_run._execute(output)
got = output.getvalue().split("\n")[:-1]
assert [". g1.a"] == got
def test_file_dep_change_checker(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
depfile.save_success(my_task)
depfile.checker = TimestampChecker()
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "processed t2\n" == got
def testProcessRunMThread(self, dependency1, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=tasks_sample())
result = cmd_run._execute(output, num_process=1, par_type='thread')
assert 0 == result
got = output.getvalue().split("\n")[:-1]
assert [". t1", ". t2", ". g1.a", ". g1.b", ". t3"] == got
def testProcessRunSingleSubtasks(self, depfile_name):
output = StringIO()
task_list = tasks_sample()
assert task_list[4].name == "g1.b"
task_list[4].task_dep = ["t3"]
cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=task_list, sel_tasks=["g1"])
cmd_run._execute(output, single=True)
got = output.getvalue().split("\n")[:-1]
# t3 is a depenendency of g1.b but not included
assert [". g1.a", ". g1.b"] == got
def test_file_dep_change_checker(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
depfile.save_success(my_task)
depfile.checker = TimestampChecker()
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "processed t2\n" == got
def testSetVerbosity(self, depfile_name):
output = StringIO()
t = Task('x', None)
used_verbosity = []
def my_execute(out, err, verbosity):
used_verbosity.append(verbosity)
t.execute = my_execute
cmd_run = CmdFactory(Run, backend='dbm', dep_file=depfile_name,
task_list=[t])
cmd_run._execute(output, verbosity=2)
assert 2 == used_verbosity[0], used_verbosity
def testProcessRunSingle(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=tasks_sample(),
sel_tasks=["t3"])
cmd_run._execute(output, single=True)
got = output.getvalue().split("\n")[:-1]
# t1 is a depenendency of t3 but not included
assert [". t3"] == got
def test_outfile(self, depfile_name):
cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample(), sel_tasks=["g1.a"])
cmd_run._execute("test.out")
try:
outfile = open("test.out", "r")
got = outfile.read()
outfile.close()
assert ". g1.a\n" == got
finally:
if os.path.exists("test.out"):
os.remove("test.out")
def testErrorStatus(self, dependency1, dep_manager):
"""Check that problematic tasks show an 'E' as status."""
task_list = tasks_bad_sample()
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, dep_manager=dep_manager,
task_list=task_list)
cmd_list._execute(status=True)
for line in output.getvalue().split('\n'):
if line:
assert line.strip().startswith('E ')
def test_file_dep(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "processed t2\n" == got
dep = list(my_task.file_dep)[0]
timestamp, size, md5 = depfile._get(my_task.name, dep)
assert get_file_md5(get_abspath("data/dependency1")) == md5
def testMP_not_available(self, dependency1, depfile_name, capsys, monkeypatch):
# make sure MRunner wont be used
monkeypatch.setattr(runner.MRunner, "available", Mock(return_value=False))
output = StringIO()
cmd_run = CmdFactory(Run, backend="dbm", dep_file=depfile_name, task_list=tasks_sample())
result = cmd_run._execute(output, num_process=1)
assert 0 == result
got = output.getvalue().split("\n")[:-1]
assert [". t1", ". t2", ". g1.a", ". g1.b", ". t3"] == got
err = capsys.readouterr()[1]
assert "WARNING:" in err
assert "parallel using threads" in err
def testPluginReporter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(
Run,
backend="dbm",
dep_file=depfile_name,
task_list=[tasks_sample()[0]],
config={"REPORTER": {"my": "tests.test_cmd_run:MyReporter"}},
)
cmd_run._execute(output, reporter="my")
got = output.getvalue().split("\n")[:-1]
assert "MyReporter.start t1" == got[0]
def test_file_dep(self, depfile, dependency1):
my_task = Task("t2", [""], file_dep=['tests/data/dependency1'])
output = StringIO()
cmd_reset = CmdFactory(ResetDep, outstream=output, task_list=[my_task],
dep_manager=depfile)
cmd_reset._execute()
got = output.getvalue()
assert "processed t2\n" == got
dep = list(my_task.file_dep)[0]
timestamp, size, md5 = depfile._get(my_task.name, dep)
assert get_file_md5(get_abspath("data/dependency1")) == md5
def testErrorStatus(self, dependency1, dep_manager):
"""Check that problematic tasks show an 'E' as status."""
task_list = tasks_bad_sample()
output = StringIO()
cmd_list = CmdFactory(List,
outstream=output,
dep_manager=dep_manager,
task_list=task_list)
cmd_list._execute(status=True)
for line in output.getvalue().split('\n'):
if line:
assert line.strip().startswith('E ')
def testDoc(self):
output = StringIO()
tasks = tasks_sample()
cmd_list = CmdFactory(List, outstream=output, task_list=tasks)
cmd_list._execute(quiet=False)
got = [line for line in output.getvalue().split('\n') if line]
expected = []
for t in sorted(tasks):
if not t.is_subtask:
expected.append([t.name, t.doc])
assert len(expected) == len(got)
for exp1, got1 in zip(expected, got):
assert exp1 == got1.split(None, 1)
def testPluginReporter(self, depfile_name):
output = StringIO()
cmd_run = CmdFactory(
Run,
backend='dbm',
dep_file=depfile_name,
task_list=[tasks_sample()[0]],
config={'REPORTER': {
'my': 'tests.test_cmd_run:MyReporter'
}})
cmd_run._execute(output, reporter='my')
got = output.getvalue().split("\n")[:-1]
assert 'MyReporter.start t1' == got[0]
def testProcessRunSingleSubtasks(self, depfile_name):
output = StringIO()
task_list = tasks_sample()
assert task_list[4].name == 'g1.b'
task_list[4].task_dep = ['t3']
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=task_list,
sel_tasks=["g1"])
cmd_run._execute(output, single=True)
got = output.getvalue().split("\n")[:-1]
# t3 is a depenendency of g1.b but not included
assert [". g1.a", ". g1.b"] == got
def testStatus(self, dependency1, depfile):
task_list = tasks_sample()
depfile.ignore(task_list[0]) # t1
depfile.save_success(task_list[1]) # t2
depfile.close()
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
backend='dbm', task_list=task_list)
cmd_list._execute(status=True)
got = [line.strip() for line in output.getvalue().split('\n') if line]
assert 'R g1' in got
assert 'I t1' in got
assert 'U t2' in got
def testStatus_result_dep_bug_gh44(self, dependency1, depfile):
# make sure task dict is passed when checking up-to-date
task_list = [Task("t1", [""], doc="t1 doc string"),
Task("t2", [""], uptodate=[result_dep('t1')]),]
depfile.save_success(task_list[0]) # t1
depfile.close()
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, dep_file=depfile.name,
backend='dbm', task_list=task_list)
cmd_list._execute(status=True)
got = [line.strip() for line in output.getvalue().split('\n') if line]
assert 'R t1' in got
assert 'R t2' in got
def test_outfile(self, depfile_name):
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=tasks_sample(),
sel_tasks=["g1.a"])
cmd_run._execute('test.out')
try:
outfile = open('test.out', 'r')
got = outfile.read()
outfile.close()
assert ". g1.a\n" == got
finally:
if os.path.exists('test.out'):
os.remove('test.out')