def test_job_with_error_is_handled_correctly(self):
# Given.
problem = EllipticalDrop(self.sim_dir, self.output_dir)
problem.cases[0].base_command += ' --xxx'
s = self._make_scheduler()
t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s)
# When.
try:
t.run(wait=1)
except RuntimeError:
pass
# Then.
# Ensure that the directories are copied over when they have errors.
sim1 = os.path.join(self.root, self.sim_dir, 'elliptical_drop',
'no_update_h')
self.assertTrue(os.path.exists(sim1))
sim2 = os.path.join(self.root, self.sim_dir, 'elliptical_drop',
'update_h')
self.assertTrue(os.path.exists(sim2))
# Ensure that all the correct but already scheduled jobs are completed.
task_status = t.task_status
status_values = list(task_status.values())
self.assertEqual(status_values.count('error'), 1)
self.assertEqual(status_values.count('done'), 1)
self.assertEqual(status_values.count('not started'), 1)
for t, s in task_status.items():
if s == 'done':
self.assertTrue(t.complete())
if s == 'error':
self.assertRaises(RuntimeError, t.complete)
def test_tasks_with_dependencies(self):
# Given
s = self._make_scheduler()
cmd = 'python -c "import time; print(time.time())"'
ct1_dir = os.path.join(self.sim_dir, '1')
ct2_dir = os.path.join(self.sim_dir, '2')
ct3_dir = os.path.join(self.sim_dir, '3')
ct1 = CommandTask(cmd, output_dir=ct1_dir)
ct2 = CommandTask(cmd, output_dir=ct2_dir, depends=[ct1])
ct3 = CommandTask(cmd, output_dir=ct3_dir, depends=[ct1, ct2])
# When
t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s)
# Then
self.assertEqual(len(t.todo), 3)
# When
t.run(wait=0.1)
wait_until(lambda: not ct3.complete())
# Then.
# Ensure that the tasks are run in the right order.
ct1_t, ct2_t, ct3_t = [
self._get_time(x) for x in (ct1_dir, ct2_dir, ct3_dir)
]
self.assertTrue(ct2_t > ct1_t)
self.assertTrue(ct3_t > ct2_t)
def test_task_runner_checks_for_error_in_running_tasks(self, m_t_cores):
# Given
s = self._make_scheduler()
cmd = 'python -c "import sys, time; time.sleep(0.1); sys.exit(1)"'
ct1_dir = os.path.join(self.sim_dir, '1')
ct2_dir = os.path.join(self.sim_dir, '2')
ct3_dir = os.path.join(self.sim_dir, '3')
job_info = dict(n_core=2, n_thread=2)
ct1 = CommandTask(cmd, output_dir=ct1_dir, job_info=job_info)
ct2 = CommandTask(cmd, output_dir=ct2_dir, job_info=job_info)
ct3 = CommandTask(cmd, output_dir=ct3_dir, job_info=job_info)
# When
t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s)
# Then
self.assertEqual(len(t.todo), 3)
# When
n_errors = t.run(wait=0.1)
# Then
# In this case, two tasks should have run and one should not have run
# as the other two had errors.
self.assertEqual(n_errors, 2)
self.assertEqual(len(t.todo), 1)
self.assertTrue(os.path.exists(ct3_dir))
self.assertTrue(os.path.exists(ct2_dir))
self.assertFalse(os.path.exists(ct1_dir))
def test_task_runner_waits_for_tasks_in_the_end(self, m_t_cores):
# Given
s = self._make_scheduler()
cmd = 'python -c "import sys, time; time.sleep(0.1); sys.exit(1)"'
ct1_dir = os.path.join(self.sim_dir, '1')
ct2_dir = os.path.join(self.sim_dir, '2')
ct3_dir = os.path.join(self.sim_dir, '3')
ct1 = CommandTask(cmd, output_dir=ct1_dir)
ct2 = CommandTask(cmd, output_dir=ct2_dir)
ct3 = CommandTask(cmd, output_dir=ct3_dir)
# When
t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s)
n_errors = t.run(wait=0.1)
# Then
# All the tasks may have been run but those that ran will fail.
self.assertEqual(n_errors + len(t.todo), 3)
self.assertTrue(n_errors > 0)
def test_nothing_is_run_when_output_exists(self):
# Given.
s = self._make_scheduler()
output = os.path.join(self.output_dir, 'elliptical_drop')
os.makedirs(output)
# When
problem = EllipticalDrop(self.sim_dir, self.output_dir)
t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s)
# Then.
self.assertEqual(len(t.todo), 0)
def test_task_runner_does_not_add_repeated_tasks(self):
# Given
s = self._make_scheduler()
cmd = 'python -c "print(1)"'
ct1 = CommandTask(cmd, output_dir=self.sim_dir)
ct2 = CommandTask(cmd, output_dir=self.sim_dir)
# When
t = TaskRunner(tasks=[ct1, ct2, ct1], scheduler=s)
# Then
self.assertEqual(len(t.todo), 1)
def test_simulation_with_dependencies(self):
# Given
class A(Problem):
def setup(self):
cmd = 'python -c "import time; print(time.time())"'
s1 = Simulation(self.input_path('1'), cmd)
s2 = Simulation(self.input_path('2'), cmd, depends=[s1])
s3 = Simulation(self.input_path('3'), cmd, depends=[s1, s2])
self.cases = [s1, s2, s3]
def run(self):
self.make_output_dir()
s = self._make_scheduler()
# When
problem = A(self.sim_dir, self.output_dir)
task = SolveProblem(problem)
t = TaskRunner(tasks=[task], scheduler=s)
# Then
self.assertEqual(len(t.todo), 4)
# Basically only one instance of CommandTask should be created.
names = [x.__class__.__name__ for x in t.todo]
self.assertEqual(names.count('CommandTask'), 3)
self.assertEqual(names.count('SolveProblem'), 1)
# When
t.run(wait=0.1)
wait_until(lambda: not task.complete())
# Then
ct1_t, ct2_t, ct3_t = [
self._get_time(problem.input_path(x)) for x in ('1', '2', '3')
]
self.assertTrue(ct2_t > ct1_t)
self.assertTrue(ct3_t > ct2_t)
def test_automation(self):
# Given.
problem = EllipticalDrop(self.sim_dir, self.output_dir)
s = self._make_scheduler()
t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s)
# When.
t.run(wait=1)
# Then.
sim1 = os.path.join(self.root, self.sim_dir, 'elliptical_drop',
'no_update_h')
self.assertTrue(os.path.exists(sim1))
sim2 = os.path.join(self.root, self.sim_dir, 'elliptical_drop',
'update_h')
self.assertTrue(os.path.exists(sim2))
results = os.path.join(self.root, self.output_dir, 'elliptical_drop',
'result.txt')
self.assertTrue(os.path.exists(results))
data = open(results).read()
self.assertTrue('no_update_h' in data)
self.assertTrue('update_h' in data)
# When.
problem = EllipticalDrop(self.sim_dir, self.output_dir)
t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s)
# Then.
self.assertEqual(len(t.todo), 0)
# When
problem.clean()
# Then.
out1 = os.path.join(self.root, self.output_dir, 'elliptical_drop',
'update_h')
out2 = os.path.join(self.root, self.output_dir, 'elliptical_drop',
'no_update_h')
self.assertFalse(os.path.exists(out1))
self.assertFalse(os.path.exists(out2))
self.assertTrue(os.path.exists(sim1))
self.assertTrue(os.path.exists(sim2))
def test_task_runner_doesnt_block_on_problem_with_error(self, m_t_cores):
# Given
class A(Problem):
def get_requires(self):
cmd = ('python -c "import sys, time; time.sleep(0.1); '
'sys.exit(1)"')
ct = CommandTask(cmd, output_dir=self.input_path())
return [('task1', ct)]
def run(self):
self.make_output_dir()
s = self._make_scheduler()
# When
task = RunAll(simulation_dir=self.sim_dir,
output_dir=self.output_dir,
problem_classes=[A])
t = TaskRunner(tasks=[task], scheduler=s)
n_error = t.run(wait=0.1)
# Then
self.assertTrue(n_error > 0)
# For the case where there is a match expression
# When
problem = A(self.sim_dir, self.output_dir)
problem.clean()
task = RunAll(simulation_dir=self.sim_dir,
output_dir=self.output_dir,
problem_classes=[A],
match='*task1')
t = TaskRunner(tasks=[task], scheduler=s)
n_error = t.run(wait=0.1)
# Then
self.assertTrue(n_error > 0)
def test_problem_depending_on_other_problems(self):
# Given
class A(Problem):
def get_requires(self):
cmd = 'python -c "print(1)"'
# Can return tasks ...
ct = CommandTask(cmd, output_dir=self.sim_dir)
return [('task1', ct)]
def run(self):
self.make_output_dir()
class B(Problem):
def get_requires(self):
# or return Problem instances ...
return [('a', A(self.sim_dir, self.out_dir))]
def run(self):
self.make_output_dir()
class C(Problem):
def get_requires(self):
# ... or Problem subclasses
return [('a', A), ('b', B)]
def run(self):
self.make_output_dir()
s = self._make_scheduler()
# When
task = RunAll(simulation_dir=self.sim_dir,
output_dir=self.output_dir,
problem_classes=[A, B, C])
t = TaskRunner(tasks=[task], scheduler=s)
# Then
self.assertEqual(len(t.todo), 5)
# Basically only one instance of CommandTask should be created.
names = [x.__class__.__name__ for x in t.todo]
problems = [x.problem for x in t.todo if isinstance(x, SolveProblem)]
self.assertEqual(names.count('RunAll'), 1)
self.assertEqual(names.count('CommandTask'), 1)
self.assertEqual(names.count('SolveProblem'), 3)
self.assertEqual(len(problems), 3)
self.assertEqual(sorted(x.__class__.__name__ for x in problems),
['A', 'B', 'C'])
# When
t.run(wait=0.1)
# Then.
self.assertEqual(t.todo, [])
# The output dirs should exist now.
for name in ('A', 'B', 'C'):
self.assertTrue(os.path.exists(os.path.join(self.output_dir,
name)))
# When
# We set force to True.
task = RunAll(simulation_dir=self.sim_dir,
output_dir=self.output_dir,
problem_classes=[A, B, C],
force=True)
# Then
# Make sure that all the output directories are deleted as this will
for name in ('A', 'B', 'C'):
self.assertFalse(
os.path.exists(os.path.join(self.output_dir, name)))