def test_generate_tasks_after_run(self):
rollout = Rollout({})
rollout.save()
create_task(rollout)
rollout.rollout()
self.assertRaises(Exception, rollout.generate_tasks)
def test_sequential_quit_and_rollback_on_failure(self):
class RecordedRollout(Rollout):
rollback_calls = []
def rollback(self):
self.rollback_calls.append(self)
super(RecordedRollout, self).rollback()
rollout = RecordedRollout({})
rollout.save()
task1 = create_task(rollout)
task_error = create_task(rollout, TestTaskFail)
task2 = create_task(rollout)
root = create_task(rollout, SequentialExecTask, [task1, task_error, task2])
rollout.rollout()
self.assertRun(task1)
self.assertRun(task_error)
self.assertNotRun(task2)
self.assertEqual(RecordedRollout.rollback_calls, [rollout])
self.assertReverted(task_error)
self.assertReverted(task1)
self.assertNotReverted(task2)
def test_signals(self):
rollouts = defaultdict(dict)
rollout_ids = defaultdict(dict)
tasks = defaultdict(dict)
classes = DelayTask, SequentialExecTask, ParallelExecTask
signals = ('abort_rollout',), ('term_rollout',), ('skip_rollback', 'term_rollout')
for cls in classes:
for sigs in signals:
rollout = Rollout({})
rollout.save()
rollouts[cls][sigs] = rollout
rollout_ids[cls][sigs] = rollout.id
if cls is DelayTask:
task = create_task(rollout, DelayTask, seconds=15)
tasks[cls][sigs] = task
else:
t1, t2, t3 = [create_task(rollout, DelayTask, seconds=15) for _ in range(3)]
task = create_task(rollout, cls, [t1, t2, t3])
tasks[cls][sigs] = t3
for cls in classes:
for sigs in signals:
rollout = rollouts[cls][sigs]
rollout.rollout_async()
# Enough time for rollouts to start
time.sleep(0.5)
for cls in classes:
for sigs in signals:
id = rollout_ids[cls][sigs]
for sig in sigs:
self.assertTrue(Rollout._can_signal(id, sig))
self.assertTrue(Rollout._do_signal(id, sig))
self.assertTrue(Rollout._is_signalling(id, sig))
# Enough time for rollouts to finish and save to db
time.sleep(2)
for cls in classes:
for sigs in signals:
rollout_id = rollout_ids[cls][sigs]
rollout = Rollout._from_id(rollout_id)
self.assertTrue(rollout.rollout_finish_dt, 'Rollout for %s not finished when sent %s' % (cls, sigs))
task = tasks[cls][sigs]
task = Task._from_id(task.id)
# Sequential exec's last task should not have run due to aborts
if cls is SequentialExecTask:
self.assertFalse(task.run_start_dt, 'Final task %s run for %s rollout when sent %s' % (task, cls, sigs))
else:
self.assertTrue(task.run_start_dt, 'Final task %s not run for %s rollout when sent %s' % (task, cls, sigs))
# If rollbacks were skipped the root task should not have reverted
if 'skip_rollback' in sigs:
self.assertFalse(rollout.root_task.revert_start_dt, 'Rollout for %s rolled back when sent %s' % (cls, sigs))
else:
self.assertTrue(rollout.root_task.revert_start_dt, 'Rollout for %s not rolled back when sent %s' % (cls, sigs))
def _test_exec_rollout(self, exec_cls):
rollout = Rollout({})
rollout.save()
task1 = create_task(rollout)
task2 = create_task(rollout)
task3 = create_task(rollout)
task4 = create_task(rollout)
root = create_task(rollout, exec_cls, [task1, task2, task3, task4])
rollout.rollout()
# Do not assert parents or root is run since otherwise we'd have to
# override their call methods
for task in task1, task2, task3, task4:
self.assertRun(task)
self.assertNotReverted(task)
def test_monitor_rolls_back(self):
monitor_wake_event = Event()
finish_step_event = Event()
def monitor(monitor_event, abort_event):
monitor_wake_event.wait()
abort_event.set()
finish_step_event.set()
class MonitoredRollout(Rollout):
monitors = {
'mon': monitor,
}
rollback_calls = []
def rollback(self):
self.rollback_calls.append(self)
super(MonitoredRollout, self).rollback()
rollout = MonitoredRollout({'monitors': ['mon']})
rollout.save()
def wake_monitor_then_wait():
monitor_wake_event.set()
finish_step_event.wait()
self.assertFalse(MonitoredRollout.rollback_calls)
class WakeMonitorWaitTask(TestTask):
@classmethod
def _run(cls, state, children, abort, term):
wake_monitor_then_wait()
task_run = create_task(rollout)
task_wake_monitor_wait = create_task(rollout, WakeMonitorWaitTask)
task_not_called = create_task(rollout)
root = create_task(rollout, SequentialExecTask, [task_run, task_wake_monitor_wait, task_not_called])
rollout.rollout()
self.assertRun(task_run)
self.assertRun(task_wake_monitor_wait)
self.assertNotRun(task_not_called)
self.assertEqual(MonitoredRollout.rollback_calls, [rollout])
self.assertReverted(task_run)
self.assertReverted(task_wake_monitor_wait)
def test_single_task_rollback(self):
rollout = Rollout({})
rollout.save()
root = create_task(rollout, TestTaskFail)
rollout.rollout()
self.assertRun(root)
self.assertReverted(root)
def test_single_task_rollout(self):
rollout = Rollout({})
rollout.save()
root = create_task(rollout)
rollout.rollout()
self.assertRun(root)
self.assertNotReverted(root)
def _generate_tasks(self):
create_task(self)