def test_default_times_retry(self):
flow = lf.Flow('flow-1',
retry.Times(3, 'r1')).add(utils.SaveOrderTask('t1'),
utils.FailingTask('t2'))
engine = self._make_engine(flow)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = [
't1', u't2 reverted(Failure: RuntimeError: Woot!)',
't1 reverted(5)', 't1',
u't2 reverted(Failure: RuntimeError: Woot!)', 't1 reverted(5)',
't1', u't2 reverted(Failure: RuntimeError: Woot!)',
't1 reverted(5)'
]
self.assertEqual(self.values, expected)
def test_linear_nested_to_parallel_revert_exception(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1', sleep=0.01),
utils.SaveOrderTask(self.values, name='task2', sleep=0.01),
lf.Flow('l-inner').add(
utils.SaveOrderTask(self.values, name='task3'),
utils.NastyTask(), utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
result = set(self.values)
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)', 'task3'
])
self.assertIsSubset(possible_result, result)
def test_flow_failures_are_passed_to_revert(self):
class CheckingTask(task.Task):
def execute(m_self):
return 'RESULT'
def revert(m_self, result, flow_failures):
self.assertEqual(result, 'RESULT')
self.assertEqual(list(flow_failures.keys()), ['fail1'])
fail = flow_failures['fail1']
self.assertIsInstance(fail, misc.Failure)
self.assertEqual(str(fail), 'Failure: RuntimeError: Woot!')
flow = lf.Flow('test').add(CheckingTask(), utils.FailingTask('fail1'))
engine = self._make_engine(flow)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
def test_default_times_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add(
utils.ProgressingTask('t1'),
utils.FailingTask('t2'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(3)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r REVERTING',
'r1.r REVERTED',
'flow-1.f REVERTED']
self.assertEqual(expected, capturer.values)
def test_correctly_reverts_children(self):
flow = lf.Flow('root-1').add(
utils.ProgressingTask('task1'),
lf.Flow('child-1').add(utils.ProgressingTask('task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
expected = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING',
'task2.t SUCCESS(5)', 'fail.t RUNNING',
'fail.t FAILURE(Failure: RuntimeError: Woot!)', 'fail.t REVERTING',
'fail.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED',
'task1.t REVERTING', 'task1.t REVERTED'
]
self.assertEqual(expected, capturer.values)
def test_failing_task_with_notifications(self):
flow = utils.FailingTask('fail')
engine = self._make_engine(flow)
utils.register_notifiers(engine, self.values)
expected = [
'flow RUNNING', 'fail RUNNING', 'fail FAILURE', 'fail REVERTING',
'fail reverted(Failure: RuntimeError: Woot!)', 'fail REVERTED',
'flow REVERTED'
]
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(self.values, expected)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
now_expected = expected + ['fail PENDING', 'flow PENDING'] + expected
self.assertEqual(self.values, now_expected)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_revert_ok_for_linear_in_unordered(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(name='task1'),
lf.Flow('p-inner').add(utils.SaveOrderTask(name='task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertIn('fail reverted(Failure: RuntimeError: Woot!)',
self.values)
# NOTE(imelnikov): if task1 was run, it should have been reverted.
if 'task1' in self.values:
task1_story = ['task1', 'task1 reverted(5)']
self.assertIsSuperAndSubsequence(self.values, task1_story)
# NOTE(imelnikov): task2 should have been run and reverted
task2_story = ['task2', 'task2 reverted(5)']
self.assertIsSuperAndSubsequence(self.values, task2_story)
def test_failing_task_with_notifications(self):
values = []
flow = utils.FailingTask('fail')
engine = self._make_engine(flow)
expected = ['fail.f RUNNING', 'fail.t RUNNING',
'fail.t FAILURE(Failure: RuntimeError: Woot!)',
'fail.t REVERTING', 'fail.t REVERTED',
'fail.f REVERTED']
with utils.CaptureListener(engine, values=values) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(expected, capturer.values)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
with utils.CaptureListener(engine, values=values) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
now_expected = list(expected)
now_expected.extend(['fail.t PENDING', 'fail.f PENDING'])
now_expected.extend(expected)
self.assertEqual(now_expected, values)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_graph_flow_four_tasks_revert(self):
flow = gf.Flow('g-4-failing').add(
utils.ProgressingTask(name='task4', provides='d', requires=['c']),
utils.ProgressingTask(name='task2', provides='b', requires=['a']),
utils.FailingTask(name='task3', provides='c', requires=['b']),
utils.ProgressingTask(name='task1', provides='a'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
expected = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING',
'task2.t SUCCESS(5)', 'task3.t RUNNING',
'task3.t FAILURE(Failure: RuntimeError: Woot!)',
'task3.t REVERTING', 'task3.t REVERTED', 'task2.t REVERTING',
'task2.t REVERTED', 'task1.t REVERTING', 'task1.t REVERTED'
]
self.assertEqual(expected, capturer.values)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_nested_parallel_revert_exception_is_reraised(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
lf.Flow('p-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.NastyTask(), utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
result = set(self.values)
# Task1, task2 may *not* have executed and also may have *not* reverted
# since the above is an unordered flow so take that into account by
# ensuring that the superset is matched.
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)',
'task3', 'task3 reverted(5)'
])
self.assertIsSubset(possible_result, result)
def test_linear_nested_to_parallel_revert(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
lf.Flow('l-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.FailingTask(self.values, name='fail', sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
result = set(self.values)
# Since this is an unordered flow we can not guarantee that task1 or
# task2 will exist and be reverted, although they may exist depending
# on how the OS thread scheduling and execution graph algorithm...
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)',
'task3', 'task3 reverted(5)',
'fail reverted(Failure: RuntimeError: Woot!)'
])
self.assertIsSubset(possible_result, result)
def test_suspend_linear_flow_on_revert(self):
flow = lf.Flow('linear').add(utils.ProgressingTask('a'),
utils.ProgressingTask('b'),
utils.FailingTask('c'))
engine = self._make_engine(flow)
with SuspendingListener(engine,
task_name='b',
task_state=states.REVERTED) as capturer:
engine.run()
self.assertEqual(states.SUSPENDED, engine.storage.get_flow_state())
expected = [
'a.t RUNNING', 'a.t SUCCESS(5)', 'b.t RUNNING', 'b.t SUCCESS(5)',
'c.t RUNNING', 'c.t FAILURE(Failure: RuntimeError: Woot!)',
'c.t REVERTING', 'c.t REVERTED(None)', 'b.t REVERTING',
'b.t REVERTED(None)'
]
self.assertEqual(expected, capturer.values)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertRaisesRegex(RuntimeError, '^Woot', engine.run)
self.assertEqual(states.REVERTED, engine.storage.get_flow_state())
expected = ['a.t REVERTING', 'a.t REVERTED(None)']
self.assertEqual(expected, capturer.values)
def test_revert_ok_for_unordered_in_linear(self):
flow = lf.Flow('p-root').add(
utils.SaveOrderTask(name='task1'),
utils.SaveOrderTask(name='task2'),
uf.Flow('p-inner').add(utils.SaveOrderTask(name='task3'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
# NOTE(imelnikov): we don't know if task 3 was run, but if it was,
# it should have been reverted in correct order.
possible_values_no_task3 = [
'task1', 'task2', 'fail reverted(Failure: RuntimeError: Woot!)',
'task2 reverted(5)', 'task1 reverted(5)'
]
self.assertIsSuperAndSubsequence(self.values, possible_values_no_task3)
if 'task3' in self.values:
possible_values_task3 = [
'task1', 'task2', 'task3', 'task3 reverted(5)',
'task2 reverted(5)', 'task1 reverted(5)'
]
self.assertIsSuperAndSubsequence(self.values,
possible_values_task3)
def test_suspend_and_resume_linear_flow_on_revert(self):
flow = lf.Flow('linear').add(
utils.SaveOrderTask('a'),
utils.SaveOrderTask('b'),
utils.FailingTask('c')
)
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.REVERTED):
engine.run()
# pretend we are resuming
engine2 = self._make_engine(flow, engine.storage._flowdetail)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run)
self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED)
self.assertEqual(
self.values,
['a',
'b',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(5)',
'a reverted(5)'])
def test_revert_ok_for_linear_in_unordered(self):
flow = uf.Flow('p-root').add(
utils.ProgressingTask(name='task1'),
lf.Flow('p-inner').add(utils.ProgressingTask(name='task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertIn('fail.t FAILURE(Failure: RuntimeError: Woot!)',
capturer.values)
# NOTE(imelnikov): if task1 was run, it should have been reverted.
if 'task1' in capturer.values:
task1_story = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task1.t REVERTED'
]
self.assertIsSuperAndSubsequence(capturer.values, task1_story)
# NOTE(imelnikov): task2 should have been run and reverted
task2_story = [
'task2.t RUNNING', 'task2.t SUCCESS(5)', 'task2.t REVERTED'
]
self.assertIsSuperAndSubsequence(capturer.values, task2_story)
def test_nested_provides_graph_reverts_correctly(self):
flow = gf.Flow("test").add(
utils.SaveOrderTask('a', requires=['x']),
lf.Flow("test2", retry=retry.Times(2)).add(
utils.SaveOrderTask('b', provides='x'),
utils.FailingTask('c')))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
engine.storage.save('test2_retry', 1)
engine.storage.save('b', 11)
engine.storage.save('a', 10)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertItemsEqual(self.values[:3], [
'a reverted(10)',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(11)',
])
# Task 'a' was or was not executed again, both cases are ok.
self.assertIsSuperAndSubsequence(
self.values[3:],
['b', 'c reverted(Failure: RuntimeError: Woot!)', 'b reverted(5)'])
self.assertEqual(engine.storage.get_flow_state(), st.REVERTED)
def test_graph_flow_four_tasks_revert(self):
flow = gf.Flow('g-4-failing').add(
utils.SaveOrderTask(self.values,
name='task4',
provides='d',
requires=['c']),
utils.SaveOrderTask(self.values,
name='task2',
provides='b',
requires=['a']),
utils.FailingTask(self.values,
name='task3',
provides='c',
requires=['b']),
utils.SaveOrderTask(self.values, name='task1', provides='a'))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertEquals(self.values, [
'task1', 'task2', 'task3 reverted(Failure: RuntimeError: Woot!)',
'task2 reverted(5)', 'task1 reverted(5)'
])
self.assertEquals(engine.storage.get_flow_state(), states.REVERTED)
def test_parallel_revert_exception_do_not_revert_linear_tasks(self):
flow = lf.Flow('l-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
uf.Flow('p-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.NastyTask(), utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
# Depending on when (and if failing task) is executed the exception
# raised could be either woot or gotcha since the above unordered
# sub-flow does not guarantee that the ordering will be maintained,
# even with sleeping.
was_nasty = False
try:
engine.run()
self.assertTrue(False)
except RuntimeError as e:
self.assertRegexpMatches(str(e), '^Gotcha|^Woot')
if 'Gotcha!' in str(e):
was_nasty = True
result = set(self.values)
possible_result = set(['task1', 'task2', 'task3', 'task3 reverted(5)'])
if not was_nasty:
possible_result.update(['task1 reverted(5)', 'task2 reverted(5)'])
self.assertIsSubset(possible_result, result)
# If the nasty task killed reverting, then task1 and task2 should not
# have reverted, but if the failing task stopped execution then task1
# and task2 should have reverted.
if was_nasty:
must_not_have = ['task1 reverted(5)', 'task2 reverted(5)']
for r in must_not_have:
self.assertNotIn(r, result)
else:
must_have = ['task1 reverted(5)', 'task2 reverted(5)']
for r in must_have:
self.assertIn(r, result)
def test_revert_exception_is_reraised(self):
flow = lf.Flow('revert-1').add(utils.NastyTask(),
utils.FailingTask(name='fail'))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
def test_restart_reverted_flow_with_retry(self):
flow = lf.Flow('test', retry=utils.OneReturnRetry(provides='x')).add(
utils.FailingTask('fail'))
engine = self._make_engine(flow)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
