def test_parallel_flow_two_tasks(self):
flow = uf.Flow('p-2').add(utils.SaveOrderTask(name='task1'),
utils.SaveOrderTask(name='task2'))
self._make_engine(flow).run()
result = set(self.values)
self.assertEqual(result, set(['task1', 'task2']))
def test_sequential_flow_two_tasks(self):
flow = lf.Flow('flow-2').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'))
self._make_engine(flow).run()
self.assertEquals(self.values, ['task1', 'task2'])
self.assertEquals(len(flow), 2)
def test_sequential_flow_iter_suspend_resume(self):
flow = lf.Flow('flow-2').add(utils.SaveOrderTask(name='task1'),
utils.SaveOrderTask(name='task2'))
_lb, fd = p_utils.temporary_flow_detail(self.backend)
e = self._make_engine(flow, flow_detail=fd)
it = e.run_iter()
gathered_states = []
suspend_it = None
while True:
try:
s = it.send(suspend_it)
gathered_states.append(s)
if s == states.WAITING:
# Stop it before task2 runs/starts.
suspend_it = True
except StopIteration:
break
self.assertTrue(len(gathered_states) > 0)
self.assertEqual(self.values, ['task1'])
self.assertEqual(states.SUSPENDED, e.storage.get_flow_state())
# Attempt to resume it and see what runs now...
#
# NOTE(harlowja): Clear all the values, but don't reset the reference.
while len(self.values):
self.values.pop()
gathered_states = list(e.run_iter())
self.assertTrue(len(gathered_states) > 0)
self.assertEqual(self.values, ['task2'])
self.assertEqual(states.SUCCESS, e.storage.get_flow_state())
def test_suspend_and_revert_even_if_task_is_gone(self):
flow = lf.Flow('linear').add(
utils.SaveOrderTask('a'),
utils.SaveOrderTask('b'),
utils.FailingTask('c')
)
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.REVERTED):
engine.run()
expected_values = ['a', 'b',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(5)']
self.assertEqual(self.values, expected_values)
# pretend we are resuming, but task 'c' gone when flow got updated
flow2 = lf.Flow('linear').add(
utils.SaveOrderTask('a'),
utils.SaveOrderTask('b')
)
engine2 = self._make_engine(flow2, engine.storage._flowdetail)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run)
self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED)
expected_values.append('a reverted(5)')
self.assertEqual(self.values, expected_values)
def test_graph_flow_two_independent_tasks(self):
flow = gf.Flow('g-2').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'))
self._make_engine(flow).run()
self.assertEquals(set(self.values), set(['task1', 'task2']))
self.assertEquals(len(flow), 2)
def test_parallel_nested_to_linear_revert(self):
flow = lf.Flow('l-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
uf.Flow('p-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
result = set(self.values)
# Task3 may or may not have executed, depending on scheduling and
# task ordering selection, so it may or may not exist in the result set
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)',
'task3', 'task3 reverted(5)'
])
self.assertIsSubset(possible_result, result)
# These must exist, since the linearity of the linear flow ensures
# that they were executed first.
must_have = [
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)'
]
for r in must_have:
self.assertIn(r, result)
def test_suspend_linear_flow_on_revert(self):
flow = lf.Flow('linear').add(
utils.SaveOrderTask('a'),
utils.SaveOrderTask('b'),
utils.FailingTask('c')
)
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.REVERTED):
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED)
self.assertEqual(
self.values,
['a', 'b',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(5)'])
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
self.assertEqual(
self.values,
['a',
'b',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(5)',
'a reverted(5)'])
def test_sequential_flow_two_tasks_iter(self):
flow = lf.Flow('flow-2').add(utils.SaveOrderTask(name='task1'),
utils.SaveOrderTask(name='task2'))
e = self._make_engine(flow)
gathered_states = list(e.run_iter())
self.assertTrue(len(gathered_states) > 0)
self.assertEqual(self.values, ['task1', 'task2'])
self.assertEqual(len(flow), 2)
def test_revert_raises_for_linear_in_unordered(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(name='task1'),
lf.Flow('p-inner').add(utils.SaveOrderTask(name='task2'),
utils.NastyFailingTask()))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertNotIn('task2 reverted(5)', self.values)
def test_correctly_reverts_children(self):
flow = lf.Flow('root-1').add(
utils.SaveOrderTask('task1'),
lf.Flow('child-1').add(utils.SaveOrderTask('task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(self.values, [
'task1', 'task2', 'fail reverted(Failure: RuntimeError: Woot!)',
'task2 reverted(5)', 'task1 reverted(5)'
])
def test_parallel_revert_exception_is_reraised_(self):
flow = lf.Flow('p-r-reraise').add(
utils.SaveOrderTask(self.values, name='task1', sleep=0.01),
utils.NastyTask(),
utils.FailingTask(sleep=0.01),
utils.SaveOrderTask(self.values,
name='task2') # this should not get reverted
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
result = set(self.values)
self.assertEquals(result, set(['task1']))
def test_graph_flow_four_tasks_revert(self):
flow = gf.Flow('g-4-failing').add(
utils.SaveOrderTask(name='task4', provides='d', requires=['c']),
utils.SaveOrderTask(name='task2', provides='b', requires=['a']),
utils.FailingTask(name='task3', provides='c', requires=['b']),
utils.SaveOrderTask(name='task1', provides='a'))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(self.values, [
'task1', 'task2', 'task3 reverted(Failure: RuntimeError: Woot!)',
'task2 reverted(5)', 'task1 reverted(5)'
])
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_parallel_revert_specific(self):
flow = uf.Flow('p-r-r').add(
utils.SaveOrderTask(self.values, name='task1', sleep=0.01),
utils.FailingTask(sleep=0.01),
utils.SaveOrderTask(self.values, name='task2', sleep=0.01))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
result = set(self.values)
# NOTE(harlowja): task 1/2 may or may not have executed, even with the
# sleeps due to the fact that the above is an unordered flow.
possible_result = set(
['task1', 'task2', 'task2 reverted(5)', 'task1 reverted(5)'])
self.assertIsSubset(possible_result, result)
def test_suspend_linear_flow(self):
flow = lf.Flow('linear').add(
utils.SaveOrderTask('a'),
utils.SaveOrderTask('b'),
utils.SaveOrderTask('c')
)
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.SUCCESS):
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED)
self.assertEqual(self.values, ['a', 'b'])
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS)
self.assertEqual(self.values, ['a', 'b', 'c'])
def test_revert_raises_for_unordered_in_linear(self):
flow = lf.Flow('p-root').add(
utils.SaveOrderTask(name='task1'),
utils.SaveOrderTask(name='task2'),
uf.Flow('p-inner').add(utils.SaveOrderTask(name='task3'),
utils.NastyFailingTask()))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
# NOTE(imelnikov): we don't know if task 3 was run, but if it was,
# it should have been reverted in correct order.
possible_values = ['task1', 'task2', 'task3', 'task3 reverted(5)']
self.assertIsSuperAndSubsequence(possible_values, self.values)
possible_values_no_task3 = ['task1', 'task2']
self.assertIsSuperAndSubsequence(self.values, possible_values_no_task3)
def test_storage_is_rechecked(self):
flow = lf.Flow('linear').add(
utils.SaveOrderTask('b', requires=['foo']),
utils.SaveOrderTask('c')
)
engine = self._make_engine(flow)
engine.storage.inject({'foo': 'bar'})
with SuspendingListener(engine, task_name='b',
task_state=states.SUCCESS):
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED)
# uninject everything:
engine.storage.save(engine.storage.injector_name,
{}, states.SUCCESS)
self.assertRaises(exc.MissingDependencies, engine.run)
def test_linear_nested_to_parallel_revert_exception(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1', sleep=0.01),
utils.SaveOrderTask(self.values, name='task2', sleep=0.01),
lf.Flow('l-inner').add(
utils.SaveOrderTask(self.values, name='task3'),
utils.NastyTask(), utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
result = set(self.values)
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)', 'task3'
])
self.assertIsSubset(possible_result, result)
def test_graph_flow_four_tasks_added_separately(self):
flow = (gf.Flow('g-4').add(
utils.SaveOrderTask(name='task4', provides='d', requires=[
'c'
])).add(
utils.SaveOrderTask(name='task2', provides='b', requires=[
'a'
])).add(
utils.SaveOrderTask(name='task3',
provides='c',
requires=['b'])).add(
utils.SaveOrderTask(name='task1',
provides='a')))
self._make_engine(flow).run()
self.assertEqual(self.values, ['task1', 'task2', 'task3', 'task4'])
def test_parallel_flow_one_task(self):
flow = uf.Flow('p-1').add(
utils.SaveOrderTask(name='task1', provides='a'))
engine = self._make_engine(flow)
engine.run()
self.assertEqual(self.values, ['task1'])
self.assertEqual(engine.storage.fetch_all(), {'a': 5})
def _pretend_to_run_a_flow_and_crash(self, when):
flow = uf.Flow('flow-1', retry.Times(3, provides='x')).add(
utils.SaveOrderTask('task1'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
# imagine we run engine
engine.storage.set_flow_state(st.RUNNING)
engine.storage.set_atom_intention('flow-1_retry', st.EXECUTE)
engine.storage.set_atom_intention('task1', st.EXECUTE)
# we execute retry
engine.storage.save('flow-1_retry', 1)
# task fails
fail = failure.Failure.from_exception(RuntimeError('foo')),
engine.storage.save('task1', fail, state=st.FAILURE)
if when == 'task fails':
return engine
# we save it's failure to retry and ask what to do
engine.storage.save_retry_failure('flow-1_retry', 'task1', fail)
if when == 'retry queried':
return engine
# it returned 'RETRY', so we update it's intention
engine.storage.set_atom_intention('flow-1_retry', st.RETRY)
if when == 'retry updated':
return engine
# we set task1 intention to REVERT
engine.storage.set_atom_intention('task1', st.REVERT)
if when == 'task updated':
return engine
# we schedule task1 for reversion
engine.storage.set_atom_state('task1', st.REVERTING)
if when == 'revert scheduled':
return engine
raise ValueError('Invalid crash point: %s' % when)
def test_nested_flow_reverts_parent_retries(self):
retry1 = retry.Times(3, 'r1', provides='x')
retry2 = retry.Times(0, 'r2', provides='x2')
flow = lf.Flow('flow-1', retry1).add(
utils.SaveOrderTask("task1"),
lf.Flow('flow-2', retry2).add(utils.ConditionalTask("task2")))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
utils.register_notifiers(engine, self.values)
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2, 'x2': 1})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1', 'task1 SUCCESS', 'r2 RUNNING', 'r2 SUCCESS',
'task2 RUNNING', 'task2', 'task2 FAILURE', 'task2 REVERTING',
u'task2 reverted(Failure: RuntimeError: Woot!)', 'task2 REVERTED',
'r2 REVERTING', 'r2 REVERTED', 'task1 REVERTING',
'task1 reverted(5)', 'task1 REVERTED', 'r1 RETRYING',
'task1 PENDING', 'r2 PENDING', 'task2 PENDING', 'r1 RUNNING',
'r1 SUCCESS', 'task1 RUNNING', 'task1', 'task1 SUCCESS',
'r2 RUNNING', 'r2 SUCCESS', 'task2 RUNNING', 'task2',
'task2 SUCCESS', 'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def test_revert_ok_for_linear_in_unordered(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(name='task1'),
lf.Flow('p-inner').add(utils.SaveOrderTask(name='task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertIn('fail reverted(Failure: RuntimeError: Woot!)',
self.values)
# NOTE(imelnikov): if task1 was run, it should have been reverted.
if 'task1' in self.values:
task1_story = ['task1', 'task1 reverted(5)']
self.assertIsSuperAndSubsequence(self.values, task1_story)
# NOTE(imelnikov): task2 should have been run and reverted
task2_story = ['task2', 'task2 reverted(5)']
self.assertIsSuperAndSubsequence(self.values, task2_story)
def test_graph_flow_four_tasks_revert_failure(self):
flow = gf.Flow('g-3-nasty').add(
utils.NastyTask(name='task2', provides='b', requires=['a']),
utils.FailingTask(name='task3', requires=['b']),
utils.SaveOrderTask(name='task1', provides='a'))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertEqual(engine.storage.get_flow_state(), states.FAILURE)
def test_run_task_with_notifications(self):
flow = utils.SaveOrderTask(name='task1')
engine = self._make_engine(flow)
utils.register_notifiers(engine, self.values)
engine.run()
self.assertEqual(self.values, [
'flow RUNNING', 'task1 RUNNING', 'task1', 'task1 SUCCESS',
'flow SUCCESS'
])
def test_retry_tasks_that_has_not_been_reverted(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ConditionalTask('c'), utils.SaveOrderTask('t1'))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
engine.run()
expected = [
'c', u'c reverted(Failure: RuntimeError: Woot!)', 'c', 't1'
]
self.assertEqual(self.values, expected)
def test_nested_parallel_revert_exception_is_reraised(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
lf.Flow('p-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.NastyTask(), utils.FailingTask(sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
result = set(self.values)
# Task1, task2 may *not* have executed and also may have *not* reverted
# since the above is an unordered flow so take that into account by
# ensuring that the superset is matched.
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)',
'task3', 'task3 reverted(5)'
])
self.assertIsSubset(possible_result, result)
def test_linear_nested_to_parallel_revert(self):
flow = uf.Flow('p-root').add(
utils.SaveOrderTask(self.values, name='task1'),
utils.SaveOrderTask(self.values, name='task2'),
lf.Flow('l-inner').add(
utils.SaveOrderTask(self.values, name='task3', sleep=0.1),
utils.FailingTask(self.values, name='fail', sleep=0.01)))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
result = set(self.values)
# Since this is an unordered flow we can not guarantee that task1 or
# task2 will exist and be reverted, although they may exist depending
# on how the OS thread scheduling and execution graph algorithm...
possible_result = set([
'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)',
'task3', 'task3 reverted(5)',
'fail reverted(Failure: RuntimeError: Woot!)'
])
self.assertIsSubset(possible_result, result)
def test_resume_flow_that_should_be_retried(self):
flow = lf.Flow('flow-1',
retry.Times(3, 'r1')).add(utils.SaveOrderTask('t1'),
utils.SaveOrderTask('t2'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
utils.register_notifiers(engine, self.values)
engine.storage.set_atom_intention('r1', st.RETRY)
engine.storage.set_atom_state('r1', st.SUCCESS)
engine.storage.set_atom_state('t1', st.REVERTED)
engine.storage.set_atom_state('t2', st.REVERTED)
engine.run()
expected = [
'flow RUNNING', 'r1 RETRYING', 't1 PENDING', 't2 PENDING',
'r1 RUNNING', 'r1 SUCCESS', 't1 RUNNING', 't1', 't1 SUCCESS',
't2 RUNNING', 't2', 't2 SUCCESS', 'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def test_suspend_one_task(self):
flow = utils.SaveOrderTask('a')
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.SUCCESS):
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS)
self.assertEqual(self.values, ['a'])
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS)
self.assertEqual(self.values, ['a'])
def test_sequential_flow_two_tasks_with_resumption(self):
flow = lf.Flow('lf-2-r').add(
utils.SaveOrderTask(name='task1', provides='x1'),
utils.SaveOrderTask(name='task2', provides='x2'))
# Create FlowDetail as if we already run task1
_lb, fd = p_utils.temporary_flow_detail(self.backend)
td = logbook.TaskDetail(name='task1', uuid='42')
td.state = states.SUCCESS
td.results = 17
fd.add(td)
with contextlib.closing(self.backend.get_connection()) as conn:
fd.update(conn.update_flow_details(fd))
td.update(conn.update_task_details(td))
engine = self._make_engine(flow, fd)
engine.run()
self.assertEqual(self.values, ['task2'])
self.assertEqual(engine.storage.fetch_all(), {'x1': 17, 'x2': 5})
