def test_revert_all_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ProgressingTask("task1"),
lf.Flow('flow-2', retry.AlwaysRevertAll('r2')).add(
utils.ConditionalTask("task2"))
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.fetch_all(), {'y': 2})
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
'task1.t RUNNING',
'task1.t SUCCESS(5)',
'r2.r RUNNING',
'r2.r SUCCESS(None)',
'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING',
'task2.t REVERTED',
'r2.r REVERTING',
'r2.r REVERTED',
'task1.t REVERTING',
'task1.t REVERTED',
'r1.r REVERTING',
'r1.r REVERTED',
'flow-1.f REVERTED']
self.assertEqual(expected, capturer.values)
def test_factory(blowup):
f = lf.Flow("test")
if not blowup:
f.add(test_utils.ProgressingTask('test1'))
else:
f.add(test_utils.FailingTask("test1"))
return f
def test_on_update_progress(self):
request = self.make_request(task=utils.ProgressingTask(), arguments={})
# create server and process request
s = self.server(reset_master_mock=True)
s._process_request(request, self.message_mock)
# check calls
master_mock_calls = [
mock.call.Response(pr.RUNNING),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.EVENT,
details={'progress': 0.0},
event_type=task_atom.EVENT_UPDATE_PROGRESS),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.EVENT,
details={'progress': 1.0},
event_type=task_atom.EVENT_UPDATE_PROGRESS),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.SUCCESS, result=5),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid)
]
self.assertEqual(master_mock_calls, self.master_mock.mock_calls)
def test_when_subflow_fails_revert_success_tasks(self):
waiting_task = utils.WaitForOneFromTask('task2', 'task1',
[st.SUCCESS, st.FAILURE])
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
utils.ProgressingTask('task1'),
lf.Flow('flow-2').add(waiting_task,
utils.ConditionalTask('task3')))
engine = self._make_engine(flow)
engine.task_notifier.register('*', waiting_task.callback)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = [
'r.r RUNNING', 'r.r SUCCESS(1)', 'task1.t RUNNING',
'task2.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t SUCCESS(5)',
'task3.t RUNNING', 'task3.t FAILURE(Failure: RuntimeError: Woot!)',
'task3.t REVERTING', 'task1.t REVERTING', 'task3.t REVERTED',
'task1.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED',
'r.r RETRYING', 'task1.t PENDING', 'task2.t PENDING',
'task3.t PENDING', 'r.r RUNNING', 'r.r SUCCESS(2)',
'task1.t RUNNING', 'task2.t RUNNING', 'task1.t SUCCESS(5)',
'task2.t SUCCESS(5)', 'task3.t RUNNING', 'task3.t SUCCESS(None)'
]
self.assertItemsEqual(capturer.values, expected)
def test_on_update_progress(self):
request = self.make_request(task=utils.ProgressingTask(), arguments={})
# create server and process request
s = self.server(reset_main_mock=True)
s._process_request(request, self.message_mock)
# check calls
main_mock_calls = [
mock.call.Response(pr.RUNNING),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.PROGRESS, progress=0.0, event_data={}),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.PROGRESS, progress=1.0, event_data={}),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid),
mock.call.Response(pr.SUCCESS, result=5),
mock.call.proxy.publish(self.response_inst_mock,
self.reply_to,
correlation_id=self.task_uuid)
]
self.assertEqual(self.main_mock.mock_calls, main_mock_calls)
def test_nested_flow_reverts_parent_retries(self):
retry1 = retry.Times(3, 'r1', provides='x')
retry2 = retry.Times(0, 'r2', provides='x2')
flow = lf.Flow('flow-1', retry1).add(
utils.ProgressingTask("task1"),
lf.Flow('flow-2', retry2).add(utils.ConditionalTask("task2")))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2, 'x2': 1})
expected = [
'flow-1.f RUNNING', 'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING', 'task2.t REVERTED', 'r2.r REVERTING',
'r2.r REVERTED', 'task1.t REVERTING', 'task1.t REVERTED',
'r1.r RETRYING', 'task1.t PENDING', 'r2.r PENDING',
'task2.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(2)',
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING', 'task2.t SUCCESS(None)',
'flow-1.f SUCCESS'
]
self.assertEqual(expected, capturer.values)
def test_retry_tasks_that_has_not_been_reverted(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ConditionalTask('c'),
utils.ProgressingTask('t1')
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
'c.t RUNNING',
'c.t FAILURE(Failure: RuntimeError: Woot!)',
'c.t REVERTING',
'c.t REVERTED',
'r1.r RETRYING',
'c.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
'c.t RUNNING',
'c.t SUCCESS(None)',
't1.t RUNNING',
't1.t SUCCESS(5)',
'flow-1.f SUCCESS']
self.assertEqual(capturer.values, expected)
def test_graph_flow_one_task(self):
flow = gf.Flow('g-1').add(utils.ProgressingTask(name='task1'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
def _pretend_to_run_a_flow_and_crash(self, when):
flow = uf.Flow('flow-1', retry.Times(3, provides='x')).add(
utils.ProgressingTask('task1'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
# imagine we run engine
engine.storage.set_flow_state(st.RUNNING)
engine.storage.set_atom_intention('flow-1_retry', st.EXECUTE)
engine.storage.set_atom_intention('task1', st.EXECUTE)
# we execute retry
engine.storage.save('flow-1_retry', 1)
# task fails
fail = failure.Failure.from_exception(RuntimeError('foo'))
engine.storage.save('task1', fail, state=st.FAILURE)
if when == 'task fails':
return engine
# we save it's failure to retry and ask what to do
engine.storage.save_retry_failure('flow-1_retry', 'task1', fail)
if when == 'retry queried':
return engine
# it returned 'RETRY', so we update it's intention
engine.storage.set_atom_intention('flow-1_retry', st.RETRY)
if when == 'retry updated':
return engine
# we set task1 intention to REVERT
engine.storage.set_atom_intention('task1', st.REVERT)
if when == 'task updated':
return engine
# we schedule task1 for reversion
engine.storage.set_atom_state('task1', st.REVERTING)
if when == 'revert scheduled':
return engine
raise ValueError('Invalid crash point: %s' % when)
def test_states_retry_success_linear_flow(self):
flow = lf.Flow('flow-1', retry.Times(4, 'r1', provides='x')).add(
utils.ProgressingTask("task1"),
utils.ConditionalTask("task2")
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = ['flow-1.f RUNNING',
'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(5)',
'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING', 'task2.t REVERTED',
'task1.t REVERTING', 'task1.t REVERTED',
'r1.r RETRYING',
'task1.t PENDING',
'task2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
'task1.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t RUNNING',
'task2.t SUCCESS(None)',
'flow-1.f SUCCESS']
self.assertEqual(expected, capturer.values)
def test_run_task_as_flow(self):
flow = utils.ProgressingTask(name='task1')
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
def test_unordered_flow_task_fails_parallel_tasks_should_be_reverted(self):
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
utils.ProgressingTask("task1"),
utils.ConditionalTask("task2")
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = ['flow-1.f RUNNING',
'r.r RUNNING',
'r.r SUCCESS(1)',
'task1.t RUNNING',
'task2.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING',
'task1.t REVERTING',
'task2.t REVERTED',
'task1.t REVERTED',
'r.r RETRYING',
'task1.t PENDING',
'task2.t PENDING',
'r.r RUNNING',
'r.r SUCCESS(2)',
'task1.t RUNNING',
'task2.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t SUCCESS(None)',
'flow-1.f SUCCESS']
self.assertItemsEqual(capturer.values, expected)
def test_correctly_reverts_children(self):
flow = lf.Flow('root-1').add(
utils.ProgressingTask('task1'),
lf.Flow('child-1').add(utils.ProgressingTask('task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
expected = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING',
'task2.t SUCCESS(5)', 'fail.t RUNNING',
'fail.t FAILURE(Failure: RuntimeError: Woot!)', 'fail.t REVERTING',
'fail.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED',
'task1.t REVERTING', 'task1.t REVERTED'
]
self.assertEqual(expected, capturer.values)
def test_basic_do_not_capture(self):
flow = lf.Flow("test")
flow.add(test_utils.ProgressingTask("task1"))
e = self._make_engine(flow)
with test_utils.CaptureListener(e, capture_task=False) as capturer:
e.run()
expected = ['test.f RUNNING', 'test.f SUCCESS']
self.assertEqual(expected, capturer.values)
def test_parallel_flow_one_task(self):
flow = uf.Flow('p-1').add(
utils.ProgressingTask(name='task1', provides='a'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
self.assertEqual(engine.storage.fetch_all(), {'a': 5})
def test_on_run_reply_failure(self):
request = self.make_request(task=utils.ProgressingTask(), arguments={})
self.proxy_inst_mock.publish.side_effect = RuntimeError('Woot!')
# create server and process request
s = self.server(reset_master_mock=True)
s._process_request(request, self.message_mock)
self.assertEqual(1, self.proxy_inst_mock.publish.call_count)
def test_overlap_sibling_expected_result(self):
flow = lf.Flow('flow-1')
flow.add(utils.ProgressingTask(provides='source'))
flow.add(utils.TaskOneReturn(provides='source'))
flow.add(utils.AddOne())
engine = self._make_engine(flow)
engine.run()
results = engine.storage.fetch_all()
self.assertEqual(2, results['result'])
def test_graph_flow_four_tasks_revert_failure(self):
flow = gf.Flow('g-3-nasty').add(
utils.NastyTask(name='task2', provides='b', requires=['a']),
utils.FailingTask(name='task3', requires=['b']),
utils.ProgressingTask(name='task1', provides='a'))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertEqual(engine.storage.get_flow_state(), states.FAILURE)
def test_suspend_linear_flow(self):
flow = lf.Flow('linear').add(utils.ProgressingTask('a'),
utils.ProgressingTask('b'),
utils.ProgressingTask('c'))
engine = self._make_engine(flow)
with SuspendingListener(engine,
task_name='b',
task_state=states.SUCCESS) as capturer:
engine.run()
self.assertEqual(states.SUSPENDED, engine.storage.get_flow_state())
expected = [
'a.t RUNNING', 'a.t SUCCESS(5)', 'b.t RUNNING', 'b.t SUCCESS(5)'
]
self.assertEqual(expected, capturer.values)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
self.assertEqual(states.SUCCESS, engine.storage.get_flow_state())
expected = ['c.t RUNNING', 'c.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
def test_graph_flow_four_tasks_added_separately(self):
flow = (gf.Flow('g-4')
.add(utils.ProgressingTask(name='task4',
provides='d', requires=['c']))
.add(utils.ProgressingTask(name='task2',
provides='b', requires=['a']))
.add(utils.ProgressingTask(name='task3',
provides='c', requires=['b']))
.add(utils.ProgressingTask(name='task1',
provides='a'))
)
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)',
'task2.t RUNNING', 'task2.t SUCCESS(5)',
'task3.t RUNNING', 'task3.t SUCCESS(5)',
'task4.t RUNNING', 'task4.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
def test_graph_flow_four_tasks_revert(self):
flow = gf.Flow('g-4-failing').add(
utils.ProgressingTask(name='task4', provides='d', requires=['c']),
utils.ProgressingTask(name='task2', provides='b', requires=['a']),
utils.FailingTask(name='task3', provides='c', requires=['b']),
utils.ProgressingTask(name='task1', provides='a'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
expected = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING',
'task2.t SUCCESS(5)', 'task3.t RUNNING',
'task3.t FAILURE(Failure: RuntimeError: Woot!)',
'task3.t REVERTING', 'task3.t REVERTED', 'task2.t REVERTING',
'task2.t REVERTED', 'task1.t REVERTING', 'task1.t REVERTED'
]
self.assertEqual(expected, capturer.values)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_inject_persistent_missing(self):
t = test_utils.ProgressingTask('my retry', requires=['x'])
s = self._get_storage()
s.ensure_atom(t)
missing = s.fetch_unsatisfied_args(t.name, t.rebind)
self.assertEqual(set(['x']), missing)
s.inject_atom_args(t.name, {'x': 2}, transient=False)
missing = s.fetch_unsatisfied_args(t.name, t.rebind)
self.assertEqual(set(), missing)
args = s.fetch_mapped_args(t.rebind, atom_name=t.name)
self.assertEqual(2, args['x'])
def test_resume_flow_that_should_be_retried(self):
flow = lf.Flow('flow-1',
retry.Times(3, 'r1')).add(utils.ProgressingTask('t1'),
utils.ProgressingTask('t2'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
with utils.CaptureListener(engine) as capturer:
engine.storage.set_atom_intention('r1', st.RETRY)
engine.storage.set_atom_state('r1', st.SUCCESS)
engine.storage.set_atom_state('t1', st.REVERTED)
engine.storage.set_atom_state('t2', st.REVERTED)
engine.run()
expected = [
'flow-1.f RUNNING', 'r1.r RETRYING', 't1.t PENDING',
't2.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(1)', 't1.t RUNNING',
't1.t SUCCESS(5)', 't2.t RUNNING', 't2.t SUCCESS(5)',
'flow-1.f SUCCESS'
]
self.assertEqual(expected, capturer.values)
def test_sequential_flow_two_tasks_with_resumption(self):
flow = lf.Flow('lf-2-r').add(
utils.ProgressingTask(name='task1', provides='x1'),
utils.ProgressingTask(name='task2', provides='x2'))
# Create FlowDetail as if we already run task1
lb, fd = p_utils.temporary_flow_detail(self.backend)
td = logbook.TaskDetail(name='task1', uuid='42')
td.state = states.SUCCESS
td.results = 17
fd.add(td)
with contextlib.closing(self.backend.get_connection()) as conn:
fd.update(conn.update_flow_details(fd))
td.update(conn.update_atom_details(td))
engine = self._make_engine(flow, fd)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['task2.t RUNNING', 'task2.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
self.assertEqual(engine.storage.fetch_all(), {'x1': 17, 'x2': 5})
def test_suspend_linear_flow_on_revert(self):
flow = lf.Flow('linear').add(utils.ProgressingTask('a'),
utils.ProgressingTask('b'),
utils.FailingTask('c'))
engine = self._make_engine(flow)
with SuspendingListener(engine,
task_name='b',
task_state=states.REVERTED) as capturer:
engine.run()
self.assertEqual(states.SUSPENDED, engine.storage.get_flow_state())
expected = [
'a.t RUNNING', 'a.t SUCCESS(5)', 'b.t RUNNING', 'b.t SUCCESS(5)',
'c.t RUNNING', 'c.t FAILURE(Failure: RuntimeError: Woot!)',
'c.t REVERTING', 'c.t REVERTED(None)', 'b.t REVERTING',
'b.t REVERTED(None)'
]
self.assertEqual(expected, capturer.values)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertRaisesRegex(RuntimeError, '^Woot', engine.run)
self.assertEqual(states.REVERTED, engine.storage.get_flow_state())
expected = ['a.t REVERTING', 'a.t REVERTED(None)']
self.assertEqual(expected, capturer.values)
def test_revert_raises_for_unordered_in_linear(self):
flow = lf.Flow('p-root').add(
utils.ProgressingTask(name='task1'),
utils.ProgressingTask(name='task2'),
uf.Flow('p-inner').add(utils.ProgressingTask(name='task3'),
utils.NastyFailingTask(name='nasty')))
engine = self._make_engine(flow)
with utils.CaptureListener(engine,
capture_flow=False,
skip_tasks=['nasty']) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
# NOTE(imelnikov): we don't know if task 3 was run, but if it was,
# it should have been reverted in correct order.
possible_values = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING',
'task2.t SUCCESS(5)', 'task3.t RUNNING', 'task3.t SUCCESS(5)',
'task3.t REVERTING', 'task3.t REVERTED'
]
self.assertIsSuperAndSubsequence(possible_values, capturer.values)
possible_values_no_task3 = ['task1.t RUNNING', 'task2.t RUNNING']
self.assertIsSuperAndSubsequence(capturer.values,
possible_values_no_task3)
def test_revert_ok_for_linear_in_unordered(self):
flow = uf.Flow('p-root').add(
utils.ProgressingTask(name='task1'),
lf.Flow('p-inner').add(utils.ProgressingTask(name='task2'),
utils.FailingTask('fail')))
engine = self._make_engine(flow)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertIn('fail.t FAILURE(Failure: RuntimeError: Woot!)',
capturer.values)
# NOTE(imelnikov): if task1 was run, it should have been reverted.
if 'task1' in capturer.values:
task1_story = [
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task1.t REVERTED'
]
self.assertIsSuperAndSubsequence(capturer.values, task1_story)
# NOTE(imelnikov): task2 should have been run and reverted
task2_story = [
'task2.t RUNNING', 'task2.t SUCCESS(5)', 'task2.t REVERTED'
]
self.assertIsSuperAndSubsequence(capturer.values, task2_story)
def test_suspend_one_task(self):
flow = utils.ProgressingTask('a')
engine = self._make_engine(flow)
with SuspendingListener(engine, task_name='b',
task_state=states.SUCCESS) as capturer:
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS)
expected = ['a.t RUNNING', 'a.t SUCCESS(5)']
self.assertEqual(expected, capturer.values)
with SuspendingListener(engine, task_name='b',
task_state=states.SUCCESS) as capturer:
engine.run()
self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS)
expected = []
self.assertEqual(expected, capturer.values)
def test_default_times_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add(
utils.ProgressingTask('t1'),
utils.FailingTask('t2'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(3)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r REVERTING',
'r1.r REVERTED',
'flow-1.f REVERTED']
self.assertEqual(expected, capturer.values)