def test_nested_flow_reverts_parent_retries(self):
retry1 = retry.Times(3, 'r1', provides='x')
retry2 = retry.Times(0, 'r2', provides='x2')
flow = lf.Flow('flow-1', retry1).add(
utils.ProgressingTask("task1"),
lf.Flow('flow-2', retry2).add(utils.ConditionalTask("task2")))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2, 'x2': 1})
expected = [
'flow-1.f RUNNING', 'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING', 'task2.t REVERTED', 'r2.r REVERTING',
'r2.r REVERTED', 'task1.t REVERTING', 'task1.t REVERTED',
'r1.r RETRYING', 'task1.t PENDING', 'r2.r PENDING',
'task2.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(2)',
'task1.t RUNNING', 'task1.t SUCCESS(5)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING', 'task2.t SUCCESS(None)',
'flow-1.f SUCCESS'
]
self.assertEqual(expected, capturer.values)
def test_states_retry_failure_parent_flow_fails(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x1')).add(
utils.TaskNoRequiresNoReturns("task1"),
lf.Flow('flow-2', retry.Times(3, 'r2', provides='x2')).add(
utils.TaskNoRequiresNoReturns("task2"),
utils.TaskNoRequiresNoReturns("task3")),
utils.ConditionalTask("task4", rebind={'x': 'x1'}))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {
'y': 2,
'x1': 2,
'x2': 1
})
expected = [
'flow-1.f RUNNING', 'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(None)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING', 'task2.t SUCCESS(None)',
'task3.t RUNNING', 'task3.t SUCCESS(None)', 'task4.t RUNNING',
'task4.t FAILURE(Failure: RuntimeError: Woot!)',
'task4.t REVERTING', 'task4.t REVERTED', 'task3.t REVERTING',
'task3.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED',
'r2.r REVERTING', 'r2.r REVERTED', 'task1.t REVERTING',
'task1.t REVERTED', 'r1.r RETRYING', 'task1.t PENDING',
'r2.r PENDING', 'task2.t PENDING', 'task3.t PENDING',
'task4.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(2)',
'task1.t RUNNING', 'task1.t SUCCESS(None)', 'r2.r RUNNING',
'r2.r SUCCESS(1)', 'task2.t RUNNING', 'task2.t SUCCESS(None)',
'task3.t RUNNING', 'task3.t SUCCESS(None)', 'task4.t RUNNING',
'task4.t SUCCESS(None)', 'flow-1.f SUCCESS'
]
self.assertEqual(expected, capturer.values)
def test_nested_flow_reverts_parent_retries(self):
retry1 = retry.Times(3, 'r1', provides='x')
retry2 = retry.Times(0, 'r2', provides='x2')
flow = lf.Flow('flow-1', retry1).add(
utils.SaveOrderTask("task1"),
lf.Flow('flow-2', retry2).add(utils.ConditionalTask("task2")))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
utils.register_notifiers(engine, self.values)
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2, 'x2': 1})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1', 'task1 SUCCESS', 'r2 RUNNING', 'r2 SUCCESS',
'task2 RUNNING', 'task2', 'task2 FAILURE', 'task2 REVERTING',
u'task2 reverted(Failure: RuntimeError: Woot!)', 'task2 REVERTED',
'r2 REVERTING', 'r2 REVERTED', 'task1 REVERTING',
'task1 reverted(5)', 'task1 REVERTED', 'r1 RETRYING',
'task1 PENDING', 'r2 PENDING', 'task2 PENDING', 'r1 RUNNING',
'r1 SUCCESS', 'task1 RUNNING', 'task1', 'task1 SUCCESS',
'r2 RUNNING', 'r2 SUCCESS', 'task2 RUNNING', 'task2',
'task2 SUCCESS', 'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def test_states_retry_failure_parent_flow_fails(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x1')).add(
utils.TaskNoRequiresNoReturns("task1"),
lf.Flow('flow-2', retry.Times(3, 'r2', provides='x2')).add(
utils.TaskNoRequiresNoReturns("task2"),
utils.TaskNoRequiresNoReturns("task3")),
utils.ConditionalTask("task4", rebind={'x': 'x1'}))
engine = self._make_engine(flow)
utils.register_notifiers(engine, self.values)
engine.storage.inject({'y': 2})
engine.run()
self.assertEqual(engine.storage.fetch_all(), {
'y': 2,
'x1': 2,
'x2': 1
})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1 SUCCESS', 'r2 RUNNING', 'r2 SUCCESS', 'task2 RUNNING',
'task2 SUCCESS', 'task3 RUNNING', 'task3 SUCCESS', 'task4 RUNNING',
'task4', 'task4 FAILURE', 'task4 REVERTING',
u'task4 reverted(Failure: RuntimeError: Woot!)', 'task4 REVERTED',
'task3 REVERTING', 'task3 REVERTED', 'task2 REVERTING',
'task2 REVERTED', 'r2 REVERTING', 'r2 REVERTED', 'task1 REVERTING',
'task1 REVERTED', 'r1 RETRYING', 'task1 PENDING', 'r2 PENDING',
'task2 PENDING', 'task3 PENDING', 'task4 PENDING', 'r1 RUNNING',
'r1 SUCCESS', 'task1 RUNNING', 'task1 SUCCESS', 'r2 RUNNING',
'r2 SUCCESS', 'task2 RUNNING', 'task2 SUCCESS', 'task3 RUNNING',
'task3 SUCCESS', 'task4 RUNNING', 'task4', 'task4 SUCCESS',
'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def test_nested_provides_graph_retried_correctly(self):
flow = gf.Flow("test").add(
utils.ProgressingTask('a', requires=['x']),
lf.Flow("test2", retry=retry.Times(2)).add(
utils.ProgressingTask('b', provides='x'),
utils.ProgressingTask('c')))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
engine.storage.save('test2_retry', 1)
engine.storage.save('b', 11)
# pretend that 'c' failed
fail = failure.Failure.from_exception(RuntimeError('Woot!'))
engine.storage.save('c', fail, st.FAILURE)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
expected = ['c.t REVERTING',
'c.t REVERTED',
'b.t REVERTING',
'b.t REVERTED']
self.assertItemsEqual(capturer.values[:4], expected)
expected = ['test2_retry.r RETRYING',
'b.t PENDING',
'c.t PENDING',
'test2_retry.r RUNNING',
'test2_retry.r SUCCESS(2)',
'b.t RUNNING',
'b.t SUCCESS(5)',
'a.t RUNNING',
'c.t RUNNING',
'a.t SUCCESS(5)',
'c.t SUCCESS(5)']
self.assertItemsEqual(expected, capturer.values[4:])
self.assertEqual(engine.storage.get_flow_state(), st.SUCCESS)
def test_states_retry_success_linear_flow(self):
flow = lf.Flow('flow-1', retry.Times(4, 'r1', provides='x')).add(
utils.ProgressingTask("task1"),
utils.ConditionalTask("task2")
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = ['flow-1.f RUNNING',
'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(5)',
'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING', 'task2.t REVERTED',
'task1.t REVERTING', 'task1.t REVERTED',
'r1.r RETRYING',
'task1.t PENDING',
'task2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
'task1.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t RUNNING',
'task2.t SUCCESS(None)',
'flow-1.f SUCCESS']
self.assertEqual(expected, capturer.values)
def test_revert_all_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ProgressingTask("task1"),
lf.Flow('flow-2', retry.AlwaysRevertAll('r2')).add(
utils.ConditionalTask("task2"))
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.fetch_all(), {'y': 2})
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
'task1.t RUNNING',
'task1.t SUCCESS(5)',
'r2.r RUNNING',
'r2.r SUCCESS(None)',
'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING',
'task2.t REVERTED',
'r2.r REVERTING',
'r2.r REVERTED',
'task1.t REVERTING',
'task1.t REVERTED',
'r1.r REVERTING',
'r1.r REVERTED',
'flow-1.f REVERTED']
self.assertEqual(expected, capturer.values)
def test_resume_flow_that_should_be_retried(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add(
utils.ProgressingTask('t1'),
utils.ProgressingTask('t2')
)
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
with utils.CaptureListener(engine) as capturer:
engine.storage.set_atom_intention('r1', st.RETRY)
engine.storage.set_atom_state('r1', st.SUCCESS)
engine.storage.set_atom_state('t1', st.REVERTED)
engine.storage.set_atom_state('t2', st.REVERTED)
engine.run()
expected = ['flow-1.f RUNNING',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t SUCCESS(5)',
'flow-1.f SUCCESS']
self.assertEqual(expected, capturer.values)
def test_when_subflow_fails_revert_running_tasks(self):
waiting_task = utils.WaitForOneFromTask('task1', 'task2',
[st.SUCCESS, st.FAILURE])
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
waiting_task,
utils.ConditionalTask('task2')
)
engine = self._make_engine(flow)
engine.task_notifier.register('*', waiting_task.callback)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine, capture_flow=False) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = ['r.r RUNNING',
'r.r SUCCESS(1)',
'task1.t RUNNING',
'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING',
'task2.t REVERTED',
'task1.t SUCCESS(5)',
'task1.t REVERTING',
'task1.t REVERTED',
'r.r RETRYING',
'task1.t PENDING',
'task2.t PENDING',
'r.r RUNNING',
'r.r SUCCESS(2)',
'task1.t RUNNING',
'task2.t RUNNING',
'task2.t SUCCESS(None)',
'task1.t SUCCESS(5)']
self.assertItemsEqual(capturer.values, expected)
def test_retry_tasks_that_has_not_been_reverted(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ConditionalTask('c'),
utils.ProgressingTask('t1')
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
'c.t RUNNING',
'c.t FAILURE(Failure: RuntimeError: Woot!)',
'c.t REVERTING',
'c.t REVERTED',
'r1.r RETRYING',
'c.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
'c.t RUNNING',
'c.t SUCCESS(None)',
't1.t RUNNING',
't1.t SUCCESS(5)',
'flow-1.f SUCCESS']
self.assertEqual(capturer.values, expected)
def get_create_amphora_for_lb_flow(self):
"""Creates a flow to create an amphora for a load balancer.
This flow is used when there are no spare amphora available
for a new load balancer. It builds an amphora and allocates
for the specific load balancer.
:returns: The The flow for creating the amphora
"""
create_amp_for_lb_flow = linear_flow.Flow(
constants.CREATE_AMPHORA_FOR_LB_FLOW)
create_amp_for_lb_flow.add(
database_tasks.CreateAmphoraInDB(provides=constants.AMPHORA_ID))
if self.REST_AMPHORA_DRIVER:
create_amp_for_lb_flow.add(
cert_task.GenerateServerPEMTask(provides=constants.SERVER_PEM))
create_amp_for_lb_flow.add(
compute_tasks.CertComputeCreate(
requires=(constants.AMPHORA_ID, constants.SERVER_PEM),
provides=constants.COMPUTE_ID))
else:
create_amp_for_lb_flow.add(
compute_tasks.ComputeCreate(requires=constants.AMPHORA_ID,
provides=constants.COMPUTE_ID))
create_amp_for_lb_flow.add(
database_tasks.UpdateAmphoraComputeId(
requires=(constants.AMPHORA_ID, constants.COMPUTE_ID)))
create_amp_for_lb_flow.add(
database_tasks.MarkAmphoraBootingInDB(
requires=(constants.AMPHORA_ID, constants.COMPUTE_ID)))
wait_flow = linear_flow.Flow(
constants.WAIT_FOR_AMPHORA,
retry=retry.Times(CONF.controller_worker.amp_active_retries))
wait_flow.add(
compute_tasks.ComputeWait(requires=constants.COMPUTE_ID,
provides=constants.COMPUTE_OBJ))
wait_flow.add(
database_tasks.UpdateAmphoraInfo(requires=(constants.AMPHORA_ID,
constants.COMPUTE_OBJ),
provides=constants.AMPHORA))
create_amp_for_lb_flow.add(wait_flow)
create_amp_for_lb_flow.add(
amphora_driver_tasks.AmphoraFinalize(requires=constants.AMPHORA))
create_amp_for_lb_flow.add(
database_tasks.MarkAmphoraAllocatedInDB(
requires=(constants.AMPHORA, constants.LOADBALANCER_ID)))
create_amp_for_lb_flow.add(
database_tasks.ReloadAmphora(requires=constants.AMPHORA_ID,
provides=constants.AMPHORA))
create_amp_for_lb_flow.add(
database_tasks.ReloadLoadBalancer(
name=constants.RELOAD_LB_AFTER_AMP_ASSOC,
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
new_LB_net_subflow = self._lb_flows.get_new_LB_networking_subflow()
create_amp_for_lb_flow.add(new_LB_net_subflow)
create_amp_for_lb_flow.add(
database_tasks.MarkLBActiveInDB(requires=constants.LOADBALANCER))
return create_amp_for_lb_flow
def test_unordered_flow_task_fails_parallel_tasks_should_be_reverted(self):
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
utils.ProgressingTask("task1"),
utils.ConditionalTask("task2")
)
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
with utils.CaptureListener(engine) as capturer:
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = ['flow-1.f RUNNING',
'r.r RUNNING',
'r.r SUCCESS(1)',
'task1.t RUNNING',
'task2.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING',
'task1.t REVERTING',
'task2.t REVERTED',
'task1.t REVERTED',
'r.r RETRYING',
'task1.t PENDING',
'task2.t PENDING',
'r.r RUNNING',
'r.r SUCCESS(2)',
'task1.t RUNNING',
'task2.t RUNNING',
'task1.t SUCCESS(5)',
'task2.t SUCCESS(None)',
'flow-1.f SUCCESS']
self.assertItemsEqual(capturer.values, expected)
def _pretend_to_run_a_flow_and_crash(self, when):
flow = uf.Flow('flow-1', retry.Times(3, provides='x')).add(
utils.ProgressingTask('task1'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
# imagine we run engine
engine.storage.set_flow_state(st.RUNNING)
engine.storage.set_atom_intention('flow-1_retry', st.EXECUTE)
engine.storage.set_atom_intention('task1', st.EXECUTE)
# we execute retry
engine.storage.save('flow-1_retry', 1)
# task fails
fail = failure.Failure.from_exception(RuntimeError('foo'))
engine.storage.save('task1', fail, state=st.FAILURE)
if when == 'task fails':
return engine
# we save it's failure to retry and ask what to do
engine.storage.save_retry_failure('flow-1_retry', 'task1', fail)
if when == 'retry queried':
return engine
# it returned 'RETRY', so we update it's intention
engine.storage.set_atom_intention('flow-1_retry', st.RETRY)
if when == 'retry updated':
return engine
# we set task1 intention to REVERT
engine.storage.set_atom_intention('task1', st.REVERT)
if when == 'task updated':
return engine
# we schedule task1 for reversion
engine.storage.set_atom_state('task1', st.REVERTING)
if when == 'revert scheduled':
return engine
raise ValueError('Invalid crash point: %s' % when)
def test_nested_provides_graph_reverts_correctly(self):
flow = gf.Flow("test").add(
utils.ProgressingTask('a', requires=['x']),
lf.Flow("test2", retry=retry.Times(2)).add(
utils.ProgressingTask('b', provides='x'),
utils.FailingTask('c')))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
engine.storage.save('test2_retry', 1)
engine.storage.save('b', 11)
engine.storage.save('a', 10)
with utils.CaptureListener(engine, capture_flow=False) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = ['c.t RUNNING',
'c.t FAILURE(Failure: RuntimeError: Woot!)',
'a.t REVERTING',
'c.t REVERTING',
'a.t REVERTED',
'c.t REVERTED',
'b.t REVERTING',
'b.t REVERTED']
self.assertItemsEqual(capturer.values[:8], expected)
# Task 'a' was or was not executed again, both cases are ok.
self.assertIsSuperAndSubsequence(capturer.values[8:], [
'b.t RUNNING',
'c.t FAILURE(Failure: RuntimeError: Woot!)',
'b.t REVERTED',
])
self.assertEqual(engine.storage.get_flow_state(), st.REVERTED)
def create_ssl_certificate():
flow = graph_flow.Flow('Creating poppy ssl certificate').add(
linear_flow.Flow("Provision poppy ssl certificate",
retry=retry.Times(5)).
add(create_ssl_certificate_tasks.CreateProviderSSLCertificateTask()),
create_ssl_certificate_tasks.SendNotificationTask(),
create_ssl_certificate_tasks.UpdateCertInfoTask())
return flow
def delete_ssl_certificate():
flow = graph_flow.Flow('Deleting poppy ssl certificate').add(
linear_flow.Flow('Update Oslo Context').add(
common.ContextUpdateTask()),
linear_flow.Flow("Deleting poppy ssl certificate",
retry=retry.Times(5)).
add(delete_ssl_certificate_tasks.DeleteProviderSSLCertificateTask()),
delete_ssl_certificate_tasks.SendNotificationTask(),
delete_ssl_certificate_tasks.DeleteStorageSSLCertificateTask())
return flow
def test_retry_tasks_that_has_not_been_reverted(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ConditionalTask('c'), utils.SaveOrderTask('t1'))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
engine.run()
expected = [
'c', u'c reverted(Failure: RuntimeError: Woot!)', 'c', 't1'
]
self.assertEqual(self.values, expected)
def recreate_ssl_certificate():
flow = linear_flow.Flow('Recreating poppy ssl certificate').add(
delete_ssl_certificate_tasks.DeleteStorageSSLCertificateTask(),
create_ssl_certificate_tasks.CreateStorageSSLCertificateTask(),
linear_flow.Flow("Provision poppy ssl certificate",
retry=retry.Times(5)).
add(create_ssl_certificate_tasks.CreateProviderSSLCertificateTask()),
create_ssl_certificate_tasks.SendNotificationTask(),
create_ssl_certificate_tasks.UpdateCertInfoTask())
return flow
def test_unordered_flow_task_fails_parallel_tasks_should_be_reverted(self):
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
utils.SaveOrderTask("task1"), utils.ConditionalTask("task2"))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = [
'task2', 'task1', u'task2 reverted(Failure: RuntimeError: Woot!)',
'task1 reverted(5)', 'task2', 'task1'
]
self.assertItemsEqual(self.values, expected)
def test_default_times_retry(self):
flow = lf.Flow('flow-1',
retry.Times(3, 'r1')).add(utils.SaveOrderTask('t1'),
utils.FailingTask('t2'))
engine = self._make_engine(flow)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = [
't1', u't2 reverted(Failure: RuntimeError: Woot!)',
't1 reverted(5)', 't1',
u't2 reverted(Failure: RuntimeError: Woot!)', 't1 reverted(5)',
't1', u't2 reverted(Failure: RuntimeError: Woot!)',
't1 reverted(5)'
]
self.assertEqual(self.values, expected)
def test_when_subflow_fails_revert_running_tasks(self):
waiting_task = utils.WaitForOneFromTask('task1', 'task2',
[st.SUCCESS, st.FAILURE])
flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add(
waiting_task, utils.ConditionalTask('task2'))
engine = self._make_engine(flow)
engine.task_notifier.register('*', waiting_task.callback)
engine.storage.inject({'y': 2})
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = [
'task2', 'task1', u'task2 reverted(Failure: RuntimeError: Woot!)',
'task1 reverted(5)', 'task2', 'task1'
]
self.assertItemsEqual(self.values, expected)
def test_default_times_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add(
utils.ProgressingTask('t1'),
utils.FailingTask('t2'))
engine = self._make_engine(flow)
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
expected = ['flow-1.f RUNNING',
'r1.r RUNNING',
'r1.r SUCCESS(1)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(2)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r RETRYING',
't1.t PENDING',
't2.t PENDING',
'r1.r RUNNING',
'r1.r SUCCESS(3)',
't1.t RUNNING',
't1.t SUCCESS(5)',
't2.t RUNNING',
't2.t FAILURE(Failure: RuntimeError: Woot!)',
't2.t REVERTING',
't2.t REVERTED',
't1.t REVERTING',
't1.t REVERTED',
'r1.r REVERTING',
'r1.r REVERTED',
'flow-1.f REVERTED']
self.assertEqual(expected, capturer.values)
def test_states_retry_failure_linear_flow(self):
flow = lf.Flow('flow-1', retry.Times(2, 'r1', provides='x')).add(
utils.NastyTask("task1"), utils.ConditionalTask("task2"))
engine = self._make_engine(flow)
engine.storage.inject({'y': 4})
with utils.CaptureListener(engine) as capturer:
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertEqual(engine.storage.fetch_all(), {'y': 4, 'x': 1})
expected = [
'flow-1.f RUNNING', 'r1.r RUNNING', 'r1.r SUCCESS(1)',
'task1.t RUNNING', 'task1.t SUCCESS(None)', 'task2.t RUNNING',
'task2.t FAILURE(Failure: RuntimeError: Woot!)',
'task2.t REVERTING', 'task2.t REVERTED', 'task1.t REVERTING',
'task1.t FAILURE', 'flow-1.f FAILURE'
]
self.assertEqual(expected, capturer.values)
def test_states_retry_failure_linear_flow(self):
flow = lf.Flow('flow-1', retry.Times(2, 'r1', provides='x')).add(
utils.NastyTask("task1"), utils.ConditionalTask("task2"))
engine = self._make_engine(flow)
utils.register_notifiers(engine, self.values)
engine.storage.inject({'y': 4})
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertEqual(engine.storage.fetch_all(), {'y': 4, 'x': 1})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1 SUCCESS', 'task2 RUNNING', 'task2', 'task2 FAILURE',
'task2 REVERTING', u'task2 reverted(Failure: RuntimeError: Woot!)',
'task2 REVERTED', 'task1 REVERTING', 'task1 FAILURE',
'flow FAILURE'
]
self.assertEqual(self.values, expected)
def test_states_retry_success_linear_flow(self):
flow = lf.Flow('flow-1', retry.Times(4, 'r1', provides='x')).add(
utils.SaveOrderTask("task1"), utils.ConditionalTask("task2"))
engine = self._make_engine(flow)
utils.register_notifiers(engine, self.values)
engine.storage.inject({'y': 2})
engine.run()
self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1', 'task1 SUCCESS', 'task2 RUNNING', 'task2',
'task2 FAILURE', 'task2 REVERTING',
u'task2 reverted(Failure: RuntimeError: Woot!)', 'task2 REVERTED',
'task1 REVERTING', 'task1 reverted(5)', 'task1 REVERTED',
'r1 RETRYING', 'task1 PENDING', 'task2 PENDING', 'r1 RUNNING',
'r1 SUCCESS', 'task1 RUNNING', 'task1', 'task1 SUCCESS',
'task2 RUNNING', 'task2', 'task2 SUCCESS', 'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def test_revert_all_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.SaveOrderTask("task1"),
lf.Flow('flow-2', retry.AlwaysRevertAll('r2')).add(
utils.ConditionalTask("task2")))
engine = self._make_engine(flow)
engine.storage.inject({'y': 2})
utils.register_notifiers(engine, self.values)
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.fetch_all(), {'y': 2})
expected = [
'flow RUNNING', 'r1 RUNNING', 'r1 SUCCESS', 'task1 RUNNING',
'task1', 'task1 SUCCESS', 'r2 RUNNING', 'r2 SUCCESS',
'task2 RUNNING', 'task2', 'task2 FAILURE', 'task2 REVERTING',
u'task2 reverted(Failure: RuntimeError: Woot!)', 'task2 REVERTED',
'r2 REVERTING', 'r2 REVERTED', 'task1 REVERTING',
'task1 reverted(5)', 'task1 REVERTED', 'r1 REVERTING',
'r1 REVERTED', 'flow REVERTED'
]
self.assertEqual(self.values, expected)
def test_resume_flow_that_should_be_retried(self):
flow = lf.Flow('flow-1',
retry.Times(3, 'r1')).add(utils.SaveOrderTask('t1'),
utils.SaveOrderTask('t2'))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
utils.register_notifiers(engine, self.values)
engine.storage.set_atom_intention('r1', st.RETRY)
engine.storage.set_atom_state('r1', st.SUCCESS)
engine.storage.set_atom_state('t1', st.REVERTED)
engine.storage.set_atom_state('t2', st.REVERTED)
engine.run()
expected = [
'flow RUNNING', 'r1 RETRYING', 't1 PENDING', 't2 PENDING',
'r1 RUNNING', 'r1 SUCCESS', 't1 RUNNING', 't1', 't1 SUCCESS',
't2 RUNNING', 't2', 't2 SUCCESS', 'flow SUCCESS'
]
self.assertEqual(self.values, expected)
def get_create_amphora_flow(self):
"""Creates a flow to create an amphora.
Ideally that should be configurable in the
config file - a db session needs to be placed
into the flow
:returns: The flow for creating the amphora
"""
create_amphora_flow = linear_flow.Flow(constants.CREATE_AMPHORA_FLOW)
create_amphora_flow.add(
database_tasks.CreateAmphoraInDB(provides=constants.AMPHORA_ID))
if self.REST_AMPHORA_DRIVER:
create_amphora_flow.add(
cert_task.GenerateServerPEMTask(provides=constants.SERVER_PEM))
create_amphora_flow.add(
compute_tasks.CertComputeCreate(
requires=(constants.AMPHORA_ID, constants.SERVER_PEM),
provides=constants.COMPUTE_ID))
else:
create_amphora_flow.add(
compute_tasks.ComputeCreate(requires=constants.AMPHORA_ID,
provides=constants.COMPUTE_ID))
create_amphora_flow.add(
database_tasks.MarkAmphoraBootingInDB(
requires=(constants.AMPHORA_ID, constants.COMPUTE_ID)))
wait_flow = linear_flow.Flow(
constants.WAIT_FOR_AMPHORA,
retry=retry.Times(CONF.controller_worker.amp_active_retries))
wait_flow.add(compute_tasks.ComputeWait(requires=constants.COMPUTE_ID))
create_amphora_flow.add(wait_flow)
create_amphora_flow.add(
database_tasks.ReloadAmphora(requires=constants.AMPHORA_ID,
provides=constants.AMPHORA))
create_amphora_flow.add(
amphora_driver_tasks.AmphoraFinalize(requires=constants.AMPHORA))
create_amphora_flow.add(
database_tasks.MarkAmphoraReadyInDB(requires=constants.AMPHORA))
return create_amphora_flow
def test_get_vm_status_flow(self):
"""Verifies GetVmStatusTask in a successful retry flow
This test simulates creating a new VM, then directly running a flow
with the 'CheckFor' task which will fail until vm_status acquired from
GetVmStatus task returns 'ACTIVE'. The new VM will return 'BUILDING'
for the first three times the VM is acquired, then will return
'ACTIVE'
"""
# configure custom vm_status list
nova.VmStatusDetails.set_vm_status(
['ACTIVE', 'BUILD', 'BUILD', 'BUILD'])
# create flow with "GetVmStatus" task
self.flow = linear_flow.Flow(
'wait for vm to become active', retry=retry.Times(10)).add(
get_vm_status.GetVmStatus(os_client=client.nova_client(),
provides='vm_status'),
common_task.CheckFor(rebind={'check_var': 'vm_status'},
check_value='ACTIVE',
retry_delay_seconds=1),
)
# create a vm
new_instance = self.nova_client.servers.create(name="vm1",
image=self.image,
flavor=self.flavor)
# set vm_id variable in TaskFlow's data store required for task
GetVmStatusTests.task_store['nova_vm_id'] = new_instance.id
# start engine to run task
result = engines.run(self.flow, store=GetVmStatusTests.task_store)
# verify vm_status key is in BUILD state
self.assertEqual('ACTIVE', result['vm_status'],
"Invalid status received")
# cleanup
self.nova_client.servers.delete(new_instance)
def test_nested_provides_graph_retried_correctly(self):
flow = gf.Flow("test").add(
utils.SaveOrderTask('a', requires=['x']),
lf.Flow("test2", retry=retry.Times(2)).add(
utils.SaveOrderTask('b', provides='x'),
utils.SaveOrderTask('c')))
engine = self._make_engine(flow)
engine.compile()
engine.prepare()
engine.storage.save('test2_retry', 1)
engine.storage.save('b', 11)
# pretend that 'c' failed
fail = failure.Failure.from_exception(RuntimeError('Woot!'))
engine.storage.save('c', fail, st.FAILURE)
engine.run()
self.assertItemsEqual(self.values[:2], [
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(11)',
])
self.assertItemsEqual(self.values[2:], ['b', 'c', 'a'])
self.assertEqual(engine.storage.get_flow_state(), st.SUCCESS)
