def test_default_times_retry(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add(utils.SaveOrderTask('t1'), utils.FailingTask('t2')) engine = self._make_engine(flow) self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) expected = [ 't1', u't2 reverted(Failure: RuntimeError: Woot!)', 't1 reverted(5)', 't1', u't2 reverted(Failure: RuntimeError: Woot!)', 't1 reverted(5)', 't1', u't2 reverted(Failure: RuntimeError: Woot!)', 't1 reverted(5)' ] self.assertEqual(self.values, expected)
def test_linear_nested_to_parallel_revert_exception(self): flow = uf.Flow('p-root').add( utils.SaveOrderTask(self.values, name='task1', sleep=0.01), utils.SaveOrderTask(self.values, name='task2', sleep=0.01), lf.Flow('l-inner').add( utils.SaveOrderTask(self.values, name='task3'), utils.NastyTask(), utils.FailingTask(sleep=0.01))) engine = self._make_engine(flow) with self.assertRaisesRegexp(RuntimeError, '^Gotcha'): engine.run() result = set(self.values) possible_result = set([ 'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)', 'task3' ]) self.assertIsSubset(possible_result, result)
def test_flow_failures_are_passed_to_revert(self): class CheckingTask(task.Task): def execute(m_self): return 'RESULT' def revert(m_self, result, flow_failures): self.assertEqual(result, 'RESULT') self.assertEqual(list(flow_failures.keys()), ['fail1']) fail = flow_failures['fail1'] self.assertIsInstance(fail, misc.Failure) self.assertEqual(str(fail), 'Failure: RuntimeError: Woot!') flow = lf.Flow('test').add(CheckingTask(), utils.FailingTask('fail1')) engine = self._make_engine(flow) self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
def test_default_times_retry(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add( utils.ProgressingTask('t1'), utils.FailingTask('t2')) engine = self._make_engine(flow) with utils.CaptureListener(engine) as capturer: self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) expected = ['flow-1.f RUNNING', 'r1.r RUNNING', 'r1.r SUCCESS(1)', 't1.t RUNNING', 't1.t SUCCESS(5)', 't2.t RUNNING', 't2.t FAILURE(Failure: RuntimeError: Woot!)', 't2.t REVERTING', 't2.t REVERTED', 't1.t REVERTING', 't1.t REVERTED', 'r1.r RETRYING', 't1.t PENDING', 't2.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(2)', 't1.t RUNNING', 't1.t SUCCESS(5)', 't2.t RUNNING', 't2.t FAILURE(Failure: RuntimeError: Woot!)', 't2.t REVERTING', 't2.t REVERTED', 't1.t REVERTING', 't1.t REVERTED', 'r1.r RETRYING', 't1.t PENDING', 't2.t PENDING', 'r1.r RUNNING', 'r1.r SUCCESS(3)', 't1.t RUNNING', 't1.t SUCCESS(5)', 't2.t RUNNING', 't2.t FAILURE(Failure: RuntimeError: Woot!)', 't2.t REVERTING', 't2.t REVERTED', 't1.t REVERTING', 't1.t REVERTED', 'r1.r REVERTING', 'r1.r REVERTED', 'flow-1.f REVERTED'] self.assertEqual(expected, capturer.values)
def test_correctly_reverts_children(self): flow = lf.Flow('root-1').add( utils.ProgressingTask('task1'), lf.Flow('child-1').add(utils.ProgressingTask('task2'), utils.FailingTask('fail'))) engine = self._make_engine(flow) with utils.CaptureListener(engine, capture_flow=False) as capturer: self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) expected = [ 'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING', 'task2.t SUCCESS(5)', 'fail.t RUNNING', 'fail.t FAILURE(Failure: RuntimeError: Woot!)', 'fail.t REVERTING', 'fail.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED', 'task1.t REVERTING', 'task1.t REVERTED' ] self.assertEqual(expected, capturer.values)
def test_failing_task_with_notifications(self): flow = utils.FailingTask('fail') engine = self._make_engine(flow) utils.register_notifiers(engine, self.values) expected = [ 'flow RUNNING', 'fail RUNNING', 'fail FAILURE', 'fail REVERTING', 'fail reverted(Failure: RuntimeError: Woot!)', 'fail REVERTED', 'flow REVERTED' ] self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) self.assertEqual(self.values, expected) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) now_expected = expected + ['fail PENDING', 'flow PENDING'] + expected self.assertEqual(self.values, now_expected) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_revert_ok_for_linear_in_unordered(self): flow = uf.Flow('p-root').add( utils.SaveOrderTask(name='task1'), lf.Flow('p-inner').add(utils.SaveOrderTask(name='task2'), utils.FailingTask('fail'))) engine = self._make_engine(flow) self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) self.assertIn('fail reverted(Failure: RuntimeError: Woot!)', self.values) # NOTE(imelnikov): if task1 was run, it should have been reverted. if 'task1' in self.values: task1_story = ['task1', 'task1 reverted(5)'] self.assertIsSuperAndSubsequence(self.values, task1_story) # NOTE(imelnikov): task2 should have been run and reverted task2_story = ['task2', 'task2 reverted(5)'] self.assertIsSuperAndSubsequence(self.values, task2_story)
def test_failing_task_with_notifications(self): values = [] flow = utils.FailingTask('fail') engine = self._make_engine(flow) expected = ['fail.f RUNNING', 'fail.t RUNNING', 'fail.t FAILURE(Failure: RuntimeError: Woot!)', 'fail.t REVERTING', 'fail.t REVERTED', 'fail.f REVERTED'] with utils.CaptureListener(engine, values=values) as capturer: self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) with utils.CaptureListener(engine, values=values) as capturer: self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) now_expected = list(expected) now_expected.extend(['fail.t PENDING', 'fail.f PENDING']) now_expected.extend(expected) self.assertEqual(now_expected, values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_graph_flow_four_tasks_revert(self): flow = gf.Flow('g-4-failing').add( utils.ProgressingTask(name='task4', provides='d', requires=['c']), utils.ProgressingTask(name='task2', provides='b', requires=['a']), utils.FailingTask(name='task3', provides='c', requires=['b']), utils.ProgressingTask(name='task1', provides='a')) engine = self._make_engine(flow) with utils.CaptureListener(engine, capture_flow=False) as capturer: self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) expected = [ 'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task2.t RUNNING', 'task2.t SUCCESS(5)', 'task3.t RUNNING', 'task3.t FAILURE(Failure: RuntimeError: Woot!)', 'task3.t REVERTING', 'task3.t REVERTED', 'task2.t REVERTING', 'task2.t REVERTED', 'task1.t REVERTING', 'task1.t REVERTED' ] self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_nested_parallel_revert_exception_is_reraised(self): flow = uf.Flow('p-root').add( utils.SaveOrderTask(self.values, name='task1'), utils.SaveOrderTask(self.values, name='task2'), lf.Flow('p-inner').add( utils.SaveOrderTask(self.values, name='task3', sleep=0.1), utils.NastyTask(), utils.FailingTask(sleep=0.01))) engine = self._make_engine(flow) with self.assertRaisesRegexp(RuntimeError, '^Gotcha'): engine.run() result = set(self.values) # Task1, task2 may *not* have executed and also may have *not* reverted # since the above is an unordered flow so take that into account by # ensuring that the superset is matched. possible_result = set([ 'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)', 'task3', 'task3 reverted(5)' ]) self.assertIsSubset(possible_result, result)
def test_linear_nested_to_parallel_revert(self): flow = uf.Flow('p-root').add( utils.SaveOrderTask(self.values, name='task1'), utils.SaveOrderTask(self.values, name='task2'), lf.Flow('l-inner').add( utils.SaveOrderTask(self.values, name='task3', sleep=0.1), utils.FailingTask(self.values, name='fail', sleep=0.01))) engine = self._make_engine(flow) with self.assertRaisesRegexp(RuntimeError, '^Woot'): engine.run() result = set(self.values) # Since this is an unordered flow we can not guarantee that task1 or # task2 will exist and be reverted, although they may exist depending # on how the OS thread scheduling and execution graph algorithm... possible_result = set([ 'task1', 'task1 reverted(5)', 'task2', 'task2 reverted(5)', 'task3', 'task3 reverted(5)', 'fail reverted(Failure: RuntimeError: Woot!)' ]) self.assertIsSubset(possible_result, result)
def test_suspend_linear_flow_on_revert(self): flow = lf.Flow('linear').add(utils.ProgressingTask('a'), utils.ProgressingTask('b'), utils.FailingTask('c')) engine = self._make_engine(flow) with SuspendingListener(engine, task_name='b', task_state=states.REVERTED) as capturer: engine.run() self.assertEqual(states.SUSPENDED, engine.storage.get_flow_state()) expected = [ 'a.t RUNNING', 'a.t SUCCESS(5)', 'b.t RUNNING', 'b.t SUCCESS(5)', 'c.t RUNNING', 'c.t FAILURE(Failure: RuntimeError: Woot!)', 'c.t REVERTING', 'c.t REVERTED(None)', 'b.t REVERTING', 'b.t REVERTED(None)' ] self.assertEqual(expected, capturer.values) with utils.CaptureListener(engine, capture_flow=False) as capturer: self.assertRaisesRegex(RuntimeError, '^Woot', engine.run) self.assertEqual(states.REVERTED, engine.storage.get_flow_state()) expected = ['a.t REVERTING', 'a.t REVERTED(None)'] self.assertEqual(expected, capturer.values)
def test_revert_ok_for_unordered_in_linear(self): flow = lf.Flow('p-root').add( utils.SaveOrderTask(name='task1'), utils.SaveOrderTask(name='task2'), uf.Flow('p-inner').add(utils.SaveOrderTask(name='task3'), utils.FailingTask('fail'))) engine = self._make_engine(flow) self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) # NOTE(imelnikov): we don't know if task 3 was run, but if it was, # it should have been reverted in correct order. possible_values_no_task3 = [ 'task1', 'task2', 'fail reverted(Failure: RuntimeError: Woot!)', 'task2 reverted(5)', 'task1 reverted(5)' ] self.assertIsSuperAndSubsequence(self.values, possible_values_no_task3) if 'task3' in self.values: possible_values_task3 = [ 'task1', 'task2', 'task3', 'task3 reverted(5)', 'task2 reverted(5)', 'task1 reverted(5)' ] self.assertIsSuperAndSubsequence(self.values, possible_values_task3)
def test_suspend_and_resume_linear_flow_on_revert(self): flow = lf.Flow('linear').add( utils.SaveOrderTask('a'), utils.SaveOrderTask('b'), utils.FailingTask('c') ) engine = self._make_engine(flow) with SuspendingListener(engine, task_name='b', task_state=states.REVERTED): engine.run() # pretend we are resuming engine2 = self._make_engine(flow, engine.storage._flowdetail) self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run) self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED) self.assertEqual( self.values, ['a', 'b', 'c reverted(Failure: RuntimeError: Woot!)', 'b reverted(5)', 'a reverted(5)'])
def test_revert_ok_for_linear_in_unordered(self): flow = uf.Flow('p-root').add( utils.ProgressingTask(name='task1'), lf.Flow('p-inner').add(utils.ProgressingTask(name='task2'), utils.FailingTask('fail'))) engine = self._make_engine(flow) with utils.CaptureListener(engine, capture_flow=False) as capturer: self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) self.assertIn('fail.t FAILURE(Failure: RuntimeError: Woot!)', capturer.values) # NOTE(imelnikov): if task1 was run, it should have been reverted. if 'task1' in capturer.values: task1_story = [ 'task1.t RUNNING', 'task1.t SUCCESS(5)', 'task1.t REVERTED' ] self.assertIsSuperAndSubsequence(capturer.values, task1_story) # NOTE(imelnikov): task2 should have been run and reverted task2_story = [ 'task2.t RUNNING', 'task2.t SUCCESS(5)', 'task2.t REVERTED' ] self.assertIsSuperAndSubsequence(capturer.values, task2_story)
def test_nested_provides_graph_reverts_correctly(self): flow = gf.Flow("test").add( utils.SaveOrderTask('a', requires=['x']), lf.Flow("test2", retry=retry.Times(2)).add( utils.SaveOrderTask('b', provides='x'), utils.FailingTask('c'))) engine = self._make_engine(flow) engine.compile() engine.prepare() engine.storage.save('test2_retry', 1) engine.storage.save('b', 11) engine.storage.save('a', 10) self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) self.assertItemsEqual(self.values[:3], [ 'a reverted(10)', 'c reverted(Failure: RuntimeError: Woot!)', 'b reverted(11)', ]) # Task 'a' was or was not executed again, both cases are ok. self.assertIsSuperAndSubsequence( self.values[3:], ['b', 'c reverted(Failure: RuntimeError: Woot!)', 'b reverted(5)']) self.assertEqual(engine.storage.get_flow_state(), st.REVERTED)
def test_graph_flow_four_tasks_revert(self): flow = gf.Flow('g-4-failing').add( utils.SaveOrderTask(self.values, name='task4', provides='d', requires=['c']), utils.SaveOrderTask(self.values, name='task2', provides='b', requires=['a']), utils.FailingTask(self.values, name='task3', provides='c', requires=['b']), utils.SaveOrderTask(self.values, name='task1', provides='a')) engine = self._make_engine(flow) with self.assertRaisesRegexp(RuntimeError, '^Woot'): engine.run() self.assertEquals(self.values, [ 'task1', 'task2', 'task3 reverted(Failure: RuntimeError: Woot!)', 'task2 reverted(5)', 'task1 reverted(5)' ]) self.assertEquals(engine.storage.get_flow_state(), states.REVERTED)
def test_parallel_revert_exception_do_not_revert_linear_tasks(self): flow = lf.Flow('l-root').add( utils.SaveOrderTask(self.values, name='task1'), utils.SaveOrderTask(self.values, name='task2'), uf.Flow('p-inner').add( utils.SaveOrderTask(self.values, name='task3', sleep=0.1), utils.NastyTask(), utils.FailingTask(sleep=0.01))) engine = self._make_engine(flow) # Depending on when (and if failing task) is executed the exception # raised could be either woot or gotcha since the above unordered # sub-flow does not guarantee that the ordering will be maintained, # even with sleeping. was_nasty = False try: engine.run() self.assertTrue(False) except RuntimeError as e: self.assertRegexpMatches(str(e), '^Gotcha|^Woot') if 'Gotcha!' in str(e): was_nasty = True result = set(self.values) possible_result = set(['task1', 'task2', 'task3', 'task3 reverted(5)']) if not was_nasty: possible_result.update(['task1 reverted(5)', 'task2 reverted(5)']) self.assertIsSubset(possible_result, result) # If the nasty task killed reverting, then task1 and task2 should not # have reverted, but if the failing task stopped execution then task1 # and task2 should have reverted. if was_nasty: must_not_have = ['task1 reverted(5)', 'task2 reverted(5)'] for r in must_not_have: self.assertNotIn(r, result) else: must_have = ['task1 reverted(5)', 'task2 reverted(5)'] for r in must_have: self.assertIn(r, result)
def test_revert_exception_is_reraised(self): flow = lf.Flow('revert-1').add(utils.NastyTask(), utils.FailingTask(name='fail')) engine = self._make_engine(flow) self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run)
def test_restart_reverted_flow_with_retry(self): flow = lf.Flow('test', retry=utils.OneReturnRetry(provides='x')).add( utils.FailingTask('fail')) engine = self._make_engine(flow) self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)