def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': { 'execution_id': 'abcde' }}) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_max_retries_reached(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} live_action_db.context['policies']['retry'] = {'retry_count': 2} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since max retries has been reached live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_no_retry_since_no_timeout_reached(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which succeeds liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_SUCCEEDED execution_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should only be 1 object since the action didn't timeout and therefore it wasn't # retried live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_SUCCEEDED)
def test_no_retry_on_non_applicable_statuses(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action in various statuses in which we shouldn't retry non_retry_statuses = [ LIVEACTION_STATUS_REQUESTED, LIVEACTION_STATUS_SCHEDULED, LIVEACTION_STATUS_DELAYED, LIVEACTION_STATUS_CANCELING, LIVEACTION_STATUS_CANCELED, ] action_ref = 'wolfpack.action-1' for status in non_retry_statuses: liveaction = LiveActionDB(action=action_ref, parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = status execution_db.status = status LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # None of the actions should have been retried live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), len(non_retry_statuses)) self.assertEqual(len(action_execution_dbs), len(non_retry_statuses))
def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': {'execution_id': 'abcde'}} ) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_first_retry_is_successful(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be two objects - original execution and retried execution live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context original_liveaction_id = action_execution_dbs[0].liveaction["id"] context = action_execution_dbs[1].context self.assertIn("policies", context) self.assertEqual(context["policies"]["retry"]["retry_count"], 1) self.assertEqual(context["policies"]["retry"]["applied_policy"], "test_policy") self.assertEqual( context["policies"]["retry"]["retried_liveaction_id"], original_liveaction_id, ) # Simulate success of second action so no it shouldn't be retried anymore live_action_db = live_action_dbs[1] live_action_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be no new object since action succeeds so no retry was attempted live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(live_action_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(live_action_dbs[1].status, LIVEACTION_STATUS_SUCCEEDED)
def test_retry_on_timeout_first_retry_is_successful(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be two objects - original execution and retried execution live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context original_liveaction_id = action_execution_dbs[0].liveaction['id'] context = action_execution_dbs[1].context self.assertTrue('policies' in context) self.assertEqual(context['policies']['retry']['retry_count'], 1) self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy') self.assertEqual(context['policies']['retry']['retried_liveaction_id'], original_liveaction_id) # Simulate success of second action so no it shouldn't be retried anymore live_action_db = live_action_dbs[1] live_action_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be no new object since action succeeds so no retry was attempted live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(live_action_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(live_action_dbs[1].status, LIVEACTION_STATUS_SUCCEEDED)
def setup_action_models(cls): action_db = ActionDB() action_db.name = 'action-1' action_db.description = 'awesomeness' action_db.enabled = True action_db.pack = 'wolfpack' action_db.entry_point = '' action_db.runner_type = {'name': 'test-runner'} action_db.parameters = { 'actionstr': {'type': 'string', 'position': 1, 'required': True}, 'actionint': {'type': 'number', 'default': 10, 'position': 0}, 'runnerdummy': {'type': 'string', 'default': 'actiondummy'} } ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db) actionexec_db = ActionExecutionDB() actionexec_db.status = 'initializing' actionexec_db.start_timestamp = datetime.datetime.utcnow() actionexec_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } actionexec_db.parameters = params ActionDBUtilsTestCase.actionexec_db = ActionExecution.add_or_update(actionexec_db)
def update_actionexecution_status(new_status, actionexec_id=None, actionexec_db=None): """ Update the status of the specified ActionExecution to the value provided in new_status. The ActionExecution may be specified using either actionexec_id, or as an actionexec_db instance. """ if (actionexec_id is None) and (actionexec_db is None): raise ValueError('Must specify an actionexec_id or an actionexec_db when ' 'calling update_actionexecution_status') if actionexec_db is None: actionexec_db = get_actionexec_by_id(actionexec_id) if new_status not in ACTIONEXEC_STATUSES: raise ValueError('Attempting to set status for ActionExecution "%s" ' 'to unknown status string. Unknown status is "%s"', actionexec_db, new_status) LOG.debug('Updating ActionExection: "%s" with status="%s"', actionexec_db, new_status) actionexec_db.status = new_status actionexec_db = ActionExecution.add_or_update(actionexec_db) LOG.debug('Updated status for ActionExecution object: %s', actionexec_db) return actionexec_db
def update_actionexecution_status(new_status, actionexec_id=None, actionexec_db=None): """ Update the status of the specified ActionExecution to the value provided in new_status. The ActionExecution may be specified using either actionexec_id, or as an actionexec_db instance. """ if (actionexec_id is None) and (actionexec_db is None): raise ValueError( 'Must specify an actionexec_id or an actionexec_db when ' 'calling update_actionexecution_status') if actionexec_db is None: actionexec_db = get_actionexec_by_id(actionexec_id) if new_status not in ACTIONEXEC_STATUSES: raise ValueError( 'Attempting to set status for ActionExecution "%s" ' 'to unknown status string. Unknown status is "%s"', actionexec_db, new_status) LOG.debug('Updating ActionExection: "%s" with status="%s"', actionexec_db, new_status) actionexec_db.status = new_status actionexec_db = ActionExecution.add_or_update(actionexec_db) LOG.debug('Updated status for ActionExecution object: %s', actionexec_db) return actionexec_db
def test_retry_on_timeout_first_retry_is_successful(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be two objects - original execution and retried execution live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Simulate success of second action so no it shouldn't be retried anymore live_action_db = live_action_dbs[1] live_action_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be no new object since action succeeds so no retry was attempted live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(live_action_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(live_action_dbs[1].status, LIVEACTION_STATUS_SUCCEEDED)
def test_dispatch_runner_failure(self): runner_container = get_runner_container() params = {'actionstr': 'bar'} actionexec_db = self._get_failingaction_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) self.assertTrue(runner_container.dispatch(actionexec_db)) # pickup updated actionexec_db actionexec_db = ActionExecution.get_by_id(actionexec_db.id) self.assertTrue('message' in actionexec_db.result) self.assertTrue('traceback' in actionexec_db.result)
def test_dispatch(self): runner_container = get_runner_container() params = {'actionstr': 'bar'} actionexec_db = self._get_action_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) # Assert that execution ran successfully. self.assertTrue(runner_container.dispatch(actionexec_db)) actionexec_db = ActionExecution.get_by_id(actionexec_db.id) result = actionexec_db.result self.assertTrue(result.get('action_params').get('actionint') == 10) self.assertTrue(result.get('action_params').get('actionstr') == 'bar')
def test_dispatch_runner_failure(self): runner_container = get_runner_container() params = { 'actionstr': 'bar' } actionexec_db = self._get_failingaction_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) self.assertTrue(runner_container.dispatch(actionexec_db)) # pickup updated actionexec_db actionexec_db = ActionExecution.get_by_id(actionexec_db.id) self.assertTrue('message' in actionexec_db.result) self.assertTrue('traceback' in actionexec_db.result)
def test_dispatch(self): runner_container = get_runner_container() params = { 'actionstr': 'bar' } actionexec_db = self._get_action_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) # Assert that execution ran successfully. self.assertTrue(runner_container.dispatch(actionexec_db)) actionexec_db = ActionExecution.get_by_id(actionexec_db.id) result = actionexec_db.result self.assertTrue(result.get('action_params').get('actionint') == 10) self.assertTrue(result.get('action_params').get('actionstr') == 'bar')
def test_dispatch_override_default_action_params(self): runner_container = get_runner_container() params = {'actionstr': 'foo', 'actionint': 20} actionexec_db = self._get_action_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) # Assert that execution ran successfully. result = runner_container.dispatch(actionexec_db) self.assertTrue(result) actionexec_db = ActionExecution.get_by_id(actionexec_db.id) result = actionexec_db.result self.assertTrue(result.get('action_params').get('actionint') == 20) self.assertTrue(result.get('action_params').get('actionstr') == 'foo')
def put(self, id, actionexecution): try: actionexec_db = ActionExecution.get_by_id(id) except: msg = 'ActionExecution by id: %s not found.' % id pecan.abort(http_client, msg) new_actionexec_db = ActionExecutionAPI.to_model(actionexecution) if actionexec_db.status != new_actionexec_db.status: actionexec_db.status = new_actionexec_db.status if actionexec_db.result != new_actionexec_db.result: actionexec_db.result = new_actionexec_db.result actionexec_db = ActionExecution.add_or_update(actionexec_db) actionexec_api = ActionExecutionAPI.from_model(actionexec_db) return actionexec_api
def test_dispatch_override_default_action_params(self): runner_container = get_runner_container() params = { 'actionstr': 'foo', 'actionint': 20 } actionexec_db = self._get_action_exec_db_model(params) actionexec_db = ActionExecution.add_or_update(actionexec_db) # Assert that execution ran successfully. result = runner_container.dispatch(actionexec_db) self.assertTrue(result) actionexec_db = ActionExecution.get_by_id(actionexec_db.id) result = actionexec_db.result self.assertTrue(result.get('action_params').get('actionint') == 20) self.assertTrue(result.get('action_params').get('actionstr') == 'foo')
def test_update_actionexecution_status_invalid(self): actionexec_db = ActionExecutionDB() actionexec_db.status = 'initializing' actionexec_db.start_timestamp = datetime.datetime.utcnow() actionexec_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } actionexec_db.parameters = params actionexec_db = ActionExecution.add_or_update(actionexec_db) # Update by id. self.assertRaises(ValueError, action_db_utils.update_actionexecution_status, 'mea culpa', actionexec_id=actionexec_db.id)
def schedule(execution): # Use the user context from the parent action execution. Subtasks in a workflow # action can be invoked by a system user and so we want to use the user context # from the original workflow action. if getattr(execution, 'context', None) and 'parent' in execution.context: parent = ActionExecution.get_by_id(execution.context['parent']) execution.context['user'] = getattr(parent, 'context', dict()).get('user') # Validate action. action_db = action_utils.get_action_by_ref(execution.action) if not action_db: raise ValueError('Action "%s" cannot be found.' % execution.action) if not action_db.enabled: raise ValueError('Unable to execute. Action "%s" is disabled.' % execution.action) runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) if not hasattr(execution, 'parameters'): execution.parameters = dict() # Validate action parameters. schema = util_schema.get_parameter_schema(action_db) validator = util_schema.get_validator() jsonschema.validate(execution.parameters, schema, validator) # validate that no immutable params are being overriden. Although possible to # ignore the override it is safer to inform the user to avoid surprises. immutables = _get_immutable_params(action_db.parameters) immutables.extend(_get_immutable_params(runnertype_db.runner_parameters)) overridden_immutables = [p for p in six.iterkeys(execution.parameters) if p in immutables] if len(overridden_immutables) > 0: raise ValueError('Override of immutable parameter(s) %s is unsupported.' % str(overridden_immutables)) # Write to database and send to message queue. execution.status = ACTIONEXEC_STATUS_SCHEDULED execution.start_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow()) execution = ActionExecution.add_or_update(execution) LOG.audit('Action execution scheduled. ActionExecution=%s.', execution) return execution
def test_state_db_creation_async_actions(self): runner_container = get_runner_container() params = { 'actionstr': 'foo', 'actionint': 20, 'async_test': True } actionexec_db = self._get_action_exec_db_model(RunnerContainerTest.async_action_db, params) actionexec_db = ActionExecution.add_or_update(actionexec_db) # Assert that execution ran without exceptions. runner_container.dispatch(actionexec_db) states = ActionExecutionState.get_all() found = None for state in states: if state.execution_id == actionexec_db.id: found = state self.assertTrue(found is not None, 'There should be a state db object.') self.assertTrue(found.query_context is not None) self.assertTrue(found.query_module is not None)
def test_update_actionexecution_status(self): actionexec_db = ActionExecutionDB() actionexec_db.status = 'initializing' actionexec_db.start_timestamp = datetime.datetime.utcnow() actionexec_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } actionexec_db.parameters = params actionexec_db = ActionExecution.add_or_update(actionexec_db) origactionexec_db = copy.copy(actionexec_db) # Update by id. newactionexec_db = action_db_utils.update_actionexecution_status( 'running', actionexec_id=actionexec_db.id) # Verify id didn't change. self.assertEqual(origactionexec_db.id, newactionexec_db.id) self.assertEqual(newactionexec_db.status, 'running')
def test_update_actionexecution_status_and_end_timestamp(self): actionexec_db = ActionExecutionDB() actionexec_db.status = 'initializing' actionexec_db.start_timestamp = datetime.datetime.utcnow() actionexec_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref actionexec_db.parameters = {} actionexec_db = ActionExecution.add_or_update(actionexec_db) origactionexec_db = copy.copy(actionexec_db) # Update by id now = datetime.datetime.now() newactionexec_db = action_db_utils.update_actionexecution_status( status='running', end_timestamp=now, actionexec_id=actionexec_db.id) # Verify id didn't change and end_timestamp has been set self.assertEqual(origactionexec_db.id, newactionexec_db.id) self.assertEqual(newactionexec_db.status, 'running') self.assertEqual(newactionexec_db.end_timestamp, now)
def test_retry_on_timeout_policy_is_retried_twice(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be two objects - original execution and retried execution live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context original_liveaction_id = action_execution_dbs[0].liveaction['id'] context = action_execution_dbs[1].context self.assertTrue('policies' in context) self.assertEqual(context['policies']['retry']['retry_count'], 1) self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy') self.assertEqual(context['policies']['retry']['retried_liveaction_id'], original_liveaction_id) # Simulate timeout of second action which should cause another retry live_action_db = live_action_dbs[1] live_action_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) execution_db = action_execution_dbs[1] execution_db.status = LIVEACTION_STATUS_TIMED_OUT ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be three objects - original execution and 2 retried executions live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 3) self.assertEqual(len(action_execution_dbs), 3) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[2].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context original_liveaction_id = action_execution_dbs[1].liveaction['id'] context = action_execution_dbs[2].context self.assertTrue('policies' in context) self.assertEqual(context['policies']['retry']['retry_count'], 2) self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy') self.assertEqual(context['policies']['retry']['retried_liveaction_id'], original_liveaction_id)