def test_disabled_policy_not_applied_on_pre_run(self): ########## # First test a scenario where policy is enabled ########## self.assertTrue(self.policy_db.enabled) # Post run hasn't been called yet, call count should be 0 self.assertEqual(policies.get_driver.call_count, 0) liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) policy_service.apply_pre_run_policies(live_action_db) # Ony policy has been applied so call count should be 1 self.assertEqual(policies.get_driver.call_count, 1) ########## # Now a scenaro with disabled policy ########## policies.get_driver.call_count = 0 self.policy_db.enabled = False self.policy_db = Policy.add_or_update(self.policy_db) self.assertFalse(self.policy_db.enabled) self.assertEqual(policies.get_driver.call_count, 0) liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) policy_service.apply_pre_run_policies(live_action_db) # Policy is disabled so call_count should stay the same as before as no policies have been # applied self.assertEqual(policies.get_driver.call_count, 0)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _run_action(self, action_node, parent_execution_id, params, wait_for_completion=True): liveaction = LiveActionDB(action=action_node.ref) liveaction.parameters = action_param_utils.cast_params( action_ref=action_node.ref, params=params) if action_node.notify: liveaction.notify = NotificationsHelper.to_model( action_node.notify) liveaction.context = { 'parent': str(parent_execution_id), 'chain': vars(action_node) } liveaction, _ = action_service.schedule(liveaction) while (wait_for_completion and liveaction.status != LIVEACTION_STATUS_SUCCEEDED and liveaction.status != LIVEACTION_STATUS_FAILED): eventlet.sleep(1) liveaction = action_db_util.get_liveaction_by_id(liveaction.id) return liveaction
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref( 'wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def setup_action_models(cls): action_db = ActionDB() action_db.name = 'action-1' action_db.description = 'awesomeness' action_db.enabled = True action_db.pack = 'wolfpack' action_db.ref = ResourceReference(name=action_db.name, pack=action_db.pack).ref action_db.entry_point = '' action_db.runner_type = {'name': 'test-runner'} action_db.parameters = { 'actionstr': {'type': 'string', 'position': 1, 'required': True}, 'actionint': {'type': 'number', 'default': 10, 'position': 0}, 'runnerdummy': {'type': 'string', 'default': 'actiondummy'} } ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db) liveaction_db = LiveActionDB() liveaction_db.status = 'initializing' liveaction_db.start_timestamp = datetime.datetime.utcnow() liveaction_db.action = ActionDBUtilsTestCase.action_db.ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } liveaction_db.parameters = params ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel") self.assertEqual(policy_db.parameters["action"], "cancel") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB( action="wolfpack.action-2", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. expected_num_exec += 0 # This request will not be scheduled for execution. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': { 'execution_id': 'abcde' }}) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': {'execution_id': 'abcde'}} ) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) # Assert the canceling state is being published. calls = [ call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) ] LiveActionPublisher.publish_state.assert_has_calls(calls) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def _get_action_exec_db_model(self, params): liveaction_db = LiveActionDB() liveaction_db.status = 'initializing' liveaction_db.start_timestamp = datetime.datetime.utcnow() liveaction_db.action = ResourceReference(name=ParamsUtilsTest.action_db.name, pack=ParamsUtilsTest.action_db.pack).ref liveaction_db.parameters = params return liveaction_db
def _get_execution_db_model(self, status=action_constants.LIVEACTION_STATUS_SCHEDULED): live_action_db = LiveActionDB() live_action_db.status = status live_action_db.start_timestamp = datetime.datetime.utcnow() live_action_db.action = ResourceReference( name='test_action', pack='test_pack').ref live_action_db.parameters = None return action.LiveAction.add_or_update(live_action_db, publish=False)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into cancel state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in [ action_constants.LIVEACTION_STATUS_CANCELING, action_constants.LIVEACTION_STATUS_CANCELED]: break # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. canceled = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workflow_with_many_workflows(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH) execution = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_launch_workflow_mistral_offline(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH) execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def _run_action(action_node, parent_execution_id, params, wait_for_completion=True): execution = LiveActionDB(action=action_node.ref) execution.parameters = action_param_utils.cast_params(action_ref=action_node.ref, params=params) execution.context = { 'parent': str(parent_execution_id), 'chain': vars(action_node) } liveaction, _ = action_service.schedule(execution) while (wait_for_completion and liveaction.status != LIVEACTION_STATUS_SUCCEEDED and liveaction.status != LIVEACTION_STATUS_FAILED): eventlet.sleep(1) liveaction = action_db_util.get_liveaction_by_id(liveaction.id) return liveaction
def test_launch_workbook_with_many_workflows_no_default(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH) execution = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_basic_execution(self): liveaction = LiveActionDB(action='core.local', parameters={'cmd': 'uname -a'}) liveaction, _ = action_service.schedule(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, {}) self.assertDictEqual(execution.trigger_type, {}) self.assertDictEqual(execution.trigger_instance, {}) self.assertDictEqual(execution.rule, {}) action = action_utils.get_action_by_ref('core.local') self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_chained_executions(self): liveaction = LiveActionDB(action='core.chain') liveaction, _ = action_service.schedule(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) action = action_utils.get_action_by_ref('core.chain') self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action) self.assertGreater(len(execution.children), 0) for child in execution.children: record = ActionExecution.get(id=child, raise_exception=True) self.assertEqual(record.parent, str(execution.id)) self.assertEqual(record.action['name'], 'local') self.assertEqual(record.runner['name'], 'run-local')
def test_retry_on_timeout_max_retries_reached(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} live_action_db.context['policies']['retry'] = {'retry_count': 2} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since max retries has been reached live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_non_applicable_statuses(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action in various statuses in which we shouldn't retry non_retry_statuses = [ LIVEACTION_STATUS_REQUESTED, LIVEACTION_STATUS_SCHEDULED, LIVEACTION_STATUS_DELAYED, LIVEACTION_STATUS_CANCELING, LIVEACTION_STATUS_CANCELED, ] action_ref = 'wolfpack.action-1' for status in non_retry_statuses: liveaction = LiveActionDB(action=action_ref, parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = status execution_db.status = status LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # None of the actions should have been retried live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), len(non_retry_statuses)) self.assertEqual(len(action_execution_dbs), len(non_retry_statuses))
def test_enforce(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_retry_on_timeout_no_retry_since_no_timeout_reached(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which succeeds liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_SUCCEEDED execution_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should only be 1 object since the action didn't timeout and therefore it wasn't # retried live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_SUCCEEDED)
def test_launch_workbook_name_mismatch(self): action_ref = 'generic.workbook_v2_name_mismatch' MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH) execution = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_enforce(self): liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED )
def test_schedule_disabled_action(self): self.actiondb.enabled = False Action.add_or_update(self.actiondb) parameters = {'hosts': 'localhost', 'cmd': 'uname -a'} execution = LiveActionDB(action=ACTION_REF, parameters=parameters) self.assertRaises(ValueError, action_service.schedule, execution) self.actiondb.enabled = True Action.add_or_update(self.actiondb)
def test_retry_on_timeout_first_retry_is_successful(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) live_action_db, execution_db = action_service.request(liveaction) live_action_db.status = LIVEACTION_STATUS_TIMED_OUT execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be two objects - original execution and retried execution live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context original_liveaction_id = action_execution_dbs[0].liveaction["id"] context = action_execution_dbs[1].context self.assertIn("policies", context) self.assertEqual(context["policies"]["retry"]["retry_count"], 1) self.assertEqual(context["policies"]["retry"]["applied_policy"], "test_policy") self.assertEqual( context["policies"]["retry"]["retried_liveaction_id"], original_liveaction_id, ) # Simulate success of second action so no it shouldn't be retried anymore live_action_db = live_action_dbs[1] live_action_db.status = LIVEACTION_STATUS_SUCCEEDED LiveAction.add_or_update(live_action_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # There should be no new object since action succeeds so no retry was attempted live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 2) self.assertEqual(len(action_execution_dbs), 2) self.assertEqual(live_action_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT) self.assertEqual(live_action_dbs[1].status, LIVEACTION_STATUS_SUCCEEDED)
def test_liveaction_create_with_notify_on_success_only(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} notify_db = NotificationSchema() notify_sub_schema = NotificationSubSchema() notify_sub_schema.message = 'Action succeeded.' notify_sub_schema.data = {'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'}} notify_db.on_success = notify_sub_schema created.notify = notify_db saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') # Assert notify settings saved are right. self.assertEqual(notify_sub_schema.message, retrieved.notify.on_success.message) self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data) self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels) self.assertEqual(retrieved.notify.on_failure, None) self.assertEqual(retrieved.notify.on_complete, None)
def test_apply(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) FakeConcurrencyApplicator.apply_before.assert_called_once_with(liveaction) RaiseExceptionApplicator.apply_before.assert_called_once_with(liveaction) FakeConcurrencyApplicator.apply_after.assert_called_once_with(liveaction) RaiseExceptionApplicator.apply_after.assert_called_once_with(liveaction)
def _run_action(action_node, parent_execution_id, params, wait_for_completion=True): execution = LiveActionDB(action=action_node.ref) execution.parameters = action_param_utils.cast_params( action_ref=action_node.ref, params=params) execution.context = { 'parent': str(parent_execution_id), 'chain': vars(action_node) } execution, _ = action_service.schedule(execution) while (wait_for_completion and execution.status != LIVEACTION_STATUS_SUCCEEDED and execution.status != LIVEACTION_STATUS_FAILED): eventlet.sleep(1) execution = action_db_util.get_liveaction_by_id(execution.id) return execution
def test_update_LiveAction_status_invalid(self): liveaction_db = LiveActionDB() liveaction_db.status = 'initializing' liveaction_db.start_timestamp = datetime.datetime.utcnow() liveaction_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } liveaction_db.parameters = params liveaction_db = LiveAction.add_or_update(liveaction_db) # Update by id. self.assertRaises(ValueError, action_db_utils.update_liveaction_status, status='mea culpa', liveaction_id=liveaction_db.id)
def test_launch_when_workbook_not_exists(self): execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def _get_action_exec_db_model(self, action_db, params): liveaction_db = LiveActionDB() liveaction_db.status = action_constants.LIVEACTION_STATUS_SCHEDULED liveaction_db.start_timestamp = datetime.datetime.utcnow() liveaction_db.action = ResourceReference(name=action_db.name, pack=action_db.pack).ref liveaction_db.parameters = params liveaction_db.context = {'user': cfg.CONF.system_user.user} return liveaction_db
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_liveaction_create_with_notify_on_success_only(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} notify_db = NotificationSchema() notify_sub_schema = NotificationSubSchema() notify_sub_schema.message = 'Action succeeded.' notify_sub_schema.data = { 'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'} } notify_db.on_success = notify_sub_schema created.notify = notify_db saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') # Assert notify settings saved are right. self.assertEqual(notify_sub_schema.message, retrieved.notify.on_success.message) self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data) self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels) self.assertEqual(retrieved.notify.on_failure, None) self.assertEqual(retrieved.notify.on_complete, None)
def test_update_liveaction_status(self): liveaction_db = LiveActionDB() liveaction_db.status = 'initializing' liveaction_db.start_timestamp = datetime.datetime.utcnow() liveaction_db.action = ResourceReference( name=ActionDBUtilsTestCase.action_db.name, pack=ActionDBUtilsTestCase.action_db.pack).ref params = { 'actionstr': 'foo', 'some_key_that_aint_exist_in_action_or_runner': 'bar', 'runnerint': 555 } liveaction_db.parameters = params liveaction_db = LiveAction.add_or_update(liveaction_db) origliveaction_db = copy.copy(liveaction_db) # Update by id. newliveaction_db = action_db_utils.update_liveaction_status( status='running', liveaction_id=liveaction_db.id) # Verify id didn't change. self.assertEqual(origliveaction_db.id, newliveaction_db.id) self.assertEqual(newliveaction_db.status, 'running') # Update status, result, context, and end timestamp. now = datetime.datetime.utcnow() status = 'succeeded' result = 'Work is done.' context = {'third_party_id': uuid.uuid4().hex} newliveaction_db = action_db_utils.update_liveaction_status( status=status, result=result, context=context, end_timestamp=now, liveaction_id=liveaction_db.id) self.assertEqual(origliveaction_db.id, newliveaction_db.id) self.assertEqual(newliveaction_db.status, status) self.assertEqual(newliveaction_db.result, result) self.assertDictEqual(newliveaction_db.context, context) self.assertEqual(newliveaction_db.end_timestamp, now)
def test_liveaction_crud_no_notify(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') self.assertEqual(retrieved.notify, None) # Test update self.assertTrue(retrieved.end_timestamp is None) retrieved.end_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow()) updated = LiveAction.add_or_update(retrieved) self.assertTrue(updated.end_timestamp == retrieved.end_timestamp) # Test delete LiveActionModelTest._delete([retrieved]) try: retrieved = LiveAction.get_by_id(saved.id) except ValueError: retrieved = None self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def _get_failingaction_exec_db_model(self, params): liveaction_db = LiveActionDB() liveaction_db.status = action_constants.LIVEACTION_STATUS_SCHEDULED liveaction_db.start_timestamp = datetime.datetime.now() liveaction_db.action = ResourceReference( name=RunnerContainerTest.failingaction_db.name, pack=RunnerContainerTest.failingaction_db.pack).ref liveaction_db.parameters = params liveaction_db.context = {'user': cfg.CONF.system_user.user} return liveaction_db
def test_notify_triggers(self): liveaction = LiveActionDB(action='core.local') liveaction.description = '' liveaction.status = 'succeeded' liveaction.parameters = {} on_success = NotificationSubSchema(message='Action succeeded.') on_failure = NotificationSubSchema(message='Action failed.') liveaction.notify = NotificationSchema(on_success=on_success, on_failure=on_failure) liveaction.start_timestamp = datetime.datetime.utcnow() dispatcher = NotifierTestCase.MockDispatcher(self) notifier = Notifier(q_connection=None, trigger_dispatcher=dispatcher) notifier.handle_action_complete(liveaction)
def test_liveaction_create_with_notify_both_on_success_and_on_error(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} on_success = NotificationSubSchema(message='Action succeeded.') on_failure = NotificationSubSchema(message='Action failed.') created.notify = NotificationSchema(on_success=on_success, on_failure=on_failure) saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') # Assert notify settings saved are right. self.assertEqual(on_success.message, retrieved.notify.on_success.message) self.assertEqual(on_failure.message, retrieved.notify.on_failure.message) self.assertEqual(retrieved.notify.on_complete, None)
FIXTURES_1 = { 'runners': ['testrunner1.yaml', 'testrunner2.yaml'], 'actions': ['action1.yaml', 'a2.yaml'], 'triggertypes': ['triggertype1.yaml'], 'triggers': ['trigger1.yaml'] } FIXTURES_2 = { 'rules': ['rule1.yaml', 'rule2.yaml'] } MOCK_TRIGGER_INSTANCE = TriggerInstanceDB() MOCK_TRIGGER_INSTANCE.id = 'triggerinstance-test' MOCK_TRIGGER_INSTANCE.payload = {'t1_p': 't1_p_v'} MOCK_TRIGGER_INSTANCE.occurrence_time = datetime.datetime.utcnow() MOCK_LIVEACTION = LiveActionDB() MOCK_LIVEACTION.id = 'liveaction-test-1.id' MOCK_LIVEACTION.name = 'liveaction-test-1.name' MOCK_LIVEACTION.status = 'scheduled' class EnforceTest(DbTestCase): models = None @classmethod def setUpClass(cls): super(EnforceTest, cls).setUpClass() # Create TriggerTypes before creation of Rule to avoid failure. Rule requires the # Trigger and therefore TriggerType to be created prior to rule creation. cls.models = FixturesLoader().save_fixtures_to_db(