def test_cancel_delayed_execution_with_parent(self): liveaction = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': { 'execution_id': uuid.uuid4().hex }}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_over_threshold(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr") self.assertGreater(policy_db.parameters["threshold"], 0) self.assertIn("actionstr", policy_db.parameters["attributes"]) for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) eventlet.spawn(action_service.request, liveaction) # Sleep here to let the threads above schedule the action execution. eventlet.sleep(1) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Sleep here to let the threads above complete the action execution. eventlet.sleep(RUN_DELAY + 1) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_basic_execution(self): liveaction = LiveActionDB(action='core.local', parameters={'cmd': 'uname -a'}) liveaction, _ = action_service.schedule(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, {}) self.assertDictEqual(execution.trigger_type, {}) self.assertDictEqual(execution.trigger_instance, {}) self.assertDictEqual(execution.rule, {}) action = action_utils.get_action_by_ref('core.local') self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_chained_executions(self): liveaction = LiveActionDB(action='core.chain') liveaction, _ = action_service.schedule(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str(liveaction.id), raise_exception=True) action = action_utils.get_action_by_ref('core.chain') self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action) self.assertGreater(len(execution.children), 0) for child in execution.children: record = ActionExecution.get(id=child, raise_exception=True) self.assertEqual(record.parent, str(execution.id)) self.assertEqual(record.action['name'], 'local') self.assertEqual(record.runner['name'], 'run-local')
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_chained_executions(self): liveaction = LiveActionDB(action='core.chain') liveaction, _ = action_service.schedule(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) action = action_utils.get_action_by_ref('core.chain') self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action) self.assertGreater(len(execution.children), 0) for child in execution.children: record = ActionExecution.get(id=child, raise_exception=True) self.assertEqual(record.parent, str(execution.id)) self.assertEqual(record.action['name'], 'local') self.assertEqual(record.runner['name'], 'run-local')
def test_cancel_delayed_execution(self): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_liveaction_crud_no_notify(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') self.assertEqual(retrieved.notify, None) # Test update self.assertTrue(retrieved.end_timestamp is None) retrieved.end_timestamp = isotime.add_utc_tz( datetime.datetime.utcnow()) updated = LiveAction.add_or_update(retrieved) self.assertTrue(updated.end_timestamp == retrieved.end_timestamp) # Test delete LiveActionModelTest._delete([retrieved]) try: retrieved = LiveAction.get_by_id(saved.id) except ValueError: retrieved = None self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_triggered_execution(self): docs = { 'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']), 'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']), 'rule': copy.deepcopy(fixture.ARTIFACTS['rule']), 'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']) } # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type']))) trigger = Trigger.add_or_update( TriggerAPI.to_model(TriggerAPI(**docs['trigger']))) rule = RuleAPI.to_model(RuleAPI(**docs['rule'])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model( TriggerInstanceAPI(**docs['trigger_instance']))) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get( context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual( execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance))) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into cancel state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in [ action_constants.LIVEACTION_STATUS_CANCELING, action_constants.LIVEACTION_STATUS_CANCELED]: break # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. canceled = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_bootstrap(self): tracker = results_tracker.ResultsTracker() tracker._bootstrap() eventlet.sleep(0.2) exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id) exec_db = LiveAction.get_by_id(exec_id) self.assertTrue(exec_db.result['called_with'][exec_id] is not None, exec_db.result) exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id) exec_db = LiveAction.get_by_id(exec_id) self.assertTrue(exec_db.result['called_with'][exec_id] is not None, exec_db.result) tracker.shutdown()
def test_basic_cancel(self): runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None) mock_runner_run = mock.Mock(return_value=runner_run_result) with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_basic_cancel(self): runner_cls = self.get_runner_class('runner') runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None) runner_cls.run = mock.Mock(return_value=runner_run_result) liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_noop_cancel(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel") self.assertEqual(policy_db.parameters["action"], "cancel") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB( action="wolfpack.action-2", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. expected_num_exec += 0 # This request will not be scheduled for execution. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workflow_mistral_offline(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH) execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def test_launch_workflow_with_many_workflows(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH) execution = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_liveaction_create_with_notify_on_success_only(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} notify_db = NotificationSchema() notify_sub_schema = NotificationSubSchema() notify_sub_schema.message = 'Action succeeded.' notify_sub_schema.data = { 'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'} } notify_db.on_success = notify_sub_schema created.notify = notify_db saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') # Assert notify settings saved are right. self.assertEqual(notify_sub_schema.message, retrieved.notify.on_success.message) self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data) self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels) self.assertEqual(retrieved.notify.on_failure, None) self.assertEqual(retrieved.notify.on_complete, None)
def _purge_action_models(execution_db): liveaction_id = execution_db.liveaction['id'] if not liveaction_id: print('Invalid LiveAction id. Skipping delete: %s', execution_db) liveaction_db = None try: liveaction_db = LiveAction.get_by_id(liveaction_id) except: print('LiveAction with id: %s not found. Skipping delete.', liveaction_id) else: global DELETED_COUNT DELETED_COUNT += 1 try: ActionExecution.delete(execution_db) except Exception as e: print('Exception deleting Execution model: %s, exception: %s', execution_db, str(e)) else: try: LiveAction.delete(liveaction_db) except Exception as e: print('Zombie LiveAction left in db: %s. Exception: %s', liveaction_db, str(e))
def test_launch_workbook_with_many_workflows_no_default(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH) execution = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_execution_creation_action_triggered_by_rule(self): # Wait for the action execution to complete and then confirm outcome. trigger_type = self.MODELS['triggertypes']['triggertype2.json'] trigger = self.MODELS['triggers']['trigger2.json'] trigger_instance = self.MODELS['triggerinstances']['trigger_instance_1.json'] test_liveaction = self.FIXTURES['liveactions']['liveaction3.json'] rule = self.MODELS['rules']['rule3.json'] # Setup LiveAction to point to right rule and trigger_instance. # XXX: We need support for dynamic fixtures. test_liveaction['context']['rule']['id'] = str(rule.id) test_liveaction['context']['trigger_instance']['id'] = str(trigger_instance.id) test_liveaction_api = LiveActionAPI(**test_liveaction) test_liveaction = LiveAction.add_or_update(LiveActionAPI.to_model(test_liveaction_api)) liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) self.assertEqual(liveaction.status, LIVEACTION_STATUS_SCHEDULED) executions_util.create_execution_object(liveaction) execution = self._get_action_execution(liveaction__id=str(liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual(execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance))) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEquals(execution.liveaction['id'], str(liveaction.id))
def test_failed_cancel(self): runner_run_result = ( action_constants.LIVEACTION_STATUS_RUNNING, { "data": "foobar" }, None, ) mock_runner_run = mock.Mock(return_value=runner_run_result) with mock.patch.object(runner.MockActionRunner, "run", mock_runner_run): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancellation failed and execution state remains "canceling". runners.ActionRunner.cancel.assert_called_once_with() liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_liveaction_create_with_notify_on_success_only(self): created = LiveActionDB() created.action = 'core.local' created.description = '' created.status = 'running' created.parameters = {} notify_db = NotificationSchema() notify_sub_schema = NotificationSubSchema() notify_sub_schema.message = 'Action succeeded.' notify_sub_schema.data = {'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'}} notify_db.on_success = notify_sub_schema created.notify = notify_db saved = LiveActionModelTest._save_liveaction(created) retrieved = LiveAction.get_by_id(saved.id) self.assertEqual(saved.action, retrieved.action, 'Same triggertype was not returned.') # Assert notify settings saved are right. self.assertEqual(notify_sub_schema.message, retrieved.notify.on_success.message) self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data) self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels) self.assertEqual(retrieved.notify.on_failure, None) self.assertEqual(retrieved.notify.on_complete, None)
def test_enforce(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) # Assert the canceling state is being published. calls = [ call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) ] LiveActionPublisher.publish_state.assert_has_calls(calls) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': { 'execution_id': 'abcde' }}) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_workflow_task(self): # Verify initial state self.assertSequenceEqual(LiveAction.get_all(), []) self.assertSequenceEqual(ActionExecution.get_all(), []) # Start a mock action which times out live_action_db = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': {'execution_id': 'abcde'}} ) live_action_db, execution_db = action_service.request(live_action_db) live_action_db = LiveAction.get_by_id(str(live_action_db.id)) self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED) # Expire the workflow instance. live_action_db.status = LIVEACTION_STATUS_TIMED_OUT live_action_db.context['policies'] = {} execution_db.status = LIVEACTION_STATUS_TIMED_OUT LiveAction.add_or_update(live_action_db) ActionExecution.add_or_update(execution_db) # Simulate policy "apply_after" run self.policy.apply_after(target=live_action_db) # Note: There should be no new objects since live action is under the context of a workflow. live_action_dbs = LiveAction.get_all() action_execution_dbs = ActionExecution.get_all() self.assertEqual(len(live_action_dbs), 1) self.assertEqual(len(action_execution_dbs), 1) self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref( 'wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_launch_workflow_with_auth(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH) execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ACTION_CONTEXT) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name')) workflow_input = copy.deepcopy(ACTION_PARAMS) workflow_input.update({'count': '3'}) env = { '__actions': { 'st2.action': { 'st2_context': { 'auth_token': TOKEN_DB.token, 'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions', 'parent': str(liveaction.id) } } } } executions.ExecutionManager.create.assert_called_with( WF1_NAME, workflow_input=workflow_input, env=env)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workbook_name_mismatch(self): action_ref = 'generic.workbook_v2_name_mismatch' MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH) execution = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_apply(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) FakeConcurrencyApplicator.apply_before.assert_called_once_with(liveaction) RaiseExceptionApplicator.apply_before.assert_called_once_with(liveaction) FakeConcurrencyApplicator.apply_after.assert_called_once_with(liveaction) RaiseExceptionApplicator.apply_after.assert_called_once_with(liveaction)
def delete(self, exec_id): """ Stops a single execution. Handles requests: DELETE /actionexecutions/<id> """ execution_api = self._get_one(id=exec_id) if not execution_api: abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % exec_id) return liveaction_id = execution_api.liveaction['id'] if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) try: liveaction_db = LiveAction.get_by_id(liveaction_id) except: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) return if liveaction_db.status == LIVEACTION_STATUS_CANCELED: abort(http_client.OK, 'Action is already in "canceled" state.') if liveaction_db.status not in CANCELABLE_STATES: abort( http_client.OK, 'Action cannot be canceled. State = %s.' % liveaction_db.status) return liveaction_db.status = 'canceled' liveaction_db.end_timestamp = isotime.add_utc_tz( datetime.datetime.utcnow()) liveaction_db.result = {'message': 'Action canceled by user.'} try: LiveAction.add_or_update(liveaction_db) except: LOG.exception( 'Failed updating status to canceled for liveaction %s.', liveaction_db.id) abort(http_client.INTERNAL_SERVER_ERROR, 'Failed canceling execution.') return execution_db = execution_service.update_execution(liveaction_db) return ActionExecutionAPI.from_model(execution_db)
def test_triggered_execution(self): docs = { 'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']), 'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']), 'rule': copy.deepcopy(fixture.ARTIFACTS['rule']), 'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])} # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type']))) trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger']))) rule = RuleAPI.to_model(RuleAPI(**docs['rule'])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance']))) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str(liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual(execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance))) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_cancel_delayed_execution(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_launch_when_workbook_not_exists(self): execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_callback(self): execution = LiveActionDB( action='core.local', parameters={'cmd': 'uname -a'}, callback={'source': 'mistral', 'url': 'http://localhost:8989/v2/tasks/12345'}) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_SUCCEEDED) requests.request.assert_called_with('PUT', liveaction.callback['url'], data=json.dumps({'state': 'SUCCESS', 'result': NON_EMPTY_RESULT}), headers={'content-type': 'application/json'})
def test_launch_when_workbook_definition_changed(self): MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH) execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS) liveaction, _ = action_service.schedule(execution) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def _update_action_results(self, execution_id, status, results): liveaction_db = LiveAction.get_by_id(execution_id) if not liveaction_db: raise Exception('No DB model for liveaction_id: %s' % execution_id) liveaction_db.result = results liveaction_db.status = status # update liveaction, update actionexecution and then publish update. updated_liveaction = LiveAction.add_or_update(liveaction_db, publish=False) executions.update_execution(updated_liveaction) LiveAction.publish_update(updated_liveaction) return updated_liveaction
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request will be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_dispatch_runner_failure(self): runner_container = get_runner_container() params = { 'actionstr': 'bar' } liveaction_db = self._get_failingaction_exec_db_model(params) liveaction_db = LiveAction.add_or_update(liveaction_db) executions.create_execution_object(liveaction_db) runner_container.dispatch(liveaction_db) # pickup updated liveaction_db liveaction_db = LiveAction.get_by_id(liveaction_db.id) self.assertTrue('message' in liveaction_db.result) self.assertTrue('traceback' in liveaction_db.result)