def test_chain_cancel_cascade_to_subworkflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_cancel_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to cancel.
liveaction, execution = action_service.request_cancellation(liveaction, USERNAME)
# Wait until the liveaction is canceling.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is canceled.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_CANCELED)
def test_keep_state_object_on_error_at_update_result(self):
tracker = self._get_tracker()
querier = tracker.get_querier('test_querymodule')
querier._delete_state_object = mock.Mock(return_value=None)
querier.delete_state_object_on_error = False
# Ensure state objects are present.
state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id)
state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id)
self.assertIsNotNone(state1)
self.assertIsNotNone(state2)
with mock.patch.object(
querier.__class__, '_update_action_results',
mock.MagicMock(side_effect=Exception('Mock update exception.'))):
tracker._bootstrap()
eventlet.sleep(1)
exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertDictEqual(exec_db.result, {})
exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertDictEqual(exec_db.result, {})
tracker.shutdown()
# Ensure deletes are not called.
querier._delete_state_object.assert_not_called()
def test_resume(self):
# Launch the workflow execution.
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_pause(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED')
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually update the liveaction from pausing to paused. The paused state
# is usually updated by the mistral querier.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
liveaction, execution = action_service.request_resume(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'RUNNING')
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
def test_cancel_subworkflow_action(self):
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction1, execution1 = action_service.request_cancellation(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'CANCELLED'),
mock.call(WF1_EXEC.get('id'), 'CANCELLED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
def test_execution_cancellation(self):
tracker = self._get_tracker()
querier = tracker.get_querier('test_querymodule')
querier._delete_state_object = mock.Mock(return_value=None)
runners_utils.invoke_post_run = mock.Mock(return_value=None)
# Ensure state objects are present.
state1 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state1.yaml'].id)
state2 = ActionExecutionState.get_by_id(ResultsTrackerTests.states['state2.yaml'].id)
self.assertIsNotNone(state1)
self.assertIsNotNone(state2)
with mock.patch.object(
querier.__class__, 'query',
mock.MagicMock(return_value=(action_constants.LIVEACTION_STATUS_CANCELED, {}))):
tracker._bootstrap()
eventlet.sleep(2)
exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertDictEqual(exec_db.result, {})
exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertDictEqual(exec_db.result, {})
tracker.shutdown()
# Ensure deletes are called.
self.assertEqual(2, querier._delete_state_object.call_count)
# Ensure invoke_post_run is called.
self.assertEqual(2, runners_utils.invoke_post_run.call_count)
def test_chained_executions(self):
with mock.patch("st2common.runners.register_runner", mock.MagicMock(return_value=action_chain_runner)):
liveaction = LiveActionDB(action="executions.chain")
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id), raise_exception=True)
action = action_utils.get_action_by_ref("executions.chain")
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type["name"])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction["callback"], liveaction.callback)
self.assertEqual(execution.liveaction["action"], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action["name"], "local")
self.assertEqual(record.runner["name"], "run-local")
def test_chained_executions(self):
liveaction = LiveActionDB(action='core.chain')
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref('core.chain')
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action['name'], 'local')
self.assertEqual(record.runner['name'], 'run-local')
def test_basic_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertDictEqual(liveaction.result, MOCK_RESULT)
def test_cancel_on_task_action_concurrency_by_attr(self):
# Delete other policies in the test pack to avoid conflicts.
required_policy = 'mistral_tests.cancel_on_concurrency_by_attr'
self._drop_all_other_policies(required_policy)
# Get threshold from the policy.
policy = Policy.get_by_ref(required_policy)
threshold = policy.parameters.get('threshold', 0)
self.assertGreater(threshold, 0)
params = {'friend': 'grande animalerie'}
# Launch instances of the workflow up to threshold.
for i in range(0, threshold):
liveaction = LiveActionDB(action=WF1_NAME, parameters=params)
liveaction, execution1 = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
liveaction = self._wait_on_status(
liveaction,
action_constants.LIVEACTION_STATUS_RUNNING
)
# Check number of running instances
running = LiveAction.count(
action=WF1_NAME, status=action_constants.LIVEACTION_STATUS_RUNNING,
parameters__friend=params['friend'])
self.assertEqual(running, threshold)
# Mock the mistral runner cancel method to assert cancel is called.
mistral_runner_cls = runners.get_runner('mistral-v2').__class__
mock_cancel_return_value = (action_constants.LIVEACTION_STATUS_CANCELING, None, None)
mock_cancel = mock.MagicMock(return_value=mock_cancel_return_value)
with mock.patch.object(mistral_runner_cls, 'cancel', mock_cancel):
# Launch another instance of the workflow with mistral callback defined
# to indicate that this is executed under a workflow.
callback = {
'source': MISTRAL_RUNNER_NAME,
'url': 'http://127.0.0.1:8989/v2/action_executions/12345'
}
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=params, callback=callback)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
# Assert cancel has been called.
liveaction2 = self._wait_on_status(
liveaction2,
action_constants.LIVEACTION_STATUS_CANCELING
)
mistral_runner_cls.cancel.assert_called_once_with()
def test_bootstrap(self):
tracker = results_tracker.get_tracker()
tracker._bootstrap()
eventlet.sleep(0.2)
exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
tracker.shutdown()
def test_failed_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_noop_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def _purge_action_models(execution_db):
liveaction_id = execution_db.liveaction['id']
if not liveaction_id:
print('Invalid LiveAction id. Skipping delete: %s', execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
print('LiveAction with id: %s not found. Skipping delete.', liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except Exception as e:
print('Exception deleting Execution model: %s, exception: %s',
execution_db, str(e))
else:
try:
LiveAction.delete(liveaction_db)
except Exception as e:
print('Zombie LiveAction left in db: %s. Exception: %s',
liveaction_db, str(e))
def test_launch_workbook_with_many_workflows_no_default(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_cancel_retry(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'CANCELLED')
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_liveaction_create_with_notify_on_success_only(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
notify_db = NotificationSchema()
notify_sub_schema = NotificationSubSchema()
notify_sub_schema.message = 'Action succeeded.'
notify_sub_schema.data = {
'foo': 'bar',
'bar': 1,
'baz': {'k1': 'v1'}
}
notify_db.on_success = notify_sub_schema
created.notify = notify_db
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(notify_sub_schema.message,
retrieved.notify.on_success.message)
self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data)
self.assertListEqual(notify_sub_schema.routes, retrieved.notify.on_success.routes)
self.assertEqual(retrieved.notify.on_failure, None)
self.assertEqual(retrieved.notify.on_complete, None)
def test_launch_workflow_mistral_offline(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def test_launch_workflow_with_many_workflows(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_resume_option_reset_tasks(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
liveaction1 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
self.assertFalse(MistralRunner.resume.called)
# Rerun the execution.
context = {
're-run': {
'ref': execution1.id,
'tasks': ['x', 'y'],
'reset': ['y']
}
}
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=context)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
task_specs = {
'x': {
'reset': False
},
'y': {
'reset': True
}
}
MistralRunner.resume.assert_called_with(ex_ref=execution1, task_specs=task_specs)
def test_launch_workflow(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'st2_action_api_url': 'http://0.0.0.0:9101/v1',
'__actions': {
'st2.action': {
'st2_context': {
'api_url': 'http://0.0.0.0:9101/v1',
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': {
'execution_id': str(execution.id)
},
'notify': {},
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def _purge_models(execution_db):
liveaction_id = execution_db.liveaction.get("id", None)
if not liveaction_id:
LOG.error("Invalid LiveAction id. Skipping delete: %s", execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
LOG.exception("LiveAction with id: %s not found. Skipping delete.", liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except:
LOG.exception("Exception deleting Execution model: %s", execution_db)
else:
if liveaction_db:
try:
LiveAction.delete(liveaction_db)
except:
LOG.exception("Zombie LiveAction left in db: %s.", liveaction_db)
def test_launch_workbook_name_mismatch(self):
action_ref = TEST_PACK + '.workbook_v2_name_mismatch'
liveaction = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Name of the workbook must be the same', liveaction.result['error'])
def test_basic_execution(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(
liveaction__id=str(liveaction.id),
raise_exception=True
)
self.assertDictEqual(execution.trigger, {})
self.assertDictEqual(execution.trigger_type, {})
self.assertDictEqual(execution.trigger_instance, {})
self.assertDictEqual(execution.rule, {})
action = action_utils.get_action_by_ref('executions.local')
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_launch_workflow_with_notifications(self):
notify_data = {'on_complete': {'channels': ['slack'],
'message': '"@channel: Action succeeded."', 'data': {}}}
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, notify=notify_data)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'__actions': {
'st2.action': {
'st2_context': {
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': str(liveaction.id),
'notify': NotificationsHelper.from_model(liveaction.notify),
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_execution_creation_action_triggered_by_rule(self):
# Wait for the action execution to complete and then confirm outcome.
trigger_type = self.MODELS['triggertypes']['triggertype2.yaml']
trigger = self.MODELS['triggers']['trigger2.yaml']
trigger_instance = self.MODELS['triggerinstances']['trigger_instance_1.yaml']
test_liveaction = self.FIXTURES['liveactions']['liveaction3.yaml']
rule = self.MODELS['rules']['rule3.yaml']
# Setup LiveAction to point to right rule and trigger_instance.
# XXX: We need support for dynamic fixtures.
test_liveaction['context']['rule']['id'] = str(rule.id)
test_liveaction['context']['trigger_instance']['id'] = str(trigger_instance.id)
test_liveaction_api = LiveActionAPI(**test_liveaction)
test_liveaction = LiveAction.add_or_update(LiveActionAPI.to_model(test_liveaction_api))
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
executions_util.create_execution_object(liveaction)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEquals(execution.liveaction['id'], str(liveaction.id))
def test_launch_workflow_with_auth(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ACTION_CONTEXT)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'__actions': {
'st2.action': {
'st2_context': {
'auth_token': TOKEN_DB.token,
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': str(liveaction.id),
'notify': {},
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_launch_workbook_name_mismatch(self):
action_ref = 'generic.workbook_v2_name_mismatch'
MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_triggered_execution(self):
docs = {
'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type'])))
trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger'])))
rule = RuleAPI.to_model(RuleAPI(**docs['rule']))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance'])))
trace_service.add_or_update_given_trace_context(
trace_context={'trace_tag': 'test_triggered_execution_trace'},
trigger_instances=[str(trigger_instance.id)])
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def _wait_for_status(self, liveaction, status, interval=0.1, retries=100):
# Wait until the liveaction reaches status.
for i in range(0, retries):
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status != status:
eventlet.sleep(interval)
continue
return liveaction
def test_launch_when_workbook_not_exists(self):
liveaction = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def test_launch_workflow_with_st2_https(self):
cfg.CONF.set_override('api_url', 'https://0.0.0.0:9101', group='auth')
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'st2_action_api_url': 'https://0.0.0.0:9101/v1',
'__actions': {
'st2.action': {
'st2_context': {
'api_url': 'https://0.0.0.0:9101/v1',
'endpoint': 'https://0.0.0.0:9101/v1/actionexecutions',
'parent': {
'execution_id': str(execution.id)
},
'notify': {},
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_worker_shutdown(self):
action_worker = actions_worker.get_worker()
# Create a temporary file that is deleted when the file is closed and then set up an
# action to wait for this file to be deleted. This allows this test to run the action
# over a separate thread, run the shutdown sequence on the main thread, and then let
# the local runner to exit gracefully and allow _run_action to finish execution.
with tempfile.NamedTemporaryFile() as fp:
params = {
'cmd': 'while [ -e \'%s\' ]; do sleep 0.1; done' % fp.name
}
liveaction_db = self._get_liveaction_model(
WorkerTestCase.local_action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
runner_thread = eventlet.spawn(action_worker._run_action,
liveaction_db)
# Wait for the worker to add the liveaction to _running_liveactions.
while len(action_worker._running_liveactions) <= 0:
eventlet.sleep(0.1)
self.assertEqual(len(action_worker._running_liveactions), 1)
# Shutdown the worker to trigger the abandon process.
action_worker.shutdown()
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
# Verify that _running_liveactions is empty and the liveaction is abandoned.
self.assertEqual(len(action_worker._running_liveactions), 0)
self.assertEqual(liveaction_db.status,
action_constants.LIVEACTION_STATUS_ABANDONED)
# Wait for the local runner to complete. This will activate the finally block in
# _run_action but will not result in KeyError because the discard method is used to
# to remove the liveaction from _running_liveactions.
runner_thread.wait()
def test_launch_workflow(self):
MistralRunner.entry_point = mock.PropertyMock(
return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'__actions': {
'st2.action': {
'st2_context': {
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': {
'execution_id': str(execution.id)
},
'notify': {},
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_dispatch(self):
runner_container = get_runner_container()
params = {
'actionstr': 'bar'
}
liveaction_db = self._get_liveaction_model(RunnerContainerTest.action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran successfully.
runner_container.dispatch(liveaction_db)
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
result = liveaction_db.result
self.assertTrue(result.get('action_params').get('actionint') == 10)
self.assertTrue(result.get('action_params').get('actionstr') == 'bar')
# Assert that context is written correctly.
context = {
'user': '******',
'third_party_system': {
'ref_id': '1234'
}
}
self.assertDictEqual(liveaction_db.context, context)
def test_callback_retry(self):
local_runner_cls = self.get_runner_class('local_runner')
local_run_result = (action_constants.LIVEACTION_STATUS_SUCCEEDED,
NON_EMPTY_RESULT, None)
local_runner_cls.run = mock.Mock(return_value=local_run_result)
liveaction = LiveActionDB(
action='core.local',
parameters={'cmd': 'uname -a'},
callback={
'source': MISTRAL_RUNNER_NAME,
'url': 'http://127.0.0.1:8989/v2/action_executions/12345'
})
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
calls = [
call('12345', state='SUCCESS', output=NON_EMPTY_RESULT)
for i in range(0, 2)
]
action_executions.ActionExecutionManager.update.assert_has_calls(calls)
def test_save_large_execution(benchmark, fixture_file: str, approach: str) -> None:
with open(os.path.join(FIXTURES_DIR, fixture_file), "r") as fp:
content = fp.read()
data = json.loads(content)
db_setup()
model_cls = get_model_class_for_approach(approach=approach)
def run_benchmark():
live_action_db = model_cls()
live_action_db.status = "succeeded"
live_action_db.action = "core.local"
live_action_db.result = data
inserted_live_action_db = LiveAction.add_or_update(live_action_db)
return inserted_live_action_db
inserted_live_action_db = benchmark(run_benchmark)
retrieved_live_action_db = LiveAction.get_by_id(inserted_live_action_db.id)
# Assert that result is correctly converted back to dict on retrieval
assert inserted_live_action_db.result == data
assert inserted_live_action_db == retrieved_live_action_db
def process(self, execution_db):
execution_id = str(execution_db.id)
extra = {'execution': execution_db}
LOG.debug('Processing action execution "%s".',
execution_id,
extra=extra)
# Get the corresponding liveaction record.
liveaction_db = LiveAction.get_by_id(execution_db.liveaction['id'])
if execution_db.status in LIVEACTION_COMPLETED_STATES:
# If the action execution is executed under an orquesta workflow, policies for the
# action execution will be applied by the workflow engine. A policy may affect the
# final state of the action execution thereby impacting the state of the workflow.
if not workflow_service.is_action_execution_under_workflow_context(
execution_db):
policy_service.apply_post_run_policies(liveaction_db)
if liveaction_db.notify is not None:
self._post_notify_triggers(liveaction_db=liveaction_db,
execution_db=execution_db)
self._post_generic_trigger(liveaction_db=liveaction_db,
execution_db=execution_db)
def test_callback_resuming_state(self):
local_runner_cls = self.get_runner_class('local_runner',
'local_shell_command_runner')
local_run_result = (action_constants.LIVEACTION_STATUS_RESUMING,
NON_EMPTY_RESULT, None)
local_runner_cls.run = mock.Mock(return_value=local_run_result)
local_resume_result = (action_constants.LIVEACTION_STATUS_RUNNING,
NON_EMPTY_RESULT, None)
local_runner_cls.resume = mock.Mock(return_value=local_resume_result)
liveaction = LiveActionDB(
action='core.local',
parameters={'cmd': 'uname -a'},
callback={
'source': MISTRAL_RUNNER_NAME,
'url': 'http://127.0.0.1:8989/v2/action_executions/12345'
})
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, local_resume_result[0])
self.assertFalse(
action_executions.ActionExecutionManager.update.called)
def test_chained_executions(self):
liveaction = LiveActionDB(action="executions.chain")
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref("executions.chain")
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type["name"])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
# NOTE: Timestamp of liveaction and execution may be a bit different, depending on how long
# it takes to persist each object in the database
self.assertEqual(
execution.end_timestamp.replace(microsecond=0),
liveaction.end_timestamp.replace(microsecond=0),
)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction["callback"], liveaction.callback)
self.assertEqual(execution.liveaction["action"], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action["name"], "local")
self.assertEqual(record.runner["name"], "local-shell-cmd")
def test_callback_success_state(self):
local_runner_cls = self.get_runner_class('local_runner',
'local_shell_command_runner')
local_run_result = (action_constants.LIVEACTION_STATUS_SUCCEEDED,
NON_EMPTY_RESULT, None)
local_runner_cls.run = mock.Mock(return_value=local_run_result)
expected_mistral_status = self.status_map[local_run_result[0]]
liveaction = LiveActionDB(
action='core.local',
parameters={'cmd': 'uname -a'},
callback={
'source': MISTRAL_RUNNER_NAME,
'url': 'http://127.0.0.1:8989/v2/action_executions/12345'
})
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
action_executions.ActionExecutionManager.update.assert_called_with(
'12345', state=expected_mistral_status, output=NON_EMPTY_RESULT)
def test_chain_pause_resume_with_init_vars(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + "." + "test_pause_resume_with_init_vars"
params = {"tempfile": path, "message": "foobar"}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(
liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 2)
self.assertEqual(liveaction.result["tasks"][1]["result"]["stdout"],
"FOOBAR")
def test_triggered_execution(self):
docs = {
"trigger_type":
copy.deepcopy(fixture.ARTIFACTS["trigger_type"]),
"trigger":
copy.deepcopy(fixture.ARTIFACTS["trigger"]),
"rule":
copy.deepcopy(fixture.ARTIFACTS["rule"]),
"trigger_instance":
copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]),
}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs["trigger_type"])))
trigger = Trigger.add_or_update(
TriggerAPI.to_model(TriggerAPI(**docs["trigger"])))
rule = RuleAPI.to_model(RuleAPI(**docs["rule"]))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(
TriggerInstanceAPI(**docs["trigger_instance"])))
trace_service.add_or_update_given_trace_context(
trace_context={"trace_tag": "test_triggered_execution_trace"},
trigger_instances=[str(trigger_instance.id)],
)
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(
context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger,
vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type,
vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(
execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)),
)
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type["name"])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
# NOTE: Timestamp of liveaction and execution may be a bit different, depending on how long
# it takes to persist each object in the database
self.assertEqual(
execution.end_timestamp.replace(microsecond=0),
liveaction.end_timestamp.replace(microsecond=0),
)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction["callback"], liveaction.callback)
self.assertEqual(execution.liveaction["action"], liveaction.action)
def test_chain_cancel_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_cancel_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to cancel.
task1_live, task1_exec = action_service.request_cancellation(
task1_live, USERNAME)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait until the parent liveaction is canceled.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_CANCELED)
def test_launch_workflow_under_parent_chain_with_jinja_parameters(self):
ac_ctx = {
'chain': {
'parameters': {
'var1': 'foobar',
'var2': '{{foobar}}',
'var3': ['{{foo}}', '{{bar}}'],
'var4': {
'foobar': '{{foobar}}'
},
}
}
}
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ac_ctx)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'st2_execution_id': str(execution.id),
'st2_liveaction_id': str(liveaction.id),
'st2_action_api_url': 'http://0.0.0.0:9101/v1',
'__actions': {
'st2.action': {
'st2_context': {
'api_url': 'http://0.0.0.0:9101/v1',
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': {
'pack': 'mistral_tests',
'execution_id': str(execution.id),
'chain': {
'parameters': {
'var1': 'foobar',
'var2': '{% raw %}{{foobar}}{% endraw %}',
'var3': [
'{% raw %}{{foo}}{% endraw %}',
'{% raw %}{{bar}}{% endraw %}'
],
'var4': {
'foobar': '{% raw %}{{foobar}}{% endraw %}'
}
}
}
},
'notify': {},
'skip_notify_tasks': []
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_chain_pause_resume_status_change(self):
"""Tests context_result is updated when last task's status changes between pause and resume
"""
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_context_result'
params = {'tempfile': path}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# This workflow runs 'core.ask' so will pause on its own. We just need to
# wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
last_task_liveaction_id = liveaction.result['tasks'][-1][
'liveaction_id']
action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
result={'foo': 'bar'},
liveaction_id=last_task_liveaction_id)
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def test_triggered_execution(self):
docs = {
'trigger_type':
copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger':
copy.deepcopy(fixture.ARTIFACTS['trigger']),
'rule':
copy.deepcopy(fixture.ARTIFACTS['rule']),
'trigger_instance':
copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])
}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type'])))
trigger = Trigger.add_or_update(
TriggerAPI.to_model(TriggerAPI(**docs['trigger'])))
rule = RuleAPI.to_model(RuleAPI(**docs['rule']))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(
TriggerInstanceAPI(**docs['trigger_instance'])))
trace_service.add_or_update_given_trace_context(
trace_context={'trace_tag': 'test_triggered_execution_trace'},
trigger_instances=[str(trigger_instance.id)])
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(
context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger,
vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type,
vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(
execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_resume_missing_subworkflow_action(self):
requester = cfg.CONF.system_user.user
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
# Pause the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_pause(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually set the liveaction status to PAUSED.
action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED)
action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [uuid.uuid4().hex]
# Resume the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_resume(liveaction1, requester)
# Includes the previous calls.
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 3)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED'),
mock.call(WF2_EXEC.get('id'), 'RUNNING'),
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
# The workflow execution will fail because the liveaction for the subworkflow
# should not be missing and we do not know what state it is in.
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, 'result', None)
liveaction_db = action_service.update_status(
liveaction_db, status, result)
actionexecution_db = ActionExecution.get(
liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
try:
if (liveaction_db.status
== action_constants.LIVEACTION_STATUS_CANCELING
and liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_CANCELING
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
elif (liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELED):
liveaction_db, actionexecution_db = action_service.request_cancellation(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif (liveaction_db.status
== action_constants.LIVEACTION_STATUS_PAUSING
and liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_PAUSING
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
elif (liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
else:
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.')
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def test_launch_workflow_with_many_workflows(self):
liveaction = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Multiple workflows is not supported.', liveaction.result['error'])
def test_launch_workflow_mistral_offline(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Connection refused', liveaction.result['error'])
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
try:
liveaction_db = action_service.update_status(liveaction_db,
liveaction_api.status,
result=getattr(
liveaction_api,
'result', None))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution.')
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
return execution_api
def test_worker_graceful_shutdown_exit_timeout(self):
cfg.CONF.set_override(name="graceful_shutdown",
override=True,
group="actionrunner")
cfg.CONF.set_override(name="exit_still_active_check",
override=5,
group="actionrunner")
action_worker = actions_worker.get_worker()
temp_file = None
# Create a temporary file that is deleted when the file is closed and then set up an
# action to wait for this file to be deleted. This allows this test to run the action
# over a separate thread, run the shutdown sequence on the main thread, and then let
# the local runner to exit gracefully and allow _run_action to finish execution.
with tempfile.NamedTemporaryFile() as fp:
temp_file = fp.name
self.assertIsNotNone(temp_file)
self.assertTrue(os.path.isfile(temp_file))
# Launch the action execution in a separate thread.
params = {
"cmd": "while [ -e '%s' ]; do sleep 0.1; done" % temp_file
}
liveaction_db = self._get_liveaction_model(
WorkerTestCase.local_action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
runner_thread = eventlet.spawn(action_worker._run_action,
liveaction_db)
# Wait for the worker up to 10s to add the liveaction to _running_liveactions.
for i in range(0, int(10 / 0.1)):
eventlet.sleep(0.1)
if len(action_worker._running_liveactions) > 0:
break
self.assertEqual(len(action_worker._running_liveactions), 1)
# Shutdown the worker to trigger the abandon process.
shutdown_thread = eventlet.spawn(action_worker.shutdown)
# Continue the excution for 5+ seconds to ensure timeout occurs.
eventlet.sleep(6)
# Make sure the temporary file has been deleted.
self.assertFalse(os.path.isfile(temp_file))
# Wait for the worker up to 10s to remove the liveaction from _running_liveactions.
for i in range(0, int(10 / 0.1)):
eventlet.sleep(0.1)
if len(action_worker._running_liveactions) < 1:
break
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
# Verify that _running_liveactions is empty and the liveaction is abandoned.
self.assertEqual(len(action_worker._running_liveactions), 0)
self.assertEqual(
liveaction_db.status,
action_constants.LIVEACTION_STATUS_ABANDONED,
str(liveaction_db),
)
# Wait for the local runner to complete. This will activate the finally block in
# _run_action but will not result in KeyError because the discard method is used to
# to remove the liveaction from _running_liveactions.
runner_thread.wait()
shutdown_thread.kill()
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + "." + "test_pause_resume_with_subworkflow"
params = {"tempfile": path, "message": "foobar"}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(
task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 1)
subworkflow = liveaction.result["tasks"][0]
self.assertEqual(len(subworkflow["result"]["tasks"]), 1)
self.assertEqual(subworkflow["state"],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(
task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 1)
subworkflow = liveaction.result["tasks"][0]
self.assertEqual(len(subworkflow["result"]["tasks"]), 1)
self.assertEqual(subworkflow["state"],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 2)
subworkflow = liveaction.result["tasks"][0]
self.assertEqual(len(subworkflow["result"]["tasks"]), 2)
self.assertEqual(subworkflow["state"],
action_constants.LIVEACTION_STATUS_SUCCEEDED)
def run_benchmark():
retrieved_live_action_db = LiveAction.get_by_id(
inserted_live_action_db.id)
return retrieved_live_action_db
def test_migrate_executions(self):
ActionExecutionDB._meta["allow_inheritance"] = True
LiveActionDB._meta["allow_inheritance"] = True
class ActionExecutionDB_OldFieldType(ActionExecutionDB):
result = stormbase.EscapedDynamicField(default={})
class LiveActionDB_OldFieldType(LiveActionDB):
result = stormbase.EscapedDynamicField(default={})
execution_dbs = ActionExecution.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(execution_dbs), 0)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 0)
# 1. Insert data in old format
liveaction_1_db = LiveActionDB_OldFieldType()
liveaction_1_db.action = "foo.bar"
liveaction_1_db.status = action_constants.LIVEACTION_STATUS_FAILED
liveaction_1_db.result = MOCK_RESULT_1
liveaction_1_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
liveaction_1_db = LiveAction.add_or_update(liveaction_1_db,
publish=False)
execution_1_db = ActionExecutionDB_OldFieldType()
execution_1_db.action = {"a": 1}
execution_1_db.runner = {"a": 1}
execution_1_db.liveaction = {"id": liveaction_1_db.id}
execution_1_db.status = action_constants.LIVEACTION_STATUS_FAILED
execution_1_db.result = MOCK_RESULT_1
execution_1_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
execution_1_db = ActionExecution.add_or_update(execution_1_db,
publish=False)
# This execution is not in a final state yet so it should not be migrated
liveaction_2_db = LiveActionDB_OldFieldType()
liveaction_2_db.action = "foo.bar2"
liveaction_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING
liveaction_2_db.result = MOCK_RESULT_2
liveaction_2_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
liveaction_2_db = LiveAction.add_or_update(liveaction_2_db,
publish=False)
execution_2_db = ActionExecutionDB_OldFieldType()
execution_2_db.action = {"a": 2}
execution_2_db.runner = {"a": 2}
execution_2_db.liveaction = {"id": liveaction_2_db.id}
execution_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING
execution_2_db.result = MOCK_RESULT_2
execution_2_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
execution_2_db = ActionExecution.add_or_update(execution_2_db,
publish=False)
# This object is older than the default threshold so it should not be migrated
execution_3_db = ActionExecutionDB_OldFieldType()
execution_3_db.action = {"a": 2}
execution_3_db.runner = {"a": 2}
execution_3_db.liveaction = {"id": liveaction_2_db.id}
execution_3_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
execution_3_db.result = MOCK_RESULT_1
execution_3_db.start_timestamp = datetime.datetime.utcfromtimestamp(
0).replace(tzinfo=datetime.timezone.utc)
execution_3_db = ActionExecution.add_or_update(execution_3_db,
publish=False)
# Verify data has been inserted in old format
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 3)
execution_dbs = ActionExecution.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(execution_dbs), 3)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(execution_dbs), 0)
liveaction_dbs = LiveAction.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(liveaction_dbs), 2)
liveaction_dbs = LiveAction.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(liveaction_dbs), 2)
liveaction_dbs = LiveAction.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(liveaction_dbs), 0)
# Update inserted documents and remove special _cls field added by mongoengine. We need to
# do that here due to how mongoengine works with subclasses.
ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
}).update(set___cls="ActionExecutionDB")
LiveAction.query(__raw__={
"result": {
"$type": "object",
},
}).update(set___cls="LiveActionDB")
# 2. Run migration
start_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2)
end_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
migration_module.migrate_executions(start_dt=start_dt, end_dt=end_dt)
# 3. Verify data has been migrated - only 1 item should have been migrated since it's in a
# completed state
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 2)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(execution_dbs), 1)
execution_db_1_retrieved = ActionExecution.get_by_id(execution_1_db.id)
self.assertEqual(execution_db_1_retrieved.result, MOCK_RESULT_1)
execution_db_2_retrieved = ActionExecution.get_by_id(execution_2_db.id)
self.assertEqual(execution_db_2_retrieved.result, MOCK_RESULT_2)
liveaction_db_1_retrieved = LiveAction.get_by_id(liveaction_1_db.id)
self.assertEqual(liveaction_db_1_retrieved.result, MOCK_RESULT_1)
liveaction_db_2_retrieved = LiveAction.get_by_id(liveaction_2_db.id)
self.assertEqual(liveaction_db_2_retrieved.result, MOCK_RESULT_2)
def test_launch_workbook_with_many_workflows_no_default(self):
liveaction = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('Default workflow cannot be determined.', liveaction.result['error'])
def test_chain_pause_resume_last_task_failed_with_no_next_task(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_last_task_failed_with_no_next_task'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(
liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_FAILED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
self.assertEqual(liveaction.result['tasks'][0]['state'],
action_constants.LIVEACTION_STATUS_FAILED)
def delete(self, id, requester_user, show_secrets=False):
"""
Stops a single execution.
Handles requests:
DELETE /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_CANCELED:
LOG.info('Action %s already in "canceled" state; \
returning execution object.' % liveaction_db.id)
return execution_api
if liveaction_db.status not in action_constants.LIVEACTION_CANCELABLE_STATES:
abort(
http_client.OK, 'Action cannot be canceled. State = %s.' %
liveaction_db.status)
try:
(liveaction_db,
execution_db) = action_service.request_cancellation(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
except:
LOG.exception('Failed requesting cancellation for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed canceling execution.')
return ActionExecutionAPI.from_model(
execution_db, mask_secrets=from_model_kwargs['mask_secrets'])
def request(liveaction):
"""
Request an action execution.
:return: (liveaction, execution)
:rtype: tuple
"""
# Use the user context from the parent action execution. Subtasks in a workflow
# action can be invoked by a system user and so we want to use the user context
# from the original workflow action.
if getattr(liveaction, 'context', None) and 'parent' in liveaction.context:
parent = LiveAction.get_by_id(liveaction.context['parent'])
liveaction.context['user'] = getattr(parent, 'context',
dict()).get('user')
# Validate action.
action_db = action_utils.get_action_by_ref(liveaction.action)
if not action_db:
raise ValueError('Action "%s" cannot be found.' % liveaction.action)
if not action_db.enabled:
raise ValueError('Unable to execute. Action "%s" is disabled.' %
liveaction.action)
runnertype_db = action_utils.get_runnertype_by_name(
action_db.runner_type['name'])
if not hasattr(liveaction, 'parameters'):
liveaction.parameters = dict()
# Validate action parameters.
schema = util_schema.get_parameter_schema(action_db)
validator = util_schema.get_validator()
util_schema.validate(liveaction.parameters,
schema,
validator,
use_default=True)
# validate that no immutable params are being overriden. Although possible to
# ignore the override it is safer to inform the user to avoid surprises.
immutables = _get_immutable_params(action_db.parameters)
immutables.extend(_get_immutable_params(runnertype_db.runner_parameters))
overridden_immutables = [
p for p in six.iterkeys(liveaction.parameters) if p in immutables
]
if len(overridden_immutables) > 0:
raise ValueError(
'Override of immutable parameter(s) %s is unsupported.' %
str(overridden_immutables))
# Set notification settings for action.
# XXX: There are cases when we don't want notifications to be sent for a particular
# execution. So we should look at liveaction.parameters['notify']
# and not set liveaction.notify.
if action_db.notify:
liveaction.notify = action_db.notify
# Write to database and send to message queue.
liveaction.status = action_constants.LIVEACTION_STATUS_REQUESTED
liveaction.start_timestamp = date_utils.get_datetime_utc_now()
# Publish creation after both liveaction and actionexecution are created.
liveaction = LiveAction.add_or_update(liveaction, publish=False)
execution = executions.create_execution_object(liveaction, publish=False)
# Assume that this is a creation.
LiveAction.publish_create(liveaction)
LiveAction.publish_status(liveaction)
ActionExecution.publish_create(execution)
extra = {'liveaction_db': liveaction, 'execution_db': execution}
LOG.audit(
'Action execution requested. LiveAction.id=%s, ActionExecution.id=%s' %
(liveaction.id, execution.id),
extra=extra)
return liveaction, execution
