def _purge_models(execution_db):
liveaction_id = execution_db.liveaction.get("id", None)
if not liveaction_id:
LOG.error("Invalid LiveAction id. Skipping delete: %s", execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
LOG.exception("LiveAction with id: %s not found. Skipping delete.", liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except:
LOG.exception("Exception deleting Execution model: %s", execution_db)
else:
if liveaction_db:
try:
LiveAction.delete(liveaction_db)
except:
LOG.exception("Zombie LiveAction left in db: %s.", liveaction_db)
def get_descendants(actionexecution_id, descendant_depth=-1, result_fmt=None):
"""
Returns all descendant executions upto the specified descendant_depth for
the supplied actionexecution_id.
"""
descendants = DESCENDANT_VIEWS.get(result_fmt, DFSDescendantView)()
children = ActionExecution.query(parent=actionexecution_id,
**{'order_by': ['start_timestamp']})
LOG.debug('Found %s children for id %s.', len(children), actionexecution_id)
current_level = [(child, 1) for child in children]
while current_level:
parent, level = current_level.pop(0)
parent_id = str(parent.id)
descendants.add(parent)
if not parent.children:
continue
if level != -1 and level == descendant_depth:
continue
children = ActionExecution.query(parent=parent_id, **{'order_by': ['start_timestamp']})
LOG.debug('Found %s children for id %s.', len(children), parent_id)
# prepend for DFS
for idx in range(len(children)):
current_level.insert(idx, (children[idx], level + 1))
return descendants.result
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
logger=LOG, timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_liveaction_gets_deleted(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
end_ts = now - timedelta(days=14)
liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
liveaction_model['start_timestamp'] = start_ts
liveaction_model['end_timestamp'] = end_ts
liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
liveaction = LiveAction.add_or_update(liveaction_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['end_timestamp'] = end_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
exec_model['liveaction']['id'] = str(liveaction.id)
ActionExecution.add_or_update(exec_model)
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(liveactions), 1)
self.assertEqual(len(executions), 1)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10))
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(executions), 0)
self.assertEqual(len(liveactions), 0)
def update_execution(liveaction_db, publish=True):
execution = ActionExecution.get(liveaction__id=str(liveaction_db.id))
decomposed = _decompose_liveaction(liveaction_db)
for k, v in six.iteritems(decomposed):
setattr(execution, k, v)
execution = ActionExecution.add_or_update(execution, publish=publish)
return execution
def _has_active_tasks(self, liveaction_db, mistral_wf_state, mistral_tasks):
# Identify if there are any active tasks in Mistral.
active_mistral_tasks = len([t for t in mistral_tasks if t['state'] in ACTIVE_STATES]) > 0
active_st2_tasks = False
execution = ActionExecution.get(liveaction__id=str(liveaction_db.id))
for child_exec_id in execution.children:
child_exec = ActionExecution.get(id=child_exec_id)
# Catch exception where a child is requested twice due to st2mistral retrying
# from a st2 API connection failure. The first child will be stuck in requested
# while the mistral workflow is already completed.
if (mistral_wf_state in DONE_STATES and
child_exec.status == action_constants.LIVEACTION_STATUS_REQUESTED):
continue
if (child_exec.status not in action_constants.LIVEACTION_COMPLETED_STATES and
child_exec.status != action_constants.LIVEACTION_STATUS_PAUSED):
active_st2_tasks = True
break
if active_mistral_tasks:
LOG.info('There are active mistral tasks for %s.', str(liveaction_db.id))
if active_st2_tasks:
LOG.info('There are active st2 tasks for %s.', str(liveaction_db.id))
return active_mistral_tasks or active_st2_tasks
def _get_runner(self, runnertype_db, action_db, liveaction_db):
runner = get_runner(runnertype_db.runner_module)
resolved_entry_point = self._get_entry_point_abs_path(action_db.pack,
action_db.entry_point)
runner.runner_type_db = runnertype_db
runner.container_service = RunnerContainerService()
runner.action = action_db
runner.action_name = action_db.name
runner.liveaction = liveaction_db
runner.liveaction_id = str(liveaction_db.id)
runner.execution = ActionExecution.get(liveaction__id=runner.liveaction_id)
runner.execution_id = str(runner.execution.id)
runner.entry_point = resolved_entry_point
runner.context = getattr(liveaction_db, 'context', dict())
runner.callback = getattr(liveaction_db, 'callback', dict())
runner.libs_dir_path = self._get_action_libs_abs_path(action_db.pack,
action_db.entry_point)
# For re-run, get the ActionExecutionDB in which the re-run is based on.
rerun_ref_id = runner.context.get('re-run', {}).get('ref')
runner.rerun_ex_ref = ActionExecution.get(id=rerun_ref_id) if rerun_ref_id else None
return runner
def _purge_action_models(execution_db):
liveaction_id = execution_db.liveaction['id']
if not liveaction_id:
print('Invalid LiveAction id. Skipping delete: %s', execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
print('LiveAction with id: %s not found. Skipping delete.', liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except Exception as e:
print('Exception deleting Execution model: %s, exception: %s',
execution_db, str(e))
else:
try:
LiveAction.delete(liveaction_db)
except Exception as e:
print('Zombie LiveAction left in db: %s. Exception: %s',
liveaction_db, str(e))
def test_chain_cancel_cascade_to_subworkflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_cancel_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to cancel.
liveaction, execution = action_service.request_cancellation(liveaction, USERNAME)
# Wait until the liveaction is canceling.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is canceled.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_CANCELED)
def create_execution_object(liveaction, action_db=None, runnertype_db=None, publish=True):
if not action_db:
action_db = action_utils.get_action_by_ref(liveaction.action)
if not runnertype_db:
runnertype_db = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runnertype_db))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
# TODO: This object initialization takes 20-30or so ms
execution = ActionExecutionDB(**attrs)
# TODO: Do 100% research this is fully safe and unique in distributed setups
execution.id = ObjectId()
execution.web_url = _get_web_url_for_execution(str(execution.id))
# NOTE: User input data is already validate as part of the API request,
# other data is set by us. Skipping validation here makes operation 10%-30% faster
execution = ActionExecution.add_or_update(execution, publish=publish, validate=False)
if parent and str(execution.id) not in parent.children:
values = {}
values['push__children'] = str(execution.id)
ActionExecution.update(parent, **values)
return execution
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions()
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_crud_complete(self):
# Create the DB record.
obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
self.assertDictEqual(model.action, self.fake_history_workflow['action'])
self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
doc = copy.deepcopy(self.fake_history_workflow['liveaction'])
doc['start_timestamp'] = doc['start_timestamp']
doc['end_timestamp'] = doc['end_timestamp']
self.assertDictEqual(model.liveaction, doc)
self.assertIsNone(getattr(model, 'parent', None))
self.assertListEqual(model.children, self.fake_history_workflow['children'])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(ValueError, ActionExecution.get_by_id, obj.id)
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=False)
# Update the web_url field in execution. Unfortunately, we need
# the execution id for constructing the URL which we only get
# after the model is written to disk.
execution.web_url = _get_web_url_for_execution(str(execution.id))
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def update_execution(liveaction_db, publish=True):
execution = ActionExecution.get(liveaction__id=str(liveaction_db.id))
decomposed = _decompose_liveaction(liveaction_db)
kw = {}
for k, v in six.iteritems(decomposed):
kw['set__' + k] = v
if liveaction_db.status != execution.status:
# Note: If the status changes we store this transition in the "log" attribute of action
# execution
kw['push__log'] = _create_execution_log_entry(liveaction_db.status)
execution = ActionExecution.update(execution, publish=publish, **kw)
return execution
def test_crud_partial(self):
# Create the DB record.
obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger, {})
self.assertDictEqual(model.trigger_type, {})
self.assertDictEqual(model.trigger_instance, {})
self.assertDictEqual(model.rule, {})
self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
doc = copy.deepcopy(self.fake_history_subtasks[0]['liveaction'])
doc['start_timestamp'] = doc['start_timestamp']
doc['end_timestamp'] = doc['end_timestamp']
self.assertDictEqual(model.liveaction, doc)
self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
self.assertListEqual(model.children, [])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(StackStormDBObjectNotFoundError, ActionExecution.get_by_id, obj.id)
def get_one(self, id, requester_user, exclude_attributes=None, include_attributes=None,
show_secrets=False):
"""
Retrieve a single execution.
Handles requests:
GET /executions/<id>[?exclude_attributes=result,trigger_instance]
:param exclude_attributes: List of attributes to exclude from the object.
:type exclude_attributes: ``list``
"""
exclude_fields = self._validate_exclude_fields(exclude_fields=exclude_attributes)
include_fields = self._validate_include_fields(include_fields=include_attributes)
from_model_kwargs = {
'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
# Special case for id == "last"
if id == 'last':
execution_db = ActionExecution.query().order_by('-id').limit(1).only('id').first()
if not execution_db:
raise ValueError('No executions found in the database')
id = str(execution_db.id)
return self._get_one_by_id(id=id, exclude_fields=exclude_fields,
include_fields=include_fields,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_VIEW)
def get_one(self, id, output_type='all', output_format='raw', existing_only=False,
requester_user=None):
# Special case for id == "last"
if id == 'last':
execution_db = ActionExecution.query().order_by('-id').limit(1).first()
if not execution_db:
raise ValueError('No executions found in the database')
id = str(execution_db.id)
execution_db = self._get_one_by_id(id=id, requester_user=requester_user,
permission_type=PermissionType.EXECUTION_VIEW)
execution_id = str(execution_db.id)
query_filters = {}
if output_type and output_type != 'all':
query_filters['output_type'] = output_type
def existing_output_iter():
# Consume and return all of the existing lines
# pylint: disable=no-member
output_dbs = ActionExecutionOutput.query(execution_id=execution_id, **query_filters)
output = ''.join([output_db.data for output_db in output_dbs])
yield six.binary_type(output.encode('utf-8'))
def make_response():
app_iter = existing_output_iter()
res = Response(content_type='text/plain', app_iter=app_iter)
return res
res = make_response()
return res
def test_chained_executions(self):
with mock.patch("st2common.runners.register_runner", mock.MagicMock(return_value=action_chain_runner)):
liveaction = LiveActionDB(action="executions.chain")
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id), raise_exception=True)
action = action_utils.get_action_by_ref("executions.chain")
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type["name"])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction["callback"], liveaction.callback)
self.assertEqual(execution.liveaction["action"], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action["name"], "local")
self.assertEqual(record.runner["name"], "run-local")
def _format_action_exec_result(self, action_node, liveaction_db, created_at, updated_at,
error=None):
"""
Format ActionExecution result so it can be used in the final action result output.
:rtype: ``dict``
"""
assert(isinstance(created_at, datetime.datetime))
assert(isinstance(updated_at, datetime.datetime))
result = {}
execution_db = None
if liveaction_db:
execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id))
result['id'] = action_node.name
result['name'] = action_node.name
result['execution_id'] = str(execution_db.id) if execution_db else None
result['workflow'] = None
result['created_at'] = isotime.format(dt=created_at)
result['updated_at'] = isotime.format(dt=updated_at)
if error or not liveaction_db:
result['state'] = LIVEACTION_STATUS_FAILED
else:
result['state'] = liveaction_db.status
if error:
result['result'] = error
else:
result['result'] = liveaction_db.result
return result
def test_update_execution(self):
"""Test ActionExecutionDb update
"""
self.assertTrue(self.executions['execution_1'].end_timestamp is None)
self.executions['execution_1'].end_timestamp = date_utils.get_datetime_utc_now()
updated = ActionExecution.add_or_update(self.executions['execution_1'])
self.assertTrue(updated.end_timestamp == self.executions['execution_1'].end_timestamp)
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain']
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def test_chained_executions(self):
liveaction = LiveActionDB(action='core.chain')
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref('core.chain')
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action['name'], 'local')
self.assertEqual(record.runner['name'], 'run-local')
def get_all(self):
"""
List all distinct filters.
Handles requests:
GET /executions/views/filters
"""
filters = {}
for name, field in six.iteritems(SUPPORTED_FILTERS):
if name not in IGNORE_FILTERS:
if isinstance(field, six.string_types):
query = '$' + field
else:
dot_notation = list(chain.from_iterable(
('$' + item, '.') for item in field
))
dot_notation.pop(-1)
query = {'$concat': dot_notation}
aggregate = ActionExecution.aggregate([
{'$match': {'parent': None}},
{'$group': {'_id': query}}
])
filters[name] = [res['_id'] for res in aggregate['result'] if res['_id']]
return filters
def test_execution_creation_chains(self):
childliveaction = self.MODELS['liveactions']['childliveaction.yaml']
child_exec = executions_util.create_execution_object(childliveaction)
parent_execution_id = childliveaction.context['parent']['execution_id']
parent_execution = ActionExecution.get_by_id(parent_execution_id)
child_execs = parent_execution.children
self.assertTrue(str(child_exec.id) in child_execs)
def _get_execution_for_liveaction(self, liveaction):
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
if not execution:
return None
return execution
def _purge_executions(timestamp=None, action_ref=None):
if not timestamp:
print('Specify a valid timestamp to purge.')
return
if not action_ref:
action_ref = ''
print('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
def should_delete(execution_db):
if action_ref != '':
return (execution_db.liveaction['action'] == action_ref
and execution_db.start_timestamp < timestamp)
else:
return execution_db.start_timestamp < timestamp
# XXX: Think about paginating this call.
filters = {'start_timestamp__lt': isotime.parse(timestamp)}
executions = ActionExecution.query(**filters)
executions_to_delete = filter(should_delete, executions)
print('#### Total number of executions to delete: %d' % len(executions_to_delete))
# Purge execution and liveaction models now
for execution_db in executions_to_delete:
_purge_action_models(execution_db)
# Print stats
print('#### Total execution models deleted: %d' % DELETED_COUNT)
def request_cancellation(liveaction, requester):
"""
Request cancellation of an action execution.
:return: (liveaction, execution)
:rtype: tuple
"""
if liveaction.status == action_constants.LIVEACTION_STATUS_CANCELING:
return liveaction
if liveaction.status not in action_constants.LIVEACTION_CANCELABLE_STATES:
raise Exception(
'Unable to cancel liveaction "%s" because it is already in a '
'completed state.' % liveaction.id
)
result = {
'message': 'Action canceled by user.',
'user': requester
}
# Run cancelation sequence for liveaction that is in running state or
# if the liveaction is operating under a workflow.
if ('parent' in liveaction.context or
liveaction.status in action_constants.LIVEACTION_STATUS_RUNNING):
status = action_constants.LIVEACTION_STATUS_CANCELING
else:
status = action_constants.LIVEACTION_STATUS_CANCELED
liveaction = update_status(liveaction, status, result=result)
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
return (liveaction, execution)
def resume(self):
mistral_ctx = self.context.get('mistral', dict())
if not mistral_ctx.get('execution_id'):
raise Exception('Unable to resume because mistral execution_id is missing.')
# If workflow is executed under another parent workflow, resume the corresponding
# action execution for the task in the parent workflow.
if 'parent' in getattr(self, 'context', {}) and mistral_ctx.get('action_execution_id'):
mistral_action_ex_id = mistral_ctx.get('action_execution_id')
self._client.action_executions.update(mistral_action_ex_id, 'RUNNING')
# Pause the main workflow execution. Any non-workflow tasks that are still
# running will be allowed to complete gracefully.
self._client.executions.update(mistral_ctx.get('execution_id'), 'RUNNING')
# Identify the list of action executions that are workflows and cascade resume.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id, raise_exception=True)
if (child_exec.runner['name'] in action_constants.WORKFLOW_RUNNER_TYPES and
child_exec.status == action_constants.LIVEACTION_STATUS_PAUSED):
action_service.request_resume(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None)
)
return (
action_constants.LIVEACTION_STATUS_RUNNING,
self.execution.result,
self.execution.context
)
def assign_parent(child):
candidates = [v for k, v in cls.refs.iteritems() if v.action["name"] == "chain"]
if candidates:
parent = random.choice(candidates)
child["parent"] = str(parent.id)
parent.children.append(child["id"])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def request_cancellation(liveaction, requester):
"""
Request cancellation of an action execution.
:return: (liveaction, execution)
:rtype: tuple
"""
if liveaction.status == action_constants.LIVEACTION_STATUS_CANCELING:
return liveaction
if liveaction.status not in action_constants.LIVEACTION_CANCELABLE_STATES:
raise Exception('Unable to cancel execution because it is already in a completed state.')
result = {
'message': 'Action canceled by user.',
'user': requester
}
# There is real work only when liveaction is still running.
status = (action_constants.LIVEACTION_STATUS_CANCELING
if liveaction.status == action_constants.LIVEACTION_STATUS_RUNNING
else action_constants.LIVEACTION_STATUS_CANCELED)
update_status(liveaction, status, result=result)
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
return (liveaction, execution)
def _get_rerun_reference(self, context):
execution_id = context.get('re-run', {}).get('ref')
return ActionExecution.get_by_id(execution_id) if execution_id else None
def setUp(self):
super(ExecutionPermissionsResolverTestCase, self).setUp()
# Create some mock users
user_1_db = UserDB(name='custom_role_unrelated_pack_action_grant')
user_1_db = User.add_or_update(user_1_db)
self.users['custom_role_unrelated_pack_action_grant'] = user_1_db
user_2_db = UserDB(
name='custom_role_pack_action_grant_unrelated_permission')
user_2_db = User.add_or_update(user_2_db)
self.users[
'custom_role_pack_action_grant_unrelated_permission'] = user_2_db
user_3_db = UserDB(name='custom_role_pack_action_view_grant')
user_3_db = User.add_or_update(user_3_db)
self.users['custom_role_pack_action_view_grant'] = user_3_db
user_4_db = UserDB(name='custom_role_action_view_grant')
user_4_db = User.add_or_update(user_4_db)
self.users['custom_role_action_view_grant'] = user_4_db
user_5_db = UserDB(name='custom_role_pack_action_execute_grant')
user_5_db = User.add_or_update(user_5_db)
self.users['custom_role_pack_action_execute_grant'] = user_5_db
user_6_db = UserDB(name='custom_role_action_execute_grant')
user_6_db = User.add_or_update(user_6_db)
self.users['custom_role_action_execute_grant'] = user_6_db
user_7_db = UserDB(name='custom_role_pack_action_all_grant')
user_7_db = User.add_or_update(user_7_db)
self.users['custom_role_pack_action_all_grant'] = user_7_db
user_8_db = UserDB(name='custom_role_action_all_grant')
user_8_db = User.add_or_update(user_8_db)
self.users['custom_role_action_all_grant'] = user_8_db
# Create some mock resources on which permissions can be granted
action_1_db = ActionDB(pack='test_pack_2',
name='action1',
entry_point='',
runner_type={'name': 'run-local'})
action_1_db = Action.add_or_update(action_1_db)
self.resources['action_1'] = action_1_db
runner = {'name': 'run-python'}
liveaction = {'action': 'test_pack_2.action1'}
status = action_constants.LIVEACTION_STATUS_REQUESTED
action = {'uid': action_1_db.get_uid(), 'pack': 'test_pack_2'}
exec_1_db = ActionExecutionDB(action=action,
runner=runner,
liveaction=liveaction,
status=status)
exec_1_db = ActionExecution.add_or_update(exec_1_db)
self.resources['exec_1'] = exec_1_db
# Create some mock roles with associated permission grants
# Custom role - one grant to an unrelated pack
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_1'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_unrelated_pack_action_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_unrelated_pack_action_grant'] = role_db
# Custom role - one grant of unrelated permission type to parent action pack
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.RULE_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(
name='custom_role_pack_action_grant_unrelated_permission',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles[
'custom_role_pack_action_grant_unrelated_permission'] = role_db
# Custom role - one grant of "action_view" to the parent pack of the action the execution
# belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_view_grant'] = role_db
# Custom role - one grant of "action_view" to the action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_view_grant'] = role_db
# Custom role - one grant of "action_execute" to the parent pack of the action the
# execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_execute_grant'] = role_db
# Custom role - one grant of "action_execute" to the the action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_execute_grant'] = role_db
# Custom role - "action_all" grant on a parent action pack the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_pack_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_pack_action_all_grant'] = role_4_db
# Custom role - "action_all" grant on action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_action_all_grant'] = role_4_db
# Create some mock role assignments
user_db = self.users['custom_role_unrelated_pack_action_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_unrelated_pack_action_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users[
'custom_role_pack_action_grant_unrelated_permission']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.
roles['custom_role_pack_action_grant_unrelated_permission'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_view_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_view_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_execute_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_execute_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_all_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_all_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo-last'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as scheduled.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Check the status changes.
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
expected_status_changes = [
'requested', 'delayed', 'requested', 'scheduled', 'running'
]
actual_status_changes = [entry['status'] for entry in execution.log]
self.assertListEqual(actual_status_changes, expected_status_changes)
def _save_execution(execution):
return ActionExecution.add_or_update(execution)
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False):
"""
Purge action executions and corresponding live action, execution output objects.
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
to_delete_execution_dbs = []
# 1. Delete ActionExecutionDB objects
try:
# Note: We call list() on the query set object because it's lazyily evaluated otherwise
to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
deleted_count = ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution models failed for query with filters: %s.',
exec_filters)
else:
logger.info('Deleted %s action execution objects' % (deleted_count))
# 2. Delete LiveActionDB objects
try:
deleted_count = LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
else:
logger.info('Deleted %s liveaction objects' % (deleted_count))
# 3. Delete ActionExecutionOutputDB objects
to_delete_exection_ids = [str(execution_db.id) for execution_db in to_delete_execution_dbs]
output_dbs_filters = {}
output_dbs_filters['execution_id'] = {'$in': to_delete_exection_ids}
try:
deleted_count = ActionExecutionOutput.delete_by_query(**output_dbs_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (output_dbs_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
output_dbs_filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
zombie_execution_instances = len(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'],
no_dereference=True,
**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.', zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
if (getattr(liveaction_api, 'result', None) is not None
and liveaction_api.status in [
action_constants.LIVEACTION_STATUS_PAUSING,
action_constants.LIVEACTION_STATUS_PAUSED,
action_constants.LIVEACTION_STATUS_RESUMING
]):
abort(
http_client.BAD_REQUEST,
'The result is not applicable for pausing and resuming execution.'
)
try:
if (liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
else:
liveaction_db = action_service.update_status(
liveaction_db,
liveaction_api.status,
result=getattr(liveaction_api, 'result', None))
actionexecution_db = ActionExecution.get(
liveaction__id=str(liveaction_db.id))
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.')
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def _do_run(self, runner, runnertype_db, action_db, liveaction_db):
resolved_entry_point = self._get_entry_point_abs_path(action_db.pack,
action_db.entry_point)
runner.container_service = RunnerContainerService()
runner.action = action_db
runner.action_name = action_db.name
runner.liveaction = liveaction_db
runner.liveaction_id = str(liveaction_db.id)
runner.execution = ActionExecution.get(liveaction__id=runner.liveaction_id)
runner.execution_id = str(runner.execution.id)
runner.entry_point = resolved_entry_point
runner.context = getattr(liveaction_db, 'context', dict())
runner.callback = getattr(liveaction_db, 'callback', dict())
runner.libs_dir_path = self._get_action_libs_abs_path(action_db.pack,
action_db.entry_point)
# Create a temporary auth token which will be available during duration of the action
# execution
runner.auth_token = self._create_auth_token(runner.context)
updated_liveaction_db = None
try:
# Finalized parameters are resolved and then rendered. This process could
# fail. Handle the exception and report the error correctly.
runner_params, action_params = param_utils.get_finalized_params(
runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters,
liveaction_db.context)
runner.runner_parameters = runner_params
LOG.debug('Performing pre-run for runner: %s', runner.runner_id)
runner.pre_run()
# Mask secret parameters in the log context
resolved_action_params = ResolvedActionParameters(action_db=action_db,
runner_type_db=runnertype_db,
runner_parameters=runner_params,
action_parameters=action_params)
extra = {'runner': runner, 'parameters': resolved_action_params}
LOG.debug('Performing run for runner: %s' % (runner.runner_id), extra=extra)
(status, result, context) = runner.run(action_params)
try:
result = json.loads(result)
except:
pass
action_completed = status in action_constants.COMPLETED_STATES
if isinstance(runner, AsyncActionRunner) and not action_completed:
self._setup_async_query(liveaction_db.id, runnertype_db, context)
except:
LOG.exception('Failed to run action.')
_, ex, tb = sys.exc_info()
# mark execution as failed.
status = action_constants.LIVEACTION_STATUS_FAILED
# include the error message and traceback to try and provide some hints.
result = {'error': str(ex), 'traceback': ''.join(traceback.format_tb(tb, 20))}
context = None
finally:
# Log action completion
extra = {'result': result, 'status': status}
LOG.debug('Action "%s" completed.' % (action_db.name), extra=extra)
# Always clean-up the auth_token
try:
LOG.debug('Setting status: %s for liveaction: %s', status, liveaction_db.id)
updated_liveaction_db = self._update_live_action_db(liveaction_db.id, status,
result, context)
except:
error = 'Cannot update LiveAction object for id: %s, status: %s, result: %s.' % (
liveaction_db.id, status, result)
LOG.exception(error)
raise
executions.update_execution(updated_liveaction_db)
extra = {'liveaction_db': updated_liveaction_db}
LOG.debug('Updated liveaction after run', extra=extra)
# Deletion of the runner generated auth token is delayed until the token expires.
# Async actions such as Mistral workflows uses the auth token to launch other
# actions in the workflow. If the auth token is deleted here, then the actions
# in the workflow will fail with unauthorized exception.
is_async_runner = isinstance(runner, AsyncActionRunner)
action_completed = status in action_constants.COMPLETED_STATES
if not is_async_runner or (is_async_runner and action_completed):
try:
self._delete_auth_token(runner.auth_token)
except:
LOG.exception('Unable to clean-up auth_token.')
LOG.debug('Performing post_run for runner: %s', runner.runner_id)
runner.post_run(status, result)
runner.container_service = None
return updated_liveaction_db
def purge_inquiries(logger):
"""Purge Inquiries that have exceeded their configured TTL
At the moment, Inquiries do not have their own database model, so this function effectively
is another, more specialized GC for executions. It will look for executions with a 'pending'
status that use the 'inquirer' runner, which is the current definition for an Inquiry.
Then it will mark those that have a nonzero TTL have existed longer than their TTL as
"timed out". It will then request that the parent workflow(s) resume, where the failure
can be handled as the user desires.
"""
# Get all existing Inquiries
filters = {
'runner__name': 'inquirer',
'status': action_constants.LIVEACTION_STATUS_PENDING
}
inquiries = list(ActionExecution.query(**filters))
gc_count = 0
# Inspect each Inquiry, and determine if TTL is expired
for inquiry in inquiries:
ttl = int(inquiry.result.get('ttl'))
if ttl <= 0:
logger.debug("Inquiry %s has a TTL of %s. Skipping." %
(inquiry.id, ttl))
continue
min_since_creation = int(
(get_datetime_utc_now() - inquiry.start_timestamp).total_seconds()
/ 60)
logger.debug(
"Inquiry %s has a TTL of %s and was started %s minute(s) ago" %
(inquiry.id, ttl, min_since_creation))
if min_since_creation > ttl:
gc_count += 1
logger.info("TTL expired for Inquiry %s. Marking as timed out." %
inquiry.id)
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_TIMED_OUT,
result=inquiry.result,
liveaction_id=inquiry.liveaction.get('id'))
executions.update_execution(liveaction_db)
# Call Inquiry runner's post_run to trigger callback to workflow
action_db = get_action_by_ref(liveaction_db.action)
invoke_post_run(liveaction_db=liveaction_db, action_db=action_db)
if liveaction_db.context.get("parent"):
# Request that root workflow resumes
root_liveaction = action_service.get_root_liveaction(
liveaction_db)
action_service.request_resume(
root_liveaction, UserDB(cfg.CONF.system_user.user))
logger.info('Marked %s ttl-expired Inquiries as "timed out".' % (gc_count))
def get_one(self, id, output_type=None, requester_user=None):
# Special case for id == "last"
if id == 'last':
execution_db = ActionExecution.query().order_by('-id').limit(
1).first()
else:
execution_db = self._get_one_by_id(
id=id,
requester_user=requester_user,
permission_type=PermissionType.EXECUTION_VIEW)
execution_id = str(execution_db.id)
query_filters = {}
if output_type and output_type != 'all':
query_filters['output_type'] = output_type
def existing_output_iter():
# Consume and return all of the existing lines
# pylint: disable=no-member
output_dbs = ActionExecutionOutput.query(execution_id=execution_id,
**query_filters)
# Note: We return all at once instead of yield line by line to avoid multiple socket
# writes and to achieve better performance
output = ''.join([output_db.data for output_db in output_dbs])
yield six.binary_type(output.encode('utf-8'))
def new_output_iter():
def noop_gen():
yield ''
# Bail out if execution has already completed / been paused
if execution_db.status in self.CLOSE_STREAM_LIVEACTION_STATES:
return noop_gen()
# Wait for and return any new line which may come in
execution_ids = [execution_id]
listener = get_listener(name='execution_output') # pylint: disable=no-member
gen = listener.generator(execution_ids=execution_ids)
def format(gen):
for pack in gen:
if not pack:
continue
else:
(_, model_api) = pack
# Note: gunicorn wsgi handler expect bytes, not unicode
# pylint: disable=no-member
if isinstance(model_api, ActionExecutionOutputAPI):
if output_type and model_api.output_type != output_type:
continue
yield six.binary_type(
model_api.data.encode('utf-8'))
elif isinstance(model_api, ActionExecutionAPI):
if model_api.status in self.CLOSE_STREAM_LIVEACTION_STATES:
yield six.binary_type('')
break
else:
LOG.debug('Unrecognized message type: %s' %
(model_api))
gen = format(gen)
return gen
def make_response():
app_iter = itertools.chain(existing_output_iter(),
new_output_iter())
res = Response(content_type='text/plain', app_iter=app_iter)
return res
res = make_response()
return res
def test_get_output_running_execution(self):
# Retrieve lister instance to avoid race with listener connection not being established
# early enough for tests to pass.
# NOTE: This only affects tests where listeners are not pre-initialized.
listener = get_listener(name='execution_output')
eventlet.sleep(1.0)
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db, publish=False)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
# Retrieve data while execution is running - endpoint return new data once it's available
# and block until the execution finishes
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
listener.shutdown()
def purge_executions(logger,
timestamp,
action_ref=None,
purge_incomplete=False):
"""
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
# TODO: Update this code to return statistics on deleted objects once we
# upgrade to newer version of MongoDB where delete_by_query actually returns
# some data
try:
ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception(
'Deletion of execution models failed for query with filters: %s.',
exec_filters)
try:
LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception(
'Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
zombie_execution_instances = len(ActionExecution.query(**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.',
zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.',
zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.',
timestamp)
def request(liveaction):
"""
Request an action execution.
:return: (liveaction, execution)
:rtype: tuple
"""
# Use the user context from the parent action execution. Subtasks in a workflow
# action can be invoked by a system user and so we want to use the user context
# from the original workflow action.
if getattr(liveaction, 'context', None) and 'parent' in liveaction.context:
parent = LiveAction.get_by_id(liveaction.context['parent'])
liveaction.context['user'] = getattr(parent, 'context',
dict()).get('user')
# Validate action.
action_db = action_utils.get_action_by_ref(liveaction.action)
if not action_db:
raise ValueError('Action "%s" cannot be found.' % liveaction.action)
if not action_db.enabled:
raise ValueError('Unable to execute. Action "%s" is disabled.' %
liveaction.action)
runnertype_db = action_utils.get_runnertype_by_name(
action_db.runner_type['name'])
if not hasattr(liveaction, 'parameters'):
liveaction.parameters = dict()
# Validate action parameters.
schema = util_schema.get_parameter_schema(action_db)
validator = util_schema.get_validator()
util_schema.validate(liveaction.parameters,
schema,
validator,
use_default=True)
# validate that no immutable params are being overriden. Although possible to
# ignore the override it is safer to inform the user to avoid surprises.
immutables = _get_immutable_params(action_db.parameters)
immutables.extend(_get_immutable_params(runnertype_db.runner_parameters))
overridden_immutables = [
p for p in six.iterkeys(liveaction.parameters) if p in immutables
]
if len(overridden_immutables) > 0:
raise ValueError(
'Override of immutable parameter(s) %s is unsupported.' %
str(overridden_immutables))
# Set notification settings for action.
# XXX: There are cases when we don't want notifications to be sent for a particular
# execution. So we should look at liveaction.parameters['notify']
# and not set liveaction.notify.
if action_db.notify:
liveaction.notify = action_db.notify
# Write to database and send to message queue.
liveaction.status = action_constants.LIVEACTION_STATUS_REQUESTED
liveaction.start_timestamp = date_utils.get_datetime_utc_now()
# Publish creation after both liveaction and actionexecution are created.
liveaction = LiveAction.add_or_update(liveaction, publish=False)
execution = executions.create_execution_object(liveaction, publish=False)
# Assume that this is a creation.
LiveAction.publish_create(liveaction)
LiveAction.publish_status(liveaction)
ActionExecution.publish_create(execution)
extra = {'liveaction_db': liveaction, 'execution_db': execution}
LOG.audit(
'Action execution requested. LiveAction.id=%s, ActionExecution.id=%s' %
(liveaction.id, execution.id),
extra=extra)
return liveaction, execution
def is_execution_canceled(execution_id):
try:
execution = ActionExecution.get_by_id(execution_id)
return execution.status == action_constants.LIVEACTION_STATUS_CANCELED
except:
return False # XXX: What to do here?
def _append_view_properties(self, rule_enforcement_apis):
"""
Method which appends corresponding execution (if available) and trigger instance object
properties.
"""
trigger_instance_ids = set([])
execution_ids = []
for rule_enforcement_api in rule_enforcement_apis:
if rule_enforcement_api.get('trigger_instance_id', None):
trigger_instance_ids.add(str(rule_enforcement_api['trigger_instance_id']))
if rule_enforcement_api.get('execution_id', None):
execution_ids.append(rule_enforcement_api['execution_id'])
# 1. Retrieve corresponding execution objects
# NOTE: Executions contain a lot of field and could contain a lot of data so we only
# retrieve fields we need
only_fields = [
'id',
'action.ref',
'action.parameters',
'runner.name',
'runner.runner_parameters',
'parameters',
'status'
]
execution_dbs = ActionExecution.query(id__in=execution_ids,
only_fields=only_fields)
execution_dbs_by_id = {}
for execution_db in execution_dbs:
execution_dbs_by_id[str(execution_db.id)] = execution_db
# 2. Retrieve corresponding trigger instance objects
trigger_instance_dbs = TriggerInstance.query(id__in=list(trigger_instance_ids))
trigger_instance_dbs_by_id = {}
for trigger_instance_db in trigger_instance_dbs:
trigger_instance_dbs_by_id[str(trigger_instance_db.id)] = trigger_instance_db
# Ammend rule enforcement objects with additional data
for rule_enforcement_api in rule_enforcement_apis:
rule_enforcement_api['trigger_instance'] = {}
rule_enforcement_api['execution'] = {}
trigger_instance_id = rule_enforcement_api.get('trigger_instance_id', None)
execution_id = rule_enforcement_api.get('execution_id', None)
trigger_instance_db = trigger_instance_dbs_by_id.get(trigger_instance_id, None)
execution_db = execution_dbs_by_id.get(execution_id, None)
if trigger_instance_db:
trigger_instance_api = TriggerInstanceAPI.from_model(trigger_instance_db)
rule_enforcement_api['trigger_instance'] = trigger_instance_api
if execution_db:
execution_api = ActionExecutionAPI.from_model(execution_db)
rule_enforcement_api['execution'] = execution_api
return rule_enforcement_apis
def test_get_output_running_execution(self):
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
pass
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
lines = resp.text.strip().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'stdout before start')
self.assertEqual(lines[1], 'stdout mid 1')
self.assertEqual(lines[2], 'stdout pre finish 1')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
lines = resp.text.strip().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'stdout before start')
self.assertEqual(lines[1], 'stdout mid 1')
self.assertEqual(lines[2], 'stdout pre finish 1')
def _get_all_executions_from_db(self):
return ActionExecution.get_all() # XXX: Paginated call.
def setUpClass(cls):
super(TestActionExecutionFilters, cls).setUpClass()
cls.dt_base = date_utils.add_utc_tz(
datetime.datetime(2014, 12, 25, 0, 0, 0))
cls.num_records = 100
cls.refs = {}
cls.start_timestamps = []
cls.fake_types = [{
'trigger':
copy.deepcopy(fixture.ARTIFACTS['trigger']),
'trigger_type':
copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger_instance':
copy.deepcopy(fixture.ARTIFACTS['trigger_instance']),
'rule':
copy.deepcopy(fixture.ARTIFACTS['rule']),
'action':
copy.deepcopy(fixture.ARTIFACTS['actions']['chain']),
'runner':
copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']),
'liveaction':
copy.deepcopy(fixture.ARTIFACTS['liveactions']['workflow']),
'context':
copy.deepcopy(fixture.ARTIFACTS['context']),
'children': []
}, {
'action':
copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
'runner':
copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
'liveaction':
copy.deepcopy(fixture.ARTIFACTS['liveactions']['task1'])
}]
def assign_parent(child):
candidates = [
v for k, v in cls.refs.items() if v.action['name'] == 'chain'
]
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(
parent.id)] = ActionExecution.add_or_update(parent)
for i in range(cls.num_records):
obj_id = str(bson.ObjectId())
timestamp = cls.dt_base + datetime.timedelta(seconds=i)
fake_type = random.choice(cls.fake_types)
data = copy.deepcopy(fake_type)
data['id'] = obj_id
data['start_timestamp'] = isotime.format(timestamp, offset=False)
data['end_timestamp'] = isotime.format(timestamp, offset=False)
data['status'] = data['liveaction']['status']
data['result'] = data['liveaction']['result']
if fake_type['action']['name'] == 'local' and random.choice(
[True, False]):
assign_parent(data)
wb_obj = ActionExecutionAPI(**data)
db_obj = ActionExecutionAPI.to_model(wb_obj)
cls.refs[obj_id] = ActionExecution.add_or_update(db_obj)
cls.start_timestamps.append(timestamp)
cls.start_timestamps = sorted(cls.start_timestamps)
def test_chain_cancel_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + "." + "test_cancel_with_subworkflow"
params = {"tempfile": path, "message": "foobar"}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to cancel.
task1_live, task1_exec = action_service.request_cancellation(
task1_live, USERNAME)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait until the parent liveaction is canceled.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 1)
subworkflow = liveaction.result["tasks"][0]
self.assertEqual(len(subworkflow["result"]["tasks"]), 1)
self.assertEqual(subworkflow["state"],
action_constants.LIVEACTION_STATUS_CANCELED)
def get_one(self, id, output_type='all', requester_user=None):
# Special case for id == "last"
if id == 'last':
execution_db = ActionExecution.query().order_by('-id').limit(
1).first()
if not execution_db:
raise ValueError('No executions found in the database')
id = str(execution_db.id)
execution_db = self._get_one_by_id(
id=id,
requester_user=requester_user,
permission_type=PermissionType.EXECUTION_VIEW)
execution_id = str(execution_db.id)
query_filters = {}
if output_type and output_type != 'all':
query_filters['output_type'] = output_type
def format_output_object(output_db_or_api):
if isinstance(output_db_or_api, ActionExecutionOutputDB):
data = ActionExecutionOutputAPI.from_model(output_db_or_api)
elif isinstance(output_db_or_api, ActionExecutionOutputAPI):
data = output_db_or_api
else:
raise ValueError('Unsupported format: %s' %
(type(output_db_or_api)))
event = 'st2.execution.output__create'
result = 'event: %s\ndata: %s\n\n' % (
event, json_encode(data, indent=None))
return result
def existing_output_iter():
# Consume and return all of the existing lines
output_dbs = ActionExecutionOutput.query(execution_id=execution_id,
**query_filters)
# Note: We return all at once instead of yield line by line to avoid multiple socket
# writes and to achieve better performance
output = [
format_output_object(output_db) for output_db in output_dbs
]
output = ''.join(output)
yield six.binary_type(output.encode('utf-8'))
def new_output_iter():
def noop_gen():
yield six.binary_type(NO_MORE_DATA_EVENT.encode('utf-8'))
# Bail out if execution has already completed / been paused
if execution_db.status in self.CLOSE_STREAM_LIVEACTION_STATES:
return noop_gen()
# Wait for and return any new line which may come in
execution_ids = [execution_id]
listener = get_listener(name='execution_output') # pylint: disable=no-member
gen = listener.generator(execution_ids=execution_ids)
def format(gen):
for pack in gen:
if not pack:
continue
else:
(_, model_api) = pack
# Note: gunicorn wsgi handler expect bytes, not unicode
# pylint: disable=no-member
if isinstance(model_api, ActionExecutionOutputAPI):
if output_type and output_type != 'all' and \
model_api.output_type != output_type:
continue
output = format_output_object(model_api).encode(
'utf-8')
yield six.binary_type(output)
elif isinstance(model_api, ActionExecutionAPI):
if model_api.status in self.CLOSE_STREAM_LIVEACTION_STATES:
yield six.binary_type(
NO_MORE_DATA_EVENT.encode('utf-8'))
break
else:
LOG.debug('Unrecognized message type: %s' %
(model_api))
gen = format(gen)
return gen
def make_response():
app_iter = itertools.chain(existing_output_iter(),
new_output_iter())
res = Response(headerlist=[("X-Accel-Buffering", "no"),
('Cache-Control', 'no-cache'),
("Content-Type",
"text/event-stream; charset=UTF-8")],
app_iter=app_iter)
return res
res = make_response()
return res
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, 'result', None)
liveaction_db = action_service.update_status(liveaction_db, status, result)
actionexecution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30 # > 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2 # < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert some mock output objects where start_timestamp > action_executions_output_ttl
new_output_count = 5
ttl_days = 15 # > 10 and < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_output_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
execs = ActionExecution.get_all()
self.assertEqual(
len(execs),
(old_executions_count + new_executions_count + new_output_count))
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(
len(stdout_dbs),
(old_executions_count + new_executions_count + new_output_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(
len(stderr_dbs),
(old_executions_count + new_executions_count + new_output_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
concurrency.sleep(15)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old executions and corresponding objects should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count + new_output_count))
# Collection for output objects older than 10 days is also enabled, so those objects
# should be deleted as well
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), (new_executions_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), (new_executions_count))
def setUp(self):
super(InquiryPermissionsResolverTestCase, self).setUp()
# Create some mock users
user_1_db = UserDB(name='custom_role_inquiry_list_grant')
user_1_db = User.add_or_update(user_1_db)
self.users['custom_role_inquiry_list_grant'] = user_1_db
user_2_db = UserDB(name='custom_role_inquiry_view_grant')
user_2_db = User.add_or_update(user_2_db)
self.users['custom_role_inquiry_view_grant'] = user_2_db
user_3_db = UserDB(name='custom_role_inquiry_respond_grant')
user_3_db = User.add_or_update(user_3_db)
self.users['custom_role_inquiry_respond_grant'] = user_3_db
user_4_db = UserDB(name='custom_role_inquiry_all_grant')
user_4_db = User.add_or_update(user_4_db)
self.users['custom_role_inquiry_all_grant'] = user_4_db
user_5_db = UserDB(name='custom_role_inquiry_inherit')
user_5_db = User.add_or_update(user_5_db)
self.users['custom_role_inquiry_inherit'] = user_5_db
# Create a workflow for testing inheritance of action_execute permission
# to inquiry_respond permission
wf_db = ActionDB(pack='examples',
name='mistral-ask-basic',
entry_point='',
runner_type={'name': 'mistral-v2'})
wf_db = Action.add_or_update(wf_db)
self.resources['wf'] = wf_db
runner = {'name': 'mistral-v2'}
liveaction = {'action': 'examples.mistral-ask-basic'}
status = action_constants.LIVEACTION_STATUS_PAUSED
# Spawn workflow
action = {'uid': wf_db.get_uid(), 'pack': 'examples'}
wf_exc_db = ActionExecutionDB(action=action,
runner=runner,
liveaction=liveaction,
status=status)
wf_exc_db = ActionExecution.add_or_update(wf_exc_db)
# Create an Inquiry on which permissions can be granted
action_1_db = ActionDB(pack='core',
name='ask',
entry_point='',
runner_type={'name': 'inquirer'})
action_1_db = Action.add_or_update(action_1_db)
self.resources['action_1'] = action_1_db
runner = {'name': 'inquirer'}
liveaction = {'action': 'core.ask'}
status = action_constants.LIVEACTION_STATUS_PENDING
# For now, Inquiries are "borrowing" the ActionExecutionDB model,
# so we have to test with that model
action = {'uid': action_1_db.get_uid(), 'pack': 'core'}
inquiry_1_db = ActionExecutionDB(action=action,
runner=runner,
liveaction=liveaction,
status=status)
# A separate inquiry that has a parent (so we can test workflow permission inheritance)
inquiry_2_db = ActionExecutionDB(action=action,
runner=runner,
liveaction=liveaction,
status=status,
parent=str(wf_exc_db.id))
# A bit gross, but it's what we have to do since Inquiries
# don't yet have their own data model
def get_uid():
return "inquiry"
inquiry_1_db.get_uid = get_uid
inquiry_2_db.get_uid = get_uid
inquiry_1_db = ActionExecution.add_or_update(inquiry_1_db)
inquiry_2_db = ActionExecution.add_or_update(inquiry_2_db)
self.resources['inquiry_1'] = inquiry_1_db
self.resources['inquiry_2'] = inquiry_2_db
############################################################
# Create some mock roles with associated permission grants #
############################################################
# Custom role - "inquiry_list" grant
grant_db = PermissionGrantDB(
resource_uid=self.resources['inquiry_1'].get_uid(),
resource_type=ResourceType.INQUIRY,
permission_types=[PermissionType.INQUIRY_LIST])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_inquiry_list_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_inquiry_list_grant'] = role_db
# Custom role - "inquiry_view" grant
grant_db = PermissionGrantDB(
resource_uid=self.resources['inquiry_1'].get_uid(),
resource_type=ResourceType.INQUIRY,
permission_types=[PermissionType.INQUIRY_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_inquiry_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_inquiry_view_grant'] = role_db
# Custom role - "inquiry_respond" grant
grant_db = PermissionGrantDB(
resource_uid=self.resources['inquiry_1'].get_uid(),
resource_type=ResourceType.INQUIRY,
permission_types=[PermissionType.INQUIRY_RESPOND])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_inquiry_respond_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_inquiry_respond_grant'] = role_db
# Custom role - "inquiry_all" grant
grant_db = PermissionGrantDB(
resource_uid=self.resources['inquiry_1'].get_uid(),
resource_type=ResourceType.INQUIRY,
permission_types=[PermissionType.INQUIRY_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_inquiry_all_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_inquiry_all_grant'] = role_db
# Custom role - inheritance grant
grant_db = PermissionGrantDB(
resource_uid=self.resources['wf'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_inquiry_inherit',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_inquiry_inherit'] = role_db
#####################################
# Create some mock role assignments #
#####################################
user_db = self.users['custom_role_inquiry_list_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_inquiry_list_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_inquiry_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_inquiry_view_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_inquiry_respond_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_inquiry_respond_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_inquiry_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_inquiry_all_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_inquiry_inherit']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_inquiry_inherit'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models["executions"]["execution1.yaml"])
exec_model["start_timestamp"] = start_ts
exec_model["status"] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model["id"] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model["id"], count=1
)
exec_model = copy.deepcopy(self.models["executions"]["execution1.yaml"])
exec_model["start_timestamp"] = start_ts
exec_model["status"] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model["id"] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model["id"], count=1
)
exec_model = copy.deepcopy(self.models["executions"]["execution1.yaml"])
exec_model["start_timestamp"] = start_ts
exec_model["status"] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model["id"] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model["id"], count=1
)
exec_model = copy.deepcopy(self.models["executions"]["execution1.yaml"])
exec_model["start_timestamp"] = start_ts
exec_model["status"] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model["id"] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model["id"], count=1
)
exec_model = copy.deepcopy(self.models["executions"]["execution1.yaml"])
exec_model["start_timestamp"] = start_ts
exec_model["status"] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model["id"] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model["id"], count=1
)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type="stdout")
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type="stderr")
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(
logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False
)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type="stdout")
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type="stderr")
self.assertEqual(len(stderr_dbs), 5)
purge_executions(
logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True
)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type="stdout")
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type="stderr")
self.assertEqual(len(stderr_dbs), 0)
def create_execution_object(liveaction,
action_db=None,
runnertype_db=None,
publish=True):
if not action_db:
action_db = action_utils.get_action_by_ref(liveaction.action)
if not runnertype_db:
runnertype_db = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runnertype_db))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule,
liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(
db_api=Trigger, ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(
TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
# TODO: This object initialization takes 20-30or so ms
execution = ActionExecutionDB(**attrs)
# TODO: Do 100% research this is fully safe and unique in distributed setups
execution.id = ObjectId()
execution.web_url = _get_web_url_for_execution(str(execution.id))
# NOTE: User input data is already validate as part of the API request,
# other data is set by us. Skipping validation here makes operation 10%-30% faster
execution = ActionExecution.add_or_update(execution,
publish=publish,
validate=False)
if parent and str(execution.id) not in parent.children:
values = {}
values['push__children'] = str(execution.id)
ActionExecution.update(parent, **values)
return execution
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(
task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(
task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 2)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_SUCCEEDED)
def _get_action_execution(self, **kwargs):
return ActionExecution.get(**kwargs)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
