def abandon_execution_if_incomplete(liveaction_id, publish=True):
"""
Marks execution as abandoned if it is still incomplete. Abandoning an
execution implies that its end state is unknown and cannot anylonger
be determined. This method should only be called if the owning process
is certain it can no longer determine status of an execution.
"""
liveaction_db = action_utils.get_liveaction_by_id(liveaction_id)
# No need to abandon and already complete action
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
raise ValueError('LiveAction %s already in a completed state %s.' %
(liveaction_id, liveaction_db.status))
# Update status to reflect execution being abandoned.
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_ABANDONED,
liveaction_db=liveaction_db,
result={})
execution_db = update_execution(liveaction_db, publish=publish)
LOG.info('Marked execution %s as %s.', execution_db.id,
action_constants.LIVEACTION_STATUS_ABANDONED)
# Invoke post run on the action to execute post run operations such as callback.
runners_utils.invoke_post_run(liveaction_db)
return execution_db
def abandon_execution_if_incomplete(liveaction_id, publish=True):
"""
Marks execution as abandoned if it is still incomplete. Abandoning an
execution implies that its end state is unknown and cannot anylonger
be determined. This method should only be called if the owning process
is certain it can no longer determine status of an execution.
"""
liveaction_db = action_utils.get_liveaction_by_id(liveaction_id)
# No need to abandon and already complete action
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
raise ValueError('LiveAction %s already in a completed state %s.' %
(liveaction_id, liveaction_db.status))
# Update status to reflect execution being abandoned.
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_ABANDONED,
liveaction_db=liveaction_db,
result={})
execution_db = update_execution(liveaction_db, publish=publish)
LOG.info('Marked execution %s as %s.', execution_db.id,
action_constants.LIVEACTION_STATUS_ABANDONED)
# Invoke post run on the action to execute post run operations such as callback.
runners_utils.invoke_post_run(liveaction_db)
return execution_db
def purge_inquiries(logger):
"""Purge Inquiries that have exceeded their configured TTL
At the moment, Inquiries do not have their own database model, so this function effectively
is another, more specialized GC for executions. It will look for executions with a 'pending'
status that use the 'inquirer' runner, which is the current definition for an Inquiry.
Then it will mark those that have a nonzero TTL have existed longer than their TTL as
"timed out". It will then request that the parent workflow(s) resume, where the failure
can be handled as the user desires.
"""
# Get all existing Inquiries
filters = {'runner__name': 'inquirer', 'status': action_constants.LIVEACTION_STATUS_PENDING}
inquiries = list(ActionExecution.query(**filters))
gc_count = 0
# Inspect each Inquiry, and determine if TTL is expired
for inquiry in inquiries:
ttl = int(inquiry.result.get('ttl'))
if ttl <= 0:
logger.debug("Inquiry %s has a TTL of %s. Skipping." % (inquiry.id, ttl))
continue
min_since_creation = int(
(get_datetime_utc_now() - inquiry.start_timestamp).total_seconds() / 60
)
logger.debug("Inquiry %s has a TTL of %s and was started %s minute(s) ago" % (
inquiry.id, ttl, min_since_creation))
if min_since_creation > ttl:
gc_count += 1
logger.info("TTL expired for Inquiry %s. Marking as timed out." % inquiry.id)
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_TIMED_OUT,
result=inquiry.result,
liveaction_id=inquiry.liveaction.get('id'))
executions.update_execution(liveaction_db)
# Call Inquiry runner's post_run to trigger callback to workflow
action_db = get_action_by_ref(liveaction_db.action)
invoke_post_run(liveaction_db=liveaction_db, action_db=action_db)
if liveaction_db.context.get("parent"):
# Request that root workflow resumes
root_liveaction = action_service.get_root_liveaction(liveaction_db)
action_service.request_resume(
root_liveaction,
UserDB(cfg.CONF.system_user.user)
)
logger.info('Marked %s ttl-expired Inquiries as "timed out".' % (gc_count))
def _query_and_save_results(self, query_context, last_query_time=None):
this_query_time = time.time()
execution_id = query_context.execution_id
actual_query_context = query_context.query_context
LOG.debug('Querying external service for results. Context: %s' %
actual_query_context)
try:
(status, results) = self.query(execution_id,
actual_query_context,
last_query_time=last_query_time)
except:
LOG.exception('Failed querying results for liveaction_id %s.',
execution_id)
if self.delete_state_object_on_error:
self._delete_state_object(query_context)
LOG.debug('Removed state object %s.', query_context)
return
liveaction_db = None
try:
liveaction_db = self._update_action_results(
execution_id, status, results)
except Exception:
LOG.exception(
'Failed updating action results for liveaction_id %s',
execution_id)
if self.delete_state_object_on_error:
self._delete_state_object(query_context)
LOG.debug('Removed state object %s.', query_context)
return
if (status in action_constants.LIVEACTION_COMPLETED_STATES
or status == action_constants.LIVEACTION_STATUS_PAUSED):
runners_utils.invoke_post_run(liveaction_db)
self._delete_state_object(query_context)
LOG.debug("Detailed workflow liveaction results - ",
extra={'liveaction_db': liveaction_db})
return
if not self._is_state_object_exist(query_context):
LOG.warning(
'Query for liveaction_id %s is not rescheduled '
'because state object %s has been deleted.', execution_id,
query_context.id)
return
self._query_contexts.put((this_query_time, query_context))
def update_status(
liveaction,
new_status,
result=None,
publish=True,
set_result_size=False,
context=None,
):
if liveaction.status == new_status:
return liveaction
old_status = liveaction.status
updates = {
"liveaction_id": liveaction.id,
"status": new_status,
"result": result,
"publish": False,
"context": context,
}
if new_status in action_constants.LIVEACTION_COMPLETED_STATES:
updates["end_timestamp"] = date_utils.get_datetime_utc_now()
liveaction = action_utils.update_liveaction_status(**updates)
action_execution = executions.update_execution(
liveaction, set_result_size=set_result_size)
msg = ("The status of action execution is changed from %s to %s. "
"<LiveAction.id=%s, ActionExecution.id=%s>" %
(old_status, new_status, liveaction.id, action_execution.id))
extra = {
"action_execution_db": action_execution,
"liveaction_db": liveaction
}
LOG.audit(msg, extra=extra)
LOG.info(msg)
# Invoke post run if liveaction status is completed or paused.
if (new_status in action_constants.LIVEACTION_COMPLETED_STATES
or new_status == action_constants.LIVEACTION_STATUS_PAUSED):
runners_utils.invoke_post_run(liveaction)
if publish:
LiveAction.publish_status(liveaction)
return liveaction
def _query_and_save_results(self, query_context, last_query_time=None):
this_query_time = time.time()
execution_id = query_context.execution_id
actual_query_context = query_context.query_context
LOG.debug('Querying external service for results. Context: %s' %
actual_query_context)
try:
(status, results) = self.query(execution_id,
actual_query_context,
last_query_time=last_query_time)
except:
LOG.exception('Failed querying results for liveaction_id %s.',
execution_id)
if self.delete_state_object_on_error:
self._delete_state_object(query_context)
LOG.debug('Removed state object %s.', query_context)
return
liveaction_db = None
try:
liveaction_db = self._update_action_results(
execution_id, status, results)
except Exception:
LOG.exception(
'Failed updating action results for liveaction_id %s',
execution_id)
if self.delete_state_object_on_error:
self._delete_state_object(query_context)
LOG.debug('Removed state object %s.', query_context)
return
if status in action_constants.LIVEACTION_COMPLETED_STATES:
if status != action_constants.LIVEACTION_STATUS_CANCELED:
runners_utils.invoke_post_run(liveaction_db)
self._delete_state_object(query_context)
return
self._query_contexts.put((this_query_time, query_context))
def update_status(liveaction, new_status, result=None, publish=True):
if liveaction.status == new_status:
return liveaction
old_status = liveaction.status
updates = {
'liveaction_id': liveaction.id,
'status': new_status,
'result': result,
'publish': False
}
if new_status in action_constants.LIVEACTION_COMPLETED_STATES:
updates['end_timestamp'] = date_utils.get_datetime_utc_now()
liveaction = action_utils.update_liveaction_status(**updates)
action_execution = executions.update_execution(liveaction)
msg = ('The status of action execution is changed from %s to %s. '
'<LiveAction.id=%s, ActionExecution.id=%s>' %
(old_status, new_status, liveaction.id, action_execution.id))
extra = {
'action_execution_db': action_execution,
'liveaction_db': liveaction
}
LOG.audit(msg, extra=extra)
LOG.info(msg)
# Invoke post run if liveaction status is completed or paused.
if (new_status in action_constants.LIVEACTION_COMPLETED_STATES
or new_status == action_constants.LIVEACTION_STATUS_PAUSED):
runners_utils.invoke_post_run(liveaction)
if publish:
LiveAction.publish_status(liveaction)
return liveaction
def update_status(liveaction, new_status, result=None, publish=True):
if liveaction.status == new_status:
return liveaction
old_status = liveaction.status
updates = {
'liveaction_id': liveaction.id,
'status': new_status,
'result': result,
'publish': False
}
if new_status in action_constants.LIVEACTION_COMPLETED_STATES:
updates['end_timestamp'] = date_utils.get_datetime_utc_now()
liveaction = action_utils.update_liveaction_status(**updates)
action_execution = executions.update_execution(liveaction)
msg = ('The status of action execution is changed from %s to %s. '
'<LiveAction.id=%s, ActionExecution.id=%s>' % (old_status,
new_status, liveaction.id, action_execution.id))
extra = {
'action_execution_db': action_execution,
'liveaction_db': liveaction
}
LOG.audit(msg, extra=extra)
LOG.info(msg)
# Invoke post run if liveaction status is completed or paused.
if (new_status in action_constants.LIVEACTION_COMPLETED_STATES or
new_status == action_constants.LIVEACTION_STATUS_PAUSED):
runners_utils.invoke_post_run(liveaction)
if publish:
LiveAction.publish_status(liveaction)
return liveaction
def update_execution_records(wf_ex_db,
conductor,
update_lv_ac_on_statuses=None,
pub_wf_ex=False,
pub_lv_ac=True,
pub_ac_ex=True):
wf_ac_ex_id = wf_ex_db.action_execution
# Determine if workflow status has changed.
wf_old_status = wf_ex_db.status
wf_ex_db.status = conductor.get_workflow_status()
status_changed = (wf_old_status != wf_ex_db.status)
if status_changed:
msg = '[%s] Updating workflow execution from status "%s" to "%s".'
LOG.info(msg, wf_ac_ex_id, wf_old_status, wf_ex_db.status)
# Update timestamp and output if workflow is completed.
if wf_ex_db.status in statuses.COMPLETED_STATUSES:
wf_ex_db.end_timestamp = date_utils.get_datetime_utc_now()
wf_ex_db.output = conductor.get_workflow_output()
# Update task flow and other attributes.
wf_ex_db.errors = copy.deepcopy(conductor.errors)
wf_ex_db.state = conductor.workflow_state.serialize()
# Write changes to the database.
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db,
publish=pub_wf_ex)
# Return if workflow execution status is not specified in update_lv_ac_on_statuses.
if (isinstance(update_lv_ac_on_statuses, list)
and wf_ex_db.status not in update_lv_ac_on_statuses):
return
# Update the corresponding liveaction and action execution for the workflow.
wf_ac_ex_db = ex_db_access.ActionExecution.get_by_id(
wf_ex_db.action_execution)
wf_lv_ac_db = action_utils.get_liveaction_by_id(
wf_ac_ex_db.liveaction['id'])
# Gather result for liveaction and action execution.
result = {'output': wf_ex_db.output or None}
if wf_ex_db.status in statuses.ABENDED_STATUSES:
result['errors'] = wf_ex_db.errors
for wf_ex_error in wf_ex_db.errors:
msg = '[%s] Workflow execution completed with errors.'
LOG.error(msg, wf_ac_ex_id, extra=wf_ex_error)
# Sync update with corresponding liveaction and action execution.
if pub_lv_ac or pub_ac_ex:
pub_lv_ac = (wf_lv_ac_db.status != wf_ex_db.status)
pub_ac_ex = pub_lv_ac
if wf_lv_ac_db.status != wf_ex_db.status:
msg = '[%s] Updating workflow liveaction from status "%s" to "%s".'
LOG.debug(msg, wf_ac_ex_id, wf_lv_ac_db.status, wf_ex_db.status)
msg = '[%s] Workflow liveaction status change %s be published.'
LOG.debug(msg, wf_ac_ex_id, 'will' if pub_lv_ac else 'will not')
msg = '[%s] Workflow action execution status change %s be published.'
LOG.debug(msg, wf_ac_ex_id, 'will' if pub_ac_ex else 'will not')
wf_lv_ac_db = action_utils.update_liveaction_status(
status=wf_ex_db.status,
result=result,
end_timestamp=wf_ex_db.end_timestamp,
liveaction_db=wf_lv_ac_db,
publish=pub_lv_ac)
ex_svc.update_execution(wf_lv_ac_db, publish=pub_ac_ex)
# Invoke post run on the liveaction for the workflow execution.
if status_changed and wf_lv_ac_db.status in ac_const.LIVEACTION_COMPLETED_STATES:
LOG.info(
'[%s] Workflow action execution is completed and invoking post run.',
wf_ac_ex_id)
runners_utils.invoke_post_run(wf_lv_ac_db)
def test_invoke_post_run_action_does_not_exist(self):
utils.invoke_post_run(self.liveaction_db)
action_db_utils.get_action_by_ref.assert_called_once()
action_db_utils.get_runnertype_by_name.assert_not_called()
def test_invoke_post_run_action_exists(self):
utils.invoke_post_run(self.liveaction_db)
def test_invoke_post_run_action_provided(self):
utils.invoke_post_run(self.liveaction_db, action_db=self.action_db)
action_db_utils.get_action_by_ref.assert_not_called()
def test_invoke_post_run_action_does_not_exist(self):
utils.invoke_post_run(self.liveaction_db)
action_db_utils.get_action_by_ref.assert_called_once()
action_db_utils.get_runnertype_by_name.assert_not_called()
def test_invoke_post_run_action_exists(self):
utils.invoke_post_run(self.liveaction_db)
def test_invoke_post_run_action_provided(self):
utils.invoke_post_run(self.liveaction_db, action_db=self.action_db)
action_db_utils.get_action_by_ref.assert_not_called()
def update_execution_records(wf_ex_db, conductor, update_lv_ac_on_statuses=None,
pub_wf_ex=False, pub_lv_ac=True, pub_ac_ex=True):
wf_ac_ex_id = wf_ex_db.action_execution
# Determine if workflow status has changed.
wf_old_status = wf_ex_db.status
wf_ex_db.status = conductor.get_workflow_status()
status_changed = (wf_old_status != wf_ex_db.status)
if status_changed:
msg = '[%s] Updating workflow execution from status "%s" to "%s".'
LOG.info(msg, wf_ac_ex_id, wf_old_status, wf_ex_db.status)
# Update timestamp and output if workflow is completed.
if wf_ex_db.status in statuses.COMPLETED_STATUSES:
wf_ex_db.end_timestamp = date_utils.get_datetime_utc_now()
wf_ex_db.output = conductor.get_workflow_output()
# Update task flow and other attributes.
wf_ex_db.errors = copy.deepcopy(conductor.errors)
wf_ex_db.state = conductor.workflow_state.serialize()
# Write changes to the database.
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db, publish=pub_wf_ex)
# Return if workflow execution status is not specified in update_lv_ac_on_statuses.
if (isinstance(update_lv_ac_on_statuses, list) and
wf_ex_db.status not in update_lv_ac_on_statuses):
return
# Update the corresponding liveaction and action execution for the workflow.
wf_ac_ex_db = ex_db_access.ActionExecution.get_by_id(wf_ex_db.action_execution)
wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction['id'])
# Gather result for liveaction and action execution.
result = {'output': wf_ex_db.output or None}
if wf_ex_db.status in statuses.ABENDED_STATUSES:
result['errors'] = wf_ex_db.errors
for wf_ex_error in wf_ex_db.errors:
msg = '[%s] Workflow execution completed with errors.'
LOG.error(msg, wf_ac_ex_id, extra=wf_ex_error)
# Sync update with corresponding liveaction and action execution.
if pub_lv_ac or pub_ac_ex:
pub_lv_ac = (wf_lv_ac_db.status != wf_ex_db.status)
pub_ac_ex = pub_lv_ac
if wf_lv_ac_db.status != wf_ex_db.status:
msg = '[%s] Updating workflow liveaction from status "%s" to "%s".'
LOG.debug(msg, wf_ac_ex_id, wf_lv_ac_db.status, wf_ex_db.status)
msg = '[%s] Workflow liveaction status change %s be published.'
LOG.debug(msg, wf_ac_ex_id, 'will' if pub_lv_ac else 'will not')
msg = '[%s] Workflow action execution status change %s be published.'
LOG.debug(msg, wf_ac_ex_id, 'will' if pub_ac_ex else 'will not')
wf_lv_ac_db = action_utils.update_liveaction_status(
status=wf_ex_db.status,
result=result,
end_timestamp=wf_ex_db.end_timestamp,
liveaction_db=wf_lv_ac_db,
publish=pub_lv_ac)
ex_svc.update_execution(wf_lv_ac_db, publish=pub_ac_ex)
# Invoke post run on the liveaction for the workflow execution.
if status_changed and wf_lv_ac_db.status in ac_const.LIVEACTION_COMPLETED_STATES:
LOG.info('[%s] Workflow action execution is completed and invoking post run.', wf_ac_ex_id)
runners_utils.invoke_post_run(wf_lv_ac_db)
