def format_event_message(name, task_type, state, result=None, exception=None,
current_retries=0, total_retries=0, postfix=None):
exception_str = utils.format_exception(exception) if exception else None
try:
message_template = {
TASK_SENDING: "Sending task '{name}'",
TASK_STARTED: "{type} started '{name}'",
TASK_SUCCEEDED: "{type} succeeded '{name}'",
TASK_RESCHEDULED: "{type} rescheduled '{name}'",
TASK_FAILED: "{type} failed '{name}'",
}[state]
except KeyError:
raise RuntimeError('unhandled task state: {0}'.format(state))
message = message_template.format(
name=name,
type='Subgraph' if task_type == 'SubgraphTask' else 'Task'
)
if state == TASK_SUCCEEDED and result is not None:
message = '{0} - {1}'.format(message, result)
if state in (TASK_RESCHEDULED, TASK_FAILED) and exception_str:
message = '{0} -> {1}'.format(message, exception_str)
if postfix:
message = '{0}{1}'.format(message, postfix)
if current_retries > 0:
retry = ' [retry {0}{1}]'.format(
current_retries,
'/{0}'.format(total_retries)
if total_retries >= 0 else '')
message = '{0}{1}'.format(message, retry)
return message
def serialize_known_exception(e):
"""
Serialize a cloudify exception into a dict
:param e: A cloudify exception
:return: A JSON serializable payload dict
"""
tb = StringIO()
traceback.print_exc(file=tb)
trace_out = tb.getvalue()
# Needed because HttpException constructor sucks
append_message = False
# Convert exception to a know exception type that can be deserialized
# by the calling process
known_exception_type_args = []
if isinstance(e, exceptions.HttpException):
known_exception_type = exceptions.HttpException
known_exception_type_args = [e.url, e.code]
append_message = True
elif isinstance(e, exceptions.NonRecoverableError):
known_exception_type = exceptions.NonRecoverableError
elif isinstance(e, exceptions.OperationRetry):
known_exception_type = exceptions.OperationRetry
known_exception_type_args = [e.retry_after]
trace_out = None
elif isinstance(e, exceptions.RecoverableError):
known_exception_type = exceptions.RecoverableError
known_exception_type_args = [e.retry_after]
elif isinstance(e, exceptions.StopAgent):
known_exception_type = exceptions.StopAgent
elif isinstance(e, exceptions.WorkflowFailed):
known_exception_type = exceptions.WorkflowFailed
trace_out = None
else:
# convert pure user exceptions to a RecoverableError
known_exception_type = exceptions.RecoverableError
try:
causes = e.causes
except AttributeError:
causes = []
payload = {
'exception_type': type(e).__name__,
'message': format_exception(e),
'known_exception_type': known_exception_type.__name__,
'known_exception_type_args': known_exception_type_args,
'known_exception_type_kwargs': {
'causes': causes or []
},
'append_message': append_message,
}
if trace_out:
payload['traceback'] = trace_out
return payload
def execution_ended(self, execution_id, error=None):
ended_at = datetime.now()
if error:
status = Execution.FAILED
else:
status = Execution.TERMINATED
executions = self.get_executions()
for execution in executions:
if execution['id'] == execution_id:
execution.update({
'status':
status,
'status_display':
self._get_status_display(status),
'ended_at':
ended_at,
'error':
utils.format_exception(error) if error else None
})
self.store_executions(executions)
def send_task_event(state, task, send_event_func, event):
"""
Send a task event delegating to 'send_event_func'
which will send events to RabbitMQ or use the workflow context logger
in local context
:param state: the task state (valid: ['sending', 'started', 'rescheduled',
'succeeded', 'failed'])
:param task: a WorkflowTask instance to send the event for
:param send_event_func: function for actually sending the event somewhere
:param event: a dict with either a result field or an exception fields
follows celery event structure but used by local tasks as
well
"""
if _filter_task(task, state):
return
if state in (tasks_api.TASK_FAILED, tasks_api.TASK_RESCHEDULED,
tasks_api.TASK_SUCCEEDED) and event is None:
raise RuntimeError('Event for task {0} is None'.format(task.name))
if event and event.get('exception'):
exception_str = utils.format_exception(event.get('exception'))
else:
exception_str = None
if state == tasks_api.TASK_SENDING:
message = "Sending task '{0}'".format(task.name)
event_type = 'sending_task'
elif state == tasks_api.TASK_STARTED:
message = "Task started '{0}'".format(task.name)
event_type = 'task_started'
elif state == tasks_api.TASK_SUCCEEDED:
result = str(event.get('result'))
suffix = ' ({0})'.format(result) if result not in ("'None'",
'None') else ''
message = "Task succeeded '{0}{1}'".format(task.name, suffix)
event_type = 'task_succeeded'
elif state == tasks_api.TASK_RESCHEDULED:
message = "Task rescheduled '{0}'".format(task.name)
if exception_str:
message = '{0} -> {1}'.format(message, exception_str)
event_type = 'task_rescheduled'
task.error = exception_str
elif state == tasks_api.TASK_FAILED:
message = "Task failed '{0}'".format(task.name)
if exception_str:
message = "{0} -> {1}".format(message, exception_str)
event_type = 'task_failed'
task.error = exception_str
else:
raise RuntimeError('unhandled event type: {0}'.format(state))
if task.current_retries > 0:
retry = ' [retry {0}{1}]'.format(
task.current_retries, '/{0}'.format(task.total_retries)
if task.total_retries >= 0 else '')
message = '{0}{1}'.format(message, retry)
additional_context = {
'task_current_retries': task.current_retries,
'task_total_retries': task.total_retries
}
if state in (tasks_api.TASK_FAILED, tasks_api.TASK_RESCHEDULED):
additional_context['task_error_causes'] = event.get('causes')
send_event_func(task=task,
event_type=event_type,
message=message,
additional_context=additional_context)
def main():
dispatch_dir = sys.argv[1]
with open(os.path.join(dispatch_dir, 'input.json')) as f:
dispatch_inputs = json.load(f)
cloudify_context = dispatch_inputs['cloudify_context']
args = dispatch_inputs['args']
kwargs = dispatch_inputs['kwargs']
dispatch_type = cloudify_context['type']
threading.current_thread().setName('Dispatch-{0}'.format(dispatch_type))
handler_cls = TASK_HANDLERS[dispatch_type]
handler = None
try:
handler = handler_cls(cloudify_context=cloudify_context,
args=args,
kwargs=kwargs)
handler.setup_logging()
payload = handler.handle()
payload_type = 'result'
except BaseException as e:
tb = StringIO.StringIO()
traceback.print_exc(file=tb)
trace_out = tb.getvalue()
# Needed because HttpException constructor sucks
append_message = False
# Convert exception to a know exception type that can be deserialized
# by the calling process
known_exception_type_args = []
if isinstance(e, exceptions.ProcessExecutionError):
known_exception_type = exceptions.ProcessExecutionError
known_exception_type_args = [e.error_type, e.traceback]
trace_out = e.traceback
elif isinstance(e, exceptions.HttpException):
known_exception_type = exceptions.HttpException
known_exception_type_args = [e.url, e.code]
append_message = True
elif isinstance(e, exceptions.NonRecoverableError):
known_exception_type = exceptions.NonRecoverableError
elif isinstance(e, exceptions.OperationRetry):
known_exception_type = exceptions.OperationRetry
known_exception_type_args = [e.retry_after]
elif isinstance(e, exceptions.RecoverableError):
known_exception_type = exceptions.RecoverableError
known_exception_type_args = [e.retry_after]
else:
# convert pure user exceptions to a RecoverableError
known_exception_type = exceptions.RecoverableError
try:
causes = e.causes
except AttributeError:
causes = []
payload_type = 'error'
payload = {
'traceback': trace_out,
'exception_type': type(e).__name__,
'message': utils.format_exception(e),
'known_exception_type': known_exception_type.__name__,
'known_exception_type_args': known_exception_type_args,
'known_exception_type_kwargs': {
'causes': causes or []
},
'append_message': append_message,
}
logger = logging.getLogger(__name__)
logger.error('Task {0}[{1}] raised:\n{2}'.format(
handler.cloudify_context['task_name'],
handler.cloudify_context.get('task_id', '<no-id>'), trace_out))
finally:
if handler:
handler.close()
with open(os.path.join(dispatch_dir, 'output.json'), 'w') as f:
json.dump({'type': payload_type, 'payload': payload}, f)
def main():
dispatch_dir = sys.argv[1]
with open(os.path.join(dispatch_dir, 'input.json')) as f:
dispatch_inputs = json.load(f)
cloudify_context = dispatch_inputs['cloudify_context']
args = dispatch_inputs['args']
kwargs = dispatch_inputs['kwargs']
dispatch_type = cloudify_context['type']
threading.current_thread().setName('Dispatch-{0}'.format(dispatch_type))
handler_cls = TASK_HANDLERS[dispatch_type]
handler = None
try:
handler = handler_cls(cloudify_context=cloudify_context,
args=args,
kwargs=kwargs)
handler.setup_logging()
payload = handler.handle()
payload_type = 'result'
except BaseException as e:
tb = StringIO.StringIO()
traceback.print_exc(file=tb)
trace_out = tb.getvalue()
# Needed because HttpException constructor sucks
append_message = False
# Convert exception to a know exception type that can be deserialized
# by the calling process
known_exception_type_args = []
if isinstance(e, exceptions.ProcessExecutionError):
known_exception_type = exceptions.ProcessExecutionError
known_exception_type_args = [e.error_type, e.traceback]
trace_out = e.traceback
elif isinstance(e, exceptions.HttpException):
known_exception_type = exceptions.HttpException
known_exception_type_args = [e.url, e.code]
append_message = True
elif isinstance(e, exceptions.NonRecoverableError):
known_exception_type = exceptions.NonRecoverableError
elif isinstance(e, exceptions.OperationRetry):
known_exception_type = exceptions.OperationRetry
known_exception_type_args = [e.retry_after]
elif isinstance(e, exceptions.RecoverableError):
known_exception_type = exceptions.RecoverableError
known_exception_type_args = [e.retry_after]
else:
# convert pure user exceptions to a RecoverableError
known_exception_type = exceptions.RecoverableError
try:
causes = e.causes
except AttributeError:
causes = []
payload_type = 'error'
payload = {
'traceback': trace_out,
'exception_type': type(e).__name__,
'message': utils.format_exception(e),
'known_exception_type': known_exception_type.__name__,
'known_exception_type_args': known_exception_type_args,
'known_exception_type_kwargs': {'causes': causes or []},
'append_message': append_message,
}
logger = logging.getLogger(__name__)
logger.error('Task {0}[{1}] raised:\n{2}'.format(
handler.cloudify_context['task_name'],
handler.cloudify_context.get('task_id', '<no-id>'),
trace_out))
finally:
if handler:
handler.close()
with open(os.path.join(dispatch_dir, 'output.json'), 'w') as f:
json.dump({
'type': payload_type,
'payload': payload
}, f)