Example #1
0
 def test_when_datetime(self):
     x = Retry('foo', KeyError(), when=datetime.utcnow())
     assert x.humanize()
Example #2
0
File: task.py Project: pv8/celery
    def retry(self, args=None, kwargs=None, exc=None, throw=True,
              eta=None, countdown=None, max_retries=None, **options):
        """Retry the task, adding it to the back of the queue.

        Example:
            >>> from imaginary_twitter_lib import Twitter
            >>> from proj.celery import app

            >>> @app.task(bind=True)
            ... def tweet(self, auth, message):
            ...     twitter = Twitter(oauth=auth)
            ...     try:
            ...         twitter.post_status_update(message)
            ...     except twitter.FailWhale as exc:
            ...         # Retry in 5 minutes.
            ...         self.retry(countdown=60 * 5, exc=exc)

        Note:
            Although the task will never return above as `retry` raises an
            exception to notify the worker, we use `raise` in front of the
            retry to convey that the rest of the block won't be executed.

        Arguments:
            args (Tuple): Positional arguments to retry with.
            kwargs (Dict): Keyword arguments to retry with.
            exc (Exception): Custom exception to report when the max retry
                limit has been exceeded (default:
                :exc:`~@MaxRetriesExceededError`).

                If this argument is set and retry is called while
                an exception was raised (``sys.exc_info()`` is set)
                it will attempt to re-raise the current exception.

                If no exception was raised it will raise the ``exc``
                argument provided.
            countdown (float): Time in seconds to delay the retry for.
            eta (~datetime.datetime): Explicit time and date to run the
                retry at.
            max_retries (int): If set, overrides the default retry limit for
                this execution.  Changes to this parameter don't propagate to
                subsequent task retry attempts.  A value of :const:`None`,
                means "use the default", so if you want infinite retries you'd
                have to set the :attr:`max_retries` attribute of the task to
                :const:`None` first.
            time_limit (int): If set, overrides the default time limit.
            soft_time_limit (int): If set, overrides the default soft
                time limit.
            throw (bool): If this is :const:`False`, don't raise the
                :exc:`~@Retry` exception, that tells the worker to mark
                the task as being retried.  Note that this means the task
                will be marked as failed if the task raises an exception,
                or successful if it returns after the retry call.
            **options (Any): Extra options to pass on to :meth:`apply_async`.

        Raises:

            celery.exceptions.Retry:
                To tell the worker that the task has been re-sent for retry.
                This always happens, unless the `throw` keyword argument
                has been explicitly set to :const:`False`, and is considered
                normal operation.
        """
        request = self.request
        retries = request.retries + 1
        max_retries = self.max_retries if max_retries is None else max_retries

        # Not in worker or emulated by (apply/always_eager),
        # so just raise the original exception.
        if request.called_directly:
            # raises orig stack if PyErr_Occurred,
            # and augments with exc' if that argument is defined.
            raise_with_context(exc or Retry('Task can be retried', None))

        if not eta and countdown is None:
            countdown = self.default_retry_delay

        is_eager = request.is_eager
        S = self.signature_from_request(
            request, args, kwargs,
            countdown=countdown, eta=eta, retries=retries,
            **options
        )

        if max_retries is not None and retries > max_retries:
            if exc:
                # On Py3: will augment any current exception with
                # the exc' argument provided (raise exc from orig)
                raise_with_context(exc)
            raise self.MaxRetriesExceededError(
                "Can't retry {}[{}] args:{} kwargs:{}".format(
                    self.name, request.id, S.args, S.kwargs
                ), task_args=S.args, task_kwargs=S.kwargs
            )

        ret = Retry(exc=exc, when=eta or countdown, is_eager=is_eager, sig=S)

        if is_eager:
            # if task was executed eagerly using apply(),
            # then the retry must also be executed eagerly in apply method
            if throw:
                raise ret
            return ret

        try:
            S.apply_async()
        except Exception as exc:
            raise Reject(exc, requeue=False)
        if throw:
            raise ret
        return ret
Example #3
0
 def test_retry_semipredicate(self):
     try:
         raise Exception('foo')
     except Exception as exc:
         ret = Retry('Retrying task', exc)
         assert ret.exc == exc
Example #4
0
 def test_trace_Retry(self, mock_traceback_clear):
     exc = Retry('foo', 'bar')
     _, info = self.trace(self.raises, (exc, ), {})
     assert info.state == states.RETRY
     assert info.retval is exc
     mock_traceback_clear.assert_called()
 def fn_exception():
     raise Retry("Task class is being retried")
Example #6
0
    def test_celery_wrong_silo(self, process_silo_retry):
        process_silo_retry.side_effect = Retry()
        silo = factories.Silo(owner=self.user, public=False)

        with self.assertRaises(ObjectDoesNotExist):
            process_silo(silo.id, -1)
Example #7
0
    def retry(self,
              args=None,
              kwargs=None,
              exc=None,
              throw=True,
              eta=None,
              countdown=None,
              max_retries=None,
              **options):
        """Retry the task.

        :param args: Positional arguments to retry with.
        :param kwargs: Keyword arguments to retry with.
        :keyword exc: Custom exception to report when the max restart
            limit has been exceeded (default:
            :exc:`~@MaxRetriesExceededError`).

            If this argument is set and retry is called while
            an exception was raised (``sys.exc_info()`` is set)
            it will attempt to reraise the current exception.

            If no exception was raised it will raise the ``exc``
            argument provided.
        :keyword countdown: Time in seconds to delay the retry for.
        :keyword eta: Explicit time and date to run the retry at
                      (must be a :class:`~datetime.datetime` instance).
        :keyword max_retries: If set, overrides the default retry limit for
            this execution. Changes to this parameter do not propagate to
            subsequent task retry attempts. A value of :const:`None`, means
            "use the default", so if you want infinite retries you would
            have to set the :attr:`max_retries` attribute of the task to
            :const:`None` first.
        :keyword time_limit: If set, overrides the default time limit.
        :keyword soft_time_limit: If set, overrides the default soft
                                  time limit.
        :keyword \*\*options: Any extra options to pass on to
                              meth:`apply_async`.
        :keyword throw: If this is :const:`False`, do not raise the
                        :exc:`~@Retry` exception,
                        that tells the worker to mark the task as being
                        retried.  Note that this means the task will be
                        marked as failed if the task raises an exception,
                        or successful if it returns.

        :raises celery.exceptions.Retry: To tell the worker that
            the task has been re-sent for retry. This always happens,
            unless the `throw` keyword argument has been explicitly set
            to :const:`False`, and is considered normal operation.

        **Example**

        .. code-block:: pycon

            >>> from imaginary_twitter_lib import Twitter
            >>> from proj.celery import app

            >>> @app.task(bind=True)
            ... def tweet(self, auth, message):
            ...     twitter = Twitter(oauth=auth)
            ...     try:
            ...         twitter.post_status_update(message)
            ...     except twitter.FailWhale as exc:
            ...         # Retry in 5 minutes.
            ...         raise self.retry(countdown=60 * 5, exc=exc)

        Although the task will never return above as `retry` raises an
        exception to notify the worker, we use `raise` in front of the retry
        to convey that the rest of the block will not be executed.

        """
        request = self.request
        retries = request.retries + 1
        max_retries = self.max_retries if max_retries is None else max_retries

        # Not in worker or emulated by (apply/always_eager),
        # so just raise the original exception.
        if request.called_directly:
            maybe_reraise()  # raise orig stack if PyErr_Occurred
            raise exc or Retry('Task can be retried', None)

        if not eta and countdown is None:
            countdown = self.default_retry_delay

        is_eager = request.is_eager
        S = self.signature_from_request(request,
                                        args,
                                        kwargs,
                                        countdown=countdown,
                                        eta=eta,
                                        retries=retries,
                                        **options)

        if max_retries is not None and retries > max_retries:
            if exc:
                # first try to reraise the original exception
                maybe_reraise()
                # or if not in an except block then raise the custom exc.
                raise exc
            raise self.MaxRetriesExceededError(
                "Can't retry {0}[{1}] args:{2} kwargs:{3}".format(
                    self.name, request.id, S.args, S.kwargs))

        ret = Retry(exc=exc, when=eta or countdown)

        if is_eager:
            # if task was executed eagerly using apply(),
            # then the retry must also be executed eagerly.
            S.apply().get()
            if throw:
                raise ret
            return ret

        try:
            S.apply_async()
        except Exception as exc:
            raise Reject(exc, requeue=False)
        if throw:
            raise ret
        return ret
Example #8
0
 def test_when_datetime(self):
     x = Retry('foo', KeyError(), when=datetime.utcnow())
     assert x.humanize()
Example #9
0
 def test_trace_Retry(self):
     exc = Retry('foo', 'bar')
     _, info = self.trace(self.raises, (exc,), {})
     assert info.state == states.RETRY
     assert info.retval is exc
Example #10
0
 def test_pickleable(self):
     x = Retry('foo', True)
     assert pickle.loads(pickle.dumps(x))
Example #11
0
 def test_pickleable(self):
     x = Retry('foo', KeyError(), when=datetime.utcnow())
     assert pickle.loads(pickle.dumps(x))
Example #12
0
 def test_trace_Retry(self):
     exc = Retry('foo', 'bar')
     _, info = self.trace(self.raises, (exc,), {})
     self.assertEqual(info.state, states.RETRY)
     self.assertIs(info.retval, exc)
Example #13
0
class TestProcessFileAsync:
    """Test process file async."""

    @patch('cern_search_rest_api.modules.cernsearch.tasks.current_processors.get_processor')
    @patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get')
    def test_process_file_async_success(
            self,
            object_version_get_mock,
            get_processor_mock,
            appctx,
            object_version
    ):
        """Test process file calls."""
        get_processor_mock.return_value.process.return_value = 'Processed'
        object_version_get_mock.return_value = object_version

        process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf')

        object_version_get_mock.assert_called_once_with('00000000-0000-0000-0000-000000000000', 'test.pdf')
        get_processor_mock.assert_called_once_with(name=TikaProcessor.id())
        get_processor_mock.return_value.process.assert_called_once_with(object_version)

    @patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get', side_effect=Exception())
    @patch('cern_search_rest_api.modules.cernsearch.tasks.process_file_async.retry', side_effect=Retry())
    def test_process_file_async_retry(self, process_file_async_retry, object_version_get_mock, object_version):
        """Test process file calls."""
        with raises(Retry):
            process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf')

    @patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get', side_effect=Exception())
    @patch('cern_search_rest_api.modules.cernsearch.tasks.process_file_async.retry',
           side_effect=MaxRetriesExceededError())
    def test_process_file_async_failure(self, process_file_async_retry_mock, object_version):
        """Test process file calls."""
        with raises(Reject):
            process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf')
Example #14
0
 def fake_retry(exc):
     assert isinstance(exc, TembaHttpError)
     raise Retry()