Exemple #1
0
 def test_GroupResult(self):
     x = self.app.GroupResult(
         uuid(),
         [self.app.AsyncResult(uuid()) for _ in range(10)],
     )
     self.assertEqual(x, from_serializable(x.serializable(), self.app))
     self.assertEqual(x, from_serializable(x, self.app))
Exemple #2
0
    def unlock_chord(group_id, callback, interval=None, propagate=True,
                     max_retries=None, result=None,
                     Result=app.AsyncResult, GroupResult=app.GroupResult,
                     from_serializable=from_serializable):
        if interval is None:
            interval = unlock_chord.default_retry_delay
        deps = GroupResult(
            group_id,
            [from_serializable(r, Result=Result) for r in result],
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        if deps.ready():
            callback = subtask(callback)
            try:
                ret = j(propagate=propagate)
            except Exception as exc:
                culprit = next(deps._failed_join_report())

                app._tasks[callback.task].backend.fail_from_current_stack(
                    callback.id, exc=ChordError('Dependency %s raised %r' % (
                        culprit.id, exc)),
                )
            else:
                callback.delay(ret)
        else:
            return unlock_chord.retry(countdown=interval,
                                      max_retries=max_retries)
Exemple #3
0
 def test_with_parent(self):
     x = self.app.AsyncResult(uuid())
     x.parent = self.app.AsyncResult(uuid())
     y = from_serializable(x.serializable(), self.app)
     self.assertEqual(y, x)
     self.assertEqual(y.parent, x.parent)
     self.assertIsInstance(y.parent, AsyncResult)
Exemple #4
0
 def test_with_parent(self):
     x = self.app.AsyncResult(uuid())
     x.parent = self.app.AsyncResult(uuid())
     y = from_serializable(x.serializable(), self.app)
     self.assertEqual(y, x)
     self.assertEqual(y.parent, x.parent)
     self.assertIsInstance(y.parent, AsyncResult)
Exemple #5
0
    def unlock_chord(group_id, callback, interval=None, propagate=None,
                     max_retries=None, result=None,
                     Result=app.AsyncResult, GroupResult=app.GroupResult,
                     from_serializable=from_serializable):
        # if propagate is disabled exceptions raised by chord tasks
        # will be sent as part of the result list to the chord callback.
        # Since 3.1 propagate will be enabled by default, and instead
        # the chord callback changes state to FAILURE with the
        # exception set to ChordError.
        propagate = default_propagate if propagate is None else propagate

        # check if the task group is ready, and if so apply the callback.
        deps = GroupResult(
            group_id,
            [from_serializable(r, Result=Result) for r in result],
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        if deps.ready():
            callback = subtask(callback)
            try:
                ret = j(propagate=propagate)
            except Exception, exc:
                culprit = deps._failed_join_report().next()
                app._tasks[callback.task].backend.fail_from_current_stack(
                    callback.id, exc=ChordError('Dependency %s raised %r' % (
                        culprit.id, exc)),
                )
            else:
                try:
                    callback.delay(ret)
                except Exception, exc:
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Call callback error: %r' % (exc, )))
Exemple #6
0
 def _restore_group(self, group_id):
     """Get task metadata for a task by id."""
     meta = self.get(self.get_key_for_group(group_id))
     # previously this was always pickled, but later this
     # was extended to support other serializers, so the
     # structure is kind of weird.
     if meta:
         meta = self.decode(meta)
         result = meta["result"]
         meta["result"] = from_serializable(result, self.app)
         return meta
Exemple #7
0
    def unlock_chord(group_id,
                     callback,
                     interval=None,
                     propagate=None,
                     max_retries=None,
                     result=None,
                     Result=app.AsyncResult,
                     GroupResult=app.GroupResult,
                     from_serializable=from_serializable):
        # if propagate is disabled exceptions raised by chord tasks
        # will be sent as part of the result list to the chord callback.
        # Since 3.1 propagate will be enabled by default, and instead
        # the chord callback changes state to FAILURE with the
        # exception set to ChordError.
        propagate = default_propagate if propagate is None else propagate
        if interval is None:
            interval = unlock_chord.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        deps = GroupResult(
            group_id,
            [from_serializable(r, app=app) for r in result],
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        if deps.ready():
            callback = subtask(callback)
            try:
                ret = j(propagate=propagate)
            except Exception as exc:
                try:
                    culprit = next(deps._failed_join_report())
                    reason = 'Dependency {0.id} raised {1!r}'.format(
                        culprit,
                        exc,
                    )
                except StopIteration:
                    reason = repr(exc)

                app._tasks[callback.task].backend.fail_from_current_stack(
                    callback.id,
                    exc=ChordError(reason),
                )
            else:
                try:
                    callback.delay(ret)
                except Exception as exc:
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        else:
            return unlock_chord.retry(countdown=interval,
                                      max_retries=max_retries)
Exemple #8
0
 def _restore_group(self, group_id):
     """Get task metadata for a task by id."""
     meta = self.get(self.get_key_for_group(group_id))
     # previously this was always pickled, but later this
     # was extended to support other serializers, so the
     # structure is kind of weird.
     if meta:
         meta = self.decode(meta)
         result = meta['result']
         meta['result'] = from_serializable(result, self.app)
         return meta
Exemple #9
0
 def _restore_taskset(self, taskset_id):
     """Get task metadata for a task by id."""
     meta = self.get(self.get_key_for_taskset(taskset_id))
     # previously this was always pickled, but later this
     # was extended to support other serializers, so the
     # structure is kind of weird.
     if meta:
         meta = self.decode(meta)
         result = meta["result"]
         if isinstance(result, (list, tuple)):
             return {"result": from_serializable(result)}
         return meta
Exemple #10
0
 def _restore_group(self, group_id):
     """Get task metadata for a task by id."""
     meta = self.get(self.get_key_for_group(group_id))
     # previously this was always pickled, but later this
     # was extended to support other serializers, so the
     # structure is kind of weird.
     if meta:
         meta = self.decode(meta)
         result = meta['result']
         if isinstance(result, (list, tuple)):
             return {'result': from_serializable(result)}
         return meta
Exemple #11
0
 def run(self, tasks, result, group_id, partial_args):
     app = self.app
     result = from_serializable(result, app)
     # any partial args are added to all tasks in the group
     taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks))
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit])
     with app.producer_or_acquire() as pub:
         [stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit]
     parent = get_current_worker_task()
     if parent:
         parent.request.children.append(result)
     return result
Exemple #12
0
 def run(self, tasks, result, setid):
     app = self.app
     result = from_serializable(result)
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.TaskSetResult(result.id,
                 [subtask(task).apply(taskset_id=setid)
                     for task in tasks])
     with app.default_producer() as pub:
         [subtask(task).apply_async(taskset_id=setid, publisher=pub)
                 for task in tasks]
     parent = get_current_task()
     if parent:
         parent.request.children.append(result)
     return result
 def run(self, tasks, result, group_id):
     app = self.app
     result = from_serializable(result)
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.GroupResult(result.id,
                 [subtask(task).apply(group_id=group_id)
                     for task in tasks])
     with app.default_producer() as pub:
         [subtask(task).apply_async(group_id=group_id, publisher=pub,
                                    add_to_parent=False)
                 for task in tasks]
     parent = get_current_worker_task()
     if parent:
         parent.request.children.append(result)
     return result
Exemple #14
0
 def run(self, tasks, result):
     app = self.app
     result = from_serializable(result)
     with app.pool.acquire(block=True) as conn:
         with app.amqp.TaskPublisher(conn) as publisher:
             res_ = [subtask(task).apply_async(
                                 taskset_id=self.request.taskset,
                                 publisher=publisher)
                         for task in tasks]
     parent = get_current_task()
     if parent:
         parent.request.children.append(result)
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.TaskSetResult(result.id, res_)
     return result
Exemple #15
0
 def run(self, tasks, result, group_id):
     app = self.app
     result = from_serializable(result)
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.GroupResult(result.id,
                 [subtask(task).apply(group_id=group_id)
                     for task in tasks])
     with app.default_producer() as pub:
         [subtask(task).apply_async(group_id=group_id, publisher=pub,
                                    add_to_parent=False)
                 for task in tasks]
     parent = get_current_worker_task()
     if parent:
         parent.request.children.append(result)
     return result
Exemple #16
0
 def run(self, tasks, result, group_id, partial_args):
     app = self.app
     result = from_serializable(result)
     # any partial args are added to all tasks in the group
     taskit = (subtask(task).clone(partial_args)
                 for i, task in enumerate(tasks))
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.GroupResult(result.id,
                 [task.apply(group_id=group_id) for task in taskit])
     with app.producer_or_acquire() as pub:
         [task.apply_async(group_id=group_id, publisher=pub,
                           add_to_parent=False) for task in taskit]
     parent = get_current_worker_task()
     if parent:
         parent.request.children.append(result)
     return result
Exemple #17
0
    def unlock_chord(group_id, callback, interval=None, propagate=None,
                     max_retries=None, result=None,
                     Result=app.AsyncResult, GroupResult=app.GroupResult,
                     from_serializable=from_serializable):
        # if propagate is disabled exceptions raised by chord tasks
        # will be sent as part of the result list to the chord callback.
        # Since 3.1 propagate will be enabled by default, and instead
        # the chord callback changes state to FAILURE with the
        # exception set to ChordError.
        propagate = default_propagate if propagate is None else propagate
        if interval is None:
            interval = unlock_chord.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        deps = GroupResult(
            group_id,
            [from_serializable(r, app=app) for r in result],
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        if deps.ready():
            callback = subtask(callback)
            try:
                ret = j(propagate=propagate)
            except Exception as exc:
                try:
                    culprit = next(deps._failed_join_report())
                    reason = 'Dependency {0.id} raised {1!r}'.format(
                        culprit, exc,
                    )
                except StopIteration:
                    reason = repr(exc)

                app._tasks[callback.task].backend.fail_from_current_stack(
                    callback.id, exc=ChordError(reason),
                )
            else:
                try:
                    callback.delay(ret)
                except Exception as exc:
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        else:
            return unlock_chord.retry(countdown=interval,
                                      max_retries=max_retries)
Exemple #18
0
 def run(self, tasks, result):
     app = self.app
     result = from_serializable(result)
     if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
         return app.TaskSetResult(result.id, [
             subtask(task).apply(taskset_id=self.request.taskset)
             for task in tasks
         ])
     with app.pool.acquire(block=True) as conn:
         with app.amqp.TaskPublisher(conn) as publisher:
             [
                 subtask(task).apply_async(
                     taskset_id=self.request.taskset,
                     publisher=publisher) for task in tasks
             ]
     parent = get_current_task()
     if parent:
         parent.request.children.append(result)
     return result
Exemple #19
0
 def test_compat(self):
     uid = uuid()
     x = from_serializable([uid, []], app=self.app)
     self.assertEqual(x.id, uid)
Exemple #20
0
 def test_GroupResult(self):
     x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)])
     self.assertEqual(x, from_serializable(x.serializable()))
     self.assertEqual(x, from_serializable(x))
Exemple #21
0
 def test_AsyncResult(self):
     x = AsyncResult(uuid())
     self.assertEqual(x, from_serializable(x.serializable()))
     self.assertEqual(x, from_serializable(x))
Exemple #22
0
 def test_GroupResult(self):
     x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)])
     self.assertEqual(x, from_serializable(x.serializable()))
     self.assertEqual(x, from_serializable(x))
Exemple #23
0
 def test_AsyncResult(self):
     x = AsyncResult(uuid())
     self.assertEqual(x, from_serializable(x.serializable()))
     self.assertEqual(x, from_serializable(x))
Exemple #24
0
 def test_GroupResult(self):
     x = self.app.GroupResult(
         uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)],
     )
     self.assertEqual(x, from_serializable(x.serializable(), self.app))
     self.assertEqual(x, from_serializable(x, self.app))
Exemple #25
0
 def test_compat(self):
     uid = uuid()
     x = from_serializable([uid, []])
     self.assertEqual(x.id, uid)