예제 #1
0
파일: builtins.py 프로젝트: xenouch/celery
 def apply_async(self,
                 args=(),
                 kwargs={},
                 task_id=None,
                 group_id=None,
                 chord=None,
                 **options):
     app = self.app
     if app.conf.CELERY_ALWAYS_EAGER:
         return self.apply(args, kwargs, **options)
     header = kwargs.pop('header')
     body = kwargs.pop('body')
     header, body = (maybe_signature(header, app=app),
                     maybe_signature(body, app=app))
     # forward certain options to body
     if chord is not None:
         body.options['chord'] = chord
     if group_id is not None:
         body.options['group_id'] = group_id
     [body.link(s) for s in options.pop('link', [])]
     [body.link_error(s) for s in options.pop('link_error', [])]
     body_result = body.freeze(task_id)
     parent = super(Chord, self).apply_async((header, body, args),
                                             kwargs, **options)
     body_result.parent = parent
     return body_result
예제 #2
0
    def unlock_chord(group_id,
                     callback,
                     interval=None,
                     propagate=None,
                     max_retries=None,
                     result=None,
                     Result=app.AsyncResult,
                     GroupResult=app.GroupResult,
                     result_from_tuple=result_from_tuple):
        # if propagate is disabled exceptions raised by chord tasks
        # will be sent as part of the result list to the chord callback.
        # Since 3.1 propagate will be enabled by default, and instead
        # the chord callback changes state to FAILURE with the
        # exception set to ChordError.
        propagate = default_propagate if propagate is None else propagate
        if interval is None:
            interval = unlock_chord.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        callback = maybe_signature(callback, app)
        deps = GroupResult(
            group_id,
            [result_from_tuple(r, app=app) for r in result],
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        if deps.ready():
            callback = maybe_signature(callback, app=app)
            try:
                with allow_join_result():
                    ret = j(timeout=3.0, propagate=propagate)
            except Exception as exc:
                try:
                    culprit = next(deps._failed_join_report())
                    reason = 'Dependency {0.id} raised {1!r}'.format(
                        culprit,
                        exc,
                    )
                except StopIteration:
                    reason = repr(exc)
                logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
                app.backend.chord_error_from_stack(callback,
                                                   ChordError(reason))
            else:
                try:
                    callback.delay(ret)
                except Exception as exc:
                    logger.error('Chord %r raised: %r',
                                 group_id,
                                 exc,
                                 exc_info=1)
                    app.backend.chord_error_from_stack(
                        callback,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        else:
            raise unlock_chord.retry(countdown=interval,
                                     max_retries=max_retries)
예제 #3
0
    def unlock_chord(self,
                     group_id,
                     callback,
                     interval=None,
                     max_retries=None,
                     result=None,
                     Result=app.AsyncResult,
                     GroupResult=app.GroupResult,
                     result_from_tuple=result_from_tuple,
                     **kwargs):
        if interval is None:
            interval = self.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        callback = maybe_signature(callback, app)
        deps = GroupResult(
            group_id,
            [result_from_tuple(r, app=app) for r in result],
            app=app,
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        try:
            ready = deps.ready()
        except Exception as exc:
            raise self.retry(
                exc=exc,
                countdown=interval,
                max_retries=max_retries,
            )
        else:
            if not ready:
                raise self.retry(countdown=interval, max_retries=max_retries)

        callback = maybe_signature(callback, app=app)
        try:
            with allow_join_result():
                ret = j(timeout=3.0, propagate=True)
        except Exception as exc:
            try:
                culprit = next(deps._failed_join_report())
                reason = 'Dependency {0.id} raised {1!r}'.format(
                    culprit,
                    exc,
                )
            except StopIteration:
                reason = repr(exc)
            logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
            app.backend.chord_error_from_stack(callback, ChordError(reason))
        else:
            try:
                callback.delay(ret)
            except Exception as exc:
                logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
                app.backend.chord_error_from_stack(
                    callback,
                    exc=ChordError('Callback error: {0!r}'.format(exc)),
                )
예제 #4
0
파일: builtins.py 프로젝트: bakaat/celery
    def unlock_chord(self, group_id, callback, interval=None, propagate=None,
                     max_retries=None, result=None,
                     Result=app.AsyncResult, GroupResult=app.GroupResult,
                     result_from_tuple=result_from_tuple):
        # if propagate is disabled exceptions raised by chord tasks
        # will be sent as part of the result list to the chord callback.
        # Since 3.1 propagate will be enabled by default, and instead
        # the chord callback changes state to FAILURE with the
        # exception set to ChordError.
        propagate = default_propagate if propagate is None else propagate
        if interval is None:
            interval = self.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        callback = maybe_signature(callback, app)
        deps = GroupResult(
            group_id,
            [result_from_tuple(r, app=app) for r in result],
            app=app,
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        try:
            ready = deps.ready()
        except Exception as exc:
            raise self.retry(
                exc=exc, countdown=interval, max_retries=max_retries,
            )
        else:
            if not ready:
                raise self.retry(countdown=interval, max_retries=max_retries)

        callback = maybe_signature(callback, app=app)
        try:
            with allow_join_result():
                ret = j(timeout=3.0, propagate=propagate)
        except Exception as exc:
            try:
                culprit = next(deps._failed_join_report())
                reason = 'Dependency {0.id} raised {1!r}'.format(
                    culprit, exc,
                )
            except StopIteration:
                reason = repr(exc)
            logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
            app.backend.chord_error_from_stack(callback,
                                               ChordError(reason))
        else:
            try:
                callback.delay(ret)
            except Exception as exc:
                logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
                app.backend.chord_error_from_stack(
                    callback,
                    exc=ChordError('Callback error: {0!r}'.format(exc)),
                )
예제 #5
0
 def chord(self, header, body, partial_args=(), interval=None,
           countdown=1, max_retries=None, eager=False, **kwargs):
     app = self.app
     # - convert back to group if serialized
     tasks = header.tasks if isinstance(header, group) else header
     header = group([
         maybe_signature(s, app=app) for s in tasks
     ], app=self.app)
     body = maybe_signature(body, app=app)
     ch = _chord(header, body)
     return ch.run(header, body, partial_args, app, interval,
                   countdown, max_retries, **kwargs)
예제 #6
0
    def unlock_chord(self, group_id, callback, interval=None,
                     max_retries=None, result=None,
                     Result=app.AsyncResult, GroupResult=app.GroupResult,
                     result_from_tuple=result_from_tuple, **kwargs):
        if interval is None:
            interval = self.default_retry_delay

        # check if the task group is ready, and if so apply the callback.
        callback = maybe_signature(callback, app)
        deps = GroupResult(
            group_id,
            [result_from_tuple(r, app=app) for r in result],
            app=app,
        )
        j = deps.join_native if deps.supports_native_join else deps.join

        try:
            ready = deps.ready()
        except Exception as exc:
            raise self.retry(
                exc=exc, countdown=interval, max_retries=max_retries,
            )
        else:
            if not ready:
                raise self.retry(countdown=interval, max_retries=max_retries)

        callback = maybe_signature(callback, app=app)
        try:
            with allow_join_result():
                ret = j(timeout=3.0, propagate=True)
        except Exception as exc:
            try:
                culprit = next(deps._failed_join_report())
                reason = 'Dependency {0.id} raised {1!r}'.format(
                    culprit, exc,
                )
            except StopIteration:
                reason = repr(exc)
            logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
            app.backend.chord_error_from_stack(callback,
                                               ChordError(reason))
        else:
            try:
                callback.delay(ret)
            except Exception as exc:
                logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
                app.backend.chord_error_from_stack(
                    callback,
                    exc=ChordError('Callback error: {0!r}'.format(exc)),
                )
예제 #7
0
        def run(self, header, body, partial_args=(), interval=None,
                countdown=1, max_retries=None, propagate=None,
                eager=False, **kwargs):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()
            AsyncResult = app.AsyncResult
            prepare_member = self._prepare_member

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group([
                maybe_signature(s, app=app).clone() for s in tasks
            ])
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            results = [AsyncResult(prepare_member(task, body, group_id))
                       for task in header.tasks]

            return self.backend.apply_chord(
                header, partial_args, group_id,
                body, interval=interval, countdown=countdown,
                max_retries=max_retries, propagate=propagate, result=results,
            )
예제 #8
0
파일: builtins.py 프로젝트: stavxyz/celery
        def run(self, header, body, partial_args=(), interval=None,
                countdown=1, max_retries=None, propagate=None,
                eager=False, **kwargs):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()
            AsyncResult = app.AsyncResult
            prepare_member = self._prepare_member

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group([
                maybe_signature(s, app=app).clone() for s in tasks
            ])
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            results = [AsyncResult(prepare_member(task, body, group_id))
                       for task in header.tasks]

            # - fallback implementations schedules the chord_unlock task here
            app.backend.on_chord_apply(group_id, body,
                                       interval=interval,
                                       countdown=countdown,
                                       max_retries=max_retries,
                                       propagate=propagate,
                                       result=results)
            # - call the header group, returning the GroupResult.
            final_res = header(*partial_args, task_id=group_id)

            return final_res
예제 #9
0
        def run(self, header, body, partial_args=(), interval=None,
                countdown=1, max_retries=None, propagate=None,
                eager=False, **kwargs):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()
            AsyncResult = app.AsyncResult
            prepare_member = self._prepare_member

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group([
                maybe_signature(s, app=app).clone() for s in tasks
            ])
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            results = [AsyncResult(prepare_member(task, body, group_id))
                       for task in header.tasks]

            return self.backend.apply_chord(
                header, partial_args, group_id,
                body, interval=interval, countdown=countdown,
                max_retries=max_retries, propagate=propagate, result=results,
            )
예제 #10
0
파일: sets.py 프로젝트: 343829084/celery
 def __init__(self, tasks=None, app=None, Publisher=None):
     self.app = app_or_default(app or self.app)
     super(TaskSet, self).__init__(
         maybe_signature(t, app=self.app) for t in tasks or []
     )
     self.Publisher = Publisher or self.app.amqp.Producer
     self.total = len(self)  # XXX compat
예제 #11
0
파일: builtins.py 프로젝트: NotSqrt/celery
        def run(self, header, body, partial_args=(), interval=None,
                countdown=1, max_retries=None, propagate=None,
                eager=False, **kwargs):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()
            AsyncResult = app.AsyncResult
            prepare_member = self._prepare_member

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group([
                maybe_signature(s, app=app).clone() for s in tasks
            ])
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            results = [AsyncResult(prepare_member(task, body, group_id))
                       for task in header.tasks]

            # - fallback implementations schedules the chord_unlock task here
            app.backend.on_chord_apply(group_id, body,
                                       interval=interval,
                                       countdown=countdown,
                                       max_retries=max_retries,
                                       propagate=propagate,
                                       result=results)
            # - call the header group, returning the GroupResult.
            final_res = header(*partial_args, task_id=group_id)

            return final_res
예제 #12
0
    def on_chord_part_return(self,
                             request,
                             state,
                             result,
                             propagate=None,
                             **kwargs):
        app = self.app
        tid, gid = request.id, request.group
        if not gid or not tid:
            return

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        result = self.encode_result(result, state)
        with client.pipeline() as pipe:
            _, readycount, totaldiff, _, _ = pipe \
                .rpush(jkey, self.encode([1, tid, state, result])) \
                .llen(jkey) \
                .get(tkey) \
                .expire(jkey, self.expires) \
                .expire(tkey, self.expires) \
                .execute()

        totaldiff = int(totaldiff or 0)

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size'] + totaldiff
            if readycount == total:
                decode, unpack = self.decode, self._unpack_chord_result
                with client.pipeline() as pipe:
                    resl, = pipe \
                        .lrange(jkey, 0, total) \
                        .execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                    with client.pipeline() as pipe:
                        _, _ = pipe \
                            .delete(jkey) \
                            .delete(tkey) \
                            .execute()
                except Exception as exc:  # pylint: disable=broad-except
                    logger.exception('Chord callback for %r raised: %r',
                                     request.group, exc)
                    return self.chord_error_from_stack(
                        callback,
                        ChordError('Callback error: {0!r}'.format(exc)),
                    )

        except ChordError as exc:
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(callback, exc)
        except Exception as exc:  # pylint: disable=broad-except
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(
                callback,
                ChordError('Join error: {0!r}'.format(exc)),
            )
예제 #13
0
    def on_chord_part_return(self, request, state, result,
                             propagate=None, **kwargs):
        app = self.app
        tid, gid = request.id, request.group
        if not gid or not tid:
            return

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        result = self.encode_result(result, state)
        with client.pipeline() as pipe:
            pipeline = pipe \
                .rpush(jkey, self.encode([1, tid, state, result])) \
                .llen(jkey) \
                .get(tkey)

            if self.expires is not None:
                pipeline = pipeline \
                    .expire(jkey, self.expires) \
                    .expire(tkey, self.expires)

            _, readycount, totaldiff = pipeline.execute()[:3]

        totaldiff = int(totaldiff or 0)

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size'] + totaldiff
            if readycount == total:
                decode, unpack = self.decode, self._unpack_chord_result
                with client.pipeline() as pipe:
                    resl, = pipe \
                        .lrange(jkey, 0, total) \
                        .execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                    with client.pipeline() as pipe:
                        _, _ = pipe \
                            .delete(jkey) \
                            .delete(tkey) \
                            .execute()
                except Exception as exc:  # pylint: disable=broad-except
                    logger.exception(
                        'Chord callback for %r raised: %r', request.group, exc)
                    return self.chord_error_from_stack(
                        callback,
                        ChordError('Callback error: {0!r}'.format(exc)),
                    )
        except ChordError as exc:
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(callback, exc)
        except Exception as exc:  # pylint: disable=broad-except
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(
                callback,
                ChordError('Join error: {0!r}'.format(exc)),
            )
예제 #14
0
파일: builtins.py 프로젝트: hughes/celery
 def apply_async(self, args=(), kwargs={}, task_id=None, group_id=None, chord=None, **options):
     app = self.app
     if app.conf.CELERY_ALWAYS_EAGER:
         return self.apply(args, kwargs, **options)
     header = kwargs.pop("header")
     body = kwargs.pop("body")
     header, body = (maybe_signature(header, app=app), maybe_signature(body, app=app))
     # forward certain options to body
     if chord is not None:
         body.options["chord"] = chord
     if group_id is not None:
         body.options["group_id"] = group_id
     [body.link(s) for s in options.pop("link", [])]
     [body.link_error(s) for s in options.pop("link_error", [])]
     body_result = body.freeze(task_id)
     parent = super(Chord, self).apply_async((header, body, args), kwargs, **options)
     body_result.parent = parent
     return body_result
예제 #15
0
    def _new_chord_return(self,
                          task,
                          state,
                          result,
                          propagate=None,
                          PROPAGATE_STATES=states.PROPAGATE_STATES):
        app = self.app
        if propagate is None:
            propagate = self.app.conf.CELERY_CHORD_PROPAGATES
        request = task.request
        tid, gid = request.id, request.group
        if not gid or not tid:
            return

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        result = self.encode_result(result, state)
        with client.pipeline() as pipe:
            _, readycount, _ = pipe                                         \
                .rpush(jkey, self.encode([1, tid, state, result]))          \
                .llen(jkey)                                                 \
                .expire(jkey, 86400)                                        \
                .execute()

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size']
            if readycount == total:
                decode, unpack = self.decode, self._unpack_chord_result
                with client.pipeline() as pipe:
                    resl, _, = pipe                 \
                        .lrange(jkey, 0, total)     \
                        .delete(jkey)               \
                        .execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                except Exception as exc:
                    error('Chord callback for %r raised: %r',
                          request.group,
                          exc,
                          exc_info=1)
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        except ChordError as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id,
                exc=exc,
            )
        except Exception as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id,
                exc=ChordError('Join error: {0!r}'.format(exc)),
            )
예제 #16
0
파일: redis.py 프로젝트: jonadiazz/celery
    def on_chord_part_return(self, request, state, result, propagate=None):
        app = self.app
        tid, gid = request.id, request.group
        if not gid or not tid:
            return

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        result = self.encode_result(result, state)
        with client.pipeline() as pipe:
            _, readycount, totaldiff, _, _ = pipe                           \
                .rpush(jkey, self.encode([1, tid, state, result]))          \
                .llen(jkey)                                                 \
                .get(tkey)                                                  \
                .expire(jkey, 86400)                                        \
                .expire(tkey, 86400)                                        \
                .execute()

        totaldiff = int(totaldiff or 0)

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size'] + totaldiff
            if readycount == total:
                decode, unpack = self.decode, self._unpack_chord_result
                with client.pipeline() as pipe:
                    resl, _, _ = pipe               \
                        .lrange(jkey, 0, total)     \
                        .delete(jkey)               \
                        .delete(tkey)               \
                        .execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                except Exception as exc:
                    error('Chord callback for %r raised: %r',
                          request.group,
                          exc,
                          exc_info=1)
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        except ChordError as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id,
                exc=exc,
            )
        except Exception as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id,
                exc=ChordError('Join error: {0!r}'.format(exc)),
            )
예제 #17
0
    def _new_chord_return(self, task, state, result, propagate=None,
                          PROPAGATE_STATES=states.PROPAGATE_STATES):
        app = self.app
        if propagate is None:
            propagate = self.app.conf.CELERY_CHORD_PROPAGATES
        request = task.request
        tid, gid = request.id, request.group
        if not gid or not tid:
            return

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        result = self.encode_result(result, state)
        _, readycount, totaldiff, _, _ = client.pipeline()              \
            .rpush(jkey, self.encode([1, tid, state, result]))          \
            .llen(jkey)                                                 \
            .get(tkey)                                                  \
            .expire(jkey, 86400)                                        \
            .expire(tkey, 86400)                                        \
            .execute()

        totaldiff = int(totaldiff or 0)

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size'] + totaldiff
            if readycount >= total:
                decode, unpack = self.decode, self._unpack_chord_result
                resl, _, _ = client.pipeline()  \
                    .lrange(jkey, 0, total)     \
                    .delete(jkey)               \
                    .delete(tkey)               \
                    .execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                except Exception as exc:
                    error('Chord callback for %r raised: %r',
                          request.group, exc, exc_info=1)
                    app._tasks[callback.task].backend.fail_from_current_stack(
                        callback.id,
                        exc=ChordError('Callback error: {0!r}'.format(exc)),
                    )
        except ChordError as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id, exc=exc,
            )
        except Exception as exc:
            error('Chord %r raised: %r', request.group, exc, exc_info=1)
            app._tasks[callback.task].backend.fail_from_current_stack(
                callback.id, exc=ChordError('Join error: {0!r}'.format(exc)),
            )
예제 #18
0
파일: __init__.py 프로젝트: psavery/cumulus
    def _on_complete_lookup(self, completed_task_name):
        """
        Private utility method to lookup can callback that may be registered
        for a give completed task.
        """
        on_complete_map = self.setdefault('_on_complete_map', {})
        to_run = on_complete_map.get(completed_task_name)

        if to_run:
            # Convert back to celery signature object
            to_run = maybe_signature(to_run)

        return to_run
예제 #19
0
 def group(self, tasks, result, group_id, partial_args, add_to_parent=True):
     app = self.app
     result = result_from_tuple(result, app)
     # any partial args are added to all tasks in the group
     taskit = (maybe_signature(task, app=app).clone(partial_args)
               for i, task in enumerate(tasks))
     with app.producer_or_acquire() as producer:
         [stask.apply_async(group_id=group_id, producer=producer,
                            add_to_parent=False) for stask in taskit]
     parent = app.current_worker_task
     if add_to_parent and parent:
         parent.add_trail(result)
     return result
예제 #20
0
파일: __init__.py 프로젝트: Kitware/cumulus
    def _on_complete_lookup(self, completed_task_name):
        """
        Private utility method to lookup can callback that may be registered
        for a give completed task.
        """
        on_complete_map = self.setdefault('_on_complete_map', {})
        to_run = on_complete_map.get(completed_task_name)

        if to_run:
            # Convert back to celery signature object
            to_run = maybe_signature(to_run)

        return to_run
예제 #21
0
파일: builtins.py 프로젝트: alcidesv/celery
        def prepare_steps(self, args, tasks):
            app = self.app
            steps = deque(tasks)
            next_step = prev_task = prev_res = None
            tasks, results = [], []
            i = 0
            while steps:
                # First task get partial args from chain.
                task = maybe_signature(steps.popleft(), app=app)
                task = task.clone() if i else task.clone(args)
                res = task.freeze()
                i += 1

                if isinstance(task, group):
                    task = maybe_unroll_group(task)
                if isinstance(task, chain):
                    # splice the chain
                    steps.extendleft(reversed(task.tasks))
                    continue

                elif isinstance(task, group) and steps and \
                        not isinstance(steps[0], group):
                    # automatically upgrade group(..) | s to chord(group, s)
                    try:
                        next_step = steps.popleft()
                        # for chords we freeze by pretending it's a normal
                        # task instead of a group.
                        res = Signature.freeze(next_step)
                        task = chord(task, body=next_step, task_id=res.task_id)
                    except IndexError:
                        pass  # no callback, so keep as group
                if prev_task:
                    # link previous task to this task.
                    prev_task.link(task)
                    # set the results parent attribute.
                    if not res.parent:
                        res.parent = prev_res

                if not isinstance(prev_task, chord):
                    results.append(res)
                    tasks.append(task)
                prev_task, prev_res = task, res

            print(tasks)

            return tasks, results
예제 #22
0
파일: builtins.py 프로젝트: xenouch/celery
        def prepare_steps(self, args, tasks):
            app = self.app
            steps = deque(tasks)
            next_step = prev_task = prev_res = None
            tasks, results = [], []
            i = 0
            while steps:
                # First task get partial args from chain.
                task = maybe_signature(steps.popleft(), app=app)
                task = task.clone() if i else task.clone(args)
                res = task.freeze()
                i += 1

                if isinstance(task, group):
                    task = maybe_unroll_group(task)
                if isinstance(task, chain):
                    # splice the chain
                    steps.extendleft(reversed(task.tasks))
                    continue

                elif isinstance(task, group) and steps and \
                        not isinstance(steps[0], group):
                    # automatically upgrade group(..) | s to chord(group, s)
                    try:
                        next_step = steps.popleft()
                        # for chords we freeze by pretending it's a normal
                        # task instead of a group.
                        res = Signature.freeze(next_step)
                        task = chord(task, body=next_step, task_id=res.task_id)
                    except IndexError:
                        pass  # no callback, so keep as group
                if prev_task:
                    # link previous task to this task.
                    prev_task.link(task)
                    # set the results parent attribute.
                    if not res.parent:
                        res.parent = prev_res

                if not isinstance(prev_task, chord):
                    results.append(res)
                    tasks.append(task)
                prev_task, prev_res = task, res

            return tasks, results
예제 #23
0
        def run(self,
                header,
                body,
                partial_args=(),
                interval=None,
                countdown=1,
                max_retries=None,
                propagate=None,
                eager=False,
                **kwargs):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group(
                [maybe_signature(s, app=app).clone() for s in tasks],
                app=self.app)
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            body['chord_size'] = len(header.tasks)
            results = header.freeze(group_id=group_id, chord=body).results

            return self.backend.apply_chord(
                header,
                partial_args,
                group_id,
                body,
                interval=interval,
                countdown=countdown,
                max_retries=max_retries,
                propagate=propagate,
                result=results,
            )
예제 #24
0
파일: builtins.py 프로젝트: hughes/celery
        def run(
            self,
            header,
            body,
            partial_args=(),
            interval=None,
            countdown=1,
            max_retries=None,
            propagate=None,
            eager=False,
            **kwargs
        ):
            app = self.app
            propagate = default_propagate if propagate is None else propagate
            group_id = uuid()

            # - convert back to group if serialized
            tasks = header.tasks if isinstance(header, group) else header
            header = group([maybe_signature(s, app=app).clone() for s in tasks], app=self.app)
            # - eager applies the group inline
            if eager:
                return header.apply(args=partial_args, task_id=group_id)

            body.setdefault("chord_size", len(header.tasks))
            results = header.freeze(group_id=group_id, chord=body).results

            return self.backend.apply_chord(
                header,
                partial_args,
                group_id,
                body,
                interval=interval,
                countdown=countdown,
                max_retries=max_retries,
                propagate=propagate,
                result=results,
            )
예제 #25
0
    def on_chord_part_return(self,
                             request,
                             state,
                             result,
                             propagate=None,
                             **kwargs):
        app = self.app
        tid, gid, group_index = request.id, request.group, request.group_index
        if not gid or not tid:
            return
        if group_index is None:
            group_index = '+inf'

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        skey = self.get_key_for_group(gid, '.s')
        result = self.encode_result(result, state)
        encoded = self.encode([1, tid, state, result])
        with client.pipeline() as pipe:
            pipeline = (pipe.zadd(jkey, {
                encoded: group_index
            }).zcount(jkey, "-inf",
                      "+inf") if self._chord_zset else pipe.rpush(
                          jkey, encoded).llen(jkey)).get(tkey).get(skey)
            if self.expires:
                pipeline = pipeline \
                    .expire(jkey, self.expires) \
                    .expire(tkey, self.expires) \
                    .expire(skey, self.expires)

            _, readycount, totaldiff, chord_size_bytes = pipeline.execute()[:4]

        totaldiff = int(totaldiff or 0)

        if chord_size_bytes:
            try:
                callback = maybe_signature(request.chord, app=app)
                total = int(chord_size_bytes) + totaldiff
                if readycount == total:
                    header_result = GroupResult.restore(gid)
                    if header_result is not None:
                        # If we manage to restore a `GroupResult`, then it must
                        # have been complex and saved by `apply_chord()` earlier.
                        #
                        # Before we can join the `GroupResult`, it needs to be
                        # manually marked as ready to avoid blocking
                        header_result.on_ready()
                        # We'll `join()` it to get the results and ensure they are
                        # structured as intended rather than the flattened version
                        # we'd construct without any other information.
                        join_func = (header_result.join_native
                                     if header_result.supports_native_join else
                                     header_result.join)
                        with allow_join_result():
                            resl = join_func(
                                timeout=app.conf.result_chord_join_timeout,
                                propagate=True)
                    else:
                        # Otherwise simply extract and decode the results we
                        # stashed along the way, which should be faster for large
                        # numbers of simple results in the chord header.
                        decode, unpack = self.decode, self._unpack_chord_result
                        with client.pipeline() as pipe:
                            if self._chord_zset:
                                pipeline = pipe.zrange(jkey, 0, -1)
                            else:
                                pipeline = pipe.lrange(jkey, 0, total)
                            resl, = pipeline.execute()
                        resl = [unpack(tup, decode) for tup in resl]
                    try:
                        callback.delay(resl)
                    except Exception as exc:  # pylint: disable=broad-except
                        logger.exception('Chord callback for %r raised: %r',
                                         request.group, exc)
                        return self.chord_error_from_stack(
                            callback,
                            ChordError(f'Callback error: {exc!r}'),
                        )
                    finally:
                        with client.pipeline() as pipe:
                            pipe \
                                .delete(jkey) \
                                .delete(tkey) \
                                .delete(skey) \
                                .execute()
            except ChordError as exc:
                logger.exception('Chord %r raised: %r', request.group, exc)
                return self.chord_error_from_stack(callback, exc)
            except Exception as exc:  # pylint: disable=broad-except
                logger.exception('Chord %r raised: %r', request.group, exc)
                return self.chord_error_from_stack(
                    callback,
                    ChordError(f'Join error: {exc!r}'),
                )
예제 #26
0
 def test_is_list(self):
     sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))]
     sigs = maybe_signature(sigs, app=self.app)
     for sig in sigs:
         self.assertIsInstance(sig, Signature)
         self.assertIs(sig.app, self.app)
예제 #27
0
 def test_is_dict(self):
     self.assertIsInstance(
         maybe_signature(dict(self.add.s()), app=self.app), Signature,
     )
예제 #28
0
 def test_is_None(self):
     self.assertIsNone(maybe_signature(None, app=self.app))
예제 #29
0
 def test_is_None(self):
     self.assertIsNone(maybe_signature(None, app=self.app))
예제 #30
0
 def test_when_sig(self):
     s = self.add.s()
     self.assertIs(maybe_signature(s, app=self.app), s)
예제 #31
0
 def test_is_dict(self):
     assert isinstance(maybe_signature(dict(self.add.s()), app=self.app),
                       Signature)
예제 #32
0
 def test_when_sig(self):
     s = self.add.s()
     assert maybe_signature(s, app=self.app) is s
예제 #33
0
 def test_when_sig(self):
     s = self.add.s()
     assert maybe_signature(s, app=self.app) is s
예제 #34
0
 def test_is_None(self):
     assert maybe_signature(None, app=self.app) is None
예제 #35
0
 def test_is_dict(self):
     assert isinstance(maybe_signature(dict(self.add.s()), app=self.app),
                       Signature)
예제 #36
0
 def test_is_None(self):
     assert maybe_signature(None, app=self.app) is None
예제 #37
0
    def on_chord_part_return(self, request, state, result,
                             propagate=None, **kwargs):
        app = self.app
        tid, gid, group_index = request.id, request.group, request.group_index
        if not gid or not tid:
            return
        if group_index is None:
            group_index = '+inf'

        client = self.client
        jkey = self.get_key_for_group(gid, '.j')
        tkey = self.get_key_for_group(gid, '.t')
        result = self.encode_result(result, state)
        encoded = self.encode([1, tid, state, result])
        with client.pipeline() as pipe:
            pipeline = (
                pipe.zadd(jkey, {encoded: group_index}).zcount(jkey, "-inf", "+inf")
                if self._chord_zset
                else pipe.rpush(jkey, encoded).llen(jkey)
            ).get(tkey)
            if self.expires is not None:
                pipeline = pipeline \
                    .expire(jkey, self.expires) \
                    .expire(tkey, self.expires)

            _, readycount, totaldiff = pipeline.execute()[:3]

        totaldiff = int(totaldiff or 0)

        try:
            callback = maybe_signature(request.chord, app=app)
            total = callback['chord_size'] + totaldiff
            if readycount == total:
                decode, unpack = self.decode, self._unpack_chord_result
                with client.pipeline() as pipe:
                    if self._chord_zset:
                        pipeline = pipe.zrange(jkey, 0, -1)
                    else:
                        pipeline = pipe.lrange(jkey, 0, total)
                    resl, = pipeline.execute()
                try:
                    callback.delay([unpack(tup, decode) for tup in resl])
                    with client.pipeline() as pipe:
                        _, _ = pipe \
                            .delete(jkey) \
                            .delete(tkey) \
                            .execute()
                except Exception as exc:  # pylint: disable=broad-except
                    logger.exception(
                        'Chord callback for %r raised: %r', request.group, exc)
                    return self.chord_error_from_stack(
                        callback,
                        ChordError(f'Callback error: {exc!r}'),
                    )
        except ChordError as exc:
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(callback, exc)
        except Exception as exc:  # pylint: disable=broad-except
            logger.exception('Chord %r raised: %r', request.group, exc)
            return self.chord_error_from_stack(
                callback,
                ChordError(f'Join error: {exc!r}'),
            )
예제 #38
0
파일: builtins.py 프로젝트: xenouch/celery
 def apply(self, args=(), kwargs={}, propagate=True, **options):
     body = kwargs['body']
     res = super(Chord, self).apply(args, dict(kwargs, eager=True),
                                    **options)
     return maybe_signature(body, app=self.app).apply(args=(res.get(
         propagate=propagate).get(), ))
예제 #39
0
 def test_when_sig(self):
     s = self.add.s()
     self.assertIs(maybe_signature(s, app=self.app), s)
예제 #40
0
 def test_is_dict(self):
     self.assertIsInstance(
         maybe_signature(dict(self.add.s()), app=self.app),
         Signature,
     )
예제 #41
0
 def __init__(self, tasks=None, app=None, Publisher=None):
     self.app = app_or_default(app or self.app)
     super(TaskSet, self).__init__(
         maybe_signature(t, app=self.app) for t in tasks or [])
     self.Publisher = Publisher or self.app.amqp.TaskProducer
     self.total = len(self)  # XXX compat
예제 #42
0
파일: builtins.py 프로젝트: NotSqrt/celery
 def prepare_member(task):
     task = maybe_signature(task, app=self.app)
     task.options['group_id'] = group_id
     return task, task.freeze()
예제 #43
0
파일: builtins.py 프로젝트: xenouch/celery
 def prepare_member(task):
     task = maybe_signature(task, app=self.app)
     task.options['group_id'] = group_id
     return task, task.freeze()
예제 #44
0
파일: builtins.py 프로젝트: NotSqrt/celery
 def apply(self, args=(), kwargs={}, propagate=True, **options):
     body = kwargs['body']
     res = super(Chord, self).apply(args, dict(kwargs, eager=True),
                                    **options)
     return maybe_signature(body, app=self.app).apply(
         args=(res.get(propagate=propagate).get(), ))
예제 #45
0
 def test_is_list(self):
     sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))]
     sigs = maybe_signature(sigs, app=self.app)
     for sig in sigs:
         self.assertIsInstance(sig, Signature)
         self.assertIs(sig.app, self.app)