def apply(self, args=(), kwargs={}, signature=maybe_signature, **options): app = self.app last, fargs = None, args # fargs passed to first task only for task in kwargs["tasks"]: res = signature(task, app=app).clone(fargs).apply(last and (last.get(),)) res.parent, last, fargs = last, res, None return last
def apply(self, args=(), kwargs={}, signature=maybe_signature, **options): app = self.app last, fargs = None, args # fargs passed to first task only for task in kwargs['tasks']: res = signature(task, app=app).clone(fargs).apply( last and (last.get(), ), ) res.parent, last, fargs = last, res, None return last
def signature(self, args=None, *starargs, **starkwargs): """Create signature. Returns: :class:`~celery.signature`: object for this task, wrapping arguments and execution options for a single task invocation. """ starkwargs.setdefault('app', self.app) return signature(self, args, *starargs, **starkwargs)
def unlock_chord(self, group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, result_from_tuple=result_from_tuple): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: interval = self.default_retry_delay # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [result_from_tuple(r, app=app) for r in result], app=app, ) j = deps.join_native if deps.supports_native_join else deps.join try: ready = deps.ready() except Exception as exc: raise self.retry( exc=exc, countdown=interval, max_retries=max_retries, ) else: if not ready: raise self.retry(countdown=interval, max_retries=max_retries) callback = signature(callback, app=app) try: with allow_join_result(): ret = j(timeout=3.0, propagate=propagate) except Exception as exc: try: culprit = next(deps._failed_join_report()) reason = 'Dependency {0.id} raised {1!r}'.format( culprit, exc, ) except StopIteration: reason = repr(exc) logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) app.backend.chord_error_from_stack(callback, ChordError(reason)) else: try: callback.delay(ret) except Exception as exc: logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) app.backend.chord_error_from_stack( callback, exc=ChordError('Callback error: {0!r}'.format(exc)), )
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: interval = unlock_chord.default_retry_delay # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [from_serializable(r, app=app) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = signature(callback, app=app) try: ret = j(propagate=propagate) except Exception as exc: try: culprit = next(deps._failed_join_report()) reason = 'Dependency {0.id} raised {1!r}'.format( culprit, exc, ) except StopIteration: reason = repr(exc) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError(reason), ) else: try: callback.delay(ret) except Exception as exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Callback error: {0!r}'.format(exc)), ) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def read_finish_continue( prevres: Union[Tuple[List[str], int, List[Dict[str, str]]], Tuple[List[Dict[str, str]]]], callback: dict, filename: str, operation_id: int, ): """ NOTE: This task should be placed after each `read_next` task Checks whether the previous task (`read_next`) returned a tuple of 3 results or 1 If previous task returned a tuple of 3 results, it means EOF has not been reached and the operation should continue with another `read_next` If previous task returned a tuple of 1 result (just the final list of dicts), EOF has been reached - initiate the given callback (should be a serialized signature) """ if len(prevres) == 3: # Continue with another `read_next`, `read_finish_continue` pair # Use the regular tappable configuration as well tappable( ( read_next.s(filename) # Pass in the same callback, filename, and operation_id | read_finish_continue.s(callback, filename, operation_id)), # Function to check whether or not operation should pause should_pause.s(operation_id), # Pause handler save_state.s(operation_id), # Start the chain with the previous result (tuple of 3 elements: see `read_next`) ).delay(prevres) # Just a dummy return to aid in logging - doesn't really serve a purpose return f"Continuing - Total rows read: {len(prevres[-1])}" else: # EOF reached, finished reading - initiate the callback task and pass it the final list of dicts signature(callback).delay(prevres[0]) # Just a dummy return to aid in logging - doesn't really serve a purpose return f"Finished Reading - Total rows read: {len(prevres[-1])}"
def run(self, tasks, result, group_id, partial_args, add_to_parent=True): app = self.app result = result_from_tuple(result, app) # any partial args are added to all tasks in the group taskit = (signature(task, app=app).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit]) with app.producer_or_acquire() as pub: [stask.apply_async(group_id=group_id, producer=pub, add_to_parent=False) for stask in taskit] parent = get_current_worker_task() if add_to_parent and parent: parent.add_trail(result) return result
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, result_from_tuple=result_from_tuple): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: interval = unlock_chord.default_retry_delay # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [result_from_tuple(r, app=app) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = signature(callback, app=app) try: ret = j(propagate=propagate) except Exception as exc: try: culprit = next(deps._failed_join_report()) reason = 'Dependency {0.id} raised {1!r}'.format( culprit, exc, ) except StopIteration: reason = repr(exc) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError(reason), ) else: try: callback.delay(ret) except Exception as exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Callback error: {0!r}'.format(exc)), ) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def test_keeping_link_error_on_chaining(self): x = self.add.s(2, 2) | self.mul.s(4) assert isinstance(x, _chain) x.link_error(SIG) assert SIG in x.options['link_error'] t = signature(SIG) z = x | t assert isinstance(z, _chain) assert t in z.tasks assert not z.options.get('link_error') assert SIG in z.tasks[0].options['link_error'] assert not z.tasks[2].options.get('link_error') assert SIG in x.options['link_error'] assert t not in x.tasks assert not x.tasks[0].options.get('link_error') z = t | x assert isinstance(z, _chain) assert t in z.tasks assert not z.options.get('link_error') assert SIG in z.tasks[1].options['link_error'] assert not z.tasks[0].options.get('link_error') assert SIG in x.options['link_error'] assert t not in x.tasks assert not x.tasks[0].options.get('link_error') y = self.add.s(4, 4) | self.div.s(2) assert isinstance(y, _chain) z = x | y assert isinstance(z, _chain) assert not z.options.get('link_error') assert SIG in z.tasks[0].options['link_error'] assert not z.tasks[2].options.get('link_error') assert SIG in x.options['link_error'] assert not x.tasks[0].options.get('link_error') z = y | x assert isinstance(z, _chain) assert not z.options.get('link_error') assert SIG in z.tasks[3].options['link_error'] assert not z.tasks[1].options.get('link_error') assert SIG in x.options['link_error'] assert not x.tasks[0].options.get('link_error')
def run(self, tasks, result, group_id, partial_args): app = self.app result = result_from_tuple(result, app) # any partial args are added to all tasks in the group taskit = (signature(task, app=app).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult( result.id, [stask.apply(group_id=group_id) for stask in taskit], ) with app.producer_or_acquire() as pub: [stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit] parent = get_current_worker_task() if parent: parent.add_trail(result) return result
def replace(self, sig): """Replace this task, with a new task inheriting the task id. Execution of the host task ends immediately and no subsequent statements will be run. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. It is best to always use ``return self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord") if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( link=self.request.callbacks, link_error=self.request.errbacks, ) if self.request.chain: for t in reversed(self.request.chain): sig |= signature(t, app=self.app) sig.set( chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, ) sig.freeze(self.request.id) if self.request.is_eager: return sig.apply().get() else: sig.delay() raise Ignore('Replaced by new task')
def replace(self, sig): """Replace the current task, with a new task inheriting the same task id. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised, so the best practice is to always use ``raise self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: if chord: chord = sig.options['chord'] | chord else: chord = sig.options['chord'] if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( chord=chord, link=self.request.callbacks, link_error=self.request.errbacks, ) chord = None if self.request.chain: for t in self.request.chain: sig |= signature(t) sig.freeze(self.request.id, group_id=self.request.group, chord=chord, root_id=self.request.root_id) sig.delay() raise Ignore('Replaced by new task')
def replace(self, sig): """Replace this task, with a new task inheriting the task id. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised, so the best practice is to always use ``raise self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord" ) if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( link=self.request.callbacks, link_error=self.request.errbacks, ) if self.request.chain: for t in reversed(self.request.chain): sig |= signature(t, app=self.app) sig.set( chord=chord, group_id=self.request.group, root_id=self.request.root_id, ) sig.freeze(self.request.id) sig.delay() raise Ignore('Replaced by new task')
def get_task_signature( self, score_name: str, player_name: str, measure_name: str, func: str, delay: int, *task_args, **task_kwargs, ) -> "Signature": if self.tuning: msg = ( "host is tuning or pending tuning; strongly advise against running new" " tasks on it or they may be interrupted without warning or recovery" ) logger.warning( gudlog(msg, score_name, player_name, None, self.name)) description = f"{score_name}.{player_name}.{self.name}.{measure_name}" task_id = str( uuid4()) # we need to know the task_id a priori for score.task_map sig_opts = { "queue": self.name, "shadow": description, "task_id": task_id, "countdown": delay, } sig = signature(func, args=task_args, kwargs=task_kwargs, options=sig_opts) msg = (f"task signature created for {func}{task_args} with" f" kwargs:\n{task_kwargs}\nand options:\n{sig_opts}") logger.log( 5, gudlog(msg, score_name, player_name, measure_name, self.name)) return sig
def test_AsyncResult_when_not_registered(self): s = signature('xxx.not.registered', app=self.app) self.assertTrue(s.AsyncResult)
def test_apply_async_when_not_registered(self): s = signature('xxx.not.registered', app=self.app) self.assertTrue(s._apply_async)
def call_task(self, task): try: signature(task, app=self.app).apply_async() except Exception as exc: error('Could not call task: %r', exc, exc_info=1)
def test_link_error(self): x = signature(SIG) x.link_error(SIG) x.link_error(SIG) self.assertIn(SIG, x.options['link_error']) self.assertEqual(len(x.options['link_error']), 1)
def test_apply_async_when_not_registered(self): s = signature('xxx.not.registered', app=self.app) assert s._apply_async
def xstarmap(task, it): task = signature(task, app=app).type return [task(*item) for item in it]
def test_reverse(self): x = chord([self.add.s(2, 2), self.add.s(4, 4)], body=self.mul.s(4)) self.assertIsInstance(signature(x), chord) self.assertIsInstance(signature(dict(x)), chord)
def test_reverse(self): x = chord([self.add.s(2, 2), self.add.s(4, 4)], body=self.mul.s(4)) assert isinstance(signature(x), chord) assert isinstance(signature(dict(x)), chord)
def test_link_error(self): x = signature(SIG) x.link_error(SIG) x.link_error(SIG) assert SIG in x.options['link_error'] assert len(x.options['link_error']) == 1
def test_reverse(self): x = group([self.add.s(2, 2), self.add.s(4, 4)]) assert isinstance(signature(x), group) assert isinstance(signature(dict(x)), group)
def test_reverse(self): x = self.add.s(2, 2) | self.add.s(2) assert isinstance(signature(x), _chain) assert isinstance(signature(dict(x)), _chain)
def deserialize_chain(serialized_ch: List[dict]): # Build task signatures from list of dicts (serialized json) return chain(signature(x) for x in serialized_ch)
def test_reverse(self): x = self.add.s(2, 2) | self.add.s(2) self.assertIsInstance(signature(x), chain) self.assertIsInstance(signature(dict(x)), chain)
def test_reverse(self): x = group([self.add.s(2, 2), self.add.s(4, 4)]) self.assertIsInstance(signature(x), group) self.assertIsInstance(signature(dict(x)), group)
def test_AsyncResult_when_not_registered(self): s = signature('xxx.not.registered', app=self.app) assert s.AsyncResult
def replace(self, sig): """Replace this task, with a new task inheriting the task id. Execution of the host task ends immediately and no subsequent statements will be run. .. versionadded:: 4.0 Arguments: sig (Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. It is best to always use ``return self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord") if isinstance(sig, _chain) and not getattr(sig, "tasks", True): raise ImproperlyConfigured("Cannot replace with an empty chain") # Ensure callbacks or errbacks from the replaced signature are retained if isinstance(sig, group): # Groups get uplifted to a chord so that we can link onto the body sig |= self.app.tasks['celery.accumulate'].s(index=0) for callback in maybe_list(self.request.callbacks) or []: sig.link(callback) for errback in maybe_list(self.request.errbacks) or []: sig.link_error(errback) # If the replacement signature is a chain, we need to push callbacks # down to the final task so they run at the right time even if we # proceed to link further tasks from the original request below if isinstance(sig, _chain) and "link" in sig.options: final_task_links = sig.tasks[-1].options.setdefault("link", []) final_task_links.extend(maybe_list(sig.options["link"])) # We need to freeze the replacement signature with the current task's # ID to ensure that we don't disassociate it from the existing task IDs # which would break previously constructed results objects. sig.freeze(self.request.id) # Ensure the important options from the original signature are retained replaced_task_nesting = self.request.get('replaced_task_nesting', 0) + 1 sig.set(chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, replaced_task_nesting=replaced_task_nesting) # If the task being replaced is part of a chain, we need to re-create # it with the replacement signature - these subsequent tasks will # retain their original task IDs as well for t in reversed(self.request.chain or []): sig |= signature(t, app=self.app) # Stamping sig with parents groups stamped_headers = self.request.stamped_headers if self.request.stamps: groups = self.request.stamps.get("groups") sig.stamp(visitor=GroupStampingVisitor( groups=groups, stamped_headers=stamped_headers)) # Finally, either apply or delay the new signature! if self.request.is_eager: return sig.apply().get() else: sig.delay() raise Ignore('Replaced by new task')
def replace(self, sig): """Replace this task, with a new task inheriting the task id. Execution of the host task ends immediately and no subsequent statements will be run. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. It is best to always use ``return self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord" ) if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( link=self.request.callbacks, link_error=self.request.errbacks, ) elif isinstance(sig, _chain): if not sig.tasks: raise ImproperlyConfigured( "Cannot replace with an empty chain" ) if self.request.chain: # We need to freeze the new signature with the current task's ID to # ensure that we don't disassociate the new chain from the existing # task IDs which would break previously constructed results # objects. sig.freeze(self.request.id) if "link" in sig.options: final_task_links = sig.tasks[-1].options.setdefault("link", []) final_task_links.extend(maybe_list(sig.options["link"])) # Construct the new remainder of the task by chaining the signature # we're being replaced by with signatures constructed from the # chain elements in the current request. for t in reversed(self.request.chain): sig |= signature(t, app=self.app) sig.set( chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, ) sig.freeze(self.request.id) if self.request.is_eager: task_result = sig.apply() with allow_join_result(): return task_result.get() else: sig.delay() raise Ignore('Replaced by new task')