def test_cleared_context(self): changes = dict(id="unique id", args=["some", 1], wibble="wobble") ctx = Context() ctx.update(changes) ctx.clear() defaults = dict(default_context, children=[]) self.assertDictEqual(get_context_as_dict(ctx), defaults) self.assertDictEqual(get_context_as_dict(Context()), defaults)
def test_updated_context(self): expected = dict(default_context) changes = dict(id='unique id', args=['some', 1], wibble='wobble') ctx = Context() expected.update(changes) ctx.update(changes) assert get_context_as_dict(ctx) == expected assert get_context_as_dict(Context()) == default_context
def test_cleared_context(self): changes = {'id': 'unique id', 'args': ['some', 1], 'wibble': 'wobble'} ctx = Context() ctx.update(changes) ctx.clear() defaults = dict(default_context, children=[]) assert get_context_as_dict(ctx) == defaults assert get_context_as_dict(Context()) == defaults
def test_updated_context(self): expected = dict(default_context) changes = dict(id='unique id', args=['some', 1], wibble='wobble') ctx = Context() expected.update(changes) ctx.update(changes) self.assertDictEqual(get_context_as_dict(ctx), expected) self.assertDictEqual(get_context_as_dict(Context()), default_context)
def test_updated_context(self): expected = dict(default_context) changes = {'id': 'unique id', 'args': ['some', 1], 'wibble': 'wobble'} ctx = Context() expected.update(changes) ctx.update(changes) assert get_context_as_dict(ctx) == expected assert get_context_as_dict(Context()) == default_context
def test_cleared_context(self): changes = dict(id='unique id', args=['some', 1], wibble='wobble') ctx = Context() ctx.update(changes) ctx.clear() defaults = dict(default_context, children=[]) assert get_context_as_dict(ctx) == defaults assert get_context_as_dict(Context()) == defaults
def test_modified_context(self): expected = dict(default_context) ctx = Context() expected['id'] = 'unique id' expected['args'] = ['some', 1] ctx.id = 'unique id' ctx.args = ['some', 1] assert get_context_as_dict(ctx) == expected assert get_context_as_dict(Context()) == default_context
def test_modified_context(self): expected = dict(default_context) ctx = Context() expected['id'] = 'unique id' expected['args'] = ['some', 1] ctx.id = 'unique id' ctx.args = ['some', 1] self.assertDictEqual(get_context_as_dict(ctx), expected) self.assertDictEqual(get_context_as_dict(Context()), default_context)
def test_context_get(self): expected = dict(default_context) changes = dict(id="unique id", args=["some", 1], wibble="wobble") ctx = Context() expected.update(changes) ctx.update(changes) ctx_dict = get_context_as_dict(ctx, getter=Context.get) self.assertDictEqual(ctx_dict, expected) self.assertDictEqual(get_context_as_dict(Context()), default_context)
def test_modified_context(self): expected = dict(default_context) ctx = Context() expected["id"] = "unique id" expected["args"] = ["some", 1] ctx.id = "unique id" ctx.args = ["some", 1] self.assertDictEqual(get_context_as_dict(ctx), expected) self.assertDictEqual(get_context_as_dict(Context()), default_context)
def chord_error_from_stack(self, callback, exc=None): app = self.app try: backend = app._tasks[callback.task].backend except KeyError: backend = self # We have to make a fake request since either the callback failed or # we're pretending it did since we don't have information about the # chord part(s) which failed. This request is constructed as a best # effort for new style errbacks and may be slightly misleading about # what really went wrong, but at least we call them! fake_request = Context({ "id": callback.options.get("task_id"), "errbacks": callback.options.get("link_error", []), "delivery_info": dict(), **callback }) try: self._call_task_errbacks(fake_request, exc, None) except Exception as eb_exc: # pylint: disable=broad-except return backend.fail_from_current_stack(callback.id, exc=eb_exc) else: return backend.fail_from_current_stack(callback.id, exc=exc)
def _create_request(self, task_id, name, args, kwargs, argsrepr=None, kwargsrepr=None, task_protocol=2): msg = self.app.amqp.task_protocols[task_protocol]( task_id=task_id, name=name, args=args, kwargs=kwargs, argsrepr=argsrepr, kwargsrepr=kwargsrepr, ) if task_protocol == 1: body, headers, _, _ = hybrid_to_proto2(msg, msg.body) properties = None sent_event = {} else: headers, properties, body, sent_event = msg context = Context( headers=headers, properties=properties, body=body, sent_event=sent_event, ) request = Request(context, decoded=True, task=name) if task_protocol == 1: assert request.argsrepr is None assert request.kwargsrepr is None else: assert request.argsrepr is not None assert request.kwargsrepr is not None return request
def test_get_request_meta(self): x = self.app.AsyncResult('1') request = Context(task='foo', children=None, args=['one', 'two'], kwargs={'kwarg1': 'three'}, hostname="foo", retries=1, delivery_info={'routing_key': 'celery'}) x.backend.store_result(task_id="1", result='foo', state=states.SUCCESS, traceback=None, request=request) assert x.name == 'foo' assert x.args == ['one', 'two'] assert x.kwargs == {'kwarg1': 'three'} assert x.worker == 'foo' assert x.retries == 1 assert x.queue == 'celery' assert isinstance(x.date_done, datetime.datetime) assert x.task_id == "1" assert x.state == "SUCCESS" result = self.app.AsyncResult(self.task4['id']) assert result.date_done is None
def test_get_result_meta(self, result_serializer, args, kwargs): self.app.conf.result_serializer = result_serializer tb = DatabaseBackend(self.uri, app=self.app) request = Context(args=args, kwargs=kwargs, task='mytask', retries=2, hostname='celery@worker_1', delivery_info={'routing_key': 'celery'}) meta = tb._get_result_meta(result={'fizz': 'buzz'}, state=states.SUCCESS, traceback=None, request=request, format_date=False, encode=True) assert meta['result'] == {'fizz': 'buzz'} assert tb.decode(meta['args']) == args assert tb.decode(meta['kwargs']) == kwargs assert meta['queue'] == 'celery' assert meta['name'] == 'mytask' assert meta['retries'] == 2 assert meta['worker'] == "celery@worker_1"
def test_store_result(self): b = AMQPBackend(self.app) tid = uuid() request = Context(args=(1, 2, 3), kwargs={'foo': 'bar'}, task_name='mytask', retries=2, hostname='celery@worker_1', delivery_info={'routing_key': 'celery'}) b.store_result(tid, {'fizz': 'buzz'}, states.SUCCESS, request=request) meta = b.get_task_meta(tid) assert meta == { 'args': [1, 2, 3], 'children': [], 'kwargs': {'foo': 'bar'}, 'name': 'mytask', 'queue': 'celery', 'result': {'fizz': 'buzz'}, 'retries': 2, 'status': 'SUCCESS', 'task_id': tid, 'traceback': None, 'worker': 'celery@worker_1', }
def test_get_result_meta_with_none(self): b1 = BaseBackend(self.app) meta = b1._get_result_meta(result=None, state=states.SUCCESS, traceback=None, request=None) assert meta['status'] == states.SUCCESS assert meta['result'] is None assert meta['traceback'] is None self.app.conf.result_extended = True args = ['a', 'b'] kwargs = {'foo': 'bar'} task_name = 'mytask' b2 = BaseBackend(self.app) request = Context(args=args, kwargs=kwargs, task=task_name, delivery_info={'routing_key': 'celery'}) meta = b2._get_result_meta(result=None, state=states.SUCCESS, traceback=None, request=request, encode=False) assert meta['name'] == task_name assert meta['args'] == args assert meta['kwargs'] == kwargs assert meta['queue'] == 'celery'
def _context(self): """Context (:class:`~celery.app.task.Context`) of this task.""" request = self._request_dict # pylint: disable=unpacking-non-sequence # payload is a property, so pylint doesn't think it's a tuple. _, _, embed = self._payload request.update(**embed or {}) return Context(request)
def test_extract_headers(self): # Should extract custom headers from the request dict request = { 'task': 'test.test_task', 'id': 'e16eeaee-1172-49bb-9098-5437a509ffd9', 'custom-header': 'custom-value', } ctx = Context(request) assert ctx.headers == {'custom-header': 'custom-value'}
def test_cleared_context(self): changes = dict(id='unique id', args=['some', 1], wibble='wobble') ctx = Context() ctx.update(changes) ctx.clear() defaults = dict(default_context, children=[]) self.assertDictEqual(get_context_as_dict(ctx), defaults) self.assertDictEqual(get_context_as_dict(Context()), defaults)
def apply_batches_task(task: "Batches", args: Tuple[List["SimpleRequest"]], loglevel: int, logfile: None) -> Any: request_stack = task.request_stack push_request = request_stack.push pop_request = request_stack.pop push_task = _task_stack.push pop_task = _task_stack.pop prerun_receivers = signals.task_prerun.receivers postrun_receivers = signals.task_postrun.receivers success_receivers = signals.task_success.receivers # Corresponds to multiple requests, so generate a new UUID. task_id = uuid() push_task(task) task_request = Context(loglevel=loglevel, logfile=logfile) push_request(task_request) try: # -*- PRE -*- if prerun_receivers: send_prerun(sender=task, task_id=task_id, task=task, args=args, kwargs={}) # -*- TRACE -*- try: result = task(*args) state = SUCCESS except Exception as exc: result = None state = FAILURE logger.error("Error: %r", exc, exc_info=True) else: if success_receivers: send_success(sender=task, result=result) finally: try: if postrun_receivers: send_postrun( sender=task, task_id=task_id, task=task, args=args, kwargs={}, retval=result, state=state, ) finally: pop_task() pop_request() return result
def test_store_result_group_id(self): tid = uuid() state = 'SUCCESS' result = 10 request = Context(group='gid', children=[]) self.b.store_result( tid, state=state, result=result, request=request, ) stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid))) assert stored_meta['group_id'] == request.group
def test_store_result_parent_id(self): tid = uuid() pid = uuid() state = 'SUCCESS' result = 10 request = Context(parent_id=pid) self.b.store_result( tid, state=state, result=result, request=request, ) stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid))) assert stored_meta['parent_id'] == request.parent_id
def apply_batches_task(task, args, loglevel, logfile): # Mimics some of the functionality found in celery.app.trace.trace_task. request_stack = task.request_stack push_request = request_stack.push pop_request = request_stack.pop push_task = _task_stack.push pop_task = _task_stack.pop prerun_receivers = signals.task_prerun.receivers postrun_receivers = signals.task_postrun.receivers success_receivers = signals.task_success.receivers # Corresponds to multiple requests, so generate a new UUID. task_id = uuid() push_task(task) task_request = Context(loglevel=loglevel, logfile=logfile) push_request(task_request) try: # -*- PRE -*- if prerun_receivers: send_prerun(sender=task, task_id=task_id, task=task, args=args, kwargs={}) # -*- TRACE -*- try: result = task(*args) state = SUCCESS except Exception as exc: result = None state = FAILURE logger.error('Error: %r', exc, exc_info=True) else: if success_receivers: send_success(sender=task, result=result) finally: try: if postrun_receivers: send_postrun(sender=task, task_id=task_id, task=task, args=args, kwargs={}, retval=result, state=state) finally: pop_task() pop_request() return result
def test_get_result_meta_encoded(self): self.app.conf.result_extended = True b1 = BaseBackend(self.app) args = ['a', 'b'] kwargs = {'foo': 'bar'} request = Context(args=args, kwargs=kwargs) meta = b1._get_result_meta(result={'fizz': 'buzz'}, state=states.SUCCESS, traceback=None, request=request, encode=True) assert meta['args'] == ensure_bytes(b1.encode(args)) assert meta['kwargs'] == ensure_bytes(b1.encode(kwargs))
def test_dont_override_headers(self): # Should not override headers if defined in the request request = { 'task': 'test.test_task', 'id': 'e16eeaee-1172-49bb-9098-5437a509ffd9', 'headers': { 'custom-header': 'custom-value' }, 'custom-header-2': 'custom-value-2', } ctx = Context(request) assert ctx.headers == {'custom-header': 'custom-value'}
def _context(self): """Context (:class:`~celery.app.task.Context`) of this task.""" request = self.request_dict # pylint: disable=unpacking-non-sequence # payload is a property, so pylint doesn't think it's a tuple. args, kwargs, embed = self._payload request.update( { 'hostname': self.hostname, 'args': args, 'kwargs': kwargs }, **embed or {}) return Context(request)
def test_store_result_race_second_write_should_ignore_if_previous_success(self): tid = uuid() state = 'SUCCESS' result = 10 request = Context(group='gid', children=[]) self.b.store_result( tid, state=state, result=result, request=request, ) self.b.store_result( tid, state=states.FAILURE, result=result, request=request, ) stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid))) assert stored_meta['status'] == states.SUCCESS
def test_store_result_parent_id(self, serializer): self.app.conf.accept_content = ('json', serializer) self.b = KVBackend(app=self.app, serializer=serializer) tid = uuid() pid = uuid() state = 'SUCCESS' result = 10 request = Context(parent_id=pid) self.b.store_result( tid, state=state, result=result, request=request, ) stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid))) assert stored_meta['parent_id'] == request.parent_id
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( "Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = body.get("utc", False) self.flags = body.get("flags", False) self.on_ack = on_ack self.hostname = hostname self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), } self.request_dict = Context(body, called_directly=False)
def mark_as_failure(self, task_id, exc, traceback=None, request=None, store_result=True, call_errbacks=True, state=states.FAILURE): """Mark task as executed with failure.""" if store_result: self.store_result(task_id, exc, state, traceback=traceback, request=request) if request: # This task may be part of a chord if request.chord: self.on_chord_part_return(request, state, exc) # It might also have chained tasks which need to be propagated to, # this is most likely to be exclusive with being a direct part of a # chord but we'll handle both cases separately. # # The `chain_data` try block here is a bit tortured since we might # have non-iterable objects here in tests and it's easier this way. try: chain_data = iter(request.chain) except (AttributeError, TypeError): chain_data = tuple() for chain_elem in chain_data: chain_elem_opts = chain_elem['options'] # If the state should be propagated, we'll do so for all # elements of the chain. This is only truly important so # that the last chain element which controls completion of # the chain itself is marked as completed to avoid stalls. if self.store_result and state in states.PROPAGATE_STATES: try: chained_task_id = chain_elem_opts['task_id'] except KeyError: pass else: self.store_result(chained_task_id, exc, state, traceback=traceback, request=chain_elem) # If the chain element is a member of a chord, we also need # to call `on_chord_part_return()` as well to avoid stalls. if 'chord' in chain_elem_opts: failed_ctx = Context(chain_elem) failed_ctx.update(failed_ctx.options) failed_ctx.id = failed_ctx.options['task_id'] failed_ctx.group = failed_ctx.options['group_id'] self.on_chord_part_return(failed_ctx, state, exc) # And finally we'll fire any errbacks if call_errbacks and request.errbacks: self._call_task_errbacks(request, exc, traceback)
def mark_as_failure(self, task_id, exc, traceback=None, request=None, store_result=True, call_errbacks=True, state=states.FAILURE): """Mark task as executed with failure.""" if store_result: self.store_result(task_id, exc, state, traceback=traceback, request=request) if request: # This task may be part of a chord if request.chord: self.on_chord_part_return(request, state, exc) # It might also have chained tasks which need to be propagated to, # this is most likely to be exclusive with being a direct part of a # chord but we'll handle both cases separately. # # The `chain_data` try block here is a bit tortured since we might # have non-iterable objects here in tests and it's easier this way. try: chain_data = iter(request.chain) except (AttributeError, TypeError): chain_data = tuple() for chain_elem in chain_data: # Reconstruct a `Context` object for the chained task which has # enough information to for backends to work with chain_elem_ctx = Context(chain_elem) chain_elem_ctx.update(chain_elem_ctx.options) chain_elem_ctx.id = chain_elem_ctx.options.get('task_id') chain_elem_ctx.group = chain_elem_ctx.options.get('group_id') # If the state should be propagated, we'll do so for all # elements of the chain. This is only truly important so # that the last chain element which controls completion of # the chain itself is marked as completed to avoid stalls. # # Some chained elements may be complex signatures and have no # task ID of their own, so we skip them hoping that not # descending through them is OK. If the last chain element is # complex, we assume it must have been uplifted to a chord by # the canvas code and therefore the condition below will ensure # that we mark something as being complete as avoid stalling. if ( store_result and state in states.PROPAGATE_STATES and chain_elem_ctx.task_id is not None ): self.store_result( chain_elem_ctx.task_id, exc, state, traceback=traceback, request=chain_elem_ctx, ) # If the chain element is a member of a chord, we also need # to call `on_chord_part_return()` as well to avoid stalls. if 'chord' in chain_elem_ctx.options: self.on_chord_part_return(chain_elem_ctx, state, exc) # And finally we'll fire any errbacks if call_errbacks and request.errbacks: self._call_task_errbacks(request, exc, traceback)
def trace_task(uuid, args, kwargs, request=None): R = I = None kwargs = kwdict(kwargs) try: push_task(task) task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) push_request(task_request) try: # -*- PRE -*- if prerun_receivers: send_prerun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs) loader_task_init(uuid, task) if track_started: store_result(uuid, { 'pid': pid, 'hostname': hostname }, STARTED) # -*- TRACE -*- try: R = retval = fun(*args, **kwargs) state = SUCCESS except Ignore, exc: I, R = Info(IGNORED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval except RetryTaskError, exc: I = Info(RETRY, exc) state, retval = I.state, I.retval R = I.handle_error_state(task, eager=eager) except Exception, exc: if propagate: raise I = Info(FAILURE, exc) state, retval = I.state, I.retval R = I.handle_error_state(task, eager=eager) [ subtask(errback).apply_async((uuid, )) for errback in task_request.errbacks or [] ]
def setUp(self): app = mock.Mock(**{ 'conf.result_serializer': 'json', 'conf.accept_content': None }) self.backend = Backend(app) errback = { "chord_size": None, "task": "waldur_core.core.tasks.ErrorStateTransitionTask", "args": ["waldur.obj:1"], "immutable": False, "subtask_type": None, "kwargs": {}, "options": {} } self.request = Context(errbacks=[errback], id='task_id', root_id='root_id')
def test_store_result(self, result_serializer, args, kwargs): self.app.conf.result_serializer = result_serializer tb = DatabaseBackend(self.uri, app=self.app) tid = uuid() request = Context(args=args, kwargs=kwargs, task='mytask', retries=2, hostname='celery@worker_1', delivery_info={'routing_key': 'celery'}) tb.store_result(tid, {'fizz': 'buzz'}, states.SUCCESS, request=request) meta = tb.get_task_meta(tid) assert meta['result'] == {'fizz': 'buzz'} assert meta['args'] == args assert meta['kwargs'] == kwargs assert meta['queue'] == 'celery' assert meta['name'] == 'mytask' assert meta['retries'] == 2 assert meta['worker'] == "celery@worker_1"
def update_sent_state(sender=None, body=None, exchange=None, routing_key=None, **kwargs): # App may not be loaded on init from django_celery_fulldbresult.models import SCHEDULED task = current_app.tasks.get(sender) save = False status = None schedule_eta = getattr( settings, "DJANGO_CELERY_FULLDBRESULT_SCHEDULE_ETA", False) track_publish = getattr( settings, "DJANGO_CELERY_FULLDBRESULT_TRACK_PUBLISH", False) ignore_result = getattr(task, "ignore_result", False) or\ getattr(settings, "CELERY_IGNORE_RESULT", False) if schedule_eta and body.get("eta") and not body.get("chord")\ and not body.get("taskset"): status = SCHEDULED save = True elif track_publish and not ignore_result: status = PENDING save = True if save: backend = task.backend if task else current_app.backend request = Context() request.update(**body) request.date_submitted = now() request.delivery_info = { "exchange": exchange, "routing_key": routing_key } backend.store_result( body["id"], None, status, traceback=None, request=request) if status == SCHEDULED: raise SchedulingStopPublishing(task_id=body["id"])
class Request(object): """A request for task execution.""" __slots__ = ("app", "name", "id", "args", "kwargs", "on_ack", "delivery_info", "hostname", "callbacks", "errbacks", "eventer", "connection_errors", "task", "eta", "expires", "flags", "request_dict", "acknowledged", "success_msg", "error_msg", "retry_msg", "time_start", "worker_pid", "_already_revoked", "_terminate_on_ack", "_tzlocal") #: Format string used to log task success. success_msg = """\ Task %(name)s[%(id)s] succeeded in %(runtime)ss: %(return_value)s """ #: Format string used to log task failure. error_msg = """\ Task %(name)s[%(id)s] raised exception: %(exc)s """ #: Format string used to log internal error. internal_error_msg = """\ Task %(name)s[%(id)s] INTERNAL ERROR: %(exc)s """ #: Format string used to log task retry. retry_msg = """Task %(name)s[%(id)s] retry: %(exc)s""" def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( "Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = body.get("utc", False) self.flags = body.get("flags", False) self.on_ack = on_ack self.hostname = hostname self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), } self.request_dict = Context(body, called_directly=False) @classmethod def from_message(cls, message, body, **kwargs): # should be deprecated return Request(body, delivery_info=getattr(message, "delivery_info", None), **kwargs) def extend_with_default_kwargs(self, loglevel, logfile): """Extend the tasks keyword arguments with standard task arguments. Currently these are `logfile`, `loglevel`, `task_id`, `task_name`, `task_retries`, and `delivery_info`. See :meth:`celery.task.base.Task.run` for more information. Magic keyword arguments are deprecated and will be removed in version 3.0. """ kwargs = dict(self.kwargs) default_kwargs = {"logfile": logfile, "loglevel": loglevel, "task_id": self.id, "task_name": self.name, "task_retries": self.request_dict.get("retries", 0), "task_is_eager": False, "delivery_info": self.delivery_info} fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs def execute_using_pool(self, pool, loglevel=None, logfile=None): """Like :meth:`execute`, but using a worker pool. :param pool: A :class:`multiprocessing.Pool` instance. :keyword loglevel: The loglevel used by the task. :keyword logfile: The logfile used by the task. """ task = self.task if self.flags & 0x004: return pool.apply_async(execute_bare, args=(self.task, self.id, self.args, self.kwargs), accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, error_callback=self.on_failure, soft_timeout=task.soft_time_limit, timeout=task.time_limit) if self.revoked(): return hostname = self.hostname kwargs = self.kwargs if self.task.accept_magic_kwargs: kwargs = self.extend_with_default_kwargs(loglevel, logfile) request = self.request_dict request.update({"loglevel": loglevel, "logfile": logfile, "hostname": hostname, "is_eager": False, "delivery_info": self.delivery_info}) result = pool.apply_async(trace_task_ret, args=(self.task, self.id, self.args, kwargs), kwargs={"hostname": hostname, "request": request}, accept_callback=self.on_accepted, timeout_callback=self.on_timeout, callback=self.on_success, error_callback=self.on_failure, soft_timeout=task.soft_time_limit, timeout=task.time_limit) return result def execute(self, loglevel=None, logfile=None): """Execute the task in a :func:`~celery.task.trace.trace_task`. :keyword loglevel: The loglevel used by the task. :keyword logfile: The logfile used by the task. """ if self.revoked(): return # acknowledge task as being processed. if not self.task.acks_late: self.acknowledge() kwargs = self.kwargs if self.task.accept_magic_kwargs: kwargs = self.extend_with_default_kwargs(loglevel, logfile) request = self.request_dict request.update({"loglevel": loglevel, "logfile": logfile, "hostname": self.hostname, "is_eager": False, "delivery_info": self.delivery_info}) retval, _ = trace_task(self.task, self.id, self.args, kwargs, **{"hostname": self.hostname, "loader": self.app.loader, "request": request}) self.acknowledge() return retval def maybe_expire(self): """If expired, mark the task as revoked.""" if self.expires and datetime.now(self.tzlocal) > self.expires: revoked_tasks.add(self.id) if self.store_errors: self.task.backend.mark_as_revoked(self.id) def terminate(self, pool, signal=None): if self.time_start: return pool.terminate_job(self.worker_pid, signal) else: self._terminate_on_ack = (True, pool, signal) def revoked(self): """If revoked, skip task and mark state.""" if self._already_revoked: return True if self.expires: self.maybe_expire() if self.id in revoked_tasks: warn("Skipping revoked task: %s[%s]", self.name, self.id) if self.eventer and self.eventer.enabled: self.eventer.send("task-revoked", uuid=self.id) self.acknowledge() self._already_revoked = True return True return False def on_accepted(self, pid, time_accepted): """Handler called when task is accepted by worker pool.""" self.worker_pid = pid self.time_start = time_accepted task_accepted(self) if not self.task.acks_late: self.acknowledge() if self.eventer and self.eventer.enabled: self.eventer.send("task-started", uuid=self.id, pid=pid) if _does_debug: debug("Task accepted: %s[%s] pid:%r", self.name, self.id, pid) if self._terminate_on_ack is not None: _, pool, signal = self._terminate_on_ack self.terminate(pool, signal) def on_timeout(self, soft, timeout): """Handler called if the task times out.""" task_ready(self) if soft: warn("Soft time limit (%ss) exceeded for %s[%s]", timeout, self.name, self.id) exc = exceptions.SoftTimeLimitExceeded(timeout) else: error("Hard time limit (%ss) exceeded for %s[%s]", timeout, self.name, self.id) exc = exceptions.TimeLimitExceeded(timeout) if self.store_errors: self.task.backend.mark_as_failure(self.id, exc) def on_success(self, ret_value, now=None): """Handler called if the task was successfully processed.""" if isinstance(ret_value, ExceptionInfo): if isinstance(ret_value.exception, ( SystemExit, KeyboardInterrupt)): raise ret_value.exception return self.on_failure(ret_value) task_ready(self) if self.task.acks_late: self.acknowledge() if self.eventer and self.eventer.enabled: now = time.time() runtime = self.time_start and (time.time() - self.time_start) or 0 self.eventer.send("task-succeeded", uuid=self.id, result=safe_repr(ret_value), runtime=runtime) if _does_info: now = now or time.time() runtime = self.time_start and (time.time() - self.time_start) or 0 info(self.success_msg.strip(), { "id": self.id, "name": self.name, "return_value": self.repr_result(ret_value), "runtime": runtime}) def on_retry(self, exc_info): """Handler called if the task should be retried.""" if self.eventer and self.eventer.enabled: self.eventer.send("task-retried", uuid=self.id, exception=safe_repr(exc_info.exception.exc), traceback=safe_str(exc_info.traceback)) if _does_info: info(self.retry_msg.strip(), { "id": self.id, "name": self.name, "exc": safe_repr(exc_info.exception.exc)}, exc_info=exc_info) def on_failure(self, exc_info): """Handler called if the task raised an exception.""" task_ready(self) if not exc_info.internal: if isinstance(exc_info.exception, exceptions.RetryTaskError): return self.on_retry(exc_info) # This is a special case as the process would not have had # time to write the result. if isinstance(exc_info.exception, exceptions.WorkerLostError) and \ self.store_errors: self.task.backend.mark_as_failure(self.id, exc_info.exception) # (acks_late) acknowledge after result stored. if self.task.acks_late: self.acknowledge() self._log_error(exc_info) def _log_error(self, einfo): exception, traceback, exc_info, internal, sargs, skwargs = ( safe_repr(einfo.exception), safe_str(einfo.traceback), einfo.exc_info, einfo.internal, safe_repr(self.args), safe_repr(self.kwargs), ) format = self.error_msg description = "raised exception" severity = logging.ERROR if self.eventer and self.eventer.enabled: self.eventer.send("task-failed", uuid=self.id, exception=exception, traceback=traceback) if internal: format = self.internal_error_msg description = "INTERNAL ERROR" severity = logging.CRITICAL context = { "hostname": self.hostname, "id": self.id, "name": self.name, "exc": exception, "traceback": traceback, "args": sargs, "kwargs": skwargs, "description": description, } logger.log(severity, format.strip(), context, exc_info=exc_info, extra={"data": {"id": self.id, "name": self.name, "args": sargs, "kwargs": skwargs, "hostname": self.hostname, "internal": internal}}) self.task.send_error_email(context, einfo.exception) def acknowledge(self): """Acknowledge task.""" if not self.acknowledged: self.on_ack(logger, self.connection_errors) self.acknowledged = True def repr_result(self, result, maxlen=46): # 46 is the length needed to fit # "the quick brown fox jumps over the lazy dog" :) return truncate(safe_repr(result), maxlen) def info(self, safe=False): return {"id": self.id, "name": self.name, "args": self.args if safe else safe_repr(self.args), "kwargs": self.kwargs if safe else safe_repr(self.kwargs), "hostname": self.hostname, "time_start": self.time_start, "acknowledged": self.acknowledged, "delivery_info": self.delivery_info, "worker_pid": self.worker_pid} def shortinfo(self): return "%s[%s]%s%s" % ( self.name, self.id, " eta:[%s]" % (self.eta, ) if self.eta else "", " expires:[%s]" % (self.expires, ) if self.expires else "") __str__ = shortinfo def __repr__(self): return '<%s %s: %s>' % (type(self).__name__, self.id, reprcall(self.name, self.args, self.kwargs)) @property def tzlocal(self): if self._tzlocal is None: self._tzlocal = tz_or_local(self.app.conf.CELERY_TIMEZONE) return self._tzlocal @property def store_errors(self): return (not self.task.ignore_result or self.task.store_errors_even_if_ignored) def _compat_get_task_id(self): return self.id def _compat_set_task_id(self, value): self.id = value def _compat_get_task_name(self): return self.name def _compat_set_task_name(self, value): self.name = value task_id = property(_compat_get_task_id, _compat_set_task_id) task_name = property(_compat_get_task_name, _compat_set_task_name)