def extend_with_default_kwargs(self, loglevel, logfile): """Extend the tasks keyword arguments with standard task arguments. Currently these are `logfile`, `loglevel`, `task_id`, `task_name`, `task_retries`, and `delivery_info`. See :meth:`celery.task.base.Task.run` for more information. Magic keyword arguments are deprecated and will be removed in version 3.0. """ kwargs = dict(self.kwargs) default_kwargs = { "logfile": logfile, "loglevel": loglevel, "task_id": self.id, "task_name": self.name, "task_retries": self.request_dict.get("retries", 0), "task_is_eager": False, "delivery_info": self.delivery_info } fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs
def extend_with_default_kwargs(self): """Extend the tasks keyword arguments with standard task arguments. Currently these are `logfile`, `loglevel`, `task_id`, `task_name`, `task_retries`, and `delivery_info`. See :meth:`celery.task.base.Task.run` for more information. Magic keyword arguments are deprecated and will be removed in version 3.0. """ kwargs = dict(self.kwargs) default_kwargs = {'logfile': None, # deprecated 'loglevel': None, # deprecated 'task_id': self.id, 'task_name': self.name, 'task_retries': self.request_dict.get('retries', 0), 'task_is_eager': False, 'delivery_info': self.delivery_info} fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs
def extend_with_default_kwargs(self, loglevel, logfile): """Extend the tasks keyword arguments with standard task arguments. Currently these are `logfile`, `loglevel`, `task_id`, `task_name`, `task_retries`, and `delivery_info`. See :meth:`celery.task.base.Task.run` for more information. """ if not self.task.accept_magic_kwargs: return self.kwargs kwargs = dict(self.kwargs) default_kwargs = { "logfile": logfile, "loglevel": loglevel, "task_id": self.task_id, "task_name": self.task_name, "task_retries": self.retries, "task_is_eager": False, "delivery_info": self.delivery_info } fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs
def _run(self, facadefqdn, method, *args, **kwargs): # Pass the job log to the facade method so that it can log to the job log. kwargs['joblog'] = self.log self.args = args self.kwargs = kwargs facadeclass = resolve(facadefqdn) facade = facadeclass(self.dmd) bound_method = getattr(facade, method) accepted = fun_takes_kwargs(bound_method, kwargs) kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted) result = bound_method(*args, **kwargs) # Expect result = {'success': boolean, 'message': string} # Some old facade method jobs return None. if result: try: if not result['success']: raise FacadeMethodJobFailed return result['message'] except FacadeMethodJobFailed: raise except (TypeError, KeyError): self.log.error( 'The output from job {} is not in the right format.'. format(self.request.id))
def apply(self, args=None, kwargs=None, link=None, link_error=None, **options): """Execute this task locally, by blocking until the task returns. :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. :rtype :class:`celery.result.EagerResult`: """ # trace imports Task, so need to import inline. from celery.app.trace import eager_trace_task app = self._get_app() args = args or () # add 'self' if this is a bound method. if self.__self__ is not None: args = (self.__self__, ) + tuple(args) kwargs = kwargs or {} task_id = options.get('task_id') or uuid() retries = options.get('retries', 0) throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS', options.pop('throw', None)) # Make sure we get the task instance, not class. task = app._tasks[self.name] request = {'id': task_id, 'retries': retries, 'is_eager': True, 'logfile': options.get('logfile'), 'loglevel': options.get('loglevel', 0), 'callbacks': maybe_list(link), 'errbacks': maybe_list(link_error), 'delivery_info': {'is_eager': True}} if self.accept_magic_kwargs: default_kwargs = {'task_name': task.name, 'task_id': task_id, 'task_retries': retries, 'task_is_eager': True, 'logfile': options.get('logfile'), 'loglevel': options.get('loglevel', 0), 'delivery_info': {'is_eager': True}} supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in items(default_kwargs) if key in supported_keys) kwargs.update(extend_with) tb = None retval, info = eager_trace_task(task, task_id, args, kwargs, app=self._get_app(), request=request, propagate=throw) if isinstance(retval, ExceptionInfo): retval, tb = retval.exception, retval.traceback state = states.SUCCESS if info is None else info.state return EagerResult(task_id, retval, state, traceback=tb)
def extend_with_default_kwargs(self, loglevel, logfile): """Extend the tasks keyword arguments with standard task arguments. Currently these are ``logfile``, ``loglevel``, ``task_id``, ``task_name``, ``task_retries``, and ``delivery_info``. See :meth:`celery.task.base.Task.run` for more information. """ if not self.task.accept_magic_kwargs: return self.kwargs kwargs = dict(self.kwargs) default_kwargs = {"logfile": logfile, "loglevel": loglevel, "task_id": self.task_id, "task_name": self.task_name, "task_retries": self.retries, "task_is_eager": False, "delivery_info": self.delivery_info} fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs
def extend_with_default_kwargs(self): """Extend the tasks keyword arguments with standard task arguments. Currently these are `logfile`, `loglevel`, `task_id`, `task_name`, `task_retries`, and `delivery_info`. See :meth:`celery.task.base.Task.run` for more information. Magic keyword arguments are deprecated and will be removed in version 4.0. """ kwargs = dict(self.kwargs) default_kwargs = { 'logfile': None, # deprecated 'loglevel': None, # deprecated 'task_id': self.id, 'task_name': self.name, 'task_retries': self.request_dict.get('retries', 0), 'task_is_eager': False, 'delivery_info': self.delivery_info } fun = self.task.run supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in items(default_kwargs) if key in supported_keys) kwargs.update(extend_with) return kwargs
def apply(task, args, kwargs, **options): """Apply the task locally. This will block until the task completes, and returns a :class:`celery.result.EagerResult` instance. """ args = args or [] kwargs = kwargs or {} task_id = options.get("task_id", gen_unique_id()) retries = options.get("retries", 0) task = tasks[task.name] # Make sure we get the instance, not class. default_kwargs = {"task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": None, "loglevel": 0} supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) trace = TaskTrace(task.name, task_id, args, kwargs, task=task) retval = trace.execute() return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def _run(self, facadefqdn, method, *args, **kwargs): """Execute a facade's method. :param str facadefqdn: classpath to facade class. :param str method: name of method on facade class. :param *args: positional arguments to method. :param **kwargs: keyword arguments to method. :return: The return value from the method. :raise FacadeMethodJobFailed: method did not succeed. """ facadeclass = resolve(facadefqdn) facade = facadeclass(self.dmd) bound_method = getattr(facade, method, _notfound) if bound_method is _notfound: raise FacadeMethodJobFailed( "No such attribute on %s: %s" % (facadeclass, method), ) if not callable(bound_method): raise FacadeMethodJobFailed( "Not a callable method: %s.%s" % (facadeclass, method), ) # Some facade methods were written to accept a 'joblog' # parameter provided by this task. kwarg_names = set(kwargs) kwarg_names.add("joblog") # fun_takes_kwargs filters out parameter names not found in the # facade method's signature. accepted = fun_takes_kwargs(bound_method, kwarg_names) # If 'joblog' was not filtered out, add it to kwargs. if "joblog" in accepted: kwargs["joblog"] = self.log # Rebuild kwargs to include only accepted parameter names. kwargs = { k: v for k, v in kwargs.iteritems() if k in accepted } result = bound_method(*args, **kwargs) # Expect result = {'success': boolean, 'message': string} # Some old facade method jobs return None. if result: try: if not result["success"]: raise FacadeMethodJobFailed(str(result)) return result["message"] except (TypeError, KeyError): self.log.warn( "The output from job %s is not in the right format: " "%s.%s returned %s", self.request.id, facadefqdn, bound_method, result, ) return result
def _run(self, facadefqdn, method, *args, **kwargs): self.args = args self.kwargs = kwargs facadeclass = resolve(facadefqdn) facade = facadeclass(self.dmd) bound_method = getattr(facade, method) accepted = fun_takes_kwargs(bound_method, kwargs) kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted) result = bound_method(*args, **kwargs)
def apply(self, args=None, kwargs=None, **options): """Execute this task locally, by blocking until the task returns. :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. :rtype :class:`celery.result.EagerResult`: """ # trace imports BaseTask, so need to import inline. from celery.task.trace import eager_trace_task app = self._get_app() args = args or [] kwargs = kwargs or {} task_id = options.get("task_id") or uuid() retries = options.get("retries", 0) throw = app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS", options.pop("throw", None)) # Make sure we get the task instance, not class. task = app._tasks[self.name] request = {"id": task_id, "retries": retries, "is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} if self.accept_magic_kwargs: default_kwargs = {"task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) retval, info = eager_trace_task(task, task_id, args, kwargs, request=request, propagate=throw) if isinstance(retval, ExceptionInfo): retval = retval.exception state, tb = states.SUCCESS, '' if info is not None: state, tb = info.state, info.strtb return EagerResult(task_id, retval, state, traceback=tb)
def apply(self, args=None, kwargs=None, **options): """Execute this task locally, by blocking until the task returns. :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. :rtype :class:`celery.result.EagerResult`: See :func:`celery.execute.apply`. """ args = args or [] kwargs = kwargs or {} task_id = options.get("task_id") or gen_unique_id() retries = options.get("retries", 0) throw = self.app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS", options.pop("throw", None)) # Make sure we get the task instance, not class. task = tasks[self.name] request = {"id": task_id, "retries": retries, "is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} if self.accept_magic_kwargs: default_kwargs = {"task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) trace = TaskTrace(task.name, task_id, args, kwargs, task=task, request=request) retval = trace.execute() if isinstance(retval, ExceptionInfo): if throw: raise retval.exception retval = retval.exception return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def apply(self, args=None, kwargs=None, **options): """Execute this task locally, by blocking until the task returns. :param args: positional arguments passed on to the task. :param kwargs: keyword arguments passed on to the task. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. :rtype :class:`celery.result.EagerResult`: """ args = args or [] kwargs = kwargs or {} task_id = options.get("task_id") or gen_unique_id() retries = options.get("retries", 0) throw = self.app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS", options.pop("throw", None)) # Make sure we get the task instance, not class. task = tasks[self.name] request = {"id": task_id, "retries": retries, "is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} if self.accept_magic_kwargs: default_kwargs = {"task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": options.get("logfile"), "loglevel": options.get("loglevel", 0), "delivery_info": {"is_eager": True}} supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) trace = TaskTrace(task.name, task_id, args, kwargs, task=task, request=request, propagate=throw) retval = trace.execute() if isinstance(retval, ExceptionInfo): retval = retval.exception return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def newf(*args, **kwargs): backend = get_backend() was_patched = backend._patched get_backend().patch() #since this function takes all keyword arguments, #we will pass only the ones the function below accepts, just as celery does supported_keys = fun_takes_kwargs(f, kwargs) new_kwargs = dict((key, val) for key, val in kwargs.items() if key in supported_keys) try: ret = f(*args, **new_kwargs) finally: local.clear() if not was_patched: get_backend().unpatch() return ret
def newf(*args, **kwargs): backend = get_backend() was_patched = backend._patched get_backend().patch() # since this function takes all keyword arguments, # we will pass only the ones the function below accepts, # just as celery does supported_keys = fun_takes_kwargs(f, kwargs) new_kwargs = dict((key, val) for key, val in kwargs.items() if key in supported_keys) try: ret = f(*args, **new_kwargs) finally: local.clear() if not was_patched: get_backend().unpatch() return ret
def apply(task, args, kwargs, **options): """Apply the task locally. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. This will block until the task completes, and returns a :class:`celery.result.EagerResult` instance. """ args = args or [] kwargs = kwargs or {} task_id = options.get("task_id") or gen_unique_id() retries = options.get("retries", 0) throw = options.pop("throw", conf.EAGER_PROPAGATES_EXCEPTIONS) task = tasks[task.name] # make sure we get the instance, not class. default_kwargs = { "task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": options.get("logfile"), "delivery_info": { "is_eager": True }, "loglevel": options.get("loglevel", 0) } supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) trace = TaskTrace(task.name, task_id, args, kwargs, task=task) retval = trace.execute() if isinstance(retval, ExceptionInfo): if throw: raise retval.exception retval = retval.exception return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def extend_with_default_kwargs(self, loglevel, logfile): """Extend the tasks keyword arguments with standard task arguments. Currently these are ``logfile``, ``loglevel``, ``task_id``, ``task_name`` and ``task_retries``. See :meth:`celery.task.base.Task.run` for more information. """ kwargs = dict(self.kwargs) default_kwargs = {"logfile": logfile, "loglevel": loglevel, "task_id": self.task_id, "task_name": self.task_name, "task_retries": self.retries} fun = getattr(self.task_func, "run", self.task_func) supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) return kwargs
def apply(task, args, kwargs, **options): """Apply the task locally. :keyword throw: Re-raise task exceptions. Defaults to the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting. This will block until the task completes, and returns a :class:`celery.result.EagerResult` instance. """ args = args or [] kwargs = kwargs or {} task_id = options.get("task_id") or gen_unique_id() retries = options.get("retries", 0) throw = options.pop("throw", conf.EAGER_PROPAGATES_EXCEPTIONS) task = tasks[task.name] # make sure we get the instance, not class. default_kwargs = { "task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": options.get("logfile"), "delivery_info": {"is_eager": True}, "loglevel": options.get("loglevel", 0), } supported_keys = fun_takes_kwargs(task.run, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) trace = TaskTrace(task.name, task_id, args, kwargs, task=task) retval = trace.execute() if isinstance(retval, ExceptionInfo): if throw: raise retval.exception retval = retval.exception return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def apply(task, args, kwargs, **options): """Apply the task locally. This will block until the task completes, and returns a :class:`celery.result.EagerResult` instance. """ args = args or [] kwargs = kwargs or {} task_id = gen_unique_id() retries = options.get("retries", 0) # If it's a Task class we need to have to instance # for it to be callable. task = inspect.isclass(task) and task() or task default_kwargs = { "task_name": task.name, "task_id": task_id, "task_retries": retries, "task_is_eager": True, "logfile": None, "loglevel": 0, } fun = getattr(task, "run", task) supported_keys = fun_takes_kwargs(fun, default_kwargs) extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys) kwargs.update(extend_with) try: ret_value = task(*args, **kwargs) status = "DONE" strtb = None except Exception, exc: type_, value_, tb = sys.exc_info() strtb = "\n".join(traceback.format_exception(type_, value_, tb)) ret_value = exc status = "FAILURE"
def _run(self, facadefqdn, method, *args, **kwargs): # Pass the job log to the facade method so that it can log to the job log. kwargs['joblog'] = self.log self.args = args self.kwargs = kwargs facadeclass = resolve(facadefqdn) facade = facadeclass(self.dmd) bound_method = getattr(facade, method) accepted = fun_takes_kwargs(bound_method, kwargs) kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted) result = bound_method(*args, **kwargs) # Expect result = {'success': boolean, 'message': string} # Some old facade method jobs return None. if result: try: if not result['success']: raise FacadeMethodJobFailed return result['message'] except FacadeMethodJobFailed: raise except (TypeError, KeyError): self.log.error('The output from job {} is not in the right format.' .format(self.request.id))