def extend_with_default_kwargs(self, loglevel, logfile):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are `logfile`, `loglevel`, `task_id`,
`task_name`, `task_retries`, and `delivery_info`.
See :meth:`celery.task.base.Task.run` for more information.
Magic keyword arguments are deprecated and will be removed
in version 3.0.
"""
kwargs = dict(self.kwargs)
default_kwargs = {
"logfile": logfile,
"loglevel": loglevel,
"task_id": self.id,
"task_name": self.name,
"task_retries": self.request_dict.get("retries", 0),
"task_is_eager": False,
"delivery_info": self.delivery_info
}
fun = self.task.run
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def extend_with_default_kwargs(self):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are `logfile`, `loglevel`, `task_id`,
`task_name`, `task_retries`, and `delivery_info`.
See :meth:`celery.task.base.Task.run` for more information.
Magic keyword arguments are deprecated and will be removed
in version 3.0.
"""
kwargs = dict(self.kwargs)
default_kwargs = {'logfile': None, # deprecated
'loglevel': None, # deprecated
'task_id': self.id,
'task_name': self.name,
'task_retries': self.request_dict.get('retries', 0),
'task_is_eager': False,
'delivery_info': self.delivery_info}
fun = self.task.run
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def extend_with_default_kwargs(self, loglevel, logfile):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are `logfile`, `loglevel`, `task_id`,
`task_name`, `task_retries`, and `delivery_info`.
See :meth:`celery.task.base.Task.run` for more information.
"""
if not self.task.accept_magic_kwargs:
return self.kwargs
kwargs = dict(self.kwargs)
default_kwargs = {
"logfile": logfile,
"loglevel": loglevel,
"task_id": self.task_id,
"task_name": self.task_name,
"task_retries": self.retries,
"task_is_eager": False,
"delivery_info": self.delivery_info
}
fun = self.task.run
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def _run(self, facadefqdn, method, *args, **kwargs):
# Pass the job log to the facade method so that it can log to the job log.
kwargs['joblog'] = self.log
self.args = args
self.kwargs = kwargs
facadeclass = resolve(facadefqdn)
facade = facadeclass(self.dmd)
bound_method = getattr(facade, method)
accepted = fun_takes_kwargs(bound_method, kwargs)
kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted)
result = bound_method(*args, **kwargs)
# Expect result = {'success': boolean, 'message': string}
# Some old facade method jobs return None.
if result:
try:
if not result['success']:
raise FacadeMethodJobFailed
return result['message']
except FacadeMethodJobFailed:
raise
except (TypeError, KeyError):
self.log.error(
'The output from job {} is not in the right format.'.
format(self.request.id))
def apply(self, args=None, kwargs=None,
link=None, link_error=None, **options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports Task, so need to import inline.
from celery.app.trace import eager_trace_task
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__, ) + tuple(args)
kwargs = kwargs or {}
task_id = options.get('task_id') or uuid()
retries = options.get('retries', 0)
throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS',
options.pop('throw', None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'delivery_info': {'is_eager': True}}
if self.accept_magic_kwargs:
default_kwargs = {'task_name': task.name,
'task_id': task_id,
'task_retries': retries,
'task_is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'delivery_info': {'is_eager': True}}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val)
for key, val in items(default_kwargs)
if key in supported_keys)
kwargs.update(extend_with)
tb = None
retval, info = eager_trace_task(task, task_id, args, kwargs,
app=self._get_app(),
request=request, propagate=throw)
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if info is None else info.state
return EagerResult(task_id, retval, state, traceback=tb)
def extend_with_default_kwargs(self, loglevel, logfile):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are ``logfile``, ``loglevel``, ``task_id``,
``task_name``, ``task_retries``, and ``delivery_info``.
See :meth:`celery.task.base.Task.run` for more information.
"""
if not self.task.accept_magic_kwargs:
return self.kwargs
kwargs = dict(self.kwargs)
default_kwargs = {"logfile": logfile,
"loglevel": loglevel,
"task_id": self.task_id,
"task_name": self.task_name,
"task_retries": self.retries,
"task_is_eager": False,
"delivery_info": self.delivery_info}
fun = self.task.run
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def extend_with_default_kwargs(self):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are `logfile`, `loglevel`, `task_id`,
`task_name`, `task_retries`, and `delivery_info`.
See :meth:`celery.task.base.Task.run` for more information.
Magic keyword arguments are deprecated and will be removed
in version 4.0.
"""
kwargs = dict(self.kwargs)
default_kwargs = {
'logfile': None, # deprecated
'loglevel': None, # deprecated
'task_id': self.id,
'task_name': self.name,
'task_retries': self.request_dict.get('retries', 0),
'task_is_eager': False,
'delivery_info': self.delivery_info
}
fun = self.task.run
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in items(default_kwargs)
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def apply(self, args=None, kwargs=None,
link=None, link_error=None, **options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports Task, so need to import inline.
from celery.app.trace import eager_trace_task
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__, ) + tuple(args)
kwargs = kwargs or {}
task_id = options.get('task_id') or uuid()
retries = options.get('retries', 0)
throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS',
options.pop('throw', None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'delivery_info': {'is_eager': True}}
if self.accept_magic_kwargs:
default_kwargs = {'task_name': task.name,
'task_id': task_id,
'task_retries': retries,
'task_is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'delivery_info': {'is_eager': True}}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val)
for key, val in items(default_kwargs)
if key in supported_keys)
kwargs.update(extend_with)
tb = None
retval, info = eager_trace_task(task, task_id, args, kwargs,
app=self._get_app(),
request=request, propagate=throw)
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if info is None else info.state
return EagerResult(task_id, retval, state, traceback=tb)
def apply(task, args, kwargs, **options):
"""Apply the task locally.
This will block until the task completes, and returns a
:class:`celery.result.EagerResult` instance.
"""
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id", gen_unique_id())
retries = options.get("retries", 0)
task = tasks[task.name] # Make sure we get the instance, not class.
default_kwargs = {"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": None,
"loglevel": 0}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
trace = TaskTrace(task.name, task_id, args, kwargs, task=task)
retval = trace.execute()
return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def _run(self, facadefqdn, method, *args, **kwargs):
"""Execute a facade's method.
:param str facadefqdn: classpath to facade class.
:param str method: name of method on facade class.
:param *args: positional arguments to method.
:param **kwargs: keyword arguments to method.
:return: The return value from the method.
:raise FacadeMethodJobFailed: method did not succeed.
"""
facadeclass = resolve(facadefqdn)
facade = facadeclass(self.dmd)
bound_method = getattr(facade, method, _notfound)
if bound_method is _notfound:
raise FacadeMethodJobFailed(
"No such attribute on %s: %s" % (facadeclass, method),
)
if not callable(bound_method):
raise FacadeMethodJobFailed(
"Not a callable method: %s.%s" % (facadeclass, method),
)
# Some facade methods were written to accept a 'joblog'
# parameter provided by this task.
kwarg_names = set(kwargs)
kwarg_names.add("joblog")
# fun_takes_kwargs filters out parameter names not found in the
# facade method's signature.
accepted = fun_takes_kwargs(bound_method, kwarg_names)
# If 'joblog' was not filtered out, add it to kwargs.
if "joblog" in accepted:
kwargs["joblog"] = self.log
# Rebuild kwargs to include only accepted parameter names.
kwargs = {
k: v
for k, v in kwargs.iteritems()
if k in accepted
}
result = bound_method(*args, **kwargs)
# Expect result = {'success': boolean, 'message': string}
# Some old facade method jobs return None.
if result:
try:
if not result["success"]:
raise FacadeMethodJobFailed(str(result))
return result["message"]
except (TypeError, KeyError):
self.log.warn(
"The output from job %s is not in the right format: "
"%s.%s returned %s",
self.request.id, facadefqdn, bound_method, result,
)
return result
def _run(self, facadefqdn, method, *args, **kwargs):
self.args = args
self.kwargs = kwargs
facadeclass = resolve(facadefqdn)
facade = facadeclass(self.dmd)
bound_method = getattr(facade, method)
accepted = fun_takes_kwargs(bound_method, kwargs)
kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted)
result = bound_method(*args, **kwargs)
def _run(self, facadefqdn, method, *args, **kwargs):
self.args = args
self.kwargs = kwargs
facadeclass = resolve(facadefqdn)
facade = facadeclass(self.dmd)
bound_method = getattr(facade, method)
accepted = fun_takes_kwargs(bound_method, kwargs)
kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted)
result = bound_method(*args, **kwargs)
def apply(self, args=None, kwargs=None, **options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports BaseTask, so need to import inline.
from celery.task.trace import eager_trace_task
app = self._get_app()
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id") or uuid()
retries = options.get("retries", 0)
throw = app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS",
options.pop("throw", None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {"id": task_id,
"retries": retries,
"is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
if self.accept_magic_kwargs:
default_kwargs = {"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val)
for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
retval, info = eager_trace_task(task, task_id, args, kwargs,
request=request, propagate=throw)
if isinstance(retval, ExceptionInfo):
retval = retval.exception
state, tb = states.SUCCESS, ''
if info is not None:
state, tb = info.state, info.strtb
return EagerResult(task_id, retval, state, traceback=tb)
def apply(self, args=None, kwargs=None, **options):
"""Execute this task locally, by blocking until the task
returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting.
:rtype :class:`celery.result.EagerResult`:
See :func:`celery.execute.apply`.
"""
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id") or gen_unique_id()
retries = options.get("retries", 0)
throw = self.app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS",
options.pop("throw", None))
# Make sure we get the task instance, not class.
task = tasks[self.name]
request = {"id": task_id,
"retries": retries,
"is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
if self.accept_magic_kwargs:
default_kwargs = {"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val)
for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
trace = TaskTrace(task.name, task_id, args, kwargs,
task=task, request=request)
retval = trace.execute()
if isinstance(retval, ExceptionInfo):
if throw:
raise retval.exception
retval = retval.exception
return EagerResult(task_id, retval, trace.status,
traceback=trace.strtb)
def apply(self, args=None, kwargs=None, **options):
"""Execute this task locally, by blocking until the task
returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id") or gen_unique_id()
retries = options.get("retries", 0)
throw = self.app.either("CELERY_EAGER_PROPAGATES_EXCEPTIONS",
options.pop("throw", None))
# Make sure we get the task instance, not class.
task = tasks[self.name]
request = {"id": task_id,
"retries": retries,
"is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
if self.accept_magic_kwargs:
default_kwargs = {"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"delivery_info": {"is_eager": True}}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val)
for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
trace = TaskTrace(task.name, task_id, args, kwargs,
task=task, request=request, propagate=throw)
retval = trace.execute()
if isinstance(retval, ExceptionInfo):
retval = retval.exception
return EagerResult(task_id, retval, trace.status,
traceback=trace.strtb)
def newf(*args, **kwargs):
backend = get_backend()
was_patched = backend._patched
get_backend().patch()
#since this function takes all keyword arguments,
#we will pass only the ones the function below accepts, just as celery does
supported_keys = fun_takes_kwargs(f, kwargs)
new_kwargs = dict((key, val) for key, val in kwargs.items()
if key in supported_keys)
try:
ret = f(*args, **new_kwargs)
finally:
local.clear()
if not was_patched:
get_backend().unpatch()
return ret
def newf(*args, **kwargs):
backend = get_backend()
was_patched = backend._patched
get_backend().patch()
# since this function takes all keyword arguments,
# we will pass only the ones the function below accepts,
# just as celery does
supported_keys = fun_takes_kwargs(f, kwargs)
new_kwargs = dict((key, val) for key, val in kwargs.items()
if key in supported_keys)
try:
ret = f(*args, **new_kwargs)
finally:
local.clear()
if not was_patched:
get_backend().unpatch()
return ret
def apply(task, args, kwargs, **options):
"""Apply the task locally.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting.
This will block until the task completes, and returns a
:class:`celery.result.EagerResult` instance.
"""
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id") or gen_unique_id()
retries = options.get("retries", 0)
throw = options.pop("throw", conf.EAGER_PROPAGATES_EXCEPTIONS)
task = tasks[task.name] # make sure we get the instance, not class.
default_kwargs = {
"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": options.get("logfile"),
"delivery_info": {
"is_eager": True
},
"loglevel": options.get("loglevel", 0)
}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
trace = TaskTrace(task.name, task_id, args, kwargs, task=task)
retval = trace.execute()
if isinstance(retval, ExceptionInfo):
if throw:
raise retval.exception
retval = retval.exception
return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def extend_with_default_kwargs(self, loglevel, logfile):
"""Extend the tasks keyword arguments with standard task arguments.
Currently these are ``logfile``, ``loglevel``, ``task_id``,
``task_name`` and ``task_retries``.
See :meth:`celery.task.base.Task.run` for more information.
"""
kwargs = dict(self.kwargs)
default_kwargs = {"logfile": logfile,
"loglevel": loglevel,
"task_id": self.task_id,
"task_name": self.task_name,
"task_retries": self.retries}
fun = getattr(self.task_func, "run", self.task_func)
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items()
if key in supported_keys)
kwargs.update(extend_with)
return kwargs
def apply(task, args, kwargs, **options):
"""Apply the task locally.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS` setting.
This will block until the task completes, and returns a
:class:`celery.result.EagerResult` instance.
"""
args = args or []
kwargs = kwargs or {}
task_id = options.get("task_id") or gen_unique_id()
retries = options.get("retries", 0)
throw = options.pop("throw", conf.EAGER_PROPAGATES_EXCEPTIONS)
task = tasks[task.name] # make sure we get the instance, not class.
default_kwargs = {
"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": options.get("logfile"),
"delivery_info": {"is_eager": True},
"loglevel": options.get("loglevel", 0),
}
supported_keys = fun_takes_kwargs(task.run, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys)
kwargs.update(extend_with)
trace = TaskTrace(task.name, task_id, args, kwargs, task=task)
retval = trace.execute()
if isinstance(retval, ExceptionInfo):
if throw:
raise retval.exception
retval = retval.exception
return EagerResult(task_id, retval, trace.status, traceback=trace.strtb)
def apply(task, args, kwargs, **options):
"""Apply the task locally.
This will block until the task completes, and returns a
:class:`celery.result.EagerResult` instance.
"""
args = args or []
kwargs = kwargs or {}
task_id = gen_unique_id()
retries = options.get("retries", 0)
# If it's a Task class we need to have to instance
# for it to be callable.
task = inspect.isclass(task) and task() or task
default_kwargs = {
"task_name": task.name,
"task_id": task_id,
"task_retries": retries,
"task_is_eager": True,
"logfile": None,
"loglevel": 0,
}
fun = getattr(task, "run", task)
supported_keys = fun_takes_kwargs(fun, default_kwargs)
extend_with = dict((key, val) for key, val in default_kwargs.items() if key in supported_keys)
kwargs.update(extend_with)
try:
ret_value = task(*args, **kwargs)
status = "DONE"
strtb = None
except Exception, exc:
type_, value_, tb = sys.exc_info()
strtb = "\n".join(traceback.format_exception(type_, value_, tb))
ret_value = exc
status = "FAILURE"
def _run(self, facadefqdn, method, *args, **kwargs):
# Pass the job log to the facade method so that it can log to the job log.
kwargs['joblog'] = self.log
self.args = args
self.kwargs = kwargs
facadeclass = resolve(facadefqdn)
facade = facadeclass(self.dmd)
bound_method = getattr(facade, method)
accepted = fun_takes_kwargs(bound_method, kwargs)
kwargs = dict((k, v) for k, v in kwargs.iteritems() if k in accepted)
result = bound_method(*args, **kwargs)
# Expect result = {'success': boolean, 'message': string}
# Some old facade method jobs return None.
if result:
try:
if not result['success']:
raise FacadeMethodJobFailed
return result['message']
except FacadeMethodJobFailed:
raise
except (TypeError, KeyError):
self.log.error('The output from job {} is not in the right format.'
.format(self.request.id))
