def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
assert trace.trace_task_ret is trace._fast_trace_task
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
assert trace.trace_task_ret is trace._trace_task_ret
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def test_trace_task_ret(self):
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
res = _trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app)
self.assertEqual(res, 4 ** 4)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertTrue(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertTrue(runtime)
self.assertIsInstance(runtime, numbers.Real)
def apply(self,
args=None,
kwargs=None,
link=None,
link_error=None,
**options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`task_eager_propagates`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports Task, so need to import inline.
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__, ) + tuple(args)
kwargs = kwargs or {}
task_id = options.get('task_id') or uuid()
retries = options.get('retries', 0)
throw = app.either('task_eager_propagates', options.pop('throw', None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {
'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'headers': options.get('headers'),
'delivery_info': {
'is_eager': True
}
}
tb = None
tracer = build_tracer(
task.name,
task,
eager=True,
propagate=throw,
app=self._get_app(),
)
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def process_initializer(app, hostname):
"""Pool child process initializer.
This will initialize a child pool process to ensure the correct
app instance is used and things like
logging works.
"""
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title('celeryd', hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
app.log.setup(int(os.environ.get('CELERY_LOG_LEVEL', 0) or 0),
os.environ.get('CELERY_LOG_FILE') or None,
bool(os.environ.get('CELERY_LOG_REDIRECT', False)),
str(os.environ.get('CELERY_LOG_REDIRECT_LEVEL')))
if os.environ.get('FORKED_BY_MULTIPROCESSING'):
# pool did execv after fork
trace.setup_worker_optimizations(app)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
# rebuild execution handler for all tasks.
from celery.app.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name, task, app.loader, hostname,
app=app)
signals.worker_process_init.send(sender=None)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name,
self.mytask,
self.app.loader,
'test',
app=self.app,
)
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {})
self.assertEqual(res, 4**4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
res = trace.trace_task_ret(
self.mytask.name,
uuid(),
[4],
{},
app=self.app,
)
self.assertEqual(res, 4**4)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_trace_task_ret(self):
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
res = _trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app)
self.assertEqual(res, 4 ** 4)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
assert trace.trace_task_ret is trace._fast_trace_task
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
assert trace.trace_task_ret is trace._trace_task_ret
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def apply(self, args=None, kwargs=None,
link=None, link_error=None,
task_id=None, retries=None, throw=None,
logfile=None, loglevel=None, headers=None, **options):
"""Execute this task locally, by blocking until the task returns.
Arguments:
args (Tuple): positional arguments passed on to the task.
kwargs (Dict): keyword arguments passed on to the task.
throw (bool): Re-raise task exceptions.
Defaults to the :setting:`task_eager_propagates` setting.
Returns:
celery.result.EagerResult: pre-evaluated result.
"""
# trace imports Task, so need to import inline.
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
kwargs = kwargs or {}
task_id = task_id or uuid()
retries = retries or 0
if throw is None:
throw = app.conf.task_eager_propagates
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {
'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': logfile,
'loglevel': loglevel or 0,
'hostname': gethostname(),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'headers': headers,
'ignore_result': options.get('ignore_result', False),
'delivery_info': {
'is_eager': True,
'exchange': options.get('exchange'),
'routing_key': options.get('routing_key'),
'priority': options.get('priority'),
},
}
tb = None
tracer = build_tracer(
task.name, task, eager=True,
propagate=throw, app=self._get_app(),
)
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
if isinstance(retval, Retry) and retval.sig is not None:
return retval.sig.apply(retries=retries + 1)
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def test_fast_trace_task__no_content_type(self):
self.app.tasks[self.add.name].__trace__ = build_tracer(
self.add.name, self.add, app=self.app,
)
_fast_trace_task(
self.add.name, 'id1', {}, ((2, 2), {}, {}), None, None,
app=self.app, _loc=[self.app.tasks, {}, 'hostname']
)
def test_fast_trace_task__no_content_type(self):
self.app.tasks[self.add.name].__trace__ = build_tracer(
self.add.name, self.add, app=self.app,
)
_fast_trace_task(
self.add.name, 'id1', {}, ((2, 2), {}, {}), None, None,
app=self.app, _loc=[self.app.tasks, {}, 'hostname']
)
def update_strategies(self):
loader = self.app.loader
for name, task in self.app.tasks.items():
self.strategies[name] = task.start_strategy(self.app, self)
task.__trace__ = build_tracer(name,
task,
loader,
self.hostname,
app=self.app)
def apply(self, args=None, kwargs=None,
link=None, link_error=None,
task_id=None, retries=None, throw=None,
logfile=None, loglevel=None, headers=None, **options):
"""Execute this task locally, by blocking until the task returns.
Arguments:
args (Tuple): positional arguments passed on to the task.
kwargs (Dict): keyword arguments passed on to the task.
throw (bool): Re-raise task exceptions.
Defaults to the :setting:`task_eager_propagates` setting.
Returns:
celery.result.EagerResult: pre-evaluated result.
"""
# trace imports Task, so need to import inline.
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__,) + tuple(args)
kwargs = kwargs or {}
task_id = task_id or uuid()
retries = retries or 0
if throw is None:
throw = app.conf.task_eager_propagates
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {
'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': logfile,
'loglevel': loglevel or 0,
'hostname': gethostname(),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'headers': headers,
'delivery_info': {'is_eager': True},
}
tb = None
tracer = build_tracer(
task.name, task, eager=True,
propagate=throw, app=self._get_app(),
)
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def test_trace_task_ret(self):
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
_, R, _ = _trace_task_ret(
self.mytask.name, tid, message.headers,
message.body, message.content_type,
message.content_encoding, app=self.app,
)
self.assertEqual(R, repr(4 ** 4))
def test_trace_task_ret(self):
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
_, R, _ = _trace_task_ret(
self.mytask.name, tid, message.headers,
message.body, message.content_type,
message.content_encoding, app=self.app,
)
assert R == repr(4 ** 4)
def apply(self, args=None, kwargs=None,
link=None, link_error=None, **options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports Task, so need to import inline.
from billiard.einfo import ExceptionInfo
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__, ) + tuple(args)
kwargs = kwargs or {}
task_id = options.get('task_id') or uuid()
retries = options.get('retries', 0)
throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS',
options.pop('throw', None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': options.get('logfile'),
'loglevel': options.get('loglevel', 0),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'headers': options.get('headers'),
'delivery_info': {'is_eager': True}}
tb = None
tracer = build_tracer(
task.name, task, eager=True,
propagate=throw, app=self._get_app(),
)
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
self.mytask.__trace__ = build_tracer(self.mytask.name, self.mytask, self.app.loader, "test", app=self.app)
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, "__trace__")
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app)
self.assertEqual(res, 4 ** 4)
def trace(app,
task,
args=(),
kwargs={},
propagate=False,
eager=True,
request=None,
**opts):
t = build_tracer(task.name,
task,
eager=eager,
propagate=propagate,
app=app,
**opts)
ret = t('id-1', args, kwargs, request)
return ret.retval, ret.info
def process_initializer(app, hostname):
"""Pool child process initializer.
Initialize the child pool process to ensure the correct
app instance is used and things like logging works.
"""
_set_task_join_will_block(True)
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title("celeryd", hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
logfile = os.environ.get("CELERY_LOG_FILE") or None
if logfile and "%i" in logfile.lower():
# logfile path will differ so need to set up logging again.
app.log.already_setup = False
app.log.setup(
int(os.environ.get("CELERY_LOG_LEVEL", 0) or 0),
logfile,
bool(os.environ.get("CELERY_LOG_REDIRECT", False)),
str(os.environ.get("CELERY_LOG_REDIRECT_LEVEL")),
hostname=hostname,
)
if os.environ.get("FORKED_BY_MULTIPROCESSING"):
# pool did execv after fork
trace.setup_worker_optimizations(app, hostname)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
# rebuild execution handler for all tasks.
from celery.app.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name,
task,
app.loader,
hostname,
app=app)
from celery.worker import state as worker_state
worker_state.reset_state()
signals.worker_process_init.send(sender=None)
def apply(self, args=None, kwargs=None, link=None, link_error=None, **options):
"""Execute this task locally, by blocking until the task returns.
:param args: positional arguments passed on to the task.
:param kwargs: keyword arguments passed on to the task.
:keyword throw: Re-raise task exceptions. Defaults to
the :setting:`task_eager_propagates`
setting.
:rtype :class:`celery.result.EagerResult`:
"""
# trace imports Task, so need to import inline.
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
# add 'self' if this is a bound method.
if self.__self__ is not None:
args = (self.__self__,) + tuple(args)
kwargs = kwargs or {}
task_id = options.get("task_id") or uuid()
retries = options.get("retries", 0)
throw = app.either("task_eager_propagates", options.pop("throw", None))
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {
"id": task_id,
"retries": retries,
"is_eager": True,
"logfile": options.get("logfile"),
"loglevel": options.get("loglevel", 0),
"callbacks": maybe_list(link),
"errbacks": maybe_list(link_error),
"headers": options.get("headers"),
"delivery_info": {"is_eager": True},
}
tb = None
tracer = build_tracer(task.name, task, eager=True, propagate=throw, app=self._get_app())
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def test_trace_task_ret(self):
mytask.__trace__ = build_tracer(mytask.name, mytask,
self.app.loader, 'test')
res = _trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
def trace(app, task, args=(), kwargs={},
propagate=False, eager=True, request=None, **opts):
t = build_tracer(task.name, task,
eager=eager, propagate=propagate, app=app, **opts)
ret = t('id-1', args, kwargs, request)
return ret.retval, ret.info
def test_trace_task_ret(self):
mytask.__trace__ = build_tracer(mytask.name, mytask, self.app.loader,
'test')
res = _trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4**4)
def update_strategies(self):
loader = self.app.loader
for name, task in items(self.app.tasks):
self.strategies[name] = task.start_strategy(self.app, self)
task.__trace__ = build_tracer(name, task, loader, self.hostname,
app=self.app)