def process_initializer(app, hostname):
"""Pool child process initializer.
This will initialize a child pool process to ensure the correct
app instance is used and things like
logging works.
"""
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title('celeryd', hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
app.log.setup(int(os.environ.get('CELERY_LOG_LEVEL', 0) or 0),
os.environ.get('CELERY_LOG_FILE') or None,
bool(os.environ.get('CELERY_LOG_REDIRECT', False)),
str(os.environ.get('CELERY_LOG_REDIRECT_LEVEL')))
if os.environ.get('FORKED_BY_MULTIPROCESSING'):
# pool did execv after fork
trace.setup_worker_optimizations(app)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
# rebuild execution handler for all tasks.
from celery.app.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name, task, app.loader, hostname,
app=app)
signals.worker_process_init.send(sender=None)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
assert trace.trace_task_ret is trace._fast_trace_task
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
assert trace.trace_task_ret is trace._trace_task_ret
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertTrue(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertTrue(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name,
self.mytask,
self.app.loader,
'test',
app=self.app,
)
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {})
self.assertEqual(res, 4**4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
res = trace.trace_task_ret(
self.mytask.name,
uuid(),
[4],
{},
app=self.app,
)
self.assertEqual(res, 4**4)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
assert trace.trace_task_ret is trace._fast_trace_task
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
assert trace.trace_task_ret is trace._trace_task_ret
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app, self.hostname)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf, options=kwargs)
check_privileges(self.app.conf.accept_content)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self,
conf=self.app.conf, options=kwargs,
)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self,
conf=self.app.conf, options=kwargs,
)
check_privileges(self.app.conf.CELERY_ACCEPT_CONTENT)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app, self.hostname)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self,
conf=self.app.conf, options=kwargs,
)
check_privileges(self.app.conf.CELERY_ACCEPT_CONTENT)
def on_before_init(self, quiet=False, **kwargs):
self.quiet = quiet
trace.setup_worker_optimizations(self.app, self.hostname)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self,
conf=self.app.conf, options=kwargs,
)
check_privileges(self.app.conf.accept_content)
def test_stackprotection(self):
setup_worker_optimizations(self.app)
try:
@self.app.task(shared=False, bind=True)
def foo(self, i):
if i:
return foo(0)
return self.request
assert foo(1).called_directly
finally:
reset_worker_optimizations()
def test_stackprotection(self):
setup_worker_optimizations(self.app)
try:
@self.app.task(shared=False, bind=True)
def foo(self, i):
if i:
return foo(0)
return self.request
self.assertTrue(foo(1).called_directly)
finally:
reset_worker_optimizations()
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
self.mytask.__trace__ = build_tracer(self.mytask.name, self.mytask, self.app.loader, "test", app=self.app)
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, "__trace__")
res = trace.trace_task_ret(self.mytask.name, uuid(), [4], {}, app=self.app)
self.assertEqual(res, 4 ** 4)
def process_initializer(app, hostname):
"""Pool child process initializer.
Initialize the child pool process to ensure the correct
app instance is used and things like logging works.
"""
_set_task_join_will_block(True)
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title("celeryd", hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
logfile = os.environ.get("CELERY_LOG_FILE") or None
if logfile and "%i" in logfile.lower():
# logfile path will differ so need to set up logging again.
app.log.already_setup = False
app.log.setup(
int(os.environ.get("CELERY_LOG_LEVEL", 0) or 0),
logfile,
bool(os.environ.get("CELERY_LOG_REDIRECT", False)),
str(os.environ.get("CELERY_LOG_REDIRECT_LEVEL")),
hostname=hostname,
)
if os.environ.get("FORKED_BY_MULTIPROCESSING"):
# pool did execv after fork
trace.setup_worker_optimizations(app, hostname)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
# rebuild execution handler for all tasks.
from celery.app.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name,
task,
app.loader,
hostname,
app=app)
from celery.worker import state as worker_state
worker_state.reset_state()
signals.worker_process_init.send(sender=None)
def on_init_blueprint(self):
self._custom_logging = self.setup_logging()
# apply task execution optimizations
# -- This will finalize the app!
trace.setup_worker_optimizations(self.app, self.hostname)
