def on_start(self):
WorkController.on_start(self)
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
warnings.warn(RuntimeWarning(
'Running celeryd with superuser privileges is discouraged!'))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
sys.__stdout__.write(
str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')) + '\n'
)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def process_initializer(app, hostname):
"""Pool child process initializer."""
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title('celeryd', hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
app.log.setup(int(os.environ.get('CELERY_LOG_LEVEL', 0)),
os.environ.get('CELERY_LOG_FILE') or None,
bool(os.environ.get('CELERY_LOG_REDIRECT', False)),
str(os.environ.get('CELERY_LOG_REDIRECT_LEVEL')))
if os.environ.get('FORKED_BY_MULTIPROCESSING'):
# pool did execv after fork
trace.setup_worker_optimizations(app)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
from celery.task.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name, task, app.loader, hostname)
signals.worker_process_init.send(sender=None)
def run(self):
self.init_queues()
self.app.loader.init_worker()
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(sender=self.hostname, instance=self,
conf=self.app.conf)
if getattr(os, 'getuid', None) and os.getuid() == 0:
warnings.warn(RuntimeWarning(
'Running celeryd with superuser privileges is discouraged!'))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')))
self.set_process_status('-active-')
self.setup_logging()
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
try:
self.run_worker()
except IGNORE_ERRORS:
pass
def run(self):
self.init_queues()
self.app.loader.init_worker()
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(sender=self.hostname, instance=self, conf=self.app.conf)
if getattr(os, "getuid", None) and os.getuid() == 0:
warnings.warn(RuntimeWarning("Running celeryd with superuser privileges is discouraged!"))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(str(self.colored.cyan(" \n", self.startup_info())) + str(self.colored.reset(self.extra_info() or "")))
self.set_process_status("-active-")
self.setup_logging()
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
try:
self.run_worker()
except IGNORE_ERRORS:
pass
def on_start(self):
WorkController.on_start(self)
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
warnings.warn(RuntimeWarning(
'Running celeryd with superuser privileges is discouraged!'))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
sys.__stdout__.write(
str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')) + '\n'
)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def process_initializer(app, hostname):
"""Pool child process initializer."""
platforms.signals.reset(*WORKER_SIGRESET)
platforms.signals.ignore(*WORKER_SIGIGNORE)
platforms.set_mp_process_title("celeryd", hostname=hostname)
# This is for Windows and other platforms not supporting
# fork(). Note that init_worker makes sure it's only
# run once per process.
app.loader.init_worker()
app.loader.init_worker_process()
app.log.setup(
int(os.environ.get("CELERY_LOG_LEVEL", 0)),
os.environ.get("CELERY_LOG_FILE") or None,
bool(os.environ.get("CELERY_LOG_REDIRECT", False)),
str(os.environ.get("CELERY_LOG_REDIRECT_LEVEL")),
)
if os.environ.get("FORKED_BY_MULTIPROCESSING"):
# pool did execv after fork
trace.setup_worker_optimizations(app)
else:
app.set_current()
set_default_app(app)
app.finalize()
trace._tasks = app._tasks # enables fast_trace_task optimization.
from celery.task.trace import build_tracer
for name, task in items(app.tasks):
task.__trace__ = build_tracer(name, task, app.loader, hostname)
signals.worker_process_init.send(sender=None)
def on_before_init(self, **kwargs):
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
def __init__(self, app=None, hostname=None, **kwargs):
self.app = app_or_default(app or self.app)
setup_worker_optimizations(self.app)
self.hostname = hostname or socket.gethostname()
self.on_before_init(**kwargs)
self._finalize = Finalize(self, self.stop, exitpriority=1)
self._shutdown_complete = Event()
self.setup_instance(**self.prepare_args(**kwargs))
def __init__(self, app=None, hostname=None, **kwargs):
self.app = app_or_default(app or self.app)
setup_worker_optimizations(self.app)
self.hostname = hostname or socket.gethostname()
self.on_before_init(**kwargs)
self._finalize = Finalize(self, self.stop, exitpriority=1)
self._shutdown_complete = Event()
self.setup_instance(**self.prepare_args(**kwargs))
def on_before_init(self, purge=False, no_color=None, **kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname, instance=self, conf=conf)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile, enabled=not no_color if no_color is not None else no_color)
def test_stackprotection(self):
setup_worker_optimizations(self.app)
try:
@self.app.task(bind=True)
def foo(self, i):
if i:
return foo(0)
return self.request
self.assertTrue(foo(1).called_directly)
finally:
reset_worker_optimizations()
def test_fast_trace_task(self):
from celery.task import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
mytask.__trace__ = build_tracer(mytask.name, mytask,
self.app.loader, 'test')
res = trace.trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(mytask, '__trace__')
res = trace.trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
def test_fast_trace_task(self):
from celery.task import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
try:
mytask.__trace__ = build_tracer(mytask.name, mytask,
self.app.loader, 'test')
res = trace.trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(mytask, '__trace__')
res = trace.trace_task_ret(mytask.name, uuid(), [4], {})
self.assertEqual(res, 4 ** 4)
def on_before_init(self, purge=False, no_color=None, **kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(
sender=self.hostname, instance=self, conf=conf,
)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile,
enabled=not no_color if no_color is not None else no_color
)
def on_before_init(self, purge=False, redirect_stdouts=None,
redirect_stdouts_level=None, **kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname, instance=self,
conf=conf)
self.purge = purge
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile)
if redirect_stdouts is None:
redirect_stdouts = conf.CELERY_REDIRECT_STDOUTS,
if redirect_stdouts_level is None:
redirect_stdouts_level = conf.CELERY_REDIRECT_STDOUTS_LEVEL
self.redirect_stdouts = redirect_stdouts
self.redirect_stdouts_level = redirect_stdouts_level
def on_before_init(self,
purge=False,
redirect_stdouts=None,
redirect_stdouts_level=None,
**kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname,
instance=self,
conf=conf)
self.purge = purge
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile)
if redirect_stdouts is None:
redirect_stdouts = conf.CELERY_REDIRECT_STDOUTS,
if redirect_stdouts_level is None:
redirect_stdouts_level = conf.CELERY_REDIRECT_STDOUTS_LEVEL
self.redirect_stdouts = redirect_stdouts
self.redirect_stdouts_level = redirect_stdouts_level
def on_init_namespace(self):
self.setup_logging()
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
def on_init_blueprint(self):
self._custom_logging = self.setup_logging()
# apply task execution optimizations
# -- This will finalize the app!
trace.setup_worker_optimizations(self.app)
def on_init_namespace(self):
self._custom_logging = self.setup_logging()
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
