def on_start(self):
if not self._custom_logging and self.redirect_stdouts:
self.app.log.redirect_stdouts(self.redirect_stdouts_level)
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
warnings.warn(RuntimeWarning(
'Running the worker with superuser privileges is discouraged!',
))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
sys.__stdout__.write(
str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')) + '\n'
)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def on_start(self):
if not self._custom_logging and self.redirect_stdouts:
self.app.log.redirect_stdouts(self.redirect_stdouts_level)
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
accept_encoding = self.app.conf.CELERY_ACCEPT_CONTENT
if ('pickle' in accept_encoding or
'application/x-python-serialize' in accept_encoding):
if not C_FORCE_ROOT:
raise RuntimeError(ROOT_DISALLOWED)
warnings.warn(RuntimeWarning(ROOT_DISCOURAGED))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
sys.__stdout__.write(
str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')) + '\n'
)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def on_start(self):
if not self._custom_logging and self.redirect_stdouts:
self.app.log.redirect_stdouts(self.redirect_stdouts_level)
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(sender=self.hostname, instance=self, conf=self.app.conf)
if not self.app.conf.value_set_for("accept_content"):
warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(
safe_str(
"".join(
[
string(self.colored.cyan(" \n", self.startup_info())),
string(self.colored.reset(self.extra_info() or "")),
]
)
),
file=sys.__stdout__,
)
self.set_process_status("-active-")
self.install_platform_tweaks(self)
def bench_consumer(n=10000):
from celery.worker import WorkController
from celery.worker import state
worker = WorkController(app=celery, pool_cls="solo")
time_start = [None]
@celery.task()
def it(i):
if not i:
time_start[0] = time.time()
elif i == n - 1:
print(time.time() - time_start[0])
@celery.task()
def shutdown_worker():
raise SystemExit()
for i in xrange(n):
it.delay(i)
shutdown_worker.delay()
try:
worker.start()
except SystemExit:
assert sum(state.total_count.values()) == n + 1
def on_start(self):
app = self.app
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=app.conf,
)
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
use_image = self._term_supports_images()
if use_image:
self.termimage(static.logo_as_base64())
print(safe_str(''.join([
string(self.colored.cyan(
' \n', self.startup_info(artlines=not use_image))),
string(self.colored.reset(self.extra_info() or '')),
])), file=sys.__stdout__)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
if not self._custom_logging and self.redirect_stdouts:
app.log.redirect_stdouts(self.redirect_stdouts_level)
def on_start(self):
WorkController.on_start(self)
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
warnings.warn(RuntimeWarning(
'Running celeryd with superuser privileges is discouraged!'))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(str(self.colored.cyan(' \n', self.startup_info())) +
str(self.colored.reset(self.extra_info() or '')))
self.set_process_status('-active-')
self.setup_logging()
self.install_platform_tweaks(self)
def test_on_timer_tick(self):
worker = WorkController(concurrency=1, loglevel=10)
worker.logger = Mock()
worker.timer_debug = worker.logger.debug
worker.on_timer_tick(30.0)
logged = worker.logger.debug.call_args[0][0]
self.assertIn("30.0", logged)
def test_on_timer_tick(self):
worker = WorkController(concurrency=1, loglevel=10)
worker.logger = Mock()
worker.on_timer_tick(30.0)
xargs = worker.logger.debug.call_args[0]
fmt, arg = xargs[0], xargs[1]
self.assertEqual(30.0, arg)
self.assertIn("Next eta %s secs", fmt)
def run_worker():
worker = WorkController(concurrency=concurrency,
loglevel=loglevel,
logfile=logfile,
is_detached=detach)
try:
worker.start()
except Exception, e:
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
e.__class__, e, traceback.format_exc()))
def test_on_timer_error(self):
worker = WorkController(concurrency=1, loglevel=0)
worker.logger = MockLogger()
try:
raise KeyError("foo")
except KeyError:
exc_info = sys.exc_info()
worker.on_timer_error(exc_info)
logged = worker.logger.logged[0]
self.assertIn("KeyError", logged)
def test_on_timer_error(self):
worker = WorkController(concurrency=1, loglevel=0)
worker.logger = Mock()
try:
raise KeyError("foo")
except KeyError:
exc_info = sys.exc_info()
worker.on_timer_error(exc_info)
msg, args = worker.logger.error.call_args[0]
self.assertIn("KeyError", msg % args)
def test_on_timer_error(self):
worker = WorkController(concurrency=1, loglevel=0)
worker.logger = Mock()
try:
raise KeyError("foo")
except KeyError:
exc_info = sys.exc_info()
worker.on_timer_error(exc_info)
logged = worker.logger.error.call_args[0][0]
self.assertIn("KeyError", logged)
def run_worker():
worker = WorkController(concurrency=concurrency,
loglevel=loglevel,
logfile=logfile,
is_detached=detach)
# Install signal handler that restarts celeryd on SIGHUP
install_restart_signal_handler(worker)
try:
worker.start()
except Exception, e:
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
e.__class__, e, traceback.format_exc()))
def test_dont_stop_or_terminate(self):
worker = WorkController(concurrency=1, loglevel=0)
worker.stop()
self.assertNotEqual(worker._state, worker.CLOSE)
worker.terminate()
self.assertNotEqual(worker._state, worker.CLOSE)
sigsafe, worker.pool.signal_safe = worker.pool.signal_safe, False
try:
worker._state = worker.RUN
worker.stop(in_sighandler=True)
self.assertNotEqual(worker._state, worker.CLOSE)
worker.terminate(in_sighandler=True)
self.assertNotEqual(worker._state, worker.CLOSE)
finally:
worker.pool.signal_safe = sigsafe
def test_with_rate_limits_disabled(self):
conf.DISABLE_RATE_LIMITS = True
try:
worker = WorkController(concurrency=1, loglevel=0)
self.assertTrue(hasattr(worker.ready_queue, "put"))
finally:
conf.DISABLE_RATE_LIMITS = False
def test_dont_stop_or_terminate(self):
worker = WorkController(concurrency=1, loglevel=0)
worker.stop()
self.assertNotEqual(worker._state, worker.CLOSE)
worker.terminate()
self.assertNotEqual(worker._state, worker.CLOSE)
sigsafe, worker.pool.signal_safe = worker.pool.signal_safe, False
try:
worker._state = worker.RUN
worker.stop(in_sighandler=True)
self.assertNotEqual(worker._state, worker.CLOSE)
worker.terminate(in_sighandler=True)
self.assertNotEqual(worker._state, worker.CLOSE)
finally:
worker.pool.signal_safe = sigsafe
def test_with_rate_limits_disabled(self):
conf.DISABLE_RATE_LIMITS = True
try:
worker = WorkController(concurrency=1, loglevel=0)
self.assertIsInstance(worker.ready_queue, FastQueue)
finally:
conf.DISABLE_RATE_LIMITS = False
def test_on_timer_tick(self):
worker = WorkController(concurrency=1, loglevel=10)
Timers(worker).on_timer_tick(30.0)
xargs = self.logger.debug.call_args[0]
fmt, arg = xargs[0], xargs[1]
self.assertEqual(30.0, arg)
self.assertIn('Next eta %s secs', fmt)
def test_on_timer_error(self):
worker = WorkController(concurrency=1, loglevel=0)
try:
raise KeyError('foo')
except KeyError, exc:
Timers(worker).on_timer_error(exc)
msg, args = self.logger.error.call_args[0]
self.assertIn('KeyError', msg % args)
def run_worker():
worker = WorkController(concurrency=concurrency,
loglevel=loglevel,
logfile=logfile,
hostname=hostname,
embed_clockservice=run_clockservice,
send_events=events)
# Install signal handler so SIGHUP restarts the worker.
install_worker_restart_handler(worker)
from celery import signals
signals.worker_init.send(sender=worker)
try:
worker.start()
except Exception, e:
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
e.__class__, e, traceback.format_exc()))
def on_start(self):
app = self.app
WorkController.on_start(self)
# this signal can be used to, for example, change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname,
instance=self,
conf=app.conf,
)
if self.purge:
self.purge_messages()
if not self.quiet:
self.emit_banner()
self.set_process_status('-active-')
self.install_platform_tweaks(self)
if not self._custom_logging and self.redirect_stdouts:
app.log.redirect_stdouts(self.redirect_stdouts_level)
# TODO: Remove the following code in Celery 6.0
# This qualifies as a hack for issue #6366.
# a hack via app.__reduce_keys__(), but that may not work properly in
# all cases
warn_deprecated = True
config_source = app._config_source
if isinstance(config_source, str):
# Don't raise the warning when the settings originate from
# django.conf:settings
warn_deprecated = config_source.lower() not in [
'django.conf:settings',
]
if warn_deprecated:
if app.conf.maybe_warn_deprecated_settings():
logger.warning(
"Please run `celery upgrade settings path/to/settings.py` "
"to avoid these warnings and to allow a smoother upgrade "
"to Celery 6.0.")
def on_start(self):
app = self.app
WorkController.on_start(self)
# this signal can be used to, for example, change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=app.conf,
)
if self.purge:
self.purge_messages()
if not self.quiet:
self.emit_banner()
self.set_process_status('-active-')
self.install_platform_tweaks(self)
if not self._custom_logging and self.redirect_stdouts:
app.log.redirect_stdouts(self.redirect_stdouts_level)
def on_start(self):
app = self.app
WorkController.on_start(self)
# this signal can be used to, for example, change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=app.conf,
)
if self.purge:
self.purge_messages()
if not self.quiet:
self.emit_banner()
self.set_process_status('-active-')
self.install_platform_tweaks(self)
if not self._custom_logging and self.redirect_stdouts:
app.log.redirect_stdouts(self.redirect_stdouts_level)
def run_worker(self):
worker = WorkController(concurrency=self.concurrency,
loglevel=self.loglevel,
logfile=self.logfile,
hostname=self.hostname,
ready_callback=self.on_listener_ready,
embed_clockservice=self.run_clockservice,
send_events=self.events)
# Install signal handler so SIGHUP restarts the worker.
install_worker_restart_handler(worker)
install_worker_term_handler(worker)
install_worker_int_handler(worker)
signals.worker_init.send(sender=worker)
try:
worker.start()
except Exception, exc:
emergency_error(self.logfile,
"celeryd raised exception %s: %s\n%s" % (
exc.__class__, exc, traceback.format_exc()))
def test_on_timer_tick(self):
worker = WorkController(concurrency=1, loglevel=10)
worker.logger = Mock()
worker.timer_debug = worker.logger.debug
worker.on_timer_tick(30.0)
logged = worker.logger.debug.call_args[0][0]
self.assertIn("30.0", logged)
def main(concurrency=DAEMON_CONCURRENCY, daemon=False,
loglevel=DAEMON_LOG_LEVEL, logfile=DAEMON_LOG_FILE, discard=False,
pidfile=DAEMON_PID_FILE, queue_wakeup_after=QUEUE_WAKEUP_AFTER):
"""Run the celery daemon."""
if settings.DATABASE_ENGINE == "sqlite3" and concurrency > 1:
import warnings
warnings.warn("The sqlite3 database engine doesn't support "
"concurrency. We'll be using a single process only.",
UserWarning)
concurrency = 1
if discard:
discarded_count = discard_all()
what = "message"
if discarded_count > 1:
what = "messages"
sys.stderr.write("Discard: Erased %d %s from the queue.\n" % (
discarded_count, what))
if daemon:
sys.stderr.write("Launching celeryd in the background...\n")
pidfile_handler = PIDFile(pidfile)
pidfile_handler.check()
daemonize(pidfile=pidfile_handler)
atexit.register(remove_pidfile, pidfile)
else:
logfile = None # log to stderr when not running as daemon.
discovery.autodiscover()
celeryd = WorkController(concurrency=concurrency,
loglevel=loglevel,
logfile=logfile,
queue_wakeup_after=queue_wakeup_after,
is_detached=daemon)
try:
celeryd.run()
except Exception, e:
emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % (
e.__class__, e, traceback.format_exc()))
def on_start(self):
if not self._custom_logging and self.redirect_stdouts:
self.app.log.redirect_stdouts(self.redirect_stdouts_level)
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname,
instance=self,
conf=self.app.conf,
)
if getattr(os, 'getuid', None) and os.getuid() == 0:
accept_encoding = self.app.conf.CELERY_ACCEPT_CONTENT
if ('pickle' in accept_encoding
or 'application/x-python-serialize' in accept_encoding):
if not C_FORCE_ROOT:
raise RuntimeError(ROOT_DISALLOWED)
warnings.warn(RuntimeWarning(ROOT_DISCOURAGED))
if not self.app.conf.value_set_for('CELERY_ACCEPT_CONTENT'):
warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED))
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(''.join([
string(self.colored.cyan(' \n', self.startup_info())),
string(self.colored.reset(self.extra_info() or '')),
]),
file=sys.__stdout__)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def on_start(self):
app = self.app
if not self._custom_logging and self.redirect_stdouts:
app.log.redirect_stdouts(self.redirect_stdouts_level)
WorkController.on_start(self)
# this signal can be used to e.g. change queues after
# the -Q option has been applied.
signals.celeryd_after_setup.send(
sender=self.hostname, instance=self, conf=app.conf,
)
if self.purge:
self.purge_messages()
# Dump configuration to screen so we have some basic information
# for when users sends bug reports.
print(safe_str(''.join([
string(self.colored.cyan(' \n', self.startup_info())),
string(self.colored.reset(self.extra_info() or '')),
])), file=sys.__stdout__)
self.set_process_status('-active-')
self.install_platform_tweaks(self)
def test_on_timer_tick(self):
worker = WorkController(concurrency=1, loglevel=10)
worker.logger = Mock()
worker.timer_debug = worker.logger.debug
worker.on_timer_tick(30.0)
xargs = worker.logger.debug.call_args[0]
fmt, arg = xargs[0], xargs[1]
self.assertEqual(30.0, arg)
self.assertIn("Next eta %s secs", fmt)
def setUp(self):
self.worker = WorkController(concurrency=1, loglevel=0)
self.worker.logger = MockLogger()
def test_with_embedded_clockservice(self):
worker = WorkController(concurrency=1,
loglevel=0,
embed_clockservice=True)
self.assertTrue(worker.clockservice)
self.assertIn(worker.clockservice, worker.components)
def on_consumer_ready(self, consumer):
signals.worker_ready.send(sender=consumer)
WorkController.on_consumer_ready(self, consumer)
print('celery@{0.hostname} has started.'.format(self))
def test_with_embedded_celerybeat(self):
worker = WorkController(concurrency=1, loglevel=0, beat=True)
self.assertTrue(worker.beat)
self.assertIn(worker.beat, worker.components)
def test_with_rate_limits_disabled(self):
worker = WorkController(concurrency=1,
loglevel=0,
disable_rate_limits=True)
self.assertTrue(hasattr(worker.ready_queue, 'put'))
def create_worker(self, **kw):
worker = WorkController(concurrency=1, loglevel=0, **kw)
worker.logger = Mock()
return worker
def create_worker(self, **kw):
worker = WorkController(concurrency=1, loglevel=0, **kw)
worker.logger = Mock()
return worker
def create_worker(self, **kw):
worker = WorkController(concurrency=1, loglevel=0, **kw)
worker._shutdown_complete.set()
worker.logger = Mock()
return worker
def create_worker(self, **kw):
worker = WorkController(concurrency=1, loglevel=0, **kw)
worker._shutdown_complete.set()
worker.logger = Mock()
return worker
def test_with_embedded_beat(self):
worker = WorkController(concurrency=1, loglevel=0, beat=True)
self.assertTrue(worker.beat)
self.assertIn(worker.beat, [w.obj for w in worker.steps])