def on_start(self): if not self._custom_logging and self.redirect_stdouts: self.app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=self.app.conf, ) if getattr(os, 'getuid', None) and os.getuid() == 0: warnings.warn(RuntimeWarning( 'Running the worker with superuser privileges is discouraged!', )) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. sys.__stdout__.write( str(self.colored.cyan(' \n', self.startup_info())) + str(self.colored.reset(self.extra_info() or '')) + '\n' ) self.set_process_status('-active-') self.install_platform_tweaks(self)
def on_start(self): if not self._custom_logging and self.redirect_stdouts: self.app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=self.app.conf, ) if getattr(os, 'getuid', None) and os.getuid() == 0: accept_encoding = self.app.conf.CELERY_ACCEPT_CONTENT if ('pickle' in accept_encoding or 'application/x-python-serialize' in accept_encoding): if not C_FORCE_ROOT: raise RuntimeError(ROOT_DISALLOWED) warnings.warn(RuntimeWarning(ROOT_DISCOURAGED)) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. sys.__stdout__.write( str(self.colored.cyan(' \n', self.startup_info())) + str(self.colored.reset(self.extra_info() or '')) + '\n' ) self.set_process_status('-active-') self.install_platform_tweaks(self)
def on_start(self): if not self._custom_logging and self.redirect_stdouts: self.app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send(sender=self.hostname, instance=self, conf=self.app.conf) if not self.app.conf.value_set_for("accept_content"): warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED)) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print( safe_str( "".join( [ string(self.colored.cyan(" \n", self.startup_info())), string(self.colored.reset(self.extra_info() or "")), ] ) ), file=sys.__stdout__, ) self.set_process_status("-active-") self.install_platform_tweaks(self)
def bench_consumer(n=10000): from celery.worker import WorkController from celery.worker import state worker = WorkController(app=celery, pool_cls="solo") time_start = [None] @celery.task() def it(i): if not i: time_start[0] = time.time() elif i == n - 1: print(time.time() - time_start[0]) @celery.task() def shutdown_worker(): raise SystemExit() for i in xrange(n): it.delay(i) shutdown_worker.delay() try: worker.start() except SystemExit: assert sum(state.total_count.values()) == n + 1
def on_start(self): app = self.app WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=app.conf, ) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. use_image = self._term_supports_images() if use_image: self.termimage(static.logo_as_base64()) print(safe_str(''.join([ string(self.colored.cyan( ' \n', self.startup_info(artlines=not use_image))), string(self.colored.reset(self.extra_info() or '')), ])), file=sys.__stdout__) self.set_process_status('-active-') self.install_platform_tweaks(self) if not self._custom_logging and self.redirect_stdouts: app.log.redirect_stdouts(self.redirect_stdouts_level)
def on_start(self): WorkController.on_start(self) # apply task execution optimizations trace.setup_worker_optimizations(self.app) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=self.app.conf, ) if getattr(os, 'getuid', None) and os.getuid() == 0: warnings.warn(RuntimeWarning( 'Running celeryd with superuser privileges is discouraged!')) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print(str(self.colored.cyan(' \n', self.startup_info())) + str(self.colored.reset(self.extra_info() or ''))) self.set_process_status('-active-') self.setup_logging() self.install_platform_tweaks(self)
def test_on_timer_tick(self): worker = WorkController(concurrency=1, loglevel=10) worker.logger = Mock() worker.timer_debug = worker.logger.debug worker.on_timer_tick(30.0) logged = worker.logger.debug.call_args[0][0] self.assertIn("30.0", logged)
def test_on_timer_tick(self): worker = WorkController(concurrency=1, loglevel=10) worker.logger = Mock() worker.on_timer_tick(30.0) xargs = worker.logger.debug.call_args[0] fmt, arg = xargs[0], xargs[1] self.assertEqual(30.0, arg) self.assertIn("Next eta %s secs", fmt)
def run_worker(): worker = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, is_detached=detach) try: worker.start() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def test_on_timer_error(self): worker = WorkController(concurrency=1, loglevel=0) worker.logger = MockLogger() try: raise KeyError("foo") except KeyError: exc_info = sys.exc_info() worker.on_timer_error(exc_info) logged = worker.logger.logged[0] self.assertIn("KeyError", logged)
def test_on_timer_error(self): worker = WorkController(concurrency=1, loglevel=0) worker.logger = Mock() try: raise KeyError("foo") except KeyError: exc_info = sys.exc_info() worker.on_timer_error(exc_info) msg, args = worker.logger.error.call_args[0] self.assertIn("KeyError", msg % args)
def test_on_timer_error(self): worker = WorkController(concurrency=1, loglevel=0) worker.logger = Mock() try: raise KeyError("foo") except KeyError: exc_info = sys.exc_info() worker.on_timer_error(exc_info) logged = worker.logger.error.call_args[0][0] self.assertIn("KeyError", logged)
def run_worker(): worker = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, is_detached=detach) # Install signal handler that restarts celeryd on SIGHUP install_restart_signal_handler(worker) try: worker.start() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def test_dont_stop_or_terminate(self): worker = WorkController(concurrency=1, loglevel=0) worker.stop() self.assertNotEqual(worker._state, worker.CLOSE) worker.terminate() self.assertNotEqual(worker._state, worker.CLOSE) sigsafe, worker.pool.signal_safe = worker.pool.signal_safe, False try: worker._state = worker.RUN worker.stop(in_sighandler=True) self.assertNotEqual(worker._state, worker.CLOSE) worker.terminate(in_sighandler=True) self.assertNotEqual(worker._state, worker.CLOSE) finally: worker.pool.signal_safe = sigsafe
def test_with_rate_limits_disabled(self): conf.DISABLE_RATE_LIMITS = True try: worker = WorkController(concurrency=1, loglevel=0) self.assertTrue(hasattr(worker.ready_queue, "put")) finally: conf.DISABLE_RATE_LIMITS = False
def test_with_rate_limits_disabled(self): conf.DISABLE_RATE_LIMITS = True try: worker = WorkController(concurrency=1, loglevel=0) self.assertIsInstance(worker.ready_queue, FastQueue) finally: conf.DISABLE_RATE_LIMITS = False
def test_on_timer_tick(self): worker = WorkController(concurrency=1, loglevel=10) Timers(worker).on_timer_tick(30.0) xargs = self.logger.debug.call_args[0] fmt, arg = xargs[0], xargs[1] self.assertEqual(30.0, arg) self.assertIn('Next eta %s secs', fmt)
def test_on_timer_error(self): worker = WorkController(concurrency=1, loglevel=0) try: raise KeyError('foo') except KeyError, exc: Timers(worker).on_timer_error(exc) msg, args = self.logger.error.call_args[0] self.assertIn('KeyError', msg % args)
def run_worker(): worker = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, hostname=hostname, embed_clockservice=run_clockservice, send_events=events) # Install signal handler so SIGHUP restarts the worker. install_worker_restart_handler(worker) from celery import signals signals.worker_init.send(sender=worker) try: worker.start() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def on_start(self): app = self.app WorkController.on_start(self) # this signal can be used to, for example, change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=app.conf, ) if self.purge: self.purge_messages() if not self.quiet: self.emit_banner() self.set_process_status('-active-') self.install_platform_tweaks(self) if not self._custom_logging and self.redirect_stdouts: app.log.redirect_stdouts(self.redirect_stdouts_level) # TODO: Remove the following code in Celery 6.0 # This qualifies as a hack for issue #6366. # a hack via app.__reduce_keys__(), but that may not work properly in # all cases warn_deprecated = True config_source = app._config_source if isinstance(config_source, str): # Don't raise the warning when the settings originate from # django.conf:settings warn_deprecated = config_source.lower() not in [ 'django.conf:settings', ] if warn_deprecated: if app.conf.maybe_warn_deprecated_settings(): logger.warning( "Please run `celery upgrade settings path/to/settings.py` " "to avoid these warnings and to allow a smoother upgrade " "to Celery 6.0.")
def on_start(self): app = self.app WorkController.on_start(self) # this signal can be used to, for example, change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=app.conf, ) if self.purge: self.purge_messages() if not self.quiet: self.emit_banner() self.set_process_status('-active-') self.install_platform_tweaks(self) if not self._custom_logging and self.redirect_stdouts: app.log.redirect_stdouts(self.redirect_stdouts_level)
def run_worker(self): worker = WorkController(concurrency=self.concurrency, loglevel=self.loglevel, logfile=self.logfile, hostname=self.hostname, ready_callback=self.on_listener_ready, embed_clockservice=self.run_clockservice, send_events=self.events) # Install signal handler so SIGHUP restarts the worker. install_worker_restart_handler(worker) install_worker_term_handler(worker) install_worker_int_handler(worker) signals.worker_init.send(sender=worker) try: worker.start() except Exception, exc: emergency_error(self.logfile, "celeryd raised exception %s: %s\n%s" % ( exc.__class__, exc, traceback.format_exc()))
def main(concurrency=DAEMON_CONCURRENCY, daemon=False, loglevel=DAEMON_LOG_LEVEL, logfile=DAEMON_LOG_FILE, discard=False, pidfile=DAEMON_PID_FILE, queue_wakeup_after=QUEUE_WAKEUP_AFTER): """Run the celery daemon.""" if settings.DATABASE_ENGINE == "sqlite3" and concurrency > 1: import warnings warnings.warn("The sqlite3 database engine doesn't support " "concurrency. We'll be using a single process only.", UserWarning) concurrency = 1 if discard: discarded_count = discard_all() what = "message" if discarded_count > 1: what = "messages" sys.stderr.write("Discard: Erased %d %s from the queue.\n" % ( discarded_count, what)) if daemon: sys.stderr.write("Launching celeryd in the background...\n") pidfile_handler = PIDFile(pidfile) pidfile_handler.check() daemonize(pidfile=pidfile_handler) atexit.register(remove_pidfile, pidfile) else: logfile = None # log to stderr when not running as daemon. discovery.autodiscover() celeryd = WorkController(concurrency=concurrency, loglevel=loglevel, logfile=logfile, queue_wakeup_after=queue_wakeup_after, is_detached=daemon) try: celeryd.run() except Exception, e: emergency_error(logfile, "celeryd raised exception %s: %s\n%s" % ( e.__class__, e, traceback.format_exc()))
def on_start(self): if not self._custom_logging and self.redirect_stdouts: self.app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=self.app.conf, ) if getattr(os, 'getuid', None) and os.getuid() == 0: accept_encoding = self.app.conf.CELERY_ACCEPT_CONTENT if ('pickle' in accept_encoding or 'application/x-python-serialize' in accept_encoding): if not C_FORCE_ROOT: raise RuntimeError(ROOT_DISALLOWED) warnings.warn(RuntimeWarning(ROOT_DISCOURAGED)) if not self.app.conf.value_set_for('CELERY_ACCEPT_CONTENT'): warnings.warn(CDeprecationWarning(W_PICKLE_DEPRECATED)) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print(''.join([ string(self.colored.cyan(' \n', self.startup_info())), string(self.colored.reset(self.extra_info() or '')), ]), file=sys.__stdout__) self.set_process_status('-active-') self.install_platform_tweaks(self)
def on_start(self): app = self.app if not self._custom_logging and self.redirect_stdouts: app.log.redirect_stdouts(self.redirect_stdouts_level) WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=app.conf, ) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print(safe_str(''.join([ string(self.colored.cyan(' \n', self.startup_info())), string(self.colored.reset(self.extra_info() or '')), ])), file=sys.__stdout__) self.set_process_status('-active-') self.install_platform_tweaks(self)
def test_on_timer_tick(self): worker = WorkController(concurrency=1, loglevel=10) worker.logger = Mock() worker.timer_debug = worker.logger.debug worker.on_timer_tick(30.0) xargs = worker.logger.debug.call_args[0] fmt, arg = xargs[0], xargs[1] self.assertEqual(30.0, arg) self.assertIn("Next eta %s secs", fmt)
def setUp(self): self.worker = WorkController(concurrency=1, loglevel=0) self.worker.logger = MockLogger()
def test_with_embedded_clockservice(self): worker = WorkController(concurrency=1, loglevel=0, embed_clockservice=True) self.assertTrue(worker.clockservice) self.assertIn(worker.clockservice, worker.components)
def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) WorkController.on_consumer_ready(self, consumer) print('celery@{0.hostname} has started.'.format(self))
def test_with_embedded_celerybeat(self): worker = WorkController(concurrency=1, loglevel=0, beat=True) self.assertTrue(worker.beat) self.assertIn(worker.beat, worker.components)
def test_with_rate_limits_disabled(self): worker = WorkController(concurrency=1, loglevel=0, disable_rate_limits=True) self.assertTrue(hasattr(worker.ready_queue, 'put'))
def create_worker(self, **kw): worker = WorkController(concurrency=1, loglevel=0, **kw) worker.logger = Mock() return worker
def create_worker(self, **kw): worker = WorkController(concurrency=1, loglevel=0, **kw) worker._shutdown_complete.set() worker.logger = Mock() return worker
def test_with_embedded_beat(self): worker = WorkController(concurrency=1, loglevel=0, beat=True) self.assertTrue(worker.beat) self.assertIn(worker.beat, [w.obj for w in worker.steps])