class Worker(WorkController): redirect_stdouts = from_config() redirect_stdouts_level = from_config() def on_before_init(self, purge=False, no_color=None, **kwargs): # apply task execution optimizations trace.setup_worker_optimizations(self.app) # this signal can be used to set up configuration for # workers by name. conf = self.app.conf signals.celeryd_init.send( sender=self.hostname, instance=self, conf=conf, ) self.purge = purge self.no_color = no_color self._isatty = isatty(sys.stdout) self.colored = self.app.log.colored( self.logfile, enabled=not no_color if no_color is not None else no_color) def on_init_namespace(self): self.setup_logging() # apply task execution optimizations trace.setup_worker_optimizations(self.app) def on_start(self): WorkController.on_start(self) # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send( sender=self.hostname, instance=self, conf=self.app.conf, ) if getattr(os, 'getuid', None) and os.getuid() == 0: warnings.warn( RuntimeWarning( 'Running the worker with superuser privileges is discouraged!', )) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. sys.__stdout__.write( str(self.colored.cyan(' \n', self.startup_info())) + str(self.colored.reset(self.extra_info() or '')) + '\n') self.set_process_status('-active-') self.install_platform_tweaks(self) def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) print('{0.hostname} ready.'.format(self)) def setup_logging(self, colorize=None): if colorize is None and self.no_color is not None: colorize = not self.no_color self.app.log.setup(self.loglevel, self.logfile, self.redirect_stdouts, self.redirect_stdouts_level, colorize=colorize) def purge_messages(self): count = self.app.control.purge() print('purge: Erased {0} {1} from the queue.\n'.format( count, pluralize(count, 'message'))) def tasklist(self, include_builtins=True): tasks = self.app.tasks if not include_builtins: tasks = (t for t in tasks if not t.startswith('celery.')) return '\n'.join(' . {0}'.format(task) for task in sorted(tasks)) def extra_info(self): if self.loglevel <= logging.INFO: include_builtins = self.loglevel <= logging.DEBUG tasklist = self.tasklist(include_builtins=include_builtins) return EXTRA_INFO_FMT.format(tasks=tasklist) def startup_info(self): app = self.app concurrency = string(self.concurrency) appr = '{0}:0x{1:x}'.format(app.main or '__main__', id(app)) if not isinstance(app.loader, AppLoader): loader = qualname(app.loader) if loader.startswith('celery.loaders'): loader = loader[14:] appr += ' ({0})'.format(loader) if self.autoscale: max, min = self.autoscale concurrency = '{{min={0}, max={1}}}'.format(min, max) pool = self.pool_cls if not isinstance(pool, string_t): pool = pool.__module__ concurrency += ' ({0})'.format(pool.split('.')[-1]) events = 'ON' if not self.send_events: events = 'OFF (enable -E to monitor this worker)' banner = BANNER.format( app=appr, hostname=self.hostname, version=VERSION_BANNER, conninfo=self.app.connection().as_uri(), concurrency=concurrency, events=events, queues=app.amqp.queues.format(indent=0, indent_first=False), ).splitlines() # integrate the ASCII art. for i, x in enumerate(banner): try: banner[i] = ' '.join([ARTLINES[i], banner[i]]) except IndexError: banner[i] = ' ' * 16 + banner[i] return '\n'.join(banner) + '\n' def install_platform_tweaks(self, worker): """Install platform specific tweaks and workarounds.""" if self.app.IS_OSX: self.osx_proxy_detection_workaround() # Install signal handler so SIGHUP restarts the worker. if not self._isatty: # only install HUP handler if detached from terminal, # so closing the terminal window doesn't restart the worker # into the background. if self.app.IS_OSX: # OS X can't exec from a process using threads. # See http://github.com/celery/celery/issues#issue/152 install_HUP_not_supported_handler(worker) else: install_worker_restart_handler(worker) install_worker_term_handler(worker) install_worker_term_hard_handler(worker) install_worker_int_handler(worker) install_cry_handler() install_rdb_handler() def osx_proxy_detection_workaround(self): """See http://github.com/celery/celery/issues#issue/161""" os.environ.setdefault('celery_dummy_proxy', 'set_by_celeryd') def set_process_status(self, info): return platforms.set_mp_process_title('celeryd', info='{0} ({1})'.format( info, platforms.strargv(sys.argv)), hostname=self.hostname)
class WorkController(configurated): """Unmanaged worker instance.""" RUN = RUN CLOSE = CLOSE TERMINATE = TERMINATE app = None concurrency = from_config() loglevel = logging.ERROR logfile = from_config('log_file') send_events = from_config() pool_cls = from_config('pool') consumer_cls = from_config('consumer') mediator_cls = from_config('mediator') timer_cls = from_config('timer') timer_precision = from_config('timer_precision') autoscaler_cls = from_config('autoscaler') autoreloader_cls = from_config('autoreloader') schedule_filename = from_config() scheduler_cls = from_config('celerybeat_scheduler') task_time_limit = from_config() task_soft_time_limit = from_config() max_tasks_per_child = from_config() pool_putlocks = from_config() pool_restarts = from_config() force_execv = from_config() prefetch_multiplier = from_config() state_db = from_config() disable_rate_limits = from_config() worker_lost_wait = from_config() _state = None _running = 0 def __init__(self, loglevel=None, hostname=None, ready_callback=noop, queues=None, app=None, pidfile=None, **kwargs): self.app = app_or_default(app or self.app) self._shutdown_complete = Event() self.setup_defaults(kwargs, namespace='celeryd') self.app.select_queues(queues) # select queues subset. # Options self.loglevel = loglevel or self.loglevel self.hostname = hostname or socket.gethostname() self.ready_callback = ready_callback self._finalize = Finalize(self, self.stop, exitpriority=1) self.pidfile = pidfile self.pidlock = None self.use_eventloop = self.should_use_eventloop() # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. task_modules = set(task.__class__.__module__ for task in self.app.tasks.itervalues()) self.app.conf.CELERY_INCLUDE = tuple( set(self.app.conf.CELERY_INCLUDE) | task_modules, ) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs) def start(self): """Starts the workers main loop.""" self._state = self.RUN if self.pidfile: self.pidlock = platforms.create_pidlock(self.pidfile) try: for i, component in enumerate(self.components): logger.debug('Starting %s...', qualname(component)) self._running = i + 1 if component: component.start() logger.debug('%s OK!', qualname(component)) except SystemTerminate: self.terminate() except Exception, exc: logger.error('Unrecoverable error: %r', exc, exc_info=True) self.stop() except (KeyboardInterrupt, SystemExit): self.stop()
class Worker(configurated): WorkController = WorkController app = None inherit_confopts = (WorkController, ) loglevel = from_config('log_level') redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, hostname=None, purge=False, beat=False, queues=None, include=None, app=None, pidfile=None, autoscale=None, autoreload=False, no_execv=False, no_color=None, **kwargs): self.app = app = app_or_default(app or self.app) self.hostname = hostname or socket.gethostname() # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf) self.setup_defaults(kwargs, namespace='celeryd') if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 self.purge = purge self.beat = beat self.use_queues = [] if queues is None else queues self.queues = None self.include = include self.pidfile = pidfile self.autoscale = None self.autoreload = autoreload self.no_color = no_color self.no_execv = no_execv if autoscale: max_c, _, min_c = autoscale.partition(',') self.autoscale = [int(max_c), min_c and int(min_c) or 0] self._isatty = isatty(sys.stdout) self.colored = app.log.colored( self.logfile, enabled=not no_color if no_color is not None else no_color ) if isinstance(self.use_queues, basestring): self.use_queues = self.use_queues.split(',') if self.include: if isinstance(self.include, basestring): self.include = self.include.split(',') app.conf.CELERY_INCLUDE = ( tuple(app.conf.CELERY_INCLUDE) + tuple(self.include)) self.loglevel = mlevel(self.loglevel) def run(self): self.init_queues() self.app.loader.init_worker() # this signal can be used to e.g. change queues after # the -Q option has been applied. signals.celeryd_after_setup.send(sender=self.hostname, instance=self, conf=self.app.conf) if getattr(os, 'getuid', None) and os.getuid() == 0: warnings.warn(RuntimeWarning( 'Running celeryd with superuser privileges is discouraged!')) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print(str(self.colored.cyan(' \n', self.startup_info())) + str(self.colored.reset(self.extra_info() or ''))) self.set_process_status('-active-') self.setup_logging() # apply task execution optimizations trace.setup_worker_optimizations(self.app) try: self.run_worker() except IGNORE_ERRORS: pass def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) print('celery@%s ready.' % safe_str(self.hostname)) def init_queues(self): try: self.app.select_queues(self.use_queues) except KeyError, exc: raise ImproperlyConfigured(UNKNOWN_QUEUE % (self.use_queues, exc)) if self.app.conf.CELERY_WORKER_DIRECT: self.app.amqp.queues.select_add(worker_direct(self.hostname))
class WorkController(configurated): """Unmanaged worker instance.""" app = None concurrency = from_config() loglevel = from_config('log_level') logfile = from_config('log_file') send_events = from_config() pool_cls = from_config('pool') consumer_cls = from_config('consumer') mediator_cls = from_config('mediator') timer_cls = from_config('timer') timer_precision = from_config('timer_precision') autoscaler_cls = from_config('autoscaler') autoreloader_cls = from_config('autoreloader') schedule_filename = from_config() scheduler_cls = from_config('celerybeat_scheduler') task_time_limit = from_config() task_soft_time_limit = from_config() max_tasks_per_child = from_config() pool_putlocks = from_config() pool_restarts = from_config() force_execv = from_config() prefetch_multiplier = from_config() state_db = from_config() disable_rate_limits = from_config() worker_lost_wait = from_config() pidlock = None def __init__(self, app=None, hostname=None, **kwargs): self.app = app_or_default(app or self.app) self.hostname = hostname or socket.gethostname() self.app.loader.init_worker() self.on_before_init(**kwargs) self._finalize = Finalize(self, self.stop, exitpriority=1) self.setup_instance(**self.prepare_args(**kwargs)) def setup_instance(self, queues=None, ready_callback=None, pidfile=None, include=None, **kwargs): self.pidfile = pidfile self.setup_defaults(kwargs, namespace='celeryd') self.setup_queues(queues) self.setup_includes(include) # Set default concurrency if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 # Options self.loglevel = mlevel(self.loglevel) self.ready_callback = ready_callback or self.on_consumer_ready self.use_eventloop = self.should_use_eventloop() signals.worker_init.send(sender=self) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app, on_start=self.on_start, on_close=self.on_close, on_stopped=self.on_stopped) self.namespace.apply(self, **kwargs) def on_before_init(self, **kwargs): pass def on_start(self): if self.pidfile: self.pidlock = platforms.create_pidlock(self.pidfile) def on_consumer_ready(self, consumer): pass def on_close(self): self.app.loader.shutdown_worker() def on_stopped(self): self.timer.stop() self.consumer.close_connection() if self.pidlock: self.pidlock.release() def setup_queues(self, queues): if isinstance(queues, basestring): queues = queues.split(',') self.queues = queues try: self.app.select_queues(queues) except KeyError as exc: raise ImproperlyConfigured(UNKNOWN_QUEUE.format(queues, exc)) if self.app.conf.CELERY_WORKER_DIRECT: self.app.amqp.queues.select_add(worker_direct(self.hostname)) def setup_includes(self, includes): # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. inc = self.app.conf.CELERY_INCLUDE if includes: if isinstance(includes, basestring): includes = includes.split(',') inc = self.app.conf.CELERY_INCLUDE = tuple(inc) + tuple(includes) self.include = includes task_modules = set(task.__class__.__module__ for task in self.app.tasks.itervalues()) self.app.conf.CELERY_INCLUDE = tuple(set(inc) | task_modules) def prepare_args(self, **kwargs): return kwargs def start(self): """Starts the workers main loop.""" try: self.namespace.start(self) except SystemTerminate: self.terminate() except Exception as exc: logger.error('Unrecoverable error: %r', exc, exc_info=True) self.stop() except (KeyboardInterrupt, SystemExit): self.stop() def process_task_sem(self, req): return self._quick_acquire(self.process_task, req) def process_task(self, req): """Process task by sending it to the pool of workers.""" try: req.execute_using_pool(self.pool) except TaskRevokedError: try: self._quick_release() # Issue 877 except AttributeError: pass except Exception as exc: logger.critical('Internal error: %r\n%s', exc, traceback.format_exc(), exc_info=True) except SystemTerminate: self.terminate() raise except BaseException as exc: self.stop() raise exc def signal_consumer_close(self): try: self.consumer.close() except AttributeError: pass def should_use_eventloop(self): return (detect_environment() == 'default' and self.app.connection().is_evented and not self.app.IS_WINDOWS) def stop(self, in_sighandler=False): """Graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=True) def terminate(self, in_sighandler=False): """Not so graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=False) def _shutdown(self, warm=True): self.namespace.stop(self, terminate=not warm) self.namespace.join() def reload(self, modules=None, reload=False, reloader=None): modules = self.app.loader.task_modules if modules is None else modules imp = self.app.loader.import_from_cwd for module in set(modules or ()): if module not in sys.modules: logger.debug('importing module %s', module) imp(module) elif reload: logger.debug('reloading module %s', module) reload_from_cwd(sys.modules[module], reloader) self.pool.restart() @property def _state(self): return self.namespace.state @property def state(self): return state
class Beat(configurated): Service = beat.Service app = None loglevel = from_config('log_level') logfile = from_config('log_file') schedule = from_config('schedule_filename') scheduler_cls = from_config('scheduler') redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, no_color=None, **kwargs): """Starts the celerybeat task scheduler.""" self.app = app = app_or_default(app or self.app) self.setup_defaults(kwargs, namespace='celerybeat') self.max_interval = max_interval self.socket_timeout = socket_timeout self.no_color = no_color self.colored = app.log.colored(self.logfile, enabled=not no_color if no_color is not None else no_color, ) self.pidfile = pidfile if not isinstance(self.loglevel, int): self.loglevel = LOG_LEVELS[self.loglevel.upper()] def run(self): print(str(self.colored.cyan( 'celerybeat v{0} is starting.'.format(VERSION_BANNER)))) self.init_loader() self.set_process_title() self.start_scheduler() def setup_logging(self, colorize=None): if colorize is None and self.no_color is not None: colorize = not self.no_color self.app.log.setup(self.loglevel, self.logfile, self.redirect_stdouts, self.redirect_stdouts_level, colorize=colorize) def start_scheduler(self): c = self.colored if self.pidfile: platforms.create_pidlock(self.pidfile) beat = self.Service(app=self.app, max_interval=self.max_interval, scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) print(str(c.blue('__ ', c.magenta('-'), c.blue(' ... __ '), c.magenta('-'), c.blue(' _\n'), c.reset(self.startup_info(beat))))) self.setup_logging() if self.socket_timeout: logger.debug('Setting default socket timeout to %r', self.socket_timeout) socket.setdefaulttimeout(self.socket_timeout) try: self.install_sync_handler(beat) beat.start() except Exception as exc: logger.critical('celerybeat raised exception %s: %r', exc.__class__, exc, exc_info=True) def init_loader(self): # Run the worker init handler. # (Usually imports task modules and such.) self.app.loader.init_worker() self.app.finalize() def startup_info(self, beat): scheduler = beat.get_scheduler(lazy=True) return STARTUP_INFO_FMT.format( conninfo=self.app.connection().as_uri(), logfile=self.logfile or '[stderr]', loglevel=LOG_LEVELS[self.loglevel], loader=qualname(self.app.loader), scheduler=qualname(scheduler), scheduler_info=scheduler.info, hmax_interval=humanize_seconds(beat.max_interval), max_interval=beat.max_interval, ) def set_process_title(self): arg_start = 'manage' in sys.argv[0] and 2 or 1 platforms.set_process_title('celerybeat', info=' '.join(sys.argv[arg_start:])) def install_sync_handler(self, beat): """Install a `SIGTERM` + `SIGINT` handler that saves the celerybeat schedule.""" def _sync(signum, frame): beat.sync() raise SystemExit() platforms.signals.update(SIGTERM=_sync, SIGINT=_sync)
class Worker(configurated): WorkController = WorkController app = None inherit_confopts = (WorkController, ) loglevel = from_config("log_level") redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, hostname=None, purge=False, beat=False, queues=None, include=None, app=None, pidfile=None, autoscale=None, autoreload=False, no_execv=False, **kwargs): self.app = app = app_or_default(app or self.app) self.hostname = hostname or socket.gethostname() # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf) self.setup_defaults(kwargs, namespace="celeryd") if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 self.purge = purge self.beat = beat self.use_queues = [] if queues is None else queues self.queues = None self.include = include self.pidfile = pidfile self.autoscale = None self.autoreload = autoreload self.no_execv = no_execv if autoscale: max_c, _, min_c = autoscale.partition(",") self.autoscale = [int(max_c), min_c and int(min_c) or 0] self._isatty = isatty(sys.stdout) self.colored = app.log.colored(self.logfile) if isinstance(self.use_queues, basestring): self.use_queues = self.use_queues.split(",") if self.include: if isinstance(self.include, basestring): self.include = self.include.split(",") app.conf.CELERY_INCLUDE = ( tuple(app.conf.CELERY_INCLUDE) + tuple(self.include)) self.loglevel = mlevel(self.loglevel) def run(self): self.init_queues() self.app.loader.init_worker() if getattr(os, "getuid", None) and os.getuid() == 0: warnings.warn(RuntimeWarning( "Running celeryd with superuser privileges is discouraged!")) if self.purge: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print(str(self.colored.cyan(" \n", self.startup_info())) + str(self.colored.reset(self.extra_info() or ""))) self.set_process_status("-active-") self.redirect_stdouts_to_logger() try: self.run_worker() except IGNORE_ERRORS: pass def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) print("celery@%s has started." % self.hostname) def init_queues(self): try: self.app.select_queues(self.use_queues) except KeyError, exc: raise ImproperlyConfigured(UNKNOWN_QUEUE % (self.use_queues, exc))
class Beat(configurated): Service = beat.Service app = None loglevel = from_config('log_level') logfile = from_config('log_file') schedule = from_config('schedule_filename') scheduler_cls = from_config('scheduler') redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, no_color=None, **kwargs): """Starts the celerybeat task scheduler.""" self.app = app = app_or_default(app or self.app) self.setup_defaults(kwargs, namespace='celerybeat') self.max_interval = max_interval self.socket_timeout = socket_timeout self.no_color = no_color self.colored = app.log.colored( self.logfile, enabled=not no_color if no_color is not None else no_color, ) self.pidfile = pidfile if not isinstance(self.loglevel, int): self.loglevel = LOG_LEVELS[self.loglevel.upper()] def run(self): print( str( self.colored.cyan('celerybeat v%s is starting.' % VERSION_BANNER))) self.init_loader() self.set_process_title() self.start_scheduler() def setup_logging(self, colorize=None): if colorize is None and self.no_color is not None: colorize = not self.no_color self.app.log.setup(self.loglevel, self.logfile, self.redirect_stdouts, self.redirect_stdouts_level, colorize=colorize) def start_scheduler(self): c = self.colored if self.pidfile: platforms.create_pidlock(self.pidfile) beat = self.Service(app=self.app, max_interval=self.max_interval, scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) print( str( c.blue('__ ', c.magenta('-'), c.blue(' ... __ '), c.magenta('-'), c.blue(' _\n'), c.reset(self.startup_info(beat))))) self.setup_logging() if self.socket_timeout: logger.debug('Setting default socket timeout to %r', self.socket_timeout) socket.setdefaulttimeout(self.socket_timeout) try: self.install_sync_handler(beat) beat.start() except Exception, exc: logger.critical('celerybeat raised exception %s: %r', exc.__class__, exc, exc_info=True)
class Beat(configurated): Service = beat.Service app = None loglevel = from_config("log_level") logfile = from_config("log_file") schedule = from_config("schedule_filename") scheduler_cls = from_config("scheduler") redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, **kwargs): """Starts the celerybeat task scheduler.""" self.app = app = app_or_default(app or self.app) self.setup_defaults(kwargs, namespace="celerybeat") self.max_interval = max_interval self.socket_timeout = socket_timeout self.colored = app.log.colored(self.logfile) self.pidfile = pidfile if not isinstance(self.loglevel, int): self.loglevel = LOG_LEVELS[self.loglevel.upper()] def run(self): self.setup_logging() print( str(self.colored.cyan("celerybeat v%s is starting." % __version__))) self.init_loader() self.set_process_title() self.start_scheduler() def setup_logging(self): handled = self.app.log.setup_logging_subsystem(loglevel=self.loglevel, logfile=self.logfile) if self.redirect_stdouts and not handled: self.app.log.redirect_stdouts_to_logger( logger, loglevel=self.redirect_stdouts_level) def start_scheduler(self): c = self.colored if self.pidfile: platforms.create_pidlock(self.pidfile) beat = self.Service(app=self.app, max_interval=self.max_interval, scheduler_cls=self.scheduler_cls, schedule_filename=self.schedule) print( str( c.blue("__ ", c.magenta("-"), c.blue(" ... __ "), c.magenta("-"), c.blue(" _\n"), c.reset(self.startup_info(beat))))) if self.socket_timeout: logger.debug("Setting default socket timeout to %r", self.socket_timeout) socket.setdefaulttimeout(self.socket_timeout) try: self.install_sync_handler(beat) beat.start() except Exception, exc: logger.critical("celerybeat raised exception %s: %r", exc.__class__, exc, exc_info=True)
class WorkController(configurated): """Unmanaged worker instance.""" RUN = RUN CLOSE = CLOSE TERMINATE = TERMINATE app = None concurrency = from_config() loglevel = logging.ERROR logfile = from_config("log_file") send_events = from_config() pool_cls = from_config("pool") consumer_cls = from_config("consumer") mediator_cls = from_config("mediator") timer_cls = from_config("timer") timer_precision = from_config("timer_precision") autoscaler_cls = from_config("autoscaler") autoreloader_cls = from_config("autoreloader") schedule_filename = from_config() scheduler_cls = from_config("celerybeat_scheduler") task_time_limit = from_config() task_soft_time_limit = from_config() max_tasks_per_child = from_config() pool_putlocks = from_config() force_execv = from_config() prefetch_multiplier = from_config() state_db = from_config() disable_rate_limits = from_config() worker_lost_wait = from_config() _state = None _running = 0 def __init__(self, loglevel=None, hostname=None, ready_callback=noop, queues=None, app=None, pidfile=None, **kwargs): self.app = app_or_default(app or self.app) # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. set_default_app(self.app) self.app.finalize() trace._tasks = self.app._tasks self._shutdown_complete = Event() self.setup_defaults(kwargs, namespace="celeryd") self.app.select_queues(queues) # select queues subset. # Options self.loglevel = loglevel or self.loglevel self.hostname = hostname or socket.gethostname() self.ready_callback = ready_callback self._finalize = Finalize(self, self.stop, exitpriority=1) self.pidfile = pidfile self.pidlock = None self.use_eventloop = (detect_environment() == "default" and self.app.broker_connection().is_evented) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs) def start(self): """Starts the workers main loop.""" self._state = self.RUN if self.pidfile: self.pidlock = platforms.create_pidlock(self.pidfile) try: for i, component in enumerate(self.components): logger.debug("Starting %s...", qualname(component)) self._running = i + 1 if component: component.start() logger.debug("%s OK!", qualname(component)) except SystemTerminate: self.terminate() except Exception, exc: logger.error("Unrecoverable error: %r", exc, exc_info=True) self.stop() except (KeyboardInterrupt, SystemExit): self.stop()
class WorkController(configurated): """Unmanaged worker instance.""" RUN = RUN CLOSE = CLOSE TERMINATE = TERMINATE app = None concurrency = from_config() loglevel = from_config('log_level') logfile = from_config('log_file') send_events = from_config() pool_cls = from_config('pool') consumer_cls = from_config('consumer') mediator_cls = from_config('mediator') timer_cls = from_config('timer') timer_precision = from_config('timer_precision') autoscaler_cls = from_config('autoscaler') autoreloader_cls = from_config('autoreloader') schedule_filename = from_config() scheduler_cls = from_config('celerybeat_scheduler') task_time_limit = from_config() task_soft_time_limit = from_config() max_tasks_per_child = from_config() pool_putlocks = from_config() pool_restarts = from_config() force_execv = from_config() prefetch_multiplier = from_config() state_db = from_config() disable_rate_limits = from_config() worker_lost_wait = from_config() _state = None _running = 0 pidlock = None def __init__(self, app=None, hostname=None, **kwargs): self.app = app_or_default(app or self.app) # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. set_default_app(self.app) self.app.finalize() trace._tasks = self.app._tasks # optimization self.hostname = hostname or socket.gethostname() self.on_before_init(**kwargs) self._finalize = Finalize(self, self.stop, exitpriority=1) self._shutdown_complete = Event() self.setup_instance(**self.prepare_args(**kwargs)) def on_before_init(self, **kwargs): pass def on_start(self): pass def on_consumer_ready(self, consumer): pass def setup_instance(self, queues=None, ready_callback=None, pidfile=None, include=None, **kwargs): self.pidfile = pidfile self.app.loader.init_worker() self.setup_defaults(kwargs, namespace='celeryd') self.setup_queues(queues) self.setup_includes(include) # Set default concurrency if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 # Options self.loglevel = mlevel(self.loglevel) self.ready_callback = ready_callback or self.on_consumer_ready self.use_eventloop = self.should_use_eventloop() signals.worker_init.send(sender=self) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs) def setup_queues(self, queues): if isinstance(queues, basestring): queues = queues.split(',') self.queues = queues try: self.app.select_queues(queues) except KeyError as exc: raise ImproperlyConfigured(UNKNOWN_QUEUE.format(queues, exc)) if self.app.conf.CELERY_WORKER_DIRECT: self.app.amqp.queues.select_add(worker_direct(self.hostname)) def setup_includes(self, includes): # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. inc = self.app.conf.CELERY_INCLUDE if includes: if isinstance(includes, basestring): includes = includes.split(',') inc = self.app.conf.CELERY_INCLUDE = tuple(inc) + tuple(includes) self.include = includes task_modules = set(task.__class__.__module__ for task in self.app.tasks.itervalues()) self.app.conf.CELERY_INCLUDE = tuple(set(inc) | task_modules) def prepare_args(self, **kwargs): return kwargs def start(self): """Starts the workers main loop.""" self.on_start() self._state = self.RUN if self.pidfile: self.pidlock = platforms.create_pidlock(self.pidfile) try: for i, component in enumerate(self.components): logger.debug('Starting %s...', qualname(component)) self._running = i + 1 if component: component.start() logger.debug('%s OK!', qualname(component)) except SystemTerminate: self.terminate() except Exception as exc: logger.error('Unrecoverable error: %r', exc, exc_info=True) self.stop() except (KeyboardInterrupt, SystemExit): self.stop() try: # Will only get here if running green, # makes sure all greenthreads have exited. self._shutdown_complete.wait() except IGNORE_ERRORS: pass run = start # XXX Compat def process_task_sem(self, req): return self._quick_acquire(self.process_task, req) def process_task(self, req): """Process task by sending it to the pool of workers.""" try: req.execute_using_pool(self.pool) except TaskRevokedError: try: self._quick_release() # Issue 877 except AttributeError: pass except Exception as exc: logger.critical('Internal error: %r\n%s', exc, traceback.format_exc(), exc_info=True) except SystemTerminate: self.terminate() raise except BaseException as exc: self.stop() raise exc def signal_consumer_close(self): try: self.consumer.close() except AttributeError: pass def should_use_eventloop(self): return (detect_environment() == 'default' and self.app.connection().is_evented and not self.app.IS_WINDOWS) def stop(self, in_sighandler=False): """Graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=True) def terminate(self, in_sighandler=False): """Not so graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=False) def _shutdown(self, warm=True): what = 'Stopping' if warm else 'Terminating' socket_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(SHUTDOWN_SOCKET_TIMEOUT) # Issue 975 if self._state in (self.CLOSE, self.TERMINATE): return self.app.loader.shutdown_worker() if self.pool: self.pool.close() if self._state != self.RUN or self._running != len(self.components): # Not fully started, can safely exit. self._state = self.TERMINATE self._shutdown_complete.set() return self._state = self.CLOSE for component in reversed(self.components): logger.debug('%s %s...', what, qualname(component)) if component: stop = component.stop if not warm: stop = getattr(component, 'terminate', None) or stop stop() self.timer.stop() self.consumer.close_connection() if self.pidlock: self.pidlock.release() self._state = self.TERMINATE socket.setdefaulttimeout(socket_timeout) self._shutdown_complete.set() def reload(self, modules=None, reload=False, reloader=None): modules = self.app.loader.task_modules if modules is None else modules imp = self.app.loader.import_from_cwd for module in set(modules or ()): if module not in sys.modules: logger.debug('importing module %s', module) imp(module) elif reload: logger.debug('reloading module %s', module) reload_from_cwd(sys.modules[module], reloader) self.pool.restart() @property def state(self): return state
class WorkController(configurated): """Unmanaged worker instance.""" app = None concurrency = from_config() loglevel = from_config('log_level') logfile = from_config('log_file') send_events = from_config() pool_cls = from_config('pool') consumer_cls = from_config('consumer') timer_cls = from_config('timer') timer_precision = from_config('timer_precision') autoscaler_cls = from_config('autoscaler') autoreloader_cls = from_config('autoreloader') schedule_filename = from_config() scheduler_cls = from_config('celerybeat_scheduler') task_time_limit = from_config() task_soft_time_limit = from_config() max_tasks_per_child = from_config() pool_putlocks = from_config() pool_restarts = from_config() force_execv = from_config() prefetch_multiplier = from_config() state_db = from_config() disable_rate_limits = from_config() worker_lost_wait = from_config() pidlock = None namespace = None pool = None semaphore = None class Namespace(bootsteps.Namespace): """Worker bootstep namespace.""" name = 'Worker' default_steps = set([ 'celery.worker.components:Hub', 'celery.worker.components:Queues', 'celery.worker.components:Pool', 'celery.worker.components:Beat', 'celery.worker.components:Timer', 'celery.worker.components:StateDB', 'celery.worker.components:Consumer', 'celery.worker.autoscale:WorkerComponent', 'celery.worker.autoreload:WorkerComponent', ]) def __init__(self, app=None, hostname=None, **kwargs): self.app = app_or_default(app or self.app) self.hostname = default_nodename(hostname) self.app.loader.init_worker() self.on_before_init(**kwargs) self._finalize = [ Finalize(self, self.stop, exitpriority=1), Finalize(self, self._send_worker_shutdown, exitpriority=10), ] self.setup_instance(**self.prepare_args(**kwargs)) def setup_instance(self, queues=None, ready_callback=None, pidfile=None, include=None, use_eventloop=None, **kwargs): self.pidfile = pidfile self.setup_defaults(kwargs, namespace='celeryd') self.setup_queues(queues) self.setup_includes(include) # Set default concurrency if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 # Options self.loglevel = mlevel(self.loglevel) self.ready_callback = ready_callback or self.on_consumer_ready # this connection is not established, only used for params self._conninfo = self.app.connection() self.use_eventloop = (self.should_use_eventloop() if use_eventloop is None else use_eventloop) self.options = kwargs signals.worker_init.send(sender=self) # Initialize bootsteps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.steps = [] self.on_init_namespace() self.namespace = self.Namespace(app=self.app, on_start=self.on_start, on_close=self.on_close, on_stopped=self.on_stopped) self.namespace.apply(self, **kwargs) def on_init_namespace(self): pass def on_before_init(self, **kwargs): pass def on_start(self): if self.pidfile: self.pidlock = platforms.create_pidlock(self.pidfile) def on_consumer_ready(self, consumer): pass def on_close(self): self.app.loader.shutdown_worker() def on_stopped(self): self.timer.stop() self.consumer.shutdown() if self.pidlock: self.pidlock.release() def setup_queues(self, queues): if isinstance(queues, string_t): queues = queues.split(',') self.queues = queues try: self.app.select_queues(queues) except KeyError as exc: raise ImproperlyConfigured(UNKNOWN_QUEUE.format(queues, exc)) if self.app.conf.CELERY_WORKER_DIRECT: self.app.amqp.queues.select_add(worker_direct(self.hostname)) def setup_includes(self, includes): # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. inc = self.app.conf.CELERY_INCLUDE if includes: if isinstance(includes, string_t): includes = includes.split(',') inc = self.app.conf.CELERY_INCLUDE = tuple(inc) + tuple(includes) self.include = includes task_modules = set(task.__class__.__module__ for task in values(self.app.tasks)) self.app.conf.CELERY_INCLUDE = tuple(set(inc) | task_modules) def prepare_args(self, **kwargs): return kwargs def _send_worker_shutdown(self): signals.worker_shutdown.send(sender=self) def start(self): """Starts the workers main loop.""" try: self.namespace.start(self) except SystemTerminate: self.terminate() except Exception as exc: logger.error('Unrecoverable error: %r', exc, exc_info=True) self.stop() except (KeyboardInterrupt, SystemExit): self.stop() def _process_task_sem(self, req): return self._quick_acquire(self._process_task, req) def _process_task(self, req): """Process task by sending it to the pool of workers.""" try: req.execute_using_pool(self.pool) except TaskRevokedError: try: self._quick_release() # Issue 877 except AttributeError: pass except Exception as exc: logger.critical('Internal error: %r\n%s', exc, traceback.format_exc(), exc_info=True) except SystemTerminate: self.terminate() raise except BaseException as exc: self.stop() raise exc def signal_consumer_close(self): try: self.consumer.close() except AttributeError: pass def should_use_eventloop(self): return (detect_environment() == 'default' and self._conninfo.is_evented and not self.app.IS_WINDOWS) def stop(self, in_sighandler=False): """Graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=True) def terminate(self, in_sighandler=False): """Not so graceful shutdown of the worker server.""" self.signal_consumer_close() if not in_sighandler or self.pool.signal_safe: self._shutdown(warm=False) def _shutdown(self, warm=True): # if namespace does not exist it means that we had an # error before the bootsteps could be initialized. if self.namespace is not None: self.namespace.stop(self, terminate=not warm) self.namespace.join() def reload(self, modules=None, reload=False, reloader=None): modules = self.app.loader.task_modules if modules is None else modules imp = self.app.loader.import_from_cwd for module in set(modules or ()): if module not in sys.modules: logger.debug('importing module %s', module) imp(module) elif reload: logger.debug('reloading module %s', module) reload_from_cwd(sys.modules[module], reloader) self.pool.restart() def info(self): return { 'total': self.state.total_count, 'pid': os.getpid(), 'clock': str(self.app.clock) } def rusage(self): s = resource.getrusage(resource.RUSAGE_SELF) return { 'utime': s.ru_utime, 'stime': s.ru_stime, 'maxrss': s.ru_maxrss, 'ixrss': s.ru_ixrss, 'idrss': s.ru_idrss, 'isrss': s.ru_isrss, 'minflt': s.ru_minflt, 'majflt': s.ru_majflt, 'nswap': s.ru_nswap, 'inblock': s.ru_inblock, 'oublock': s.ru_oublock, 'msgsnd': s.ru_msgsnd, 'msgrcv': s.ru_msgrcv, 'nsignals': s.ru_nsignals, 'nvcsw': s.ru_nvcsw, 'nivcsw': s.ru_nivcsw, } def stats(self): info = self.info() info.update(self.namespace.info(self)) info.update(self.consumer.namespace.info(self.consumer)) info.update(rusage=self.rusage()) return info @property def _state(self): return self.namespace.state @property def state(self): return state
class Worker(configurated): WorkController = WorkController app = None inherit_confopts = (WorkController, ) loglevel = from_config("log_level") redirect_stdouts = from_config() redirect_stdouts_level = from_config() def __init__(self, hostname=None, discard=False, embed_clockservice=False, queues=None, include=None, app=None, pidfile=None, autoscale=None, autoreload=False, **kwargs): self.app = app = app_or_default(app or self.app) self.hostname = hostname or socket.gethostname() # this signal can be used to set up configuration for # workers by name. signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf) self.setup_defaults(kwargs, namespace="celeryd") if not self.concurrency: try: self.concurrency = cpu_count() except NotImplementedError: self.concurrency = 2 self.discard = discard self.embed_clockservice = embed_clockservice if self.app.IS_WINDOWS and self.embed_clockservice: self.die("-B option does not work on Windows. " "Please run celerybeat as a separate service.") self.use_queues = [] if queues is None else queues self.queues = None self.include = [] if include is None else include self.pidfile = pidfile self.autoscale = None self.autoreload = autoreload if autoscale: max_c, _, min_c = autoscale.partition(",") self.autoscale = [int(max_c), min_c and int(min_c) or 0] self._isatty = isatty(sys.stdout) self.colored = app.log.colored(self.logfile) if isinstance(self.use_queues, basestring): self.use_queues = self.use_queues.split(",") if isinstance(self.include, basestring): self.include = self.include.split(",") try: self.loglevel = mlevel(self.loglevel) except KeyError: self.die("Unknown level %r. Please use one of %s." % (self.loglevel, "|".join(l for l in LOG_LEVELS.keys() if isinstance(l, basestring)))) def run(self): self.init_loader() self.init_queues() self.worker_init() self.redirect_stdouts_to_logger() if getattr(os, "getuid", None) and os.getuid() == 0: warnings.warn( RuntimeWarning( "Running celeryd with superuser privileges is discouraged!" )) if self.discard: self.purge_messages() # Dump configuration to screen so we have some basic information # for when users sends bug reports. print( str(self.colored.cyan(" \n", self.startup_info())) + str(self.colored.reset(self.extra_info()))) self.set_process_status("-active-") try: self.run_worker() except IGNORE_ERRORS: pass def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) print("celery@%s has started." % self.hostname) def init_queues(self): try: self.app.select_queues(self.use_queues) except KeyError, exc: raise ImproperlyConfigured(UNKNOWN_QUEUE_ERROR % (self.use_queues, exc))