class CeleryZenJobs(ZenDaemon): mname = 'zenjobs' # For logging def __init__(self, *args, **kwargs): ZenDaemon.__init__(self, *args, **kwargs) self.setup_celery() signal.signal(signal.SIGTERM, self.sigTerm) signal.signal(signal.SIGINT, self.sigTerm) def setup_celery(self): current_app.main = self.mname current_app.db_options = self.options current_app.add_defaults({ constants.NUM_WORKERS: self.options.num_workers, constants.MAX_TASKS_PER_PROCESS: self.options.max_jobs_per_worker, }) def run(self): self.log.info('Daemon %s starting up', type(self).__name__) freeze_support() kwargs = {} if self.options.daemon: kwargs['logfile'] = zenPath('log', 'zenjobs.log') kwargs['loglevel'] = self.options.logseverity kwargs["pool_cls"] = concurrency.get_implementation( kwargs.get("pool_cls") or current_app.conf.CELERYD_POOL) self.worker = CeleryZenWorker(**kwargs) self.worker.run() # blocking call self.log.info('Daemon %s has shut down', type(self).__name__) def sigTerm(self, *args, **kwargs): from celery.worker import state setattr(state, 'should_terminate', True) super(CeleryZenJobs, self).sigTerm(*args, **kwargs) # Raise SystemTerminate to shutdown without waiting... raise SystemTerminate() def buildOptions(self): """ Adds our command line options to ZCmdBase command line options. """ ZenDaemon.buildOptions(self) self.parser.add_option('--job-log-path', dest='job_log_path', default=zenPath("log", "jobs"), help='Directory in which to store individual job log files') self.parser.add_option('--max-jobs-per-worker', dest='max_jobs_per_worker', type='int', default=1, help='Number of jobs a worker process runs before it shuts down') self.parser.add_option('--concurrent-jobs', dest='num_workers', type='int', default=2, help='Number of jobs to process concurrently') connectionFactory = getUtility(IZodbFactoryLookup).get() connectionFactory.buildOptions(self.parser)
def run(self): self.log.info('Daemon %s starting up', type(self).__name__) freeze_support() kwargs = {} if self.options.daemon: kwargs['logfile'] = zenPath('log', 'zenjobs.log') kwargs['loglevel'] = self.options.logseverity kwargs["pool_cls"] = concurrency.get_implementation( kwargs.get("pool_cls") or current_app.conf.CELERYD_POOL) self.worker = CeleryZenWorker(**kwargs) self.worker.run() # blocking call self.log.info('Daemon %s has shut down', type(self).__name__)