class CeleryZenJobs(ZenDaemon):
mname = 'zenjobs' # For logging
def __init__(self, *args, **kwargs):
ZenDaemon.__init__(self, *args, **kwargs)
self.setup_celery()
signal.signal(signal.SIGTERM, self.sigTerm)
signal.signal(signal.SIGINT, self.sigTerm)
def setup_celery(self):
current_app.main = self.mname
current_app.db_options = self.options
current_app.add_defaults({
constants.NUM_WORKERS: self.options.num_workers,
constants.MAX_TASKS_PER_PROCESS: self.options.max_jobs_per_worker,
})
def run(self):
self.log.info('Daemon %s starting up', type(self).__name__)
freeze_support()
kwargs = {}
if self.options.daemon:
kwargs['logfile'] = zenPath('log', 'zenjobs.log')
kwargs['loglevel'] = self.options.logseverity
kwargs["pool_cls"] = concurrency.get_implementation(
kwargs.get("pool_cls") or current_app.conf.CELERYD_POOL)
self.worker = CeleryZenWorker(**kwargs)
self.worker.run() # blocking call
self.log.info('Daemon %s has shut down', type(self).__name__)
def sigTerm(self, *args, **kwargs):
from celery.worker import state
setattr(state, 'should_terminate', True)
super(CeleryZenJobs, self).sigTerm(*args, **kwargs)
# Raise SystemTerminate to shutdown without waiting...
raise SystemTerminate()
def buildOptions(self):
"""
Adds our command line options to ZCmdBase command line options.
"""
ZenDaemon.buildOptions(self)
self.parser.add_option('--job-log-path', dest='job_log_path',
default=zenPath("log", "jobs"),
help='Directory in which to store individual job log files')
self.parser.add_option('--max-jobs-per-worker',
dest='max_jobs_per_worker', type='int', default=1,
help='Number of jobs a worker process runs before it shuts down')
self.parser.add_option('--concurrent-jobs',
dest='num_workers', type='int', default=2,
help='Number of jobs to process concurrently')
connectionFactory = getUtility(IZodbFactoryLookup).get()
connectionFactory.buildOptions(self.parser)
def run(self):
self.log.info('Daemon %s starting up', type(self).__name__)
freeze_support()
kwargs = {}
if self.options.daemon:
kwargs['logfile'] = zenPath('log', 'zenjobs.log')
kwargs['loglevel'] = self.options.logseverity
kwargs["pool_cls"] = concurrency.get_implementation(
kwargs.get("pool_cls") or current_app.conf.CELERYD_POOL)
self.worker = CeleryZenWorker(**kwargs)
self.worker.run() # blocking call
self.log.info('Daemon %s has shut down', type(self).__name__)