def __init__(self, name, listen_keys=None, worker_num=2, timezone=None):
BaseWorker.__init__(self, name)
self.server = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)
self.listen_keys = []
if listen_keys:
self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys]
self.timezone = timezone or get_localzone()
self.executor = ProcessPoolExecutor(worker_num, self.log)
self._stopped = True
class RQWorker(BaseWorker):
def __init__(self, name, listen_keys=None, worker_num=2, timezone=None):
BaseWorker.__init__(self, name)
self.server = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)
self.listen_keys = []
if listen_keys:
self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys]
self.timezone = timezone or get_localzone()
self.executor = ProcessPoolExecutor(worker_num, self.log)
self._stopped = True
# def _install_signal_handlers(self):
# """
# Installs signal handlers for handling SIGINT and SIGTERM gracefully.
# quote from python-rq
# """
# def request_stop(signum, frame):
# self.log.debug('Got signal %s.' % signal_name(signum))
# raise StopRequested()
#
# signal.signal(signal.SIGINT, request_stop)
# signal.signal(signal.SIGTERM, request_stop)
def start(self):
#self._install_signal_handlers()
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('elric worker running..')
while self.running:
key, serialized_job = RedisJobQueue.dequeue_any(self.server, self.listen_keys)
job = Job.deserialize(serialized_job)
self.log.debug('get job id=[%s] func=[%s]from key %s' % (job.id, job.func, key))
self.executor.execute_job(job)
def submit_job(self, func, job_key, args=None, kwargs=None, trigger=None, job_id=None,
replace_exist=False, filter_key='', filter_value='', **trigger_args):
"""
submit job to master through rpc
:type func: str or callable obj or unicode
:type job_key: str or unicode
:type args: tuple or list
:type kwargs: dict
:type trigger: str or unicode
:type job_id: str or unicode
:type replace_exist: bool
:type trigger_args: dict
"""
job_key = '%s:%s' % (self.name, job_key)
# use worker's timezone if trigger don't provide specific `timezone` configuration
trigger_args['timezone'] = self.timezone
job_in_dict = {
'id': job_id,
'func': func,
'args': args,
'trigger': create_trigger(trigger, trigger_args) if trigger else None,
'kwargs': kwargs,
'filter_key': '%s_%s' % (self.name, filter_key),
'filter_value': filter_value,
}
job = Job(**job_in_dict)
rpc_client_call('submit_job', Binary(job.serialize()),
job_key, job.id, replace_exist)
def update_job(self, job_id, job_key, next_run_time, serialized_job):
"""
send update job request to master through rpc
:type job_id: str
:type job_key: str
:type next_run_time: datetime.datetime
:type serialized_job: str
"""
job_key = '%s:%s' % (self.name, job_key)
rpc_client_call('update_job', job_id, job_key,
next_run_time, Binary(serialized_job))
def remove_job(self, job_id):
"""
send remove job request to master through rpc
:type job_id: str
"""
rpc_client_call('remove_job', job_id)
@property
def running(self):
return not self._stopped
def stop(self):
self.log.debug('Worker is quiting')
self._stopped = True
self.executor.shutdown()