def __init__(self, name, listen_keys=None, worker_num=2, timezone=None, logger_name='elric.worker'):
BaseWorker.__init__(self, name, logger_name)
self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG)
self.listen_keys = []
if listen_keys:
self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys]
self.timezone = timezone or get_localzone()
self.internal_job_queue = Queue(maxsize=worker_num)
self.executor = ProcessPoolExecutor(worker_num, self)
self._stopped = True
class RQWorker(BaseWorker):
def __init__(self, name, listen_keys=None, worker_num=2, timezone=None, logger_name='elric.worker'):
BaseWorker.__init__(self, name, logger_name)
self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG)
self.listen_keys = []
if listen_keys:
self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys]
self.timezone = timezone or get_localzone()
self.internal_job_queue = Queue(maxsize=worker_num)
self.executor = ProcessPoolExecutor(worker_num, self)
self._stopped = True
def start(self):
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('elric worker running..')
while self.running:
try:
# grab job from job queue only if internal_job_queue has space
self.internal_job_queue.put("#", True)
job_key, serialized_job = self.jobqueue.dequeue_any(self.listen_keys)
job = Job.deserialize(serialized_job)
self.log.debug('get job id=[%s] func=[%s]from key %s' % (job.id, job.func, job.job_key))
self.executor.execute_job(job)
except TypeError as e:
self.log.error(e)
time.sleep(60)
continue
def submit_job(self, func, job_key, args=None, kwargs=None, trigger=None, job_id=None,
replace_exist=False, need_filter=False, **trigger_args):
"""
submit job to master through redis queue
:type func: str or callable obj or unicode
:type job_key: str or unicode
:type args: tuple or list
:type kwargs: dict
:type trigger: str or unicode
:type job_id: str or unicode
:type replace_exist: bool
:type trigger_args: dict
"""
# use worker's timezone if trigger don't provide specific `timezone` configuration
trigger_args['timezone'] = self.timezone
job_in_dict = {
'id': "%s:%s" % (self.name, job_id) if job_id else None,
'func': func,
'args': args,
'trigger': create_trigger(trigger, trigger_args) if trigger else None,
'kwargs': kwargs,
'job_key': '%s:%s' % (self.name, job_key),
'need_filter': need_filter,
'replace_exist': replace_exist
}
job = Job(**job_in_dict)
job.check()
self.jobqueue.enqueue('__elric_submit_channel__', job.serialize())
def remove_job(self, job_id):
"""
send remove job request to master through redis queue
:type job_id: str
"""
job_in_dict = {
'id': "%s:%s" % (self.name, job_id)
}
job = Job(**job_in_dict)
self.jobqueue.enqueue('__elric_remove_channel__', job.serialize())
def finish_job(self, job_id, is_success, details, job_key, need_filter):
job_in_dict = {
'id': job_id,
'job_key': job_key,
'is_success': is_success,
'details': details,
'need_filter': need_filter
}
job = Job(**job_in_dict)
self.jobqueue.enqueue('__elric_finish_channel__', job.serialize())
@property
def running(self):
return not self._stopped
def stop(self):
self.log.debug('Worker is quiting')
self._stopped = True
self.executor.shutdown()
