def __init__(self, timezone=None): BaseMaster.__init__(self) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.jobqueue_lock = RLock() self.timezone = timezone or get_localzone() self._event = Event() self._stopped = True self.jobstore = MongoJobStore(self, **settings.JOB_STORE_CONFIG) #self.jobstore = MemoryJobStore(self) self._internal_buffer_queues = {} self._job_maximum_buffer_time = settings.JOB_QUEUE_CONFIG[ 'buffer_time']
def __init__(self, name, listen_keys=None, worker_num=2, timezone=None, logger_name='elric.worker'): BaseWorker.__init__(self, name, logger_name) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.listen_keys = [] if listen_keys: self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys] self.timezone = timezone or get_localzone() self.internal_job_queue = Queue(maxsize=worker_num) self.executor = ProcessPoolExecutor(worker_num, self) self._stopped = True
def __init__(self, timezone=None): BaseMaster.__init__(self) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.jobqueue_lock = RLock() self.timezone = timezone or get_localzone() self._event = Event() self._stopped = True self.jobstore = MongoJobStore(self, **settings.JOB_STORE_CONFIG) #self.jobstore = MemoryJobStore(self) self._internal_buffer_queues = {} self._job_maximum_buffer_time = settings.JOB_QUEUE_CONFIG['buffer_time']
class RQMasterBase(BaseMaster): MIN_WAIT_TIME = 5 def __init__(self, timezone=None): BaseMaster.__init__(self) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.jobqueue_lock = RLock() self.timezone = timezone or get_localzone() self._event = Event() self._stopped = True self.jobstore = MongoJobStore(self, **settings.JOB_STORE_CONFIG) #self.jobstore = MemoryJobStore(self) self._internal_buffer_queues = {} self._job_maximum_buffer_time = settings.JOB_QUEUE_CONFIG['buffer_time'] def submit_job(self, job): """ process submit_job request from worker. :type job: Job """ self.log.debug('client submit job, id=%s, key=%s' % (job.id, job.job_key)) # if job doesn't contains trigger, then enqueue it into job queue immediately if not job.trigger: self._enqueue_job(job.job_key, job.serialize()) # else store job into job store first else: with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG): try: self.jobstore.add_job(job) except JobAlreadyExist as e: if job.replace_exist: self.update_job(job) else: self.log.error(e) # wake up when new job has store into job store self.wake_up() def update_job(self, job): """ update job in jobstore :type job: Job """ try: self.jobstore.update_job(job) except JobDoesNotExist as e: self.log.error(e) def remove_job(self, job): """ remove job from jobstore :type job: Job """ try: self.jobstore.remove_job(job) except JobDoesNotExist: self.log.error('remove job error. job id %s does not exist' % job.id) def finish_job(self, job): """ process finish_job request from worker :type job: Job """ self.jobstore.save_execute_record(job) def _enqueue_buffer_queue(self, key, job): self.log.debug("job queue [%s] is full, put job into buffer queue" % key) try: self._internal_buffer_queues[key].put((job, datetime.now())) except KeyError: self._internal_buffer_queues[key] = Queue() self._internal_buffer_queues[key].put((job, datetime.now())) self.start_process_buffer_job(key) def _enqueue_job(self, key, job): """ enqueue job into jobqueue :type key: str :type job: str or xmlrpc.Binary """ self.log.debug('enqueue job key=[%s]' % key) with self.jobqueue_lock: # check whether job queue is full if not self.jobqueue.is_full(key): self.jobqueue.enqueue(key, job) else: self._enqueue_buffer_queue(key, job) def start(self): """ Start elric master. Select all due jobs from jobstore and enqueue them into redis queue. Then update due jobs' information into jobstore. :return: """ if self.running: raise AlreadyRunningException self._stopped = False self.log.debug('eric master start...') self.start_subscribe_thread() while True: now = datetime.now(self.timezone) wait_seconds = None with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG): for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(now): # enqueue due job into redis queue self._enqueue_job(job_key, serialized_job) # update job's information, such as next_run_time job = Job.deserialize(serialized_job) last_run_time = Job.get_serial_run_times(job, now) if last_run_time: next_run_time = Job.get_next_trigger_time(job, last_run_time[-1]) if next_run_time: job.next_run_time = next_run_time self.update_job(job) else: # if job has no next run time, then remove it from jobstore self.remove_job(job) # get next closet run time job from jobstore and set it to be wake up time closest_run_time = self.jobstore.get_closest_run_time() if closest_run_time is not None: wait_seconds = min(max(timedelta_seconds(closest_run_time - now), 0), self.MIN_WAIT_TIME) self.log.debug('Next wakeup is due at %s (in %f seconds)' % (closest_run_time, wait_seconds)) self._event.wait(wait_seconds if wait_seconds is not None else self.MIN_WAIT_TIME) self._event.clear() def wake_up(self): self._event.set() @property def running(self): return not self._stopped def start_process_buffer_job(self, job_key): """ Start filter data serialization thread """ self.log.debug('start process buffer job... job key=[%s]' % job_key) thd = threading.Thread(target=self.process_buffer_job, args=(job_key, )) thd.setDaemon(True) thd.start() def process_buffer_job(self, job_key): while True: job, buffer_time = self._internal_buffer_queues[job_key].get() with self.jobqueue_lock: if not self.jobqueue.is_full(job_key): self.jobqueue.enqueue(job_key, job) continue if (datetime.now() - buffer_time).total_seconds() < self._job_maximum_buffer_time: self.log.debug("requeue into buffer") self._internal_buffer_queues[job_key].put((job, buffer_time)) time.sleep(1.0) else: self.log.warning("timeout, discard job...") def subscribe_mq(self): while self.running: try: # grab job from job queue only if internal_job_queue has space key, serialized_job = self.jobqueue.dequeue_any(['__elric_submit_channel__', '__elric_remove_channel__', '__elric_finish_channel__']) Job.deserialize(serialized_job) self.func_map[key](Job.deserialize(serialized_job)) except TypeError as e: self.log.error(e) time.sleep(60) continue
class RQMasterBase(BaseMaster): MIN_WAIT_TIME = 5 def __init__(self, timezone=None): BaseMaster.__init__(self) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.jobqueue_lock = RLock() self.timezone = timezone or get_localzone() self._event = Event() self._stopped = True self.jobstore = MongoJobStore(self, **settings.JOB_STORE_CONFIG) #self.jobstore = MemoryJobStore(self) self._internal_buffer_queues = {} self._job_maximum_buffer_time = settings.JOB_QUEUE_CONFIG[ 'buffer_time'] def submit_job(self, job): """ process submit_job request from worker. :type job: Job """ self.log.debug('client submit job, id=%s, key=%s' % (job.id, job.job_key)) # if job doesn't contains trigger, then enqueue it into job queue immediately if not job.trigger: self._enqueue_job(job.job_key, job.serialize()) # else store job into job store first else: with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG): try: self.jobstore.add_job(job) except JobAlreadyExist as e: if job.replace_exist: self.update_job(job) else: self.log.error(e) # wake up when new job has store into job store self.wake_up() def update_job(self, job): """ update job in jobstore :type job: Job """ try: self.jobstore.update_job(job) except JobDoesNotExist as e: self.log.error(e) def remove_job(self, job): """ remove job from jobstore :type job: Job """ try: self.jobstore.remove_job(job) except JobDoesNotExist: self.log.error('remove job error. job id %s does not exist' % job.id) def finish_job(self, job): """ process finish_job request from worker :type job: Job """ self.jobstore.save_execute_record(job) def _enqueue_buffer_queue(self, key, job): self.log.debug("job queue [%s] is full, put job into buffer queue" % key) try: self._internal_buffer_queues[key].put((job, datetime.now())) except KeyError: self._internal_buffer_queues[key] = Queue() self._internal_buffer_queues[key].put((job, datetime.now())) self.start_process_buffer_job(key) def _enqueue_job(self, key, job): """ enqueue job into jobqueue :type key: str :type job: str or xmlrpc.Binary """ self.log.debug('enqueue job key=[%s]' % key) with self.jobqueue_lock: # check whether job queue is full if not self.jobqueue.is_full(key): self.jobqueue.enqueue(key, job) else: self._enqueue_buffer_queue(key, job) def start(self): """ Start elric master. Select all due jobs from jobstore and enqueue them into redis queue. Then update due jobs' information into jobstore. :return: """ if self.running: raise AlreadyRunningException self._stopped = False self.log.debug('eric master start...') self.start_subscribe_thread() while True: now = datetime.now(self.timezone) wait_seconds = None with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG): for job_id, job_key, serialized_job in self.jobstore.get_due_jobs( now): # enqueue due job into redis queue self._enqueue_job(job_key, serialized_job) # update job's information, such as next_run_time job = Job.deserialize(serialized_job) last_run_time = Job.get_serial_run_times(job, now) if last_run_time: next_run_time = Job.get_next_trigger_time( job, last_run_time[-1]) if next_run_time: job.next_run_time = next_run_time self.update_job(job) else: # if job has no next run time, then remove it from jobstore self.remove_job(job) # get next closet run time job from jobstore and set it to be wake up time closest_run_time = self.jobstore.get_closest_run_time() if closest_run_time is not None: wait_seconds = min( max(timedelta_seconds(closest_run_time - now), 0), self.MIN_WAIT_TIME) self.log.debug('Next wakeup is due at %s (in %f seconds)' % (closest_run_time, wait_seconds)) self._event.wait(wait_seconds if wait_seconds is not None else self .MIN_WAIT_TIME) self._event.clear() def wake_up(self): self._event.set() @property def running(self): return not self._stopped def start_process_buffer_job(self, job_key): """ Start filter data serialization thread """ self.log.debug('start process buffer job... job key=[%s]' % job_key) thd = threading.Thread(target=self.process_buffer_job, args=(job_key, )) thd.setDaemon(True) thd.start() def process_buffer_job(self, job_key): while True: job, buffer_time = self._internal_buffer_queues[job_key].get() with self.jobqueue_lock: if not self.jobqueue.is_full(job_key): self.jobqueue.enqueue(job_key, job) continue if (datetime.now() - buffer_time ).total_seconds() < self._job_maximum_buffer_time: self.log.debug("requeue into buffer") self._internal_buffer_queues[job_key].put((job, buffer_time)) time.sleep(1.0) else: self.log.warning("timeout, discard job...") def subscribe_mq(self): while self.running: try: # grab job from job queue only if internal_job_queue has space key, serialized_job = self.jobqueue.dequeue_any([ '__elric_submit_channel__', '__elric_remove_channel__', '__elric_finish_channel__' ]) Job.deserialize(serialized_job) self.func_map[key](Job.deserialize(serialized_job)) except TypeError as e: self.log.error(e) time.sleep(60) continue
class RQWorker(BaseWorker): def __init__(self, name, listen_keys=None, worker_num=2, timezone=None, logger_name='elric.worker'): BaseWorker.__init__(self, name, logger_name) self.jobqueue = RedisJobQueue(self, **settings.JOB_QUEUE_CONFIG) self.listen_keys = [] if listen_keys: self.listen_keys = ['%s:%s' % (self.name, listen_key) for listen_key in listen_keys] self.timezone = timezone or get_localzone() self.internal_job_queue = Queue(maxsize=worker_num) self.executor = ProcessPoolExecutor(worker_num, self) self._stopped = True def start(self): if self.running: raise AlreadyRunningException self._stopped = False self.log.debug('elric worker running..') while self.running: try: # grab job from job queue only if internal_job_queue has space self.internal_job_queue.put("#", True) job_key, serialized_job = self.jobqueue.dequeue_any(self.listen_keys) job = Job.deserialize(serialized_job) self.log.debug('get job id=[%s] func=[%s]from key %s' % (job.id, job.func, job.job_key)) self.executor.execute_job(job) except TypeError as e: self.log.error(e) time.sleep(60) continue def submit_job(self, func, job_key, args=None, kwargs=None, trigger=None, job_id=None, replace_exist=False, need_filter=False, **trigger_args): """ submit job to master through redis queue :type func: str or callable obj or unicode :type job_key: str or unicode :type args: tuple or list :type kwargs: dict :type trigger: str or unicode :type job_id: str or unicode :type replace_exist: bool :type trigger_args: dict """ # use worker's timezone if trigger don't provide specific `timezone` configuration trigger_args['timezone'] = self.timezone job_in_dict = { 'id': "%s:%s" % (self.name, job_id) if job_id else None, 'func': func, 'args': args, 'trigger': create_trigger(trigger, trigger_args) if trigger else None, 'kwargs': kwargs, 'job_key': '%s:%s' % (self.name, job_key), 'need_filter': need_filter, 'replace_exist': replace_exist } job = Job(**job_in_dict) job.check() self.jobqueue.enqueue('__elric_submit_channel__', job.serialize()) def remove_job(self, job_id): """ send remove job request to master through redis queue :type job_id: str """ job_in_dict = { 'id': "%s:%s" % (self.name, job_id) } job = Job(**job_in_dict) self.jobqueue.enqueue('__elric_remove_channel__', job.serialize()) def finish_job(self, job_id, is_success, details, job_key, need_filter): job_in_dict = { 'id': job_id, 'job_key': job_key, 'is_success': is_success, 'details': details, 'need_filter': need_filter } job = Job(**job_in_dict) self.jobqueue.enqueue('__elric_finish_channel__', job.serialize()) @property def running(self): return not self._stopped def stop(self): self.log.debug('Worker is quiting') self._stopped = True self.executor.shutdown()