def __init__(self, timezone=None):
BaseMaster.__init__(self)
self.queue_list = {}
self.server = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)
self.timezone = timezone or get_localzone()
self._event = Event()
self._stopped = True
self.jobstore = MemoryJobStore(self.log)
self.jobstore_lock = RLock()
class RQMaster(BaseMaster):
MAX_WAIT_TIME = 4294967 # Maximum value accepted by Event.wait() on Windows
def __init__(self, timezone=None):
BaseMaster.__init__(self)
self.queue_list = {}
self.queue_lock = RLock()
self.server = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)
self.timezone = timezone or get_localzone()
self._event = Event()
self._stopped = True
self.jobstore = MemoryJobStore(self.log)
self.jobstore_lock = RLock()
def submit_job(self, serialized_job, job_key, job_id, replace_exist):
"""
Receive submit_job rpc request from worker.
:type serialized_job str or xmlrpclib.Binary
:type job_key str
:type job_id str
:type replace_exist bool
"""
self.log.debug('client call submit job, id=%s, key=%s' % (job_id, job_key))
if isinstance(serialized_job, Binary):
serialized_job = serialized_job.data
job_in_dict = Job.deserialize_to_dict(serialized_job)
# if job doesn't contains trigger, then enqueue it into job queue immediately
if not job_in_dict['trigger']:
self._enqueue_job(job_key, serialized_job)
# else store job into job store first
else:
# should I need a lock here?
with self.jobstore_lock:
try:
self.jobstore.add_job(job_id, job_key, job_in_dict['next_run_time'], serialized_job)
except JobAlreadyExist:
if replace_exist:
self.jobstore.update_job(job_id, job_key, job_in_dict['next_run_time'], serialized_job)
else:
self.log.warning('submit job error. job id%s already exist' % job_id)
# wake up when new job has store into job store
self.wake_up()
def update_job(self, job_id, job_key, next_run_time, serialized_job):
"""
Receive update_job rpc request from worker
:type job_id: str
:type job_key: str
:type next_run_time datetime.datetime
:type serialized_job str or xmlrpclib.Binary
"""
if isinstance(serialized_job, Binary):
serialized_job = serialized_job.data
with self.jobstore_lock:
try:
self.jobstore.update_job(job_id, job_key=job_key, next_run_time=next_run_time,
serialized_job=serialized_job)
except JobDoesNotExist:
self.log.error('update job error. job id %s does not exist' % job_id)
def remove_job(self, job_id):
"""
Receive remove_job rpc request from worker
:type job_id: str
"""
with self.jobstore_lock:
try:
self.jobstore.remove_job(job_id)
except JobDoesNotExist:
self.log.error('remove job error. job id %s does not exist' % job_id)
def _enqueue_job(self, key, job):
"""
enqueue job into redis queue
:type key: str
:type job: str or xmlrpc.Binary
"""
with self.queue_lock:
try:
self.queue_list[key].enqueue(job)
except KeyError:
self.queue_list[key] = RedisJobQueue(self.server, key)
self.queue_list[key].enqueue(job)
def start(self):
"""
Start elric master. Select all due jobs from jobstore and enqueue them into redis queue.
Then update due jobs' information into jobstore.
:return:
"""
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('eric master start...')
while True:
now = datetime.now(self.timezone)
wait_seconds = None
with self.jobstore_lock:
for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(now):
# enqueue due job into redis queue
self._enqueue_job(job_key, serialized_job)
# update job's information, such as next_run_time
job_in_dict = Job.deserialize_to_dict(serialized_job)
last_run_time = Job.get_serial_run_times(job_in_dict, now)
if last_run_time:
next_run_time = Job.get_next_trigger_time(job_in_dict, last_run_time[-1])
if next_run_time:
job_in_dict['next_run_time'] = next_run_time
self.update_job(job_id, job_key, next_run_time, Job.dict_to_serialization(job_in_dict))
else:
# if job has no next run time, then remove it from jobstore
self.remove_job(job_id=job_id)
# get next closet run time job from jobstore and set it to be wake up time
closest_run_time = self.jobstore.get_closest_run_time()
if closest_run_time is not None:
wait_seconds = max(timedelta_seconds(closest_run_time - now), 0)
self.log.debug('Next wakeup is due at %s (in %f seconds)' % (closest_run_time, wait_seconds))
self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME)
self._event.clear()
def wake_up(self):
self._event.set()
@property
def running(self):
return not self._stopped
