def subscribe_mq(self):
while self.running:
try:
# grab job from job queue only if internal_job_queue has space
key, serialized_job = self.jobqueue.dequeue_any(['__elric_submit_channel__',
'__elric_remove_channel__',
'__elric_finish_channel__'])
Job.deserialize(serialized_job)
self.func_map[key](Job.deserialize(serialized_job))
except TypeError as e:
self.log.error(e)
time.sleep(60)
continue
def subscribe_mq(self):
while self.running:
try:
# grab job from job queue only if internal_job_queue has space
key, serialized_job = self.jobqueue.dequeue_any([
'__elric_submit_channel__', '__elric_remove_channel__',
'__elric_finish_channel__'
])
Job.deserialize(serialized_job)
self.func_map[key](Job.deserialize(serialized_job))
except TypeError as e:
self.log.error(e)
time.sleep(60)
continue
def start(self):
"""
Start elric master. Select all due jobs from jobstore and enqueue them into redis queue.
Then update due jobs' information into jobstore.
:return:
"""
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('eric master start...')
self.start_subscribe_thread()
while True:
now = datetime.now(self.timezone)
wait_seconds = None
with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG):
for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(
now):
# enqueue due job into redis queue
self._enqueue_job(job_key, serialized_job)
# update job's information, such as next_run_time
job = Job.deserialize(serialized_job)
last_run_time = Job.get_serial_run_times(job, now)
if last_run_time:
next_run_time = Job.get_next_trigger_time(
job, last_run_time[-1])
if next_run_time:
job.next_run_time = next_run_time
self.update_job(job)
else:
# if job has no next run time, then remove it from jobstore
self.remove_job(job)
# get next closet run time job from jobstore and set it to be wake up time
closest_run_time = self.jobstore.get_closest_run_time()
if closest_run_time is not None:
wait_seconds = min(
max(timedelta_seconds(closest_run_time - now), 0),
self.MIN_WAIT_TIME)
self.log.debug('Next wakeup is due at %s (in %f seconds)' %
(closest_run_time, wait_seconds))
self._event.wait(wait_seconds if wait_seconds is not None else self
.MIN_WAIT_TIME)
self._event.clear()
def start(self):
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('elric worker running..')
while self.running:
try:
# grab job from job queue only if internal_job_queue has space
self.internal_job_queue.put("#", True)
job_key, serialized_job = self.jobqueue.dequeue_any(self.listen_keys)
job = Job.deserialize(serialized_job)
self.log.debug('get job id=[%s] func=[%s]from key %s' % (job.id, job.func, job.job_key))
self.executor.execute_job(job)
except TypeError as e:
self.log.error(e)
time.sleep(60)
continue
def start(self):
"""
Start elric master. Select all due jobs from jobstore and enqueue them into redis queue.
Then update due jobs' information into jobstore.
:return:
"""
if self.running:
raise AlreadyRunningException
self._stopped = False
self.log.debug('eric master start...')
self.start_subscribe_thread()
while True:
now = datetime.now(self.timezone)
wait_seconds = None
with distributed_lock(**settings.DISTRIBUTED_LOCK_CONFIG):
for job_id, job_key, serialized_job in self.jobstore.get_due_jobs(now):
# enqueue due job into redis queue
self._enqueue_job(job_key, serialized_job)
# update job's information, such as next_run_time
job = Job.deserialize(serialized_job)
last_run_time = Job.get_serial_run_times(job, now)
if last_run_time:
next_run_time = Job.get_next_trigger_time(job, last_run_time[-1])
if next_run_time:
job.next_run_time = next_run_time
self.update_job(job)
else:
# if job has no next run time, then remove it from jobstore
self.remove_job(job)
# get next closet run time job from jobstore and set it to be wake up time
closest_run_time = self.jobstore.get_closest_run_time()
if closest_run_time is not None:
wait_seconds = min(max(timedelta_seconds(closest_run_time - now), 0), self.MIN_WAIT_TIME)
self.log.debug('Next wakeup is due at %s (in %f seconds)' % (closest_run_time, wait_seconds))
self._event.wait(wait_seconds if wait_seconds is not None else self.MIN_WAIT_TIME)
self._event.clear()
