def get_raw_result(self,
blocking=False,
timeout=None,
backoff=1.15,
max_delay=1.0,
revoke_on_timeout=False,
preserve=False):
if not blocking:
res = self._get(preserve)
if res is not EmptyData:
return res
else:
start = time_clock()
delay = .1
while self._result is EmptyData:
if timeout and time_clock() - start >= timeout:
if revoke_on_timeout:
self.revoke()
raise HueyException('timed out waiting for result')
if delay > max_delay:
delay = max_delay
if self._get(preserve) is EmptyData:
time.sleep(delay)
delay *= backoff
return self._result
def __init__(self, huey, interval, periodic):
super(Scheduler, self).__init__(huey)
self.interval = max(min(interval, 60), 1)
self.periodic = periodic
self._next_loop = time_clock()
self._next_periodic = time_clock()
def _run(self):
logger.info('task runner started.')
while not self._shutdown.is_set():
start = time_clock()
now = datetime.datetime.now()
if self._last_check + self._periodic_interval <= now:
logger.debug('checking periodic task schedule')
self._last_check = now
for validate_func, fn in self._periodic_tasks:
if validate_func(now):
self._enqueue(fn)
if self._scheduled_tasks:
logger.debug('checking scheduled tasks')
# The 0-th item of a heap is always the smallest.
while self._scheduled_tasks and \
self._scheduled_tasks[0][0] <= now:
eta, fn, args, kwargs, async_result = (heapq.heappop(
self._scheduled_tasks))
self._enqueue(fn, args, kwargs, async_result)
# Wait for most of the remained of the time remaining.
remaining = self._interval - (time_clock() - start)
if remaining > 0:
if not self._shutdown.wait(remaining * 0.9):
gevent.sleep(self._interval - (time_clock() - start))
logger.info('exiting task runner')
def _execute(self, fn, args, kwargs, async_result):
args = args or ()
kwargs = kwargs or {}
start = time_clock()
try:
ret = fn(*args, **kwargs)
except Exception as exc:
logger.exception('task %s failed' % fn.__name__)
async_result.set_exception(exc)
raise
else:
duration = time_clock() - start
if async_result is not None:
async_result.set(ret)
logger.info('executed %s in %0.3fs', fn.__name__, duration)
def sleep_for_interval(self, start_ts, nseconds):
"""
Sleep for a given interval with respect to the start timestamp.
So, if the start timestamp is 1337 and nseconds is 10, the method will
actually sleep for nseconds - (current_timestamp - start_timestamp). So
if the current timestamp is 1340, we'll only sleep for 7 seconds (the
goal being to sleep until 1347, or 1337 + 10).
"""
sleep_time = nseconds - (time_clock() - start_ts)
if sleep_time <= 0:
return
self._logger.debug('Sleeping for %s', sleep_time)
# Recompute time to sleep to improve accuracy in case the process was
# pre-empted by the kernel while logging.
sleep_time = nseconds - (time_clock() - start_ts)
if sleep_time > 0:
time.sleep(sleep_time)
def loop(self, now=None):
current = self._next_loop
self._next_loop += self.interval
if self._next_loop < time_clock():
self._logger.debug('scheduler skipping iteration to avoid race.')
return
try:
task_list = self.huey.read_schedule(now)
except Exception:
self._logger.exception('Error reading schedule.')
else:
for task in task_list:
self._logger.debug('Enqueueing %s', task)
self.huey.enqueue(task)
if self.periodic and self._next_periodic <= time_clock():
self._next_periodic += self.periodic_task_seconds
self.enqueue_periodic_tasks(now)
self.sleep_for_interval(current, self.interval)
def run(self):
"""
Run the consumer.
"""
self.start()
timeout = self._stop_flag_timeout
health_check_ts = time_clock()
while True:
try:
self.stop_flag.wait(timeout=timeout)
except KeyboardInterrupt:
self._logger.info('Received SIGINT')
self.stop(graceful=True)
except:
self._logger.exception('Error in consumer.')
self.stop()
else:
if self._received_signal:
self.stop(graceful=self._graceful)
if self.stop_flag.is_set():
break
if self._health_check:
now = time_clock()
if now >= health_check_ts + self._health_check_interval:
health_check_ts = now
self.check_worker_health()
self.huey.notify_interrupted_tasks()
if self._restart:
self._logger.info('Consumer will restart.')
python = sys.executable
os.execl(python, python, *sys.argv)
else:
self._logger.info('Consumer exiting.')
def _execute(self, task, timestamp):
if self._pre_execute:
try:
self._run_pre_execute(task)
except CancelExecution:
self._emit(S.SIGNAL_CANCELED, task)
return
start = time_clock()
exception = None
task_value = None
try:
self._tasks_in_flight.add(task)
try:
task_value = task.execute()
finally:
self._tasks_in_flight.remove(task)
duration = time_clock() - start
except TaskLockedException as exc:
logger.warning('Task %s not run, unable to acquire lock.', task.id)
exception = exc
self._emit(S.SIGNAL_LOCKED, task)
except RetryTask as exc:
logger.info('Task %s raised RetryTask, retrying.', task.id)
task.retries += 1
exception = exc
except CancelExecution as exc:
if exc.retry or (exc.retry is None and task.retries):
task.retries = max(task.retries, 1)
msg = '(task will be retried)'
else:
task.retries = 0
msg = '(aborted, will not be retried)'
logger.warning('Task %s raised CancelExecution %s.', task.id, msg)
self._emit(S.SIGNAL_CANCELED, task)
exception = exc
except KeyboardInterrupt:
logger.warning('Received exit signal, %s did not finish.', task.id)
self._emit(S.SIGNAL_INTERRUPTED, task)
return
except Exception as exc:
logger.exception('Unhandled exception in task %s.', task.id)
exception = exc
self._emit(S.SIGNAL_ERROR, task, exc)
else:
logger.info('%s executed in %0.3fs', task, duration)
if self.results and not isinstance(task, PeriodicTask):
if exception is not None:
error_data = self.build_error_result(task, exception)
self.put_result(task.id, Error(error_data))
elif task_value is not None or self.store_none:
self.put_result(task.id, task_value)
if self._post_execute:
self._run_post_execute(task, task_value, exception)
if exception is None:
# Task executed successfully, send the COMPLETE signal.
self._emit(S.SIGNAL_COMPLETE, task)
if task.on_complete and exception is None:
next_task = task.on_complete
next_task.extend_data(task_value)
self.enqueue(next_task)
elif task.on_error and exception is not None:
next_task = task.on_error
next_task.extend_data(exception)
self.enqueue(next_task)
if exception is not None and task.retries:
self._emit(S.SIGNAL_RETRYING, task)
self._requeue_task(task, self._get_timestamp())
return task_value
def _execute(self, task, timestamp):
if self._pre_execute:
try:
self._run_pre_execute(task)
except CancelExecution:
self._emit(S.SIGNAL_CANCELED, task)
return
start = time_clock()
exception = None
task_value = None
try:
try:
task_value = task.execute()
finally:
duration = time_clock() - start
except TaskLockedException as exc:
logger.warning('Task %s not run, unable to acquire lock.', task.id)
exception = exc
self._emit(S.SIGNAL_LOCKED, task)
except RetryTask as exc:
logger.info('Task %s raised RetryTask, retrying.', task.id)
task.retries += 1
exception = exc
except KeyboardInterrupt:
logger.warning('Received exit signal, %s did not finish.', task.id)
return
except Exception as exc:
logger.exception('Unhandled exception in task %s.', task.id)
exception = exc
self._emit(S.SIGNAL_ERROR, task, exc)
else:
logger.info('%s executed in %0.3fs', task, duration)
if self.results and not isinstance(task, PeriodicTask):
if exception is not None:
try:
tb = traceback.format_exc()
except AttributeError: # Seems to only happen on 3.4.
tb = '- unable to resolve traceback on Python 3.4 -'
self.put_result(
task.id,
Error({
'error': repr(exception),
'retries': task.retries,
'traceback': tb,
'task_id': task.id,
}))
elif task_value is not None or self.store_none:
self.put_result(task.id, task_value)
if self._post_execute:
self._run_post_execute(task, task_value, exception)
if exception is None:
# Task executed successfully, send the COMPLETE signal.
self._emit(S.SIGNAL_COMPLETE, task)
if task.on_complete and exception is None:
next_task = task.on_complete
next_task.extend_data(task_value)
self.enqueue(next_task)
elif task.on_error and exception is not None:
next_task = task.on_error
next_task.extend_data(exception)
self.enqueue(next_task)
if exception is not None and task.retries:
self._emit(S.SIGNAL_RETRYING, task)
self._requeue_task(task, self._get_timestamp())
return task_value
def schedule_tasks(self, consumer, now=None):
scheduler = consumer._create_scheduler()
scheduler._next_loop = time_clock() + 60
scheduler._next_periodic = time_clock() - 60
scheduler.loop(now)
