def get_raw_result(self, blocking=False, timeout=None, backoff=1.15, max_delay=1.0, revoke_on_timeout=False, preserve=False): if not blocking: res = self._get(preserve) if res is not EmptyData: return res else: start = time_clock() delay = .1 while self._result is EmptyData: if timeout and time_clock() - start >= timeout: if revoke_on_timeout: self.revoke() raise HueyException('timed out waiting for result') if delay > max_delay: delay = max_delay if self._get(preserve) is EmptyData: time.sleep(delay) delay *= backoff return self._result
def __init__(self, huey, interval, periodic): super(Scheduler, self).__init__(huey) self.interval = max(min(interval, 60), 1) self.periodic = periodic self._next_loop = time_clock() self._next_periodic = time_clock()
def _run(self): logger.info('task runner started.') while not self._shutdown.is_set(): start = time_clock() now = datetime.datetime.now() if self._last_check + self._periodic_interval <= now: logger.debug('checking periodic task schedule') self._last_check = now for validate_func, fn in self._periodic_tasks: if validate_func(now): self._enqueue(fn) if self._scheduled_tasks: logger.debug('checking scheduled tasks') # The 0-th item of a heap is always the smallest. while self._scheduled_tasks and \ self._scheduled_tasks[0][0] <= now: eta, fn, args, kwargs, async_result = (heapq.heappop( self._scheduled_tasks)) self._enqueue(fn, args, kwargs, async_result) # Wait for most of the remained of the time remaining. remaining = self._interval - (time_clock() - start) if remaining > 0: if not self._shutdown.wait(remaining * 0.9): gevent.sleep(self._interval - (time_clock() - start)) logger.info('exiting task runner')
def _execute(self, fn, args, kwargs, async_result): args = args or () kwargs = kwargs or {} start = time_clock() try: ret = fn(*args, **kwargs) except Exception as exc: logger.exception('task %s failed' % fn.__name__) async_result.set_exception(exc) raise else: duration = time_clock() - start if async_result is not None: async_result.set(ret) logger.info('executed %s in %0.3fs', fn.__name__, duration)
def sleep_for_interval(self, start_ts, nseconds): """ Sleep for a given interval with respect to the start timestamp. So, if the start timestamp is 1337 and nseconds is 10, the method will actually sleep for nseconds - (current_timestamp - start_timestamp). So if the current timestamp is 1340, we'll only sleep for 7 seconds (the goal being to sleep until 1347, or 1337 + 10). """ sleep_time = nseconds - (time_clock() - start_ts) if sleep_time <= 0: return self._logger.debug('Sleeping for %s', sleep_time) # Recompute time to sleep to improve accuracy in case the process was # pre-empted by the kernel while logging. sleep_time = nseconds - (time_clock() - start_ts) if sleep_time > 0: time.sleep(sleep_time)
def loop(self, now=None): current = self._next_loop self._next_loop += self.interval if self._next_loop < time_clock(): self._logger.debug('scheduler skipping iteration to avoid race.') return try: task_list = self.huey.read_schedule(now) except Exception: self._logger.exception('Error reading schedule.') else: for task in task_list: self._logger.debug('Enqueueing %s', task) self.huey.enqueue(task) if self.periodic and self._next_periodic <= time_clock(): self._next_periodic += self.periodic_task_seconds self.enqueue_periodic_tasks(now) self.sleep_for_interval(current, self.interval)
def run(self): """ Run the consumer. """ self.start() timeout = self._stop_flag_timeout health_check_ts = time_clock() while True: try: self.stop_flag.wait(timeout=timeout) except KeyboardInterrupt: self._logger.info('Received SIGINT') self.stop(graceful=True) except: self._logger.exception('Error in consumer.') self.stop() else: if self._received_signal: self.stop(graceful=self._graceful) if self.stop_flag.is_set(): break if self._health_check: now = time_clock() if now >= health_check_ts + self._health_check_interval: health_check_ts = now self.check_worker_health() self.huey.notify_interrupted_tasks() if self._restart: self._logger.info('Consumer will restart.') python = sys.executable os.execl(python, python, *sys.argv) else: self._logger.info('Consumer exiting.')
def _execute(self, task, timestamp): if self._pre_execute: try: self._run_pre_execute(task) except CancelExecution: self._emit(S.SIGNAL_CANCELED, task) return start = time_clock() exception = None task_value = None try: self._tasks_in_flight.add(task) try: task_value = task.execute() finally: self._tasks_in_flight.remove(task) duration = time_clock() - start except TaskLockedException as exc: logger.warning('Task %s not run, unable to acquire lock.', task.id) exception = exc self._emit(S.SIGNAL_LOCKED, task) except RetryTask as exc: logger.info('Task %s raised RetryTask, retrying.', task.id) task.retries += 1 exception = exc except CancelExecution as exc: if exc.retry or (exc.retry is None and task.retries): task.retries = max(task.retries, 1) msg = '(task will be retried)' else: task.retries = 0 msg = '(aborted, will not be retried)' logger.warning('Task %s raised CancelExecution %s.', task.id, msg) self._emit(S.SIGNAL_CANCELED, task) exception = exc except KeyboardInterrupt: logger.warning('Received exit signal, %s did not finish.', task.id) self._emit(S.SIGNAL_INTERRUPTED, task) return except Exception as exc: logger.exception('Unhandled exception in task %s.', task.id) exception = exc self._emit(S.SIGNAL_ERROR, task, exc) else: logger.info('%s executed in %0.3fs', task, duration) if self.results and not isinstance(task, PeriodicTask): if exception is not None: error_data = self.build_error_result(task, exception) self.put_result(task.id, Error(error_data)) elif task_value is not None or self.store_none: self.put_result(task.id, task_value) if self._post_execute: self._run_post_execute(task, task_value, exception) if exception is None: # Task executed successfully, send the COMPLETE signal. self._emit(S.SIGNAL_COMPLETE, task) if task.on_complete and exception is None: next_task = task.on_complete next_task.extend_data(task_value) self.enqueue(next_task) elif task.on_error and exception is not None: next_task = task.on_error next_task.extend_data(exception) self.enqueue(next_task) if exception is not None and task.retries: self._emit(S.SIGNAL_RETRYING, task) self._requeue_task(task, self._get_timestamp()) return task_value
def _execute(self, task, timestamp): if self._pre_execute: try: self._run_pre_execute(task) except CancelExecution: self._emit(S.SIGNAL_CANCELED, task) return start = time_clock() exception = None task_value = None try: try: task_value = task.execute() finally: duration = time_clock() - start except TaskLockedException as exc: logger.warning('Task %s not run, unable to acquire lock.', task.id) exception = exc self._emit(S.SIGNAL_LOCKED, task) except RetryTask as exc: logger.info('Task %s raised RetryTask, retrying.', task.id) task.retries += 1 exception = exc except KeyboardInterrupt: logger.warning('Received exit signal, %s did not finish.', task.id) return except Exception as exc: logger.exception('Unhandled exception in task %s.', task.id) exception = exc self._emit(S.SIGNAL_ERROR, task, exc) else: logger.info('%s executed in %0.3fs', task, duration) if self.results and not isinstance(task, PeriodicTask): if exception is not None: try: tb = traceback.format_exc() except AttributeError: # Seems to only happen on 3.4. tb = '- unable to resolve traceback on Python 3.4 -' self.put_result( task.id, Error({ 'error': repr(exception), 'retries': task.retries, 'traceback': tb, 'task_id': task.id, })) elif task_value is not None or self.store_none: self.put_result(task.id, task_value) if self._post_execute: self._run_post_execute(task, task_value, exception) if exception is None: # Task executed successfully, send the COMPLETE signal. self._emit(S.SIGNAL_COMPLETE, task) if task.on_complete and exception is None: next_task = task.on_complete next_task.extend_data(task_value) self.enqueue(next_task) elif task.on_error and exception is not None: next_task = task.on_error next_task.extend_data(exception) self.enqueue(next_task) if exception is not None and task.retries: self._emit(S.SIGNAL_RETRYING, task) self._requeue_task(task, self._get_timestamp()) return task_value
def schedule_tasks(self, consumer, now=None): scheduler = consumer._create_scheduler() scheduler._next_loop = time_clock() + 60 scheduler._next_periodic = time_clock() - 60 scheduler.loop(now)