def __init__(self, name='huey', blocking=True, read_timeout=1, client_name=None, **connection_params): if 'conn' not in self: raise ConfigurationError('No connection in RedisBaseStorage') self._pop = self.conn.register_script(SCHEDULE_POP_LUA) self.name = self.clean_name(name) self.queue_key = 'huey.redis.%s' % self.name self.schedule_key = 'huey.schedule.%s' % self.name self.result_key = 'huey.results.%s' % self.name self.error_key = 'huey.errors.%s' % self.name if client_name is not None: self.conn.client_setname(client_name) self.blocking = blocking self.read_timeout = read_timeout
def start(self): """ Start all consumer processes and register signal handlers. """ if self.huey.immediate: raise ConfigurationError( 'Consumer cannot be run with Huey instances where immediate ' 'is enabled. Please check your configuration and ensure that ' '"huey.immediate = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s at %s', self.workers, self.worker_type, os.getpid(), self.huey._get_timestamp()) self._logger.info('Scheduler runs every %s second(s).', self.scheduler_interval) self._logger.info('Periodic tasks are %s.', 'enabled' if self.periodic else 'disabled') self._set_signal_handlers() msg = ['The following commands are available:'] for command in self.huey._registry._registry: msg.append('+ %s' % command) self._logger.info('\n'.join(msg)) # We'll temporarily ignore SIGINT and SIGHUP (so that it is inherited # by the child-processes). Once the child processes are created, we # restore the handler. original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) if hasattr(signal, 'SIGHUP'): original_sighup_handler = signal.signal(signal.SIGHUP, signal.SIG_IGN) self.scheduler.start() for _, worker_process in self.worker_threads: worker_process.start() signal.signal(signal.SIGINT, original_sigint_handler) if hasattr(signal, 'SIGHUP'): signal.signal(signal.SIGHUP, original_sighup_handler)
def start(self): """ Start all consumer processes and register signal handlers. Don't init scheduler. """ if self.huey.always_eager: raise ConfigurationError( 'Consumer cannot be run with Huey instances where always_eager' ' is enabled. Please check your configuration and ensure that' ' "huey.always_eager = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s', self.workers, self.worker_type, os.getpid()) self._logger.info('Scheduler disabled') self._logger.info('Health checker is %s', 'enabled' if self._health_check else 'disabled') self._logger.info('Periodic tasks are %s.', 'enabled' if self.periodic else 'disabled') self._logger.info('UTC is %s.', 'enabled' if self.utc else 'disabled') for _, worker_process in self.worker_threads: worker_process.start()
def __init__(self, huey, workers=1, periodic=True, initial_delay=0.1, backoff=1.15, max_delay=10.0, scheduler_interval=1, worker_type=WORKER_THREAD, check_worker_health=True, health_check_interval=10, flush_locks=False, extra_locks=None): self._logger = logging.getLogger('huey.consumer') if huey.immediate: self._logger.warning('Consumer initialized with Huey instance ' 'that has "immediate" mode enabled. This ' 'must be disabled before the consumer can ' 'be run.') self.huey = huey self.workers = workers # Number of workers. self.periodic = periodic # Enable periodic task scheduler? self.default_delay = initial_delay # Default queue polling interval. self.backoff = backoff # Exponential backoff factor when queue empty. self.max_delay = max_delay # Maximum interval between polling events. # Ensure that the scheduler runs at an interval between 1 and 60s. self.scheduler_interval = max(min(scheduler_interval, 60), 1) if 60 % self.scheduler_interval != 0: raise ConfigurationError('Scheduler interval must be a factor ' 'of 60, e.g. 1, 2, 3, 4, 5, 6, 10, 12...') if worker_type == 'gevent': worker_type = WORKER_GREENLET self.worker_type = worker_type # What process model are we using? # Configure health-check and consumer main-loop attributes. self._stop_flag_timeout = 0.1 self._health_check = check_worker_health self._health_check_interval = float(health_check_interval) # Create the execution environment helper. self.environment = self.get_environment(self.worker_type) # Create the event used to signal the process should terminate. We'll # also store a boolean flag to indicate whether we should restart after # the processes are cleaned up. self._received_signal = False self._restart = False self._graceful = True self.stop_flag = self.environment.get_stop_flag() # In the event the consumer was killed while running a task that held # a lock, this ensures that all locks are flushed before starting. if flush_locks or extra_locks: lock_names = extra_locks.split(',') if extra_locks else () self.flush_locks(*lock_names) # Create the scheduler process (but don't start it yet). scheduler = self._create_scheduler() self.scheduler = self._create_process(scheduler, 'Scheduler') # Create the worker process(es) (also not started yet). self.worker_threads = [] for i in range(workers): worker = self._create_worker() process = self._create_process(worker, 'Worker-%d' % (i + 1)) # The worker threads are stored as [(worker impl, worker_t), ...]. # The worker impl is not currently referenced in any consumer code, # but it is referenced in the test-suite. self.worker_threads.append((worker, process))
def __init__(self, *args, **kwargs): raise ConfigurationError('Cannot initialize "%s", %s module not ' 'installed.' % (name, library))
def __init__(self, compression=False, compression_level=6): self.comp = compression self.comp_level = compression_level if self.comp and gzip is None: raise ConfigurationError('Cannot enable compression, gzip module ' 'not found.')