def __init__(self, name='huey', blocking=True, read_timeout=1, client_name=None,
**connection_params):
if 'conn' not in self:
raise ConfigurationError('No connection in RedisBaseStorage')
self._pop = self.conn.register_script(SCHEDULE_POP_LUA)
self.name = self.clean_name(name)
self.queue_key = 'huey.redis.%s' % self.name
self.schedule_key = 'huey.schedule.%s' % self.name
self.result_key = 'huey.results.%s' % self.name
self.error_key = 'huey.errors.%s' % self.name
if client_name is not None:
self.conn.client_setname(client_name)
self.blocking = blocking
self.read_timeout = read_timeout
def start(self):
"""
Start all consumer processes and register signal handlers.
"""
if self.huey.immediate:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where immediate '
'is enabled. Please check your configuration and ensure that '
'"huey.immediate = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s at %s',
self.workers, self.worker_type, os.getpid(),
self.huey._get_timestamp())
self._logger.info('Scheduler runs every %s second(s).',
self.scheduler_interval)
self._logger.info('Periodic tasks are %s.',
'enabled' if self.periodic else 'disabled')
self._set_signal_handlers()
msg = ['The following commands are available:']
for command in self.huey._registry._registry:
msg.append('+ %s' % command)
self._logger.info('\n'.join(msg))
# We'll temporarily ignore SIGINT and SIGHUP (so that it is inherited
# by the child-processes). Once the child processes are created, we
# restore the handler.
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(signal, 'SIGHUP'):
original_sighup_handler = signal.signal(signal.SIGHUP,
signal.SIG_IGN)
self.scheduler.start()
for _, worker_process in self.worker_threads:
worker_process.start()
signal.signal(signal.SIGINT, original_sigint_handler)
if hasattr(signal, 'SIGHUP'):
signal.signal(signal.SIGHUP, original_sighup_handler)
def start(self):
"""
Start all consumer processes and register signal handlers.
Don't init scheduler.
"""
if self.huey.always_eager:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where always_eager'
' is enabled. Please check your configuration and ensure that'
' "huey.always_eager = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s',
self.workers, self.worker_type, os.getpid())
self._logger.info('Scheduler disabled')
self._logger.info('Health checker is %s',
'enabled' if self._health_check else 'disabled')
self._logger.info('Periodic tasks are %s.',
'enabled' if self.periodic else 'disabled')
self._logger.info('UTC is %s.', 'enabled' if self.utc else 'disabled')
for _, worker_process in self.worker_threads:
worker_process.start()
def __init__(self, huey, workers=1, periodic=True, initial_delay=0.1,
backoff=1.15, max_delay=10.0, scheduler_interval=1,
worker_type=WORKER_THREAD, check_worker_health=True,
health_check_interval=10, flush_locks=False,
extra_locks=None):
self._logger = logging.getLogger('huey.consumer')
if huey.immediate:
self._logger.warning('Consumer initialized with Huey instance '
'that has "immediate" mode enabled. This '
'must be disabled before the consumer can '
'be run.')
self.huey = huey
self.workers = workers # Number of workers.
self.periodic = periodic # Enable periodic task scheduler?
self.default_delay = initial_delay # Default queue polling interval.
self.backoff = backoff # Exponential backoff factor when queue empty.
self.max_delay = max_delay # Maximum interval between polling events.
# Ensure that the scheduler runs at an interval between 1 and 60s.
self.scheduler_interval = max(min(scheduler_interval, 60), 1)
if 60 % self.scheduler_interval != 0:
raise ConfigurationError('Scheduler interval must be a factor '
'of 60, e.g. 1, 2, 3, 4, 5, 6, 10, 12...')
if worker_type == 'gevent': worker_type = WORKER_GREENLET
self.worker_type = worker_type # What process model are we using?
# Configure health-check and consumer main-loop attributes.
self._stop_flag_timeout = 0.1
self._health_check = check_worker_health
self._health_check_interval = float(health_check_interval)
# Create the execution environment helper.
self.environment = self.get_environment(self.worker_type)
# Create the event used to signal the process should terminate. We'll
# also store a boolean flag to indicate whether we should restart after
# the processes are cleaned up.
self._received_signal = False
self._restart = False
self._graceful = True
self.stop_flag = self.environment.get_stop_flag()
# In the event the consumer was killed while running a task that held
# a lock, this ensures that all locks are flushed before starting.
if flush_locks or extra_locks:
lock_names = extra_locks.split(',') if extra_locks else ()
self.flush_locks(*lock_names)
# Create the scheduler process (but don't start it yet).
scheduler = self._create_scheduler()
self.scheduler = self._create_process(scheduler, 'Scheduler')
# Create the worker process(es) (also not started yet).
self.worker_threads = []
for i in range(workers):
worker = self._create_worker()
process = self._create_process(worker, 'Worker-%d' % (i + 1))
# The worker threads are stored as [(worker impl, worker_t), ...].
# The worker impl is not currently referenced in any consumer code,
# but it is referenced in the test-suite.
self.worker_threads.append((worker, process))
def __init__(self, *args, **kwargs):
raise ConfigurationError('Cannot initialize "%s", %s module not '
'installed.' % (name, library))
def __init__(self, compression=False, compression_level=6):
self.comp = compression
self.comp_level = compression_level
if self.comp and gzip is None:
raise ConfigurationError('Cannot enable compression, gzip module '
'not found.')