def __init__(self, name='huey', blocking=True, read_timeout=1,
connection_pool=None, url=None, client_name=None,
**connection_params):
if Redis is None:
raise ConfigurationError('"redis" python module not found, cannot '
'use Redis storage backend. Run "pip '
'install redis" to install.')
# Drop common empty values from the connection_params.
for p in ('host', 'port', 'db'):
if p in connection_params and connection_params[p] is None:
del connection_params[p]
if sum(1 for p in (url, connection_pool, connection_params) if p) > 1:
raise ConfigurationError(
'The connection configuration is over-determined. '
'Please specify only one of the following: '
'"url", "connection_pool", or "connection_params"')
if url:
connection_pool = ConnectionPool.from_url(
url, decode_components=True)
elif connection_pool is None:
connection_pool = ConnectionPool(**connection_params)
self.pool = connection_pool
self.conn = self.redis_client(connection_pool=connection_pool)
self.connection_params = connection_params
super(RedisStorage, self).__init__(name, blocking, read_timeout, **connection_params)
def __init__(self, compression=False, compression_level=6, use_zlib=False):
self.comp = compression
self.comp_level = compression_level
self.use_zlib = use_zlib
if self.comp:
if self.use_zlib and zlib is None:
raise ConfigurationError('use_zlib specified, but zlib module '
'not found.')
elif gzip is None:
raise ConfigurationError('gzip module required to enable '
'compression.')
def start(self):
if self.huey.always_eager:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where always_eager'
' is enabled. Please check your configuration and ensure that'
' "huey.always_eager = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s' %
(self.workers, self.worker_type, os.getpid()))
self._logger.info('Scheduler runs every %s seconds.' %
(self.scheduler_interval))
self._logger.info('Periodic tasks are %s.' %
('enabled' if self.periodic else 'disabled'))
self._logger.info('UTC is %s.' %
('enabled' if self.utc else 'disabled'))
self._set_signal_handler()
msg = ['The following commands are available:']
for command in registry._registry:
msg.append('+ %s' % command.replace('queuecmd_', ''))
self._logger.info('\n'.join(msg))
# We'll temporarily ignore SIGINT (so that it is inherited by the
# child-processes). Once the child processes are created, we restore
# the handler.
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
self.scheduler.start()
for _, worker_process in self.worker_threads:
worker_process.start()
signal.signal(signal.SIGINT, original_sigint_handler)
def start(self):
"""
Start all consumer processes and register signal handlers.
"""
if self.huey.immediate:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where immediate '
'is enabled. Please check your configuration and ensure that '
'"huey.immediate = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s at %s',
self.workers, self.worker_type, os.getpid(),
self.huey._get_timestamp())
self._logger.info('Scheduler runs every %s second(s).',
self.scheduler_interval)
self._logger.info('Periodic tasks are %s.',
'enabled' if self.periodic else 'disabled')
msg = ['The following commands are available:']
for command in self.huey._registry._registry:
msg.append('+ %s' % command)
self._logger.info('\n'.join(msg))
# Start the scheduler and workers.
self.scheduler.start()
for _, worker_process in self.worker_threads:
worker_process.start()
# Finally set the signal handlers for main process.
self._set_signal_handlers()
def __init__(self,
compression=False,
compression_level=6,
use_zlib=False,
pickle_protocol=pickle.HIGHEST_PROTOCOL):
self.comp = compression
self.comp_level = compression_level
self.use_zlib = use_zlib
self.pickle_protocol = pickle_protocol or pickle.HIGHEST_PROTOCOL
if self.comp:
if self.use_zlib and zlib is None:
raise ConfigurationError('use_zlib specified, but zlib module '
'not found.')
elif gzip is None:
raise ConfigurationError('gzip module required to enable '
'compression.')
def start(self):
if self.huey.always_eager:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where always_eager'
' is enabled. Please check your configuration and ensure that'
' "huey.always_eager = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s' %
(self.workers, self.worker_type, os.getpid()))
self._logger.info('Scheduler runs every %s seconds.' %
(self.scheduler_interval))
self._logger.info('Periodic tasks are %s.' %
('enabled' if self.periodic else 'disabled'))
self._set_signal_handler()
msg = ['The following commands are available:']
for command in registry._registry:
msg.append('+ %s' % command.replace('queuecmd_', ''))
self._logger.info('\n'.join(msg))
self.scheduler.start()
for _, worker_process in self.worker_threads:
worker_process.start()
def __init__(self,
name='huey',
blocking=True,
read_timeout=1,
connection_pool=None,
url=None,
client_name=None,
**connection_params):
if Redis is None:
raise ConfigurationError('"redis" python module not found, cannot '
'use Redis storage backend. Run "pip '
'install redis" to install.')
# Drop common empty values from the connection_params.
for p in ('host', 'port', 'db'):
if p in connection_params and connection_params[p] is None:
del connection_params[p]
if sum(1 for p in (url, connection_pool, connection_params) if p) > 1:
raise ConfigurationError(
'The connection configuration is over-determined. '
'Please specify only one of the the following: '
'"url", "connection_pool", or "connection_params"')
if url:
connection_pool = ConnectionPool.from_url(url,
decode_components=True)
elif connection_pool is None:
connection_pool = ConnectionPool(**connection_params)
self.pool = connection_pool
self.conn = self.redis_client(connection_pool=connection_pool)
self.connection_params = connection_params
self._pop = self.conn.register_script(SCHEDULE_POP_LUA)
self.name = self.clean_name(name)
self.queue_key = 'huey.redis.%s' % self.name
self.schedule_key = 'huey.schedule.%s' % self.name
self.result_key = 'huey.results.%s' % self.name
self.error_key = 'huey.errors.%s' % self.name
if client_name is not None:
self.conn.client_setname(client_name)
self.blocking = blocking
self.read_timeout = read_timeout
def __init__(self, secret=None, salt='huey', **kwargs):
super(SignedSerializer, self).__init__(**kwargs)
if not secret or not salt:
raise ConfigurationError('The secret and salt parameters are '
'required by %r' % type(self))
self.secret = encode(secret)
self.salt = encode(salt)
self.separator = b':'
self._key = hashlib.sha1(self.salt + self.secret).digest()
def __init__(self, name='huey', blocking=True, read_timeout=1,
startup_nodes=None, **connection_params):
if RedisCluster is None:
raise ConfigurationError('"redis-py-cluster" python module not found, cannot '
'use RedisCluster storage backend. Run "pip '
'install redis-py-cluster" to install.')
if startup_nodes is None:
raise ConfigurationError('startup_nodes parameter required')
decode_responses = False
if sys.version_info >= (3, 0, 0):
decode_responses = True
self.conn = self.redis_client(startup_nodes=startup_nodes, decode_responses=decode_responses)
super(RedisClusterStorage, self).__init__(name, blocking, read_timeout, **connection_params)
def __init__(self, name='huey', database=None, **kwargs):
super(SqlStorage, self).__init__(name)
if database is None:
raise ConfigurationError('Use of SqlStorage requires a '
'database= argument, which should be a '
'peewee database or a connection string.')
if isinstance(database, Database):
self.database = database
else:
# Treat database argument as a URL connection string.
self.database = db_url_connect(database)
self.KV, self.Schedule, self.Task = self.create_models()
self.create_tables()
def __init__(self,
name: str = "huey",
engine: Union[Engine, str, None] = None,
**kwargs: Dict) -> None:
super().__init__(name)
if engine is None:
raise ConfigurationError(
"Use of SQLAlchemyStorage requires an "
"engine= argument, which should be a "
"SQLAlchemy engine or a connection string.")
if isinstance(engine, Engine):
self.engine = engine
else:
# Treat database argument as a URL connection string.
self.engine = create_engine(engine)
self.create_tables()
def __init__(self, name='huey', blocking=True, read_timeout=1, client_name=None,
**connection_params):
if 'conn' not in self:
raise ConfigurationError('No connection in RedisBaseStorage')
self._pop = self.conn.register_script(SCHEDULE_POP_LUA)
self.name = self.clean_name(name)
self.queue_key = 'huey.redis.%s' % self.name
self.schedule_key = 'huey.schedule.%s' % self.name
self.result_key = 'huey.results.%s' % self.name
self.error_key = 'huey.errors.%s' % self.name
if client_name is not None:
self.conn.client_setname(client_name)
self.blocking = blocking
self.read_timeout = read_timeout
def start(self):
"""
Start all consumer processes and register signal handlers.
"""
if self.huey.immediate:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where immediate '
'is enabled. Please check your configuration and ensure that '
'"huey.immediate = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s at %s',
self.workers, self.worker_type, os.getpid(),
self.huey._get_timestamp())
self._logger.info('Scheduler runs every %s second(s).',
self.scheduler_interval)
self._logger.info('Periodic tasks are %s.',
'enabled' if self.periodic else 'disabled')
self._set_signal_handlers()
msg = ['The following commands are available:']
for command in self.huey._registry._registry:
msg.append('+ %s' % command)
self._logger.info('\n'.join(msg))
# We'll temporarily ignore SIGINT and SIGHUP (so that it is inherited
# by the child-processes). Once the child processes are created, we
# restore the handler.
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(signal, 'SIGHUP'):
original_sighup_handler = signal.signal(signal.SIGHUP,
signal.SIG_IGN)
self.scheduler.start()
for _, worker_process in self.worker_threads:
worker_process.start()
signal.signal(signal.SIGINT, original_sigint_handler)
if hasattr(signal, 'SIGHUP'):
signal.signal(signal.SIGHUP, original_sighup_handler)
def start(self):
"""
Start all consumer processes and register signal handlers.
Don't init scheduler.
"""
if self.huey.always_eager:
raise ConfigurationError(
'Consumer cannot be run with Huey instances where always_eager'
' is enabled. Please check your configuration and ensure that'
' "huey.always_eager = False".')
# Log startup message.
self._logger.info('Huey consumer started with %s %s, PID %s',
self.workers, self.worker_type, os.getpid())
self._logger.info('Scheduler disabled')
self._logger.info('Health checker is %s',
'enabled' if self._health_check else 'disabled')
self._logger.info('Periodic tasks are %s.',
'enabled' if self.periodic else 'disabled')
self._logger.info('UTC is %s.', 'enabled' if self.utc else 'disabled')
for _, worker_process in self.worker_threads:
worker_process.start()
def __init__(self, huey, workers=1, periodic=True, initial_delay=0.1,
backoff=1.15, max_delay=10.0, scheduler_interval=1,
worker_type=WORKER_THREAD, check_worker_health=True,
health_check_interval=10, flush_locks=False,
extra_locks=None):
self._logger = logging.getLogger('huey.consumer')
if huey.immediate:
self._logger.warning('Consumer initialized with Huey instance '
'that has "immediate" mode enabled. This '
'must be disabled before the consumer can '
'be run.')
self.huey = huey
self.workers = workers # Number of workers.
self.periodic = periodic # Enable periodic task scheduler?
self.default_delay = initial_delay # Default queue polling interval.
self.backoff = backoff # Exponential backoff factor when queue empty.
self.max_delay = max_delay # Maximum interval between polling events.
# Ensure that the scheduler runs at an interval between 1 and 60s.
self.scheduler_interval = max(min(scheduler_interval, 60), 1)
if 60 % self.scheduler_interval != 0:
raise ConfigurationError('Scheduler interval must be a factor '
'of 60, e.g. 1, 2, 3, 4, 5, 6, 10, 12...')
if worker_type == 'gevent': worker_type = WORKER_GREENLET
self.worker_type = worker_type # What process model are we using?
# Configure health-check and consumer main-loop attributes.
self._stop_flag_timeout = 0.1
self._health_check = check_worker_health
self._health_check_interval = float(health_check_interval)
# Create the execution environment helper.
self.environment = self.get_environment(self.worker_type)
# Create the event used to signal the process should terminate. We'll
# also store a boolean flag to indicate whether we should restart after
# the processes are cleaned up.
self._received_signal = False
self._restart = False
self._graceful = True
self.stop_flag = self.environment.get_stop_flag()
# In the event the consumer was killed while running a task that held
# a lock, this ensures that all locks are flushed before starting.
if flush_locks or extra_locks:
lock_names = extra_locks.split(',') if extra_locks else ()
self.flush_locks(*lock_names)
# Create the scheduler process (but don't start it yet).
scheduler = self._create_scheduler()
self.scheduler = self._create_process(scheduler, 'Scheduler')
# Create the worker process(es) (also not started yet).
self.worker_threads = []
for i in range(workers):
worker = self._create_worker()
process = self._create_process(worker, 'Worker-%d' % (i + 1))
# The worker threads are stored as [(worker impl, worker_t), ...].
# The worker impl is not currently referenced in any consumer code,
# but it is referenced in the test-suite.
self.worker_threads.append((worker, process))
def __init__(self, *args, **kwargs):
raise ConfigurationError('Cannot initialize "%s", %s module not '
'installed.' % (name, library))
def __init__(self, compression=False, compression_level=6):
self.comp = compression
self.comp_level = compression_level
if self.comp and gzip is None:
raise ConfigurationError('Cannot enable compression, gzip module '
'not found.')
