def __init__(self, name='huey', blocking=True, read_timeout=1, connection_pool=None, url=None, client_name=None, **connection_params): if Redis is None: raise ConfigurationError('"redis" python module not found, cannot ' 'use Redis storage backend. Run "pip ' 'install redis" to install.') # Drop common empty values from the connection_params. for p in ('host', 'port', 'db'): if p in connection_params and connection_params[p] is None: del connection_params[p] if sum(1 for p in (url, connection_pool, connection_params) if p) > 1: raise ConfigurationError( 'The connection configuration is over-determined. ' 'Please specify only one of the following: ' '"url", "connection_pool", or "connection_params"') if url: connection_pool = ConnectionPool.from_url( url, decode_components=True) elif connection_pool is None: connection_pool = ConnectionPool(**connection_params) self.pool = connection_pool self.conn = self.redis_client(connection_pool=connection_pool) self.connection_params = connection_params super(RedisStorage, self).__init__(name, blocking, read_timeout, **connection_params)
def __init__(self, compression=False, compression_level=6, use_zlib=False): self.comp = compression self.comp_level = compression_level self.use_zlib = use_zlib if self.comp: if self.use_zlib and zlib is None: raise ConfigurationError('use_zlib specified, but zlib module ' 'not found.') elif gzip is None: raise ConfigurationError('gzip module required to enable ' 'compression.')
def start(self): if self.huey.always_eager: raise ConfigurationError( 'Consumer cannot be run with Huey instances where always_eager' ' is enabled. Please check your configuration and ensure that' ' "huey.always_eager = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s' % (self.workers, self.worker_type, os.getpid())) self._logger.info('Scheduler runs every %s seconds.' % (self.scheduler_interval)) self._logger.info('Periodic tasks are %s.' % ('enabled' if self.periodic else 'disabled')) self._logger.info('UTC is %s.' % ('enabled' if self.utc else 'disabled')) self._set_signal_handler() msg = ['The following commands are available:'] for command in registry._registry: msg.append('+ %s' % command.replace('queuecmd_', '')) self._logger.info('\n'.join(msg)) # We'll temporarily ignore SIGINT (so that it is inherited by the # child-processes). Once the child processes are created, we restore # the handler. original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) self.scheduler.start() for _, worker_process in self.worker_threads: worker_process.start() signal.signal(signal.SIGINT, original_sigint_handler)
def start(self): """ Start all consumer processes and register signal handlers. """ if self.huey.immediate: raise ConfigurationError( 'Consumer cannot be run with Huey instances where immediate ' 'is enabled. Please check your configuration and ensure that ' '"huey.immediate = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s at %s', self.workers, self.worker_type, os.getpid(), self.huey._get_timestamp()) self._logger.info('Scheduler runs every %s second(s).', self.scheduler_interval) self._logger.info('Periodic tasks are %s.', 'enabled' if self.periodic else 'disabled') msg = ['The following commands are available:'] for command in self.huey._registry._registry: msg.append('+ %s' % command) self._logger.info('\n'.join(msg)) # Start the scheduler and workers. self.scheduler.start() for _, worker_process in self.worker_threads: worker_process.start() # Finally set the signal handlers for main process. self._set_signal_handlers()
def __init__(self, compression=False, compression_level=6, use_zlib=False, pickle_protocol=pickle.HIGHEST_PROTOCOL): self.comp = compression self.comp_level = compression_level self.use_zlib = use_zlib self.pickle_protocol = pickle_protocol or pickle.HIGHEST_PROTOCOL if self.comp: if self.use_zlib and zlib is None: raise ConfigurationError('use_zlib specified, but zlib module ' 'not found.') elif gzip is None: raise ConfigurationError('gzip module required to enable ' 'compression.')
def start(self): if self.huey.always_eager: raise ConfigurationError( 'Consumer cannot be run with Huey instances where always_eager' ' is enabled. Please check your configuration and ensure that' ' "huey.always_eager = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s' % (self.workers, self.worker_type, os.getpid())) self._logger.info('Scheduler runs every %s seconds.' % (self.scheduler_interval)) self._logger.info('Periodic tasks are %s.' % ('enabled' if self.periodic else 'disabled')) self._set_signal_handler() msg = ['The following commands are available:'] for command in registry._registry: msg.append('+ %s' % command.replace('queuecmd_', '')) self._logger.info('\n'.join(msg)) self.scheduler.start() for _, worker_process in self.worker_threads: worker_process.start()
def __init__(self, name='huey', blocking=True, read_timeout=1, connection_pool=None, url=None, client_name=None, **connection_params): if Redis is None: raise ConfigurationError('"redis" python module not found, cannot ' 'use Redis storage backend. Run "pip ' 'install redis" to install.') # Drop common empty values from the connection_params. for p in ('host', 'port', 'db'): if p in connection_params and connection_params[p] is None: del connection_params[p] if sum(1 for p in (url, connection_pool, connection_params) if p) > 1: raise ConfigurationError( 'The connection configuration is over-determined. ' 'Please specify only one of the the following: ' '"url", "connection_pool", or "connection_params"') if url: connection_pool = ConnectionPool.from_url(url, decode_components=True) elif connection_pool is None: connection_pool = ConnectionPool(**connection_params) self.pool = connection_pool self.conn = self.redis_client(connection_pool=connection_pool) self.connection_params = connection_params self._pop = self.conn.register_script(SCHEDULE_POP_LUA) self.name = self.clean_name(name) self.queue_key = 'huey.redis.%s' % self.name self.schedule_key = 'huey.schedule.%s' % self.name self.result_key = 'huey.results.%s' % self.name self.error_key = 'huey.errors.%s' % self.name if client_name is not None: self.conn.client_setname(client_name) self.blocking = blocking self.read_timeout = read_timeout
def __init__(self, secret=None, salt='huey', **kwargs): super(SignedSerializer, self).__init__(**kwargs) if not secret or not salt: raise ConfigurationError('The secret and salt parameters are ' 'required by %r' % type(self)) self.secret = encode(secret) self.salt = encode(salt) self.separator = b':' self._key = hashlib.sha1(self.salt + self.secret).digest()
def __init__(self, name='huey', blocking=True, read_timeout=1, startup_nodes=None, **connection_params): if RedisCluster is None: raise ConfigurationError('"redis-py-cluster" python module not found, cannot ' 'use RedisCluster storage backend. Run "pip ' 'install redis-py-cluster" to install.') if startup_nodes is None: raise ConfigurationError('startup_nodes parameter required') decode_responses = False if sys.version_info >= (3, 0, 0): decode_responses = True self.conn = self.redis_client(startup_nodes=startup_nodes, decode_responses=decode_responses) super(RedisClusterStorage, self).__init__(name, blocking, read_timeout, **connection_params)
def __init__(self, name='huey', database=None, **kwargs): super(SqlStorage, self).__init__(name) if database is None: raise ConfigurationError('Use of SqlStorage requires a ' 'database= argument, which should be a ' 'peewee database or a connection string.') if isinstance(database, Database): self.database = database else: # Treat database argument as a URL connection string. self.database = db_url_connect(database) self.KV, self.Schedule, self.Task = self.create_models() self.create_tables()
def __init__(self, name: str = "huey", engine: Union[Engine, str, None] = None, **kwargs: Dict) -> None: super().__init__(name) if engine is None: raise ConfigurationError( "Use of SQLAlchemyStorage requires an " "engine= argument, which should be a " "SQLAlchemy engine or a connection string.") if isinstance(engine, Engine): self.engine = engine else: # Treat database argument as a URL connection string. self.engine = create_engine(engine) self.create_tables()
def __init__(self, name='huey', blocking=True, read_timeout=1, client_name=None, **connection_params): if 'conn' not in self: raise ConfigurationError('No connection in RedisBaseStorage') self._pop = self.conn.register_script(SCHEDULE_POP_LUA) self.name = self.clean_name(name) self.queue_key = 'huey.redis.%s' % self.name self.schedule_key = 'huey.schedule.%s' % self.name self.result_key = 'huey.results.%s' % self.name self.error_key = 'huey.errors.%s' % self.name if client_name is not None: self.conn.client_setname(client_name) self.blocking = blocking self.read_timeout = read_timeout
def start(self): """ Start all consumer processes and register signal handlers. """ if self.huey.immediate: raise ConfigurationError( 'Consumer cannot be run with Huey instances where immediate ' 'is enabled. Please check your configuration and ensure that ' '"huey.immediate = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s at %s', self.workers, self.worker_type, os.getpid(), self.huey._get_timestamp()) self._logger.info('Scheduler runs every %s second(s).', self.scheduler_interval) self._logger.info('Periodic tasks are %s.', 'enabled' if self.periodic else 'disabled') self._set_signal_handlers() msg = ['The following commands are available:'] for command in self.huey._registry._registry: msg.append('+ %s' % command) self._logger.info('\n'.join(msg)) # We'll temporarily ignore SIGINT and SIGHUP (so that it is inherited # by the child-processes). Once the child processes are created, we # restore the handler. original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) if hasattr(signal, 'SIGHUP'): original_sighup_handler = signal.signal(signal.SIGHUP, signal.SIG_IGN) self.scheduler.start() for _, worker_process in self.worker_threads: worker_process.start() signal.signal(signal.SIGINT, original_sigint_handler) if hasattr(signal, 'SIGHUP'): signal.signal(signal.SIGHUP, original_sighup_handler)
def start(self): """ Start all consumer processes and register signal handlers. Don't init scheduler. """ if self.huey.always_eager: raise ConfigurationError( 'Consumer cannot be run with Huey instances where always_eager' ' is enabled. Please check your configuration and ensure that' ' "huey.always_eager = False".') # Log startup message. self._logger.info('Huey consumer started with %s %s, PID %s', self.workers, self.worker_type, os.getpid()) self._logger.info('Scheduler disabled') self._logger.info('Health checker is %s', 'enabled' if self._health_check else 'disabled') self._logger.info('Periodic tasks are %s.', 'enabled' if self.periodic else 'disabled') self._logger.info('UTC is %s.', 'enabled' if self.utc else 'disabled') for _, worker_process in self.worker_threads: worker_process.start()
def __init__(self, huey, workers=1, periodic=True, initial_delay=0.1, backoff=1.15, max_delay=10.0, scheduler_interval=1, worker_type=WORKER_THREAD, check_worker_health=True, health_check_interval=10, flush_locks=False, extra_locks=None): self._logger = logging.getLogger('huey.consumer') if huey.immediate: self._logger.warning('Consumer initialized with Huey instance ' 'that has "immediate" mode enabled. This ' 'must be disabled before the consumer can ' 'be run.') self.huey = huey self.workers = workers # Number of workers. self.periodic = periodic # Enable periodic task scheduler? self.default_delay = initial_delay # Default queue polling interval. self.backoff = backoff # Exponential backoff factor when queue empty. self.max_delay = max_delay # Maximum interval between polling events. # Ensure that the scheduler runs at an interval between 1 and 60s. self.scheduler_interval = max(min(scheduler_interval, 60), 1) if 60 % self.scheduler_interval != 0: raise ConfigurationError('Scheduler interval must be a factor ' 'of 60, e.g. 1, 2, 3, 4, 5, 6, 10, 12...') if worker_type == 'gevent': worker_type = WORKER_GREENLET self.worker_type = worker_type # What process model are we using? # Configure health-check and consumer main-loop attributes. self._stop_flag_timeout = 0.1 self._health_check = check_worker_health self._health_check_interval = float(health_check_interval) # Create the execution environment helper. self.environment = self.get_environment(self.worker_type) # Create the event used to signal the process should terminate. We'll # also store a boolean flag to indicate whether we should restart after # the processes are cleaned up. self._received_signal = False self._restart = False self._graceful = True self.stop_flag = self.environment.get_stop_flag() # In the event the consumer was killed while running a task that held # a lock, this ensures that all locks are flushed before starting. if flush_locks or extra_locks: lock_names = extra_locks.split(',') if extra_locks else () self.flush_locks(*lock_names) # Create the scheduler process (but don't start it yet). scheduler = self._create_scheduler() self.scheduler = self._create_process(scheduler, 'Scheduler') # Create the worker process(es) (also not started yet). self.worker_threads = [] for i in range(workers): worker = self._create_worker() process = self._create_process(worker, 'Worker-%d' % (i + 1)) # The worker threads are stored as [(worker impl, worker_t), ...]. # The worker impl is not currently referenced in any consumer code, # but it is referenced in the test-suite. self.worker_threads.append((worker, process))
def __init__(self, *args, **kwargs): raise ConfigurationError('Cannot initialize "%s", %s module not ' 'installed.' % (name, library))
def __init__(self, compression=False, compression_level=6): self.comp = compression self.comp_level = compression_level if self.comp and gzip is None: raise ConfigurationError('Cannot enable compression, gzip module ' 'not found.')