def __init__(self, config):
self._config = config['AMQP']['AutoScaling']
mgmt_cfg = config['AMQP']['Management']
self.min_workers = self._config['min_workers']
self.max_workers= self._config['max_workers']
self.mps_max = self._config['mps_max']
self.max_err = self._config['max_err']
self.spawn_depth = self._config['spawn_depth']
self.requeue_on_err = self._config['requeue_on_err']
self.broker = Server("%s:%s" % (mgmt_cfg['host'], mgmt_cfg['port']),
mgmt_cfg['user'], mgmt_cfg['password'])
# Passed to workers
self.worker_config = config['AMQP']
# worker instances.
self.workers = []
self.worker_class = PikaClient
class PoolManager(object):
"""
Takes in the AMQP config, which includes the info needed to connect and
consume from the broker, as well as the min/max number of consumer
threads to run, spawn/kill threshholds, etc.
It doesn't connect or consume itself. It passes that on to the workers.
This object's responsibility is limited to thread management and providing
a singular interface with with to communicate w/ threads & get information
about them.
To get queue depths & anything else it needs to determine when/if to
spawn/kill threads, it uses the HTTP interface of the RabbitMQ
management plugin, so that's an external dependency.
"""
def __init__(self, config):
self._config = config['AMQP']['AutoScaling']
mgmt_cfg = config['AMQP']['Management']
self.min_workers = self._config['min_workers']
self.max_workers= self._config['max_workers']
self.mps_max = self._config['mps_max']
self.max_err = self._config['max_err']
self.spawn_depth = self._config['spawn_depth']
self.requeue_on_err = self._config['requeue_on_err']
self.broker = Server("%s:%s" % (mgmt_cfg['host'], mgmt_cfg['port']),
mgmt_cfg['user'], mgmt_cfg['password'])
# Passed to workers
self.worker_config = config['AMQP']
# worker instances.
self.workers = []
self.worker_class = PikaClient
def spawn_workers(self):
"""Starts a new worker, and registers it in self.workers.
"""
try:
new_worker = self.worker_class(self._config['AMQP'])
new_worker.start()
except Exception:
raise
else:
self.workers.append(new_worker)
logging.info("New worker added (%s workers)", len(self.workers))
return True
def reap_workers(self):
"""Loops over the workers and removes any that have died.
"""
for worker in self.workers:
if worker.is_alive():
continue
else:
logging.info("Removing dead worker: %s", worker.name)
self.workers.pop(self.workers.index(worker))
logging.info("%s workers running", len(self.workers))
def poll_depth(self, vhost, queue):
"""Asks RabbitMQ about queue depths. Used to determine if we need to
spawn more workers. Returns an integer.
"""
depth = self.broker.get_queue_depth(vhost, queue)
return depth
def start(self):
"""This is called by the end user's processor module to kick off
the fireworks. It's basically this class's 'main()'.
"""
worker = PikaClient(self.worker_config)
logging.info("Starting a new worker")
worker.start()
logging.info("Worker started")
