Esempio n. 1
0
    def __init__(self, config):
        self._config = config['AMQP']['AutoScaling']
        mgmt_cfg = config['AMQP']['Management']
        self.min_workers = self._config['min_workers']
        self.max_workers= self._config['max_workers']
        self.mps_max = self._config['mps_max']
        self.max_err = self._config['max_err']
        self.spawn_depth = self._config['spawn_depth']
        self.requeue_on_err = self._config['requeue_on_err']
        self.broker = Server("%s:%s" % (mgmt_cfg['host'], mgmt_cfg['port']),
                             mgmt_cfg['user'], mgmt_cfg['password'])

        # Passed to workers
        self.worker_config = config['AMQP']

        # worker instances.
        self.workers = []
        self.worker_class = PikaClient
Esempio n. 2
0
class PoolManager(object):
    """
    Takes in the AMQP config, which includes the info needed to connect and
    consume from the broker, as well as the min/max number of consumer
    threads to run, spawn/kill threshholds, etc.

    It doesn't connect or consume itself. It passes that on to the workers.
    This object's responsibility is limited to thread management and providing
    a singular interface with with to communicate w/ threads & get information
    about them.

    To get queue depths & anything else it needs to determine when/if to
    spawn/kill threads, it uses the HTTP interface of the RabbitMQ
    management plugin, so that's an external dependency.
    """
    def __init__(self, config):
        self._config = config['AMQP']['AutoScaling']
        mgmt_cfg = config['AMQP']['Management']
        self.min_workers = self._config['min_workers']
        self.max_workers= self._config['max_workers']
        self.mps_max = self._config['mps_max']
        self.max_err = self._config['max_err']
        self.spawn_depth = self._config['spawn_depth']
        self.requeue_on_err = self._config['requeue_on_err']
        self.broker = Server("%s:%s" % (mgmt_cfg['host'], mgmt_cfg['port']),
                             mgmt_cfg['user'], mgmt_cfg['password'])

        # Passed to workers
        self.worker_config = config['AMQP']

        # worker instances.
        self.workers = []
        self.worker_class = PikaClient

    def spawn_workers(self):
        """Starts a new worker, and registers it in self.workers.

        """
        try:
            new_worker = self.worker_class(self._config['AMQP'])
            new_worker.start()
        except Exception:
            raise
        else:
            self.workers.append(new_worker)
            logging.info("New worker added (%s workers)", len(self.workers))
            return True

    def reap_workers(self):
        """Loops over the workers and removes any that have died.

        """
        for worker in self.workers:
            if worker.is_alive():
                continue
            else:
                logging.info("Removing dead worker: %s", worker.name)
                self.workers.pop(self.workers.index(worker))
                logging.info("%s workers running", len(self.workers))

    def poll_depth(self, vhost, queue):
        """Asks RabbitMQ about queue depths. Used to determine if we need to
        spawn more workers. Returns an integer.

        """
        depth = self.broker.get_queue_depth(vhost, queue)
        return depth

    def start(self):
        """This is called by the end user's processor module to kick off
        the fireworks. It's basically this class's 'main()'.

        """

        worker = PikaClient(self.worker_config)
        logging.info("Starting a new worker")
        worker.start()
        logging.info("Worker started")