Exemple #1
0
    def __init__(self, conf_file):
        self.connection = None
        self.exchange = None
        self.queues = []
        self.config = Config()

        self.config.load(conf_file)
        self.stat_saver = StatSaver(self.config)
        self._init_rabbitmq()
    def __init__(self, conf_file):
        self.connection = None
        self.exchange = None
        self.queues = []
        self.config = Config()

        self.config.load(conf_file)
        self.stat_saver = StatSaver(self.config)
        self._init_rabbitmq()
class StatPersistor(ConsumerMixin):

    """
    this is the service who handle the persistence of statistiques send by
    rabbitmq
    """

    def __init__(self, conf_file):
        self.connection = None
        self.exchange = None
        self.queues = []
        self.config = Config()

        self.config.load(conf_file)
        self.stat_saver = StatSaver(self.config)
        self._init_rabbitmq()

    def _init_rabbitmq(self):
        """
        connect to rabbitmq and init the queues
        """
        self.connection = kombu.Connection(self.config.broker_url)
        exchange_name = self.config.exchange_name
        exchange = kombu.Exchange(exchange_name, type="topic")
        logging.getLogger('stat_persistor').info("listen following exchange: %s",
                                         self.config.exchange_name)

        queue_name = self.config.queue_name
        queue = kombu.Queue(queue_name, exchange=exchange, routing_key='#', durable=True)
        self.queues.append(queue)

    def get_consumers(self, Consumer, channel):
        consumers = [Consumer(queues=self.queues, callbacks=[self.process_task])]
        for c in consumers:
            c.qos(prefetch_count=1000)
        return consumers

    def handle_statistics(self, stat_hit):
        if stat_hit.IsInitialized():
            try:
                self.stat_saver.persist_stat(self.config, stat_hit)
            except (FunctionalError) as e:
                logging.getLogger('stat_persistor').warn("error while preparing stats to save: {}".format(str(e)))
        else:
            logging.getLogger('stat_persistor').warn("protobuff query not initialized,"
                                                     " no stat logged")

    def process_task(self, body, message):
        logging.getLogger('stat_persistor').debug("Message received")
        stat_request = stat_persistor.stat_pb2.StatRequest()
        try:
            stat_request.ParseFromString(body)
            logging.getLogger('stat_persistor').debug('query received: {}'.format(str(stat_request)))
        except DecodeError as e:
            logging.getLogger('stat_persistor').warn("message is not a valid "
                                             "protobuf task: {}".format(str(e)))
            message.ack()
            return

        try:
            self.handle_statistics(stat_request)
            message.ack()
        except (TechnicalError) as e:
                logging.getLogger('stat_persistor').warn("error while saving stats: {}".format(str(e)))
                # on technical error (like a database KO) we retry this task later
                # and we sleep 10 seconds
                message.requeue()
                time.sleep(10)

    def __del__(self):
        self.close()

    def close(self):
        if self.connection and self.connection.connected:
            self.connection.release()
Exemple #4
0
class StatPersistor(ConsumerMixin):
    """
    this is the service who handle the persistence of statistiques send by
    rabbitmq
    """
    def __init__(self, conf_file):
        self.connection = None
        self.exchange = None
        self.queues = []
        self.config = Config()

        self.config.load(conf_file)
        self.stat_saver = StatSaver(self.config)
        self._init_rabbitmq()

    def _init_rabbitmq(self):
        """
        connect to rabbitmq and init the queues
        """
        self.connection = kombu.Connection(self.config.broker_url)
        exchange_name = self.config.exchange_name
        exchange = kombu.Exchange(exchange_name, type="direct")
        logging.getLogger('stat_persistor').info(
            "listen following exchange: %s", self.config.exchange_name)

        queue_name = self.config.queue_name
        queue = kombu.Queue(queue_name, exchange=exchange, durable=True)
        self.queues.append(queue)

    def get_consumers(self, Consumer, channel):
        return [Consumer(queues=self.queues, callbacks=[self.process_task])]

    def handle_statistics(self, stat_hit):
        if stat_hit.IsInitialized():
            try:
                self.stat_saver.persist_stat(self.config, stat_hit)
            except (FunctionalError) as e:
                logging.getLogger('stat_persistor').warn(
                    "error while preparing stats to save: {}".format(str(e)))
        else:
            logging.getLogger('stat_persistor').warn(
                "protobuff query not initialized,"
                " no stat logged")

    def process_task(self, body, message):
        logging.getLogger('stat_persistor').debug("Message received")
        stat_request = navitiacommon.stat_pb2.StatRequest()
        try:
            stat_request.ParseFromString(body)
            logging.getLogger('stat_persistor').debug(
                'query received: {}'.format(str(stat_request)))
        except DecodeError as e:
            logging.getLogger('stat_persistor').warn(
                "message is not a valid "
                "protobuf task: {}".format(str(e)))
            message.ack()
            return

        try:
            self.handle_statistics(stat_request)
            message.ack()
        except (TechnicalError) as e:
            logging.getLogger('stat_persistor').warn(
                "error while saving stats: {}".format(str(e)))
            # on technical error (like a database KO) we retry this task later
            # and we sleep 10 seconds
            message.requeue()
            time.sleep(10)

    def __del__(self):
        self.close()

    def close(self):
        if self.connection and self.connection.connected:
            self.connection.release()