class StatPersistor(ConsumerMixin):
"""
this is the service who handle the persistence of statistiques send by
rabbitmq
"""
def __init__(self, conf_file):
self.connection = None
self.exchange = None
self.queues = []
self.config = Config()
self.config.load(conf_file)
self.stat_saver = StatSaver(self.config)
self._init_rabbitmq()
def _init_rabbitmq(self):
"""
connect to rabbitmq and init the queues
"""
self.connection = kombu.Connection(self.config.broker_url)
exchange_name = self.config.exchange_name
exchange = kombu.Exchange(exchange_name, type="topic")
logging.getLogger('stat_persistor').info("listen following exchange: %s",
self.config.exchange_name)
queue_name = self.config.queue_name
queue = kombu.Queue(queue_name, exchange=exchange, routing_key='#', durable=True)
self.queues.append(queue)
def get_consumers(self, Consumer, channel):
consumers = [Consumer(queues=self.queues, callbacks=[self.process_task])]
for c in consumers:
c.qos(prefetch_count=1000)
return consumers
def handle_statistics(self, stat_hit):
if stat_hit.IsInitialized():
try:
self.stat_saver.persist_stat(self.config, stat_hit)
except (FunctionalError) as e:
logging.getLogger('stat_persistor').warn("error while preparing stats to save: {}".format(str(e)))
else:
logging.getLogger('stat_persistor').warn("protobuff query not initialized,"
" no stat logged")
def process_task(self, body, message):
logging.getLogger('stat_persistor').debug("Message received")
stat_request = stat_persistor.stat_pb2.StatRequest()
try:
stat_request.ParseFromString(body)
logging.getLogger('stat_persistor').debug('query received: {}'.format(str(stat_request)))
except DecodeError as e:
logging.getLogger('stat_persistor').warn("message is not a valid "
"protobuf task: {}".format(str(e)))
message.ack()
return
try:
self.handle_statistics(stat_request)
message.ack()
except (TechnicalError) as e:
logging.getLogger('stat_persistor').warn("error while saving stats: {}".format(str(e)))
# on technical error (like a database KO) we retry this task later
# and we sleep 10 seconds
message.requeue()
time.sleep(10)
def __del__(self):
self.close()
def close(self):
if self.connection and self.connection.connected:
self.connection.release()
class StatPersistor(ConsumerMixin):
"""
this is the service who handle the persistence of statistiques send by
rabbitmq
"""
def __init__(self, conf_file):
self.connection = None
self.exchange = None
self.queues = []
self.config = Config()
self.config.load(conf_file)
self.stat_saver = StatSaver(self.config)
self._init_rabbitmq()
def _init_rabbitmq(self):
"""
connect to rabbitmq and init the queues
"""
self.connection = kombu.Connection(self.config.broker_url)
exchange_name = self.config.exchange_name
exchange = kombu.Exchange(exchange_name, type="direct")
logging.getLogger('stat_persistor').info(
"listen following exchange: %s", self.config.exchange_name)
queue_name = self.config.queue_name
queue = kombu.Queue(queue_name, exchange=exchange, durable=True)
self.queues.append(queue)
def get_consumers(self, Consumer, channel):
return [Consumer(queues=self.queues, callbacks=[self.process_task])]
def handle_statistics(self, stat_hit):
if stat_hit.IsInitialized():
try:
self.stat_saver.persist_stat(self.config, stat_hit)
except (FunctionalError) as e:
logging.getLogger('stat_persistor').warn(
"error while preparing stats to save: {}".format(str(e)))
else:
logging.getLogger('stat_persistor').warn(
"protobuff query not initialized,"
" no stat logged")
def process_task(self, body, message):
logging.getLogger('stat_persistor').debug("Message received")
stat_request = navitiacommon.stat_pb2.StatRequest()
try:
stat_request.ParseFromString(body)
logging.getLogger('stat_persistor').debug(
'query received: {}'.format(str(stat_request)))
except DecodeError as e:
logging.getLogger('stat_persistor').warn(
"message is not a valid "
"protobuf task: {}".format(str(e)))
message.ack()
return
try:
self.handle_statistics(stat_request)
message.ack()
except (TechnicalError) as e:
logging.getLogger('stat_persistor').warn(
"error while saving stats: {}".format(str(e)))
# on technical error (like a database KO) we retry this task later
# and we sleep 10 seconds
message.requeue()
time.sleep(10)
def __del__(self):
self.close()
def close(self):
if self.connection and self.connection.connected:
self.connection.release()