class ExperimentListener(threading.Thread):
def __init__(self, kafka_topic, kafka_servers, kafka_group_id, logger, db):
threading.Thread.__init__(self)
self.consumer = Consumer({
'bootstrap.servers': kafka_servers,
'group.id': kafka_group_id,
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
})
self.topic = kafka_topic
self.experiment_id = kafka_topic.split('-')[
0] # Get experiment ID from topic name.
self.log = logger
self.parser = MessageParser(self.log)
self.kafkaTimeout = 5
self.db = db
def run(self):
self.log.info("[0x%x] Thread started, topic: %s",
threading.get_ident(), self.topic)
self.consumer.subscribe([self.topic])
self.consume()
def consume(self):
running = True
self.log.info("[0x%x] Listening to %s", threading.get_ident(),
self.topic)
writer = mongoWriter(self.db, self.log, self.topic)
while running:
msg = self.consumer.poll(timeout=self.kafkaTimeout)
if msg != None:
if not msg.error():
msgText = msg.value().decode('utf-8')
self.log.debug('[0x%x] Message received: %s',
threading.get_ident(), msgText)
metrics = self.parser.parse_message(msgText)
if metrics != None:
self.log.debug('Metrics recieved - Writing metrics')
writer.write_metrics(metrics)
if msgText in expected_termination_messages:
self.log.info(
"[0x%x] Closing thread. Experiment %s ended because of: %s",
threading.get_ident(), self.experiment_id, msgText)
if msgText == "<EXPERIMENT_COMPLETED>":
writer.update_experiment_state(
self.experiment_id, "completed")
running = False
elif msg.error().code() != KafkaError._PARTITION_EOF:
self.log.error(msg.error())
running = False
else:
time.sleep(5)
self.consumer.close()
