def subscribe(self, topic):
def on_assign(consumer, partitions):
log.info('subscribed')
try:
self.kafka_subscriber.subscribe([topic], on_assign=on_assign)
while True:
message = self.kafka_subscriber.poll(timeout=1.0)
if self.killer.killed:
break
if message is None:
continue
if message.error():
log.error('read message error')
# commit message
self.kafka_subscriber.commit(asynchronous=False)
log.info('received message from topic {t}'.format(t=message.topic()))
print(message.value().decode('utf-8'))
except:
log.error('error subscribing to brokers')
self.close()
def run(self):
log.info('start kafka subscriber inside thread {tn}'.format(tn=self.name))
self.subscribe(self.topic)
def on_assign(consumer, partitions):
log.info('subscribed')
def on_delivery(err, msg):
if err is not None:
log.info('Failed to deliver message: %s: %s' %
(str(msg), str(err)))
else:
log.info('Message produced: %s' % (str(msg)))
def kill(self, signum, frame):
log.info('signum: {s} received'.format(s=signum))
self.killed = True