def _process_message(self, msg: KafkaMessage):
err = msg.error()
if err:
if err.code() == KafkaError._PARTITION_EOF:
return None
else:
monitoring.got_counter("kafka_consumer_exception")
params = {
"code": err.code(),
"pid": os.getpid(),
"topic": msg.topic(),
"partition": msg.partition(),
"offset": msg.offset(),
log_const.KEY_NAME: log_const.EXCEPTION_VALUE
}
log(
"KafkaConsumer Error %(code)s at pid %(pid)s: topic=%(topic)s partition=[%(partition)s] "
"reached end at offset %(offset)s\n",
params=params,
level="WARNING")
raise KafkaException(err)
if msg.value():
if msg.headers() is None:
msg.set_headers([])
return msg
def _message_produced(self, error: KafkaError, message: Message) -> None:
"""TODO"""
topic = message.topic()
msg = message.value().decode(Charset.UTF_8.value)
if error is not None:
print(f"Failed to deliver message: {msg}: {error.str()}")
else:
self.bus.emit(MSG_PROD_EVT, message=msg, topic=topic)
def delivery_success_callback(msg: Message):
pprint({
'topic': msg.topic(),
'partition': msg.partition(),
'timestamp': msg.timestamp(),
'key': msg.key(),
'value': msg.value(),
})
def delivery_report(err: str, msg: Message):
"""Called once for each message produced to indicate delivery result.
Triggered by poll() or flush()."""
if err is not None:
print("Message delivery failed: {}".format(err))
else:
print("Message delivered to {} [{}]".format(msg.topic(),
msg.partition()))