def decode_message_from_avro(self, message: Message):
key_schema_id, decoded_key = self.decode_bytes(message.key())
value_schema_id, decoded_value = self.decode_bytes(message.value())
headers = []
if message.headers():
for header_key, header_value in message.headers():
headers.append(
MessageHeader(key=header_key,
value=header_value.decode("utf-8")
if header_value else None))
decoded_message = DecodedAvroMessage(decoded_key,
decoded_value,
message.partition(),
key_schema_id,
value_schema_id,
headers=headers)
schema_version = it.count(1)
serializable_message = {
"key":
decoded_message.key,
"value":
decoded_message.value,
"partition":
decoded_message.partition,
"schema_directory_name":
f"{next(schema_version):04}_{key_schema_id}_{value_schema_id}",
}
return key_schema_id, value_schema_id, decoded_message, serializable_message
def delivery_success_callback(msg: Message):
pprint({
'topic': msg.topic(),
'partition': msg.partition(),
'timestamp': msg.timestamp(),
'key': msg.key(),
'value': msg.value(),
})
def update_callback(self, err: Optional[cimpl.KafkaError],
msg: cimpl.Message):
assert err is None, f"Received KafkaError {err}."
self.binary_value = msg.value()
self.binary_key = msg.key()
self.partition = msg.partition()
self.offset = msg.offset()
self.timestamp = msg.timestamp()[1]
def decode_message(message: Message) -> DecodedMessage:
if message.key() is None:
decoded_key = None
else:
decoded_key = message.key().decode("utf-8")
decoded_value = message.value().decode("utf-8")
headers = []
if message.headers():
for header_key, header_value in message.headers():
headers.append(MessageHeader(key=header_key, value=header_value.decode("utf-8") if header_value else None))
return DecodedMessage(
key=decoded_key,
value=decoded_value,
partition=message.partition(),
offset=message.offset(),
timestamp=str(message.timestamp()),
headers=headers,
)
def _deserializer_msg(self, msg: Message):
if self._value_deserializer:
try:
msg.set_value(self._value_deserializer(msg.value()))
except Exception as se:
raise ValueDeserializationError(exception=se, message=msg)
if self._key_deserializer:
try:
msg.set_key(self._key_deserializer(msg.key()))
except Exception as se:
raise KeyDeserializationError(exception=se, message=msg)
return msg
def write_message_to_file(self, message: Message):
key_schema_id, decoded_key = self.decode_bytes(message.key())
value_schema_id, decoded_value = self.decode_bytes(message.value())
decoded_message = DecodedAvroMessage(decoded_key, decoded_value,
message.partition(),
key_schema_id, value_schema_id)
if self.schema_changed(
decoded_message) or self.schema_dir_name is None:
self.schema_dir_name = f"{next(self.schema_version):04}_{key_schema_id}_{value_schema_id}"
self.current_key_schema_id = key_schema_id
self.current_value_schema_id = value_schema_id
self._dump_schemata(key_schema_id, value_schema_id)
serializable_message = {
"key": decoded_message.key,
"value": decoded_message.value,
"partition": decoded_message.partition,
"schema_directory_name": self.schema_dir_name,
}
pickle.dump(serializable_message, self.file)
def decode_message(message: Message) -> DecodedMessage:
decoded_key = message.key().decode("utf-8")
decoded_value = message.value().decode("utf-8")
return DecodedMessage(decoded_key, decoded_value, message.partition())
def on_message_received(self, msg: Message) -> None:
# noinspection PyArgumentList
self.message_buffer.append((msg.key(), msg.value()))
self.__idle_count = 0