def _unpack_message_set(self, tp, messages):
for offset, size, msg in messages:
if self._check_crcs and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
# If relative offset is used, we need to decompress the entire
# message first to compute the absolute offset.
inner_mset = msg.decompress()
if msg.magic > 0:
last_offset, _, _ = inner_mset[-1]
absolute_base_offset = offset - last_offset
else:
absolute_base_offset = -1
for inner_offset, inner_size, inner_msg in inner_mset:
if msg.magic > 0:
# When magic value is greater than 0, the timestamp
# of a compressed message depends on the
# typestamp type of the wrapper message:
if msg.timestamp_type == 0: # CREATE_TIME (0)
inner_timestamp = inner_msg.timestamp
elif msg.timestamp_type == 1: # LOG_APPEND_TIME (1)
inner_timestamp = msg.timestamp
else:
raise ValueError('Unknown timestamp type: {0}'
.format(msg.timestamp_type))
else:
inner_timestamp = msg.timestamp
if absolute_base_offset >= 0:
inner_offset += absolute_base_offset
key, value = self._deserialize(inner_msg)
yield ConsumerRecord(
tp.topic, tp.partition, inner_offset,
inner_timestamp, msg.timestamp_type,
key, value, inner_msg.crc,
len(inner_msg.key)
if inner_msg.key is not None else -1,
len(inner_msg.value)
if inner_msg.value is not None else -1)
else:
key, value = self._deserialize(msg)
yield ConsumerRecord(
tp.topic, tp.partition, offset,
msg.timestamp, msg.timestamp_type,
key, value, msg.crc,
len(msg.key) if msg.key is not None else -1,
len(msg.value) if msg.value is not None else -1)
def _consumer_record(self, tp, record):
key_size = len(record.key) if record.key is not None else -1
value_size = \
len(record.value) if record.value is not None else -1
if self._key_deserializer:
key = self._key_deserializer(record.key)
else:
key = record.key
if self._value_deserializer:
value = self._value_deserializer(record.value)
else:
value = record.value
return ConsumerRecord(tp.topic, tp.partition, record.offset,
record.timestamp, record.timestamp_type, key,
value, record.checksum, key_size, value_size)
def _unpack_records(self, tp, records):
# NOTE: if the batch is not compressed it's equal to 1 record in
# v0 and v1.
deserialize = self._deserialize
check_crcs = self._check_crcs
while records.has_next():
next_batch = records.next_batch()
if check_crcs and not next_batch.validate_crc():
# This iterator will be closed after the exception, so we don't
# try to drain other batches here. They will be refetched.
raise Errors.CorruptRecordException("Invalid CRC")
for record in next_batch:
# Save encoded sizes
key_size = len(record.key) if record.key is not None else -1
value_size = \
len(record.value) if record.value is not None else -1
key, value = deserialize(record)
yield ConsumerRecord(tp.topic, tp.partition, record.offset,
record.timestamp, record.timestamp_type,
key, value, record.checksum, key_size,
value_size)
def to_consumer_record(self) -> ConsumerRecord:
# We need to convert the value back into bytes before giving this back to the consumer
data = self.dict()
data["value"] = data["value"].encode("utf-8")
data["headers"] = [(k, v.encode()) for k, v in data["headers"]]
return ConsumerRecord(**data) # type: ignore