def _unpack_message_set(self, tp, messages):
for offset, size, msg in messages:
if self._check_crcs and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
yield from self._unpack_message_set(tp, msg.decompress())
else:
key, value = self._deserialize(msg)
yield ConsumerRecord(tp.topic, tp.partition, offset, key,
value)
def _unpack_message_set(self, tp, messages):
for offset, size, msg in messages:
if self._check_crcs and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
# If relative offset is used, we need to decompress the entire
# message first to compute the absolute offset.
inner_mset = msg.decompress()
if msg.magic > 0:
last_offset, _, _ = inner_mset[-1]
absolute_base_offset = offset - last_offset
else:
absolute_base_offset = -1
for inner_offset, inner_size, inner_msg in inner_mset:
if msg.magic > 0:
# When magic value is greater than 0, the timestamp
# of a compressed message depends on the
# typestamp type of the wrapper message:
if msg.timestamp_type == 0: # CREATE_TIME (0)
inner_timestamp = inner_msg.timestamp
elif msg.timestamp_type == 1: # LOG_APPEND_TIME (1)
inner_timestamp = msg.timestamp
else:
raise ValueError(
'Unknown timestamp type: {0}'.format(
msg.timestamp_type))
else:
inner_timestamp = msg.timestamp
if absolute_base_offset >= 0:
inner_offset += absolute_base_offset
key, value = self._deserialize(inner_msg)
yield ConsumerRecord(
tp.topic, tp.partition, inner_offset, inner_timestamp,
msg.timestamp_type, key, value, inner_msg.crc,
len(inner_msg.key)
if inner_msg.key is not None else -1,
len(inner_msg.value)
if inner_msg.value is not None else -1)
else:
key, value = self._deserialize(msg)
yield ConsumerRecord(
tp.topic, tp.partition, offset, msg.timestamp,
msg.timestamp_type, key, value, msg.crc,
len(msg.key) if msg.key is not None else -1,
len(msg.value) if msg.value is not None else -1)
def _unpack_message_set(self, tp, messages):
for offset, size, msg in messages:
if self.config['check_crcs'] and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
for record in self._unpack_message_set(tp, msg.decompress()):
yield record
else:
try:
key, value = self._deserialize(msg)
# If the deserializer raises StopIteration, it is erroneously
# caught by the generator. We want all exceptions to be raised
# back to the user. See Issue 545
except StopIteration as e:
log.exception('Deserializer raised StopIteration: %s', e)
raise Exception('Deserializer raised StopIteration')
yield ConsumerRecord(tp.topic, tp.partition, offset, key,
value)
