def _unpack_message_set(self, tp, messages, relative_offset=0):
try:
for offset, size, msg in messages:
if self.config['check_crcs'] and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
mset = msg.decompress()
# new format uses relative offsets for compressed messages
if msg.magic > 0:
last_offset, _, _ = mset[-1]
relative = offset - last_offset
else:
relative = 0
for record in self._unpack_message_set(tp, mset, relative):
yield record
else:
# Message v1 adds timestamp
if msg.magic > 0:
timestamp = msg.timestamp
timestamp_type = msg.timestamp_type
else:
timestamp = timestamp_type = None
key, value = self._deserialize(msg)
yield ConsumerRecord(tp.topic, tp.partition,
offset + relative_offset, timestamp,
timestamp_type, key, value)
# If unpacking raises StopIteration, it is erroneously
# caught by the generator. We want all exceptions to be raised
# back to the user. See Issue 545
except StopIteration as e:
log.exception('StopIteration raised unpacking messageset: %s', e)
raise Exception('StopIteration raised unpacking messageset')
def _unpack_message_set(self, tp, messages):
try:
for offset, size, msg in messages:
if self.config['check_crcs'] and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
for record in self._unpack_message_set(
tp, msg.decompress()):
yield record
else:
key, value = self._deserialize(msg)
yield ConsumerRecord(tp.topic, tp.partition, offset, key,
value)
# If unpacking raises StopIteration, it is erroneously
# caught by the generator. We want all exceptions to be raised
# back to the user. See Issue 545
except StopIteration as e:
log.exception('StopIteration raised unpacking messageset: %s', e)
raise Exception('StopIteration raised unpacking messageset')
def _unpack_message_set(self, tp, messages):
try:
for offset, size, msg in messages:
if self.config['check_crcs'] and not msg.validate_crc():
raise Errors.InvalidMessageError(msg)
elif msg.is_compressed():
# If relative offset is used, we need to decompress the entire message first to compute
# the absolute offset.
inner_mset = msg.decompress()
# There should only ever be a single layer of compression
if inner_mset[0][-1].is_compressed():
log.warning(
'MessageSet at %s offset %d appears '
' double-compressed. This should not'
' happen -- check your producers!', tp, offset)
if self.config['skip_double_compressed_messages']:
log.warning(
'Skipping double-compressed message at'
' %s %d', tp, offset)
continue
if msg.magic > 0:
last_offset, _, _ = inner_mset[-1]
absolute_base_offset = offset - last_offset
else:
absolute_base_offset = -1
for inner_offset, inner_size, inner_msg in inner_mset:
if msg.magic > 0:
# When magic value is greater than 0, the timestamp
# of a compressed message depends on the
# typestamp type of the wrapper message:
if msg.timestamp_type == 0: # CREATE_TIME (0)
inner_timestamp = inner_msg.timestamp
elif msg.timestamp_type == 1: # LOG_APPEND_TIME (1)
inner_timestamp = msg.timestamp
else:
raise ValueError(
'Unknown timestamp type: {0}'.format(
msg.timestamp_type))
else:
inner_timestamp = msg.timestamp
if absolute_base_offset >= 0:
inner_offset += absolute_base_offset
key, value = self._deserialize(inner_msg)
yield ConsumerRecord(
tp.topic, tp.partition, inner_offset,
inner_timestamp, msg.timestamp_type, key, value,
inner_msg.crc,
len(inner_msg.key)
if inner_msg.key is not None else -1,
len(inner_msg.value)
if inner_msg.value is not None else -1)
else:
key, value = self._deserialize(msg)
yield ConsumerRecord(
tp.topic, tp.partition, offset, msg.timestamp,
msg.timestamp_type, key, value, msg.crc,
len(msg.key) if msg.key is not None else -1,
len(msg.value) if msg.value is not None else -1)
# If unpacking raises StopIteration, it is erroneously
# caught by the generator. We want all exceptions to be raised
# back to the user. See Issue 545
except StopIteration as e:
log.exception('StopIteration raised unpacking messageset: %s', e)
raise Exception('StopIteration raised unpacking messageset')
