def _unpack_message_set(self, tp, messages, relative_offset=0): try: for offset, size, msg in messages: if self.config['check_crcs'] and not msg.validate_crc(): raise Errors.InvalidMessageError(msg) elif msg.is_compressed(): mset = msg.decompress() # new format uses relative offsets for compressed messages if msg.magic > 0: last_offset, _, _ = mset[-1] relative = offset - last_offset else: relative = 0 for record in self._unpack_message_set(tp, mset, relative): yield record else: # Message v1 adds timestamp if msg.magic > 0: timestamp = msg.timestamp timestamp_type = msg.timestamp_type else: timestamp = timestamp_type = None key, value = self._deserialize(msg) yield ConsumerRecord(tp.topic, tp.partition, offset + relative_offset, timestamp, timestamp_type, key, value) # If unpacking raises StopIteration, it is erroneously # caught by the generator. We want all exceptions to be raised # back to the user. See Issue 545 except StopIteration as e: log.exception('StopIteration raised unpacking messageset: %s', e) raise Exception('StopIteration raised unpacking messageset')
def _unpack_message_set(self, tp, messages): try: for offset, size, msg in messages: if self.config['check_crcs'] and not msg.validate_crc(): raise Errors.InvalidMessageError(msg) elif msg.is_compressed(): for record in self._unpack_message_set( tp, msg.decompress()): yield record else: key, value = self._deserialize(msg) yield ConsumerRecord(tp.topic, tp.partition, offset, key, value) # If unpacking raises StopIteration, it is erroneously # caught by the generator. We want all exceptions to be raised # back to the user. See Issue 545 except StopIteration as e: log.exception('StopIteration raised unpacking messageset: %s', e) raise Exception('StopIteration raised unpacking messageset')
def _unpack_message_set(self, tp, messages): try: for offset, size, msg in messages: if self.config['check_crcs'] and not msg.validate_crc(): raise Errors.InvalidMessageError(msg) elif msg.is_compressed(): # If relative offset is used, we need to decompress the entire message first to compute # the absolute offset. inner_mset = msg.decompress() # There should only ever be a single layer of compression if inner_mset[0][-1].is_compressed(): log.warning( 'MessageSet at %s offset %d appears ' ' double-compressed. This should not' ' happen -- check your producers!', tp, offset) if self.config['skip_double_compressed_messages']: log.warning( 'Skipping double-compressed message at' ' %s %d', tp, offset) continue if msg.magic > 0: last_offset, _, _ = inner_mset[-1] absolute_base_offset = offset - last_offset else: absolute_base_offset = -1 for inner_offset, inner_size, inner_msg in inner_mset: if msg.magic > 0: # When magic value is greater than 0, the timestamp # of a compressed message depends on the # typestamp type of the wrapper message: if msg.timestamp_type == 0: # CREATE_TIME (0) inner_timestamp = inner_msg.timestamp elif msg.timestamp_type == 1: # LOG_APPEND_TIME (1) inner_timestamp = msg.timestamp else: raise ValueError( 'Unknown timestamp type: {0}'.format( msg.timestamp_type)) else: inner_timestamp = msg.timestamp if absolute_base_offset >= 0: inner_offset += absolute_base_offset key, value = self._deserialize(inner_msg) yield ConsumerRecord( tp.topic, tp.partition, inner_offset, inner_timestamp, msg.timestamp_type, key, value, inner_msg.crc, len(inner_msg.key) if inner_msg.key is not None else -1, len(inner_msg.value) if inner_msg.value is not None else -1) else: key, value = self._deserialize(msg) yield ConsumerRecord( tp.topic, tp.partition, offset, msg.timestamp, msg.timestamp_type, key, value, msg.crc, len(msg.key) if msg.key is not None else -1, len(msg.value) if msg.value is not None else -1) # If unpacking raises StopIteration, it is erroneously # caught by the generator. We want all exceptions to be raised # back to the user. See Issue 545 except StopIteration as e: log.exception('StopIteration raised unpacking messageset: %s', e) raise Exception('StopIteration raised unpacking messageset')