def decode_consumer_metadata_response(cls, data): """ Decode bytes to a ConsumerMetadataResponse Arguments: data: bytes to decode """ ((correlation_id, error, nodeId), cur) = relative_unpack('>ihi', data, 0) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) return ConsumerMetadataResponse(error, nodeId, host, port)
def _decode_message(cls, data, offset): """ Decode a single Message The only caller of this method is decode_message_set_iter. They are decoupled to support nested messages (compressed MessageSets). The offset is actually read from decode_message_set_iter (it is part of the MessageSet payload). """ ((crc, magic, att), cur) = relative_unpack('>IBB', data, 0) if crc != crc32(data[4:]): raise ChecksumError("Message checksum failed") (key, cur) = read_int_string(data, cur) (value, cur) = read_int_string(data, cur) codec = att & ATTRIBUTE_CODEC_MASK if codec == CODEC_NONE: yield (offset, Message(magic, att, key, value)) elif codec == CODEC_GZIP: gz = gzip_decode(value) for (offset, msg) in KafkaProtocol._decode_message_set_iter(gz): yield (offset, msg) elif codec == CODEC_SNAPPY: snp = snappy_decode(value) for (offset, msg) in KafkaProtocol._decode_message_set_iter(snp): yield (offset, msg)
def _decode_message_set_iter(cls, data): """ Iteratively decode a MessageSet Reads repeated elements of (offset, message), calling decode_message to decode a single message. Since compressed messages contain futher MessageSets, these two methods have been decoupled so that they may recurse easily. """ cur = 0 read_message = False while cur < len(data): try: ((offset, ), cur) = relative_unpack('>q', data, cur) (msg, cur) = read_int_string(data, cur) for (offset, message) in KafkaProtocol._decode_message(msg, offset): read_message = True yield OffsetAndMessage(offset, message) except BufferUnderflowError: # NOTE: Not sure this is correct error handling: # Is it possible to get a BUE if the message set is somewhere # in the middle of the fetch response? If so, we probably have # an issue that's not fetch size too small. # Aren't we ignoring errors if we fail to unpack data by # raising StopIteration()? # If _decode_message() raises a ChecksumError, couldn't that # also be due to the fetch size being too small? if read_message is False: # If we get a partial read of a message, but haven't # yielded anything there's a problem raise ConsumerFetchSizeTooSmall() else: raise StopIteration()
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Arguments: data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = [] for _ in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers.append(BrokerMetadata(nodeId, host, port)) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = [] for _ in range(num_topics): ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = [] for _ in range(num_partitions): ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata.append( PartitionMetadata(topic_name, partition, leader, replicas, isr, partition_error_code) ) topic_metadata.append( TopicMetadata(topic_name, topic_error, partition_metadata) ) return MetadataResponse(brokers, topic_metadata)
def decode_offset_commit_response(cls, data): """ Decode bytes to an OffsetCommitResponse Arguments: data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) ((num_topics,), cur) = relative_unpack('>i', data, cur) for _ in xrange(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for _ in xrange(num_partitions): ((partition, error), cur) = relative_unpack('>ih', data, cur) yield OffsetCommitResponse(topic, partition, error)
def decode_produce_response(cls, data): """ Decode bytes to a ProduceResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for _ in range(num_topics): ((strlen,), cur) = relative_unpack('>h', data, cur) topic = data[cur:cur + strlen] cur += strlen ((num_partitions,), cur) = relative_unpack('>i', data, cur) for _ in range(num_partitions): ((partition, error, offset), cur) = relative_unpack('>ihq', data, cur) yield ProduceResponse(topic, partition, error, offset)
def decode_offset_fetch_response(cls, data): """ Decode bytes to an OffsetFetchResponse Arguments: data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) ((num_topics,), cur) = relative_unpack('>i', data, cur) for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for _ in range(num_partitions): ((partition, offset), cur) = relative_unpack('>iq', data, cur) (metadata, cur) = read_short_string(data, cur) ((error,), cur) = relative_unpack('>h', data, cur) yield OffsetFetchResponse(topic, partition, offset, metadata, error)
def decode_fetch_response(cls, data): """ Decode bytes to a FetchResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) for j in range(num_partitions): ((partition, error, highwater_mark_offset), cur) = \ relative_unpack('>ihq', data, cur) (message_set, cur) = read_int_string(data, cur) yield FetchResponse( topic, partition, error, highwater_mark_offset, KafkaProtocol._decode_message_set_iter(message_set))
def decode_fetch_response(cls, data): """ Decode bytes to a FetchResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for j in range(num_partitions): ((partition, error, highwater_mark_offset), cur) = \ relative_unpack('>ihq', data, cur) (message_set, cur) = read_int_string(data, cur) yield FetchResponse( topic, partition, error, highwater_mark_offset, KafkaProtocol._decode_message_set_iter(message_set))
def decode_offset_response(cls, data): """ Decode bytes to an OffsetResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for _ in range(num_partitions): ((partition, error, num_offsets,), cur) = \ relative_unpack('>ihi', data, cur) offsets = [] for k in range(num_offsets): ((offset,), cur) = relative_unpack('>q', data, cur) offsets.append(offset) yield OffsetResponse(topic, partition, error, tuple(offsets))