def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numBrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numBrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topicMetadata = {} for i in range(num_topics): ((topicError,), cur) = relative_unpack('>h', data, cur) (topicName, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partitionMetadata = {} for j in range(num_partitions): ((partitionErrorCode, partition, leader, numReplicas), cur) = relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((numIsr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % numIsr, data, cur) partitionMetadata[partition] = PartitionMetadata(topicName, partition, leader, replicas, isr) topicMetadata[topicName] = partitionMetadata return (brokers, topicMetadata)
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = {} for i in range(num_topics): # NOTE: topic_error is discarded. Should probably be returned with # the topic metadata. ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = {} for j in range(num_partitions): # NOTE: partition_error_code is discarded. Should probably be # returned with the partition metadata. ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata[partition] = \ PartitionMetadata( topic_name, partition, leader, replicas, isr) topic_metadata[topic_name] = partition_metadata return brokers, topic_metadata
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port, ), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics, ), cur) = relative_unpack('>i', data, cur) topic_metadata = {} for i in range(num_topics): # NOTE: topic_error is discarded. Should probably be returned with # the topic metadata. ((topic_error, ), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) partition_metadata = {} for j in range(num_partitions): # NOTE: partition_error_code is discarded. Should probably be # returned with the partition metadata. ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((num_isr, ), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata[partition] = \ PartitionMetadata( topic_name, partition, leader, replicas, isr) topic_metadata[topic_name] = partition_metadata return brokers, topic_metadata
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = [] for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers.append(BrokerMetadata(nodeId, host, port)) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = [] for i in range(num_topics): ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = [] for j in range(num_partitions): ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata.append( PartitionMetadata(topic_name, partition, leader, replicas, isr, partition_error_code) ) topic_metadata.append( TopicMetadata(topic_name, topic_error, partition_metadata) ) return MetadataResponse(brokers, topic_metadata)
def parse_consumer_offset_message(self, message): key = bytearray(message.key) ((key_schema,), cur) = relative_unpack('>h', key, 0) if key_schema not in [0, 1]: raise InvalidMessageException() # This is not an offset commit message (group, cur) = read_short_string(key, cur) (topic, cur) = read_short_string(key, cur) ((partition,), cur) = relative_unpack('>l', key, cur) if message.value: value = bytearray(message.value) ((value_schema,), cur) = relative_unpack('>h', value, 0) if value_schema not in [0, 1]: raise InvalidMessageException() # Unrecognized message value ((offset,), cur) = relative_unpack('>q', value, cur) else: offset = None # Offset was deleted return str(group), str(topic), partition, offset
def parse_consumer_offset_message(self, message): key = message.key ((key_schema,), cur) = relative_unpack(b'>h', key, 0) if key_schema not in [0, 1]: raise InvalidMessageException() # This is not an offset commit message (group, cur) = read_short_string(key, cur) (topic, cur) = read_short_string(key, cur) ((partition,), cur) = relative_unpack(b'>l', key, cur) if message.value: value = message.value ((value_schema,), cur) = relative_unpack(b'>h', value, 0) if value_schema not in [0, 1]: raise InvalidMessageException() # Unrecognized message value ((offset,), cur) = relative_unpack(b'>q', value, cur) else: offset = None # Offset was deleted return group.decode(), topic.decode(), partition, offset
def decode_offset_commit_response(cls, data): """ Decode bytes to an OffsetCommitResponse Params ====== data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) (client_id, cur) = read_short_string(data, cur) ((num_topics,), cur) = relative_unpack('>i', data, cur) for i in xrange(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in xrange(num_partitions): ((partition, error), cur) = relative_unpack('>ih', data, cur) yield OffsetCommitResponse(topic, partition, error)
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numBrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numBrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port, ), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics, ), cur) = relative_unpack('>i', data, cur) topicMetadata = {} for i in range(num_topics): ((topicError, ), cur) = relative_unpack('>h', data, cur) (topicName, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) partitionMetadata = {} for j in range(num_partitions): ((partitionErrorCode, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((numIsr, ), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % numIsr, data, cur) partitionMetadata[partition] = \ PartitionMetadata(topicName, partition, leader, replicas, isr) topicMetadata[topicName] = partitionMetadata return (brokers, topicMetadata)
def decode_offset_commit_response(cls, data): """ Decode bytes to an OffsetCommitResponse Params ====== data: bytes to decode """ ((correlation_id, ), cur) = relative_unpack('>i', data, 0) (client_id, cur) = read_short_string(data, cur) ((num_topics, ), cur) = relative_unpack('>i', data, cur) for i in xrange(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) for i in xrange(num_partitions): ((partition, error), cur) = relative_unpack('>ih', data, cur) yield OffsetCommitResponse(topic, partition, error)
def decode_offset_fetch_response(cls, data): """ Decode bytes to an OffsetFetchResponse Params ====== data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) (client_id, cur) = read_short_string(data, cur) ((num_topics,), cur) = relative_unpack('>i', data, cur) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, offset), cur) = relative_unpack('>iq', data, cur) (metadata, cur) = read_short_string(data, cur) ((error,), cur) = relative_unpack('>h', data, cur) yield OffsetFetchResponse(topic, partition, offset, metadata, error)
def decode_consumer_metadata_response(cls, data): """ Decode bytes to a ConsumerMetadataResponse Arguments: data: bytes to decode """ ((correlation_id, error, nodeId), cur) = relative_unpack('>ihi', data, 0) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) return ConsumerMetadataResponse(error, nodeId, host, port)
def decode_offset_fetch_response(cls, data): """ Decode bytes to an OffsetFetchResponse Arguments: data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) ((num_topics,), cur) = relative_unpack('>i', data, cur) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, offset), cur) = relative_unpack('>iq', data, cur) (metadata, cur) = read_short_string(data, cur) ((error,), cur) = relative_unpack('>h', data, cur) yield OffsetFetchResponse(topic, partition, offset, metadata, error)
def decode_fetch_response(cls, data): """ Decode bytes to a FetchResponse Params ====== data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, error, highwater_mark_offset), cur) = relative_unpack('>ihq', data, cur) (message_set, cur) = read_int_string(data, cur) yield FetchResponse(topic, partition, error, highwater_mark_offset, KafkaProtocol._decode_message_set_iter(message_set))
def decode_offset_response(cls, data): """ Decode bytes to an OffsetResponse Params ====== data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, error, num_offsets,), cur) = relative_unpack('>ihi', data, cur) offsets = [] for j in range(num_offsets): ((offset,), cur) = relative_unpack('>q', data, cur) offsets.append(offset) yield OffsetResponse(topic, partition, error, tuple(offsets))
def decode_fetch_response(cls, data): """ Decode bytes to a FetchResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) for j in range(num_partitions): ((partition, error, highwater_mark_offset), cur) = \ relative_unpack('>ihq', data, cur) (message_set, cur) = read_int_string(data, cur) yield FetchResponse( topic, partition, error, highwater_mark_offset, KafkaProtocol._decode_message_set_iter(message_set))
def decode_offset_response(cls, data): """ Decode bytes to an OffsetResponse Arguments: data: bytes to decode """ ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, error, num_offsets,), cur) = \ relative_unpack('>ihi', data, cur) offsets = [] for j in range(num_offsets): ((offset,), cur) = relative_unpack('>q', data, cur) offsets.append(offset) yield OffsetResponse(topic, partition, error, tuple(offsets))