def test_decode_offset_fetch_response(self): encoded = "".join([ struct.pack(">i", 42), # Correlation ID struct.pack(">i", 1), # One topics struct.pack(">h6s", 6, "topic1"), # First topic struct.pack(">i", 2), # Two partitions struct.pack(">i", 2), # Partition 2 struct.pack(">q", 4), # Offset 4 struct.pack(">h4s", 4, "meta"), # Metadata struct.pack(">h", 0), # No error struct.pack(">i", 4), # Partition 4 struct.pack(">q", 8), # Offset 8 struct.pack(">h4s", 4, "meta"), # Metadata struct.pack(">h", 0), # No error ]) results = KafkaProtocol.decode_offset_fetch_response(encoded) self.assertEqual( set(results), set([ OffsetFetchResponse(topic='topic1', partition=2, offset=4, error=0, metadata="meta"), OffsetFetchResponse(topic='topic1', partition=4, offset=8, error=0, metadata="meta"), ]))
def _send_offset_fetch_request_either(self, group, payloads, fail_on_error, callback): return [ callback( OffsetFetchResponse( req.topic, req.partition, self.group_offsets[req.topic].get(req.partition, -1), None, 0 if req.partition in self.group_offsets[req.topic] else 3), ) for req in payloads ]
def test_offset_distance_unknown_group(self, kafka_client_mock): with mock.patch.object( MyKafkaClient, 'send_offset_fetch_request', side_effect=lambda group, payloads, fail_on_error, callback: [ callback( OffsetFetchResponse(req.topic.decode(), req.partition, -1, None, 3)) for req in payloads ]): assert self.high_offsets['topic1'] == offset_distance( kafka_client_mock, 'derp', 'topic1', )
def pluck_topic_offset_or_zero_on_unknown(resp): try: check_error(resp) except UnknownTopicOrPartitionError: # If the server doesn't have any commited offsets by this group for # this topic, assume it's zero. pass # The API spec says server wont set an error, but 0.8.1.1 does. The actual # check is if the offset is -1. if resp.offset == -1: return OffsetFetchResponse( resp.topic, resp.partition, 0, resp.metadata, 0, ) return resp
def decode_offset_fetch_response(cls, data): """ Decode bytes to an OffsetFetchResponse Arguments: data: bytes to decode """ ((correlation_id,), cur) = relative_unpack('>i', data, 0) ((num_topics,), cur) = relative_unpack('>i', data, cur) for i in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) for i in range(num_partitions): ((partition, offset), cur) = relative_unpack('>iq', data, cur) (metadata, cur) = read_short_string(data, cur) ((error,), cur) = relative_unpack('>h', data, cur) yield OffsetFetchResponse(topic, partition, offset, metadata, error)
def mock_offset_fetch_request(group, payloads, **kwargs): return [OffsetFetchResponse(p.topic, p.partition, 0, b'', 0) for p in payloads]