def test_decode_offset_fetch_response(self):
encoded = "".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topics
struct.pack(">h6s", 6, "topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 4), # Offset 4
struct.pack(">h4s", 4, "meta"), # Metadata
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">q", 8), # Offset 8
struct.pack(">h4s", 4, "meta"), # Metadata
struct.pack(">h", 0), # No error
])
results = KafkaProtocol.decode_offset_fetch_response(encoded)
self.assertEqual(
set(results),
set([
OffsetFetchResponse(topic='topic1',
partition=2,
offset=4,
error=0,
metadata="meta"),
OffsetFetchResponse(topic='topic1',
partition=4,
offset=8,
error=0,
metadata="meta"),
]))
def _send_offset_fetch_request_either(self, group, payloads, fail_on_error,
callback):
return [
callback(
OffsetFetchResponse(
req.topic, req.partition,
self.group_offsets[req.topic].get(req.partition,
-1), None, 0 if
req.partition in self.group_offsets[req.topic] else 3), )
for req in payloads
]
def test_offset_distance_unknown_group(self, kafka_client_mock):
with mock.patch.object(
MyKafkaClient,
'send_offset_fetch_request',
side_effect=lambda group, payloads, fail_on_error, callback: [
callback(
OffsetFetchResponse(req.topic.decode(), req.partition,
-1, None, 3)) for req in payloads
]):
assert self.high_offsets['topic1'] == offset_distance(
kafka_client_mock,
'derp',
'topic1',
)
def pluck_topic_offset_or_zero_on_unknown(resp):
try:
check_error(resp)
except UnknownTopicOrPartitionError:
# If the server doesn't have any commited offsets by this group for
# this topic, assume it's zero.
pass
# The API spec says server wont set an error, but 0.8.1.1 does. The actual
# check is if the offset is -1.
if resp.offset == -1:
return OffsetFetchResponse(
resp.topic,
resp.partition,
0,
resp.metadata,
0,
)
return resp
def decode_offset_fetch_response(cls, data):
"""
Decode bytes to an OffsetFetchResponse
Arguments:
data: bytes to decode
"""
((correlation_id,), cur) = relative_unpack('>i', data, 0)
((num_topics,), cur) = relative_unpack('>i', data, cur)
for i in range(num_topics):
(topic, cur) = read_short_string(data, cur)
((num_partitions,), cur) = relative_unpack('>i', data, cur)
for i in range(num_partitions):
((partition, offset), cur) = relative_unpack('>iq', data, cur)
(metadata, cur) = read_short_string(data, cur)
((error,), cur) = relative_unpack('>h', data, cur)
yield OffsetFetchResponse(topic, partition, offset,
metadata, error)
def mock_offset_fetch_request(group, payloads, **kwargs):
return [OffsetFetchResponse(p.topic, p.partition, 0, b'', 0) for p in payloads]
