def test_makeRequest(self):
id1 = 54321
id2 = 76543
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testmakeRequest', 9092, 'clientId')
request = KafkaCodec.encode_fetch_request(b'testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testmakeRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
fail1 = eb1.call_args[0][0] # The actual failure sent to errback
self.assertTrue(fail1.check(CancelledError))
def test_decode_fetch_response(self):
t1 = "topic1"
t2 = "topic2"
msgs = map(create_message, ["message1", "hi", "boo", "foo", "so fun!"])
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
encoded = struct.pack(
'>iih%dsiihqi%dsihqi%dsh%dsiihqi%ds' %
(len(t1), len(ms1), len(ms2), len(t2), len(ms3)), 4, 2,
len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1, 1, 20, len(ms2), ms2,
len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
responses = list(KafkaCodec.decode_fetch_response(encoded))
def expand_messages(response):
return FetchResponse(response.topic, response.partition,
response.error, response.highwaterMark,
list(response.messages))
expanded_responses = map(expand_messages, responses)
expect = [
FetchResponse(
t1, 0, 0, 10,
[OffsetAndMessage(0, msgs[0]),
OffsetAndMessage(0, msgs[1])]),
FetchResponse(t1, 1, 1, 20, [OffsetAndMessage(0, msgs[2])]),
FetchResponse(
t2, 0, 0, 30,
[OffsetAndMessage(0, msgs[3]),
OffsetAndMessage(0, msgs[4])])
]
self.assertEqual(expanded_responses, expect)
def test_makeRequest(self):
id1 = 54321
id2 = 76543
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testmakeRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
fail1 = eb1.call_args[0][0] # The actual failure sent to errback
self.assertTrue(fail1.check(CancelledError))
def test_decode_fetch_response(self):
t1 = "topic1"
t2 = u"topic2"
msgs = [create_message(m) for m in [b"message1", b"hi", b"boo", b"foo", b"so fun!"]]
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
encoded = struct.pack('>iih%dsiihqi%dsihqi%dsh%dsiihqi%ds' %
(len(t1), len(ms1), len(ms2), len(t2), len(ms3)),
4, 2, len(t1), t1.encode(), 2, 0, 0, 10, len(ms1), ms1, 1,
1, 20, len(ms2), ms2, len(t2), t2.encode(), 1, 0, 0, 30,
len(ms3), ms3)
responses = list(KafkaCodec.decode_fetch_response(encoded))
def expand_messages(response):
return FetchResponse(response.topic, response.partition,
response.error, response.highwaterMark,
list(response.messages))
expanded_responses = [expand_messages(r) for r in responses]
expect = [FetchResponse(t1, 0, 0, 10, [OffsetAndMessage(0, msgs[0]),
OffsetAndMessage(0, msgs[1])]),
FetchResponse(t1, 1, 1, 20, [OffsetAndMessage(0, msgs[2])]),
FetchResponse(t2, 0, 0, 30, [OffsetAndMessage(0, msgs[3]),
OffsetAndMessage(0, msgs[4])])]
self.assertEqual(expanded_responses, expect)
def test_encode_produce_request(self):
requests = [
ProduceRequest("topic1", 0, [
create_message((-1, b"a")),
create_message((-1, b"b")),
]),
ProduceRequest(u"topic2", 1, [
create_message((-1, b"c")),
]),
]
msg_a_binary = KafkaCodec._encode_message(create_message((-1, b"a")))
msg_b_binary = KafkaCodec._encode_message(create_message((-1, b"b")))
msg_c_binary = KafkaCodec._encode_message(create_message((-1, b"c")))
header = b"".join([
struct.pack('>h', 0), # Msg Header, Message type = Produce
struct.pack('>h', 1), # Msg Header, API version
struct.pack('>i', 2), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>h', 2), # Num acks required
struct.pack('>i', 100), # Request Timeout
struct.pack('>i', 2), # The number of requests
])
total_len = len(msg_a_binary) + len(msg_b_binary)
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 0), # Partition 0
struct.pack('>i', total_len + 24), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_a_binary)), # Length of message
msg_a_binary, # Actual message
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_b_binary)), # Length of message
msg_b_binary, # Actual message
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 1), # Partition 1
struct.pack('>i',
len(msg_c_binary) + 12), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_c_binary)), # Length of message
msg_c_binary, # Actual message
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_produce_request(b"client1", 2, requests, 2,
100)
self.assertIn(encoded, [expected1, expected2])
def test_encode_produce_request(self):
requests = [
ProduceRequest("topic1", 0, [
create_message(b"a"),
create_message(b"b"),
]),
ProduceRequest(u"topic2", 1, [
create_message(b"c"),
]),
]
msg_a_binary = KafkaCodec._encode_message(create_message(b"a"))
msg_b_binary = KafkaCodec._encode_message(create_message(b"b"))
msg_c_binary = KafkaCodec._encode_message(create_message(b"c"))
header = b"".join([
struct.pack('>h', 0), # Msg Header, Message type = Produce
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 2), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>h', 2), # Num acks required
struct.pack('>i', 100), # Request Timeout
struct.pack('>i', 2), # The number of requests
])
total_len = len(msg_a_binary) + len(msg_b_binary)
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 0), # Partition 0
struct.pack('>i', total_len + 24), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_a_binary)), # Length of message
msg_a_binary, # Actual message
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_b_binary)), # Length of message
msg_b_binary, # Actual message
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 1), # Partition 1
struct.pack('>i', len(msg_c_binary) + 12), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_c_binary)), # Length of message
msg_c_binary, # Actual message
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_produce_request(
b"client1", 2, requests, 2, 100)
self.assertIn(encoded, [expected1, expected2])
def make_fetch_response(id):
t1 = "topic1"
t2 = "topic2"
msgs = map(create_message,
["message1", "hi", "boo", "foo", "so fun!"])
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
len(t1), len(ms1), len(ms2), len(t2), len(ms3))
return struct.pack(packFmt, id, 2, len(t1), t1, 2, 0, 0, 10,
len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2),
t2, 1, 0, 0, 30, len(ms3), ms3)
def make_fetch_response(id):
t1 = "topic1"
t2 = "topic2"
msgs = map(
create_message, ["message1", "hi", "boo", "foo", "so fun!"])
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
len(t1), len(ms1), len(ms2), len(t2), len(ms3))
return struct.pack(
packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1,
1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
def test_decode_metadata_response(self):
node_brokers = {
0: BrokerMetadata(0, "brokers1.afkak.rdio.com", 1000),
1: BrokerMetadata(1, "brokers1.afkak.rdio.com", 1001),
3: BrokerMetadata(3, "brokers2.afkak.rdio.com", 1000),
}
topic_partitions = {
"topic1":
TopicMetadata(
'topic1',
0,
{
0: PartitionMetadata(u"topic1", 0, 0, 1, (0, 2), (2, )),
1: PartitionMetadata("topic1", 1, 1, 3, (0, 1), (0, 1)),
},
),
u"topic2":
TopicMetadata(
u'topic2',
1,
{
0: PartitionMetadata(u"topic2", 0, 0, 0, (), ()),
},
),
}
encoded = create_encoded_metadata_response(node_brokers,
topic_partitions)
decoded = KafkaCodec.decode_metadata_response(encoded)
self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_makeUnconnectedRequest(self):
"""
Ensure that sending a request when not connected will attempt to bring
up a connection if one isn't already in the process of being brought up
"""
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testmakeUnconnectedRequest', 9092,
'clientId')
request = KafkaCodec.encode_fetch_request(
b'testmakeUnconnectedRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(1.0)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
def test_decode_offset_fetch_response(self):
encoded = b"".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topics
struct.pack(">h6s", 6, b"topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 4), # Offset 4
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">q", 8), # Offset 8
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
])
results = KafkaCodec.decode_offset_fetch_response(encoded)
self.assertEqual(set(results), set([
OffsetFetchResponse(topic=u'topic1', partition=2, offset=4,
error=0, metadata=b"meta"),
OffsetFetchResponse(topic='topic1', partition=4, offset=8,
error=0, metadata=b"meta"),
]))
def test_encode_offset_fetch_request(self):
header = b"".join([
struct.pack('>h', 9), # Message type = offset fetch
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">i", 1), # Partition 1
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partitions
struct.pack(">i", 2), # Partition 2
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_fetch_request(
b"client_id", 42, b"group_id", [
OffsetFetchRequest(b"topic1", 0),
OffsetFetchRequest(b"topic1", 1),
OffsetFetchRequest(b"topic2", 2),
])
self.assertIn(encoded, [expected1, expected2])
def test_decode_offset_fetch_response(self):
encoded = b"".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topics
struct.pack(">h6s", 6, b"topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 4), # Offset 4
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">q", 8), # Offset 8
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
])
results = KafkaCodec.decode_offset_fetch_response(encoded)
self.assertEqual(
set(results),
set([
OffsetFetchResponse(topic=b'topic1',
partition=2,
offset=4,
error=0,
metadata=b"meta"),
OffsetFetchResponse(topic=b'topic1',
partition=4,
offset=8,
error=0,
metadata=b"meta"),
]))
def test_encode_message_set(self):
message_set = [
create_message((-1, b"v1"), b"k1"),
create_message((-1, b"v2"), b"k2"),
]
encoded = KafkaCodec._encode_message_set(message_set)
expect = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 26), # Msg Size
struct.pack(">i", -634178223), # CRC
struct.pack(">bbq", 1, 0, -1), # Magic, flags, timestamp
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 26), # Msg Size
struct.pack(">i", 1926397558), # CRC
struct.pack(">bbq", 1, 0, -1), # Magic, flags, timestamp
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
])
self.assertEqual(encoded, expect)
def test_encode_offset_fetch_request(self):
header = b"".join([
struct.pack('>h', 9), # Message type = offset fetch
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">i", 1), # Partition 1
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partitions
struct.pack(">i", 2), # Partition 2
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_fetch_request(
b"client_id", 42, u"group_id", [
OffsetFetchRequest("topic1", 0),
OffsetFetchRequest(u"topic1", 1),
OffsetFetchRequest("topic2", 2),
])
self.assertIn(encoded, [expected1, expected2])
def test_makeUnconnectedRequest(self):
""" test_makeUnconnectedRequest
Ensure that sending a request when not connected will attempt to bring
up a connection if one isn't already in the process of being brought up
"""
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeUnconnectedRequest',
reactor=reactor)
request = KafkaCodec.encode_fetch_request(
'testmakeUnconnectedRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(1.0)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
def test_decode_message_set_stop_iteration(self):
encoded = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 1), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
b"@1$%(Y!", # Random padding
])
msgs = list(KafkaCodec._decode_message_set_iter(encoded))
self.assertEqual(len(msgs), 2)
msg1, msg2 = msgs
returned_offset1, decoded_message1 = msg1
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message(b"v1", b"k1"))
self.assertEqual(returned_offset2, 1)
self.assertEqual(decoded_message2, create_message(b"v2", b"k2"))
def test_encode_offset_request__no_payload(self):
expected = b"".join([
struct.pack('>h', 2), # Message type = offset fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 1), # Num topics
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 3), # Partition 3
struct.pack(">q", -1), # No time offset
struct.pack(">i", 1), # One offset requested
struct.pack(">i", 4), # Partition 3
struct.pack(">q", -1), # No time offset
struct.pack(">i", 1), # One offset requested
])
encoded = KafkaCodec.encode_offset_request(b"cid", 4, [
OffsetRequest('topic1', 3, -1, 1),
OffsetRequest(u'topic1', 4, -1, 1),
])
self.assertEqual(encoded, expected)
def test_decode_message_set_stop_iteration(self):
encoded = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 1), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
b"@1$%(Y!", # Random padding
])
msgs = list(KafkaCodec._decode_message_set_iter(encoded))
self.assertEqual(len(msgs), 2)
msg1, msg2 = msgs
returned_offset1, decoded_message1 = msg1
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message(b"v1", b"k1"))
self.assertEqual(returned_offset2, 1)
self.assertEqual(decoded_message2, create_message(b"v2", b"k2"))
def test_encode_message_set(self):
message_set = [
create_message(b"v1", b"k1"),
create_message(b"v2", b"k2")
]
encoded = KafkaCodec._encode_message_set(message_set)
expect = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
])
self.assertEqual(encoded, expect)
def test_encode_message_set(self):
message_set = [
create_message(b"v1", b"k1"),
create_message(b"v2", b"k2"),
]
encoded = KafkaCodec._encode_message_set(message_set)
expect = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
])
self.assertEqual(encoded, expect)
def test_decode_offset_response(self):
encoded = b"".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topics
struct.pack(">h6s", 6, b"topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">h", 0), # No error
struct.pack(">i", 1), # One offset
struct.pack(">q", 4), # Offset 4
struct.pack(">i", 4), # Partition 4
struct.pack(">h", 0), # No error
struct.pack(">i", 1), # One offset
struct.pack(">q", 8), # Offset 8
])
results = KafkaCodec.decode_offset_response(encoded)
self.assertEqual(
set(results),
set([
OffsetResponse(topic=b'topic1',
partition=2,
error=0,
offsets=(4, )),
OffsetResponse(topic=b'topic1',
partition=4,
error=0,
offsets=(8, )),
]))
def test_get_response_correlation_id(self):
t1 = b"topic1"
t2 = b"topic2"
corrID = 30372
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)),
corrID, 2, len(t1), t1, 2, 0, 0, 10, 1, 1, 20,
len(t2), t2, 1, 0, 0, 30)
self.assertEqual(
corrID, KafkaCodec.get_response_correlation_id(encoded))
def test_get_response_correlation_id(self):
t1 = b"topic1"
t2 = b"topic2"
corrID = 30372
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)),
corrID, 2, len(t1), t1, 2, 0, 0, 10, 1, 1, 20,
len(t2), t2, 1, 0, 0, 30)
self.assertEqual(corrID,
KafkaCodec.get_response_correlation_id(encoded))
def generate_assignments(self, members, topic_partitions):
"""
Assign topic partitions to members.
This is called on the leader once all group members have joined.
:param members:
Member join requests, as returned by
:meth:`join_group_protocols()`. These requests encode the topics
members are interested in.
:type members: List[_JoinGroupResponseMember]
:param topic_partitions: mapping of topic names to partition IDs
:type topic_partitions: Mapping[str, List[int]]
:returns: Member assignments.
:rtype: List[_SyncGroupRequestMember]
:raises _NeedTopicPartitions: when *topic_partitions* doesn't contain
partitions for a topic one of the members has requested. The
*topics* attribute indicates the topics that must be loaded for
assignment to succeed.
"""
member_metadata = {}
for member in members:
member_metadata[
member.
member_id] = KafkaCodec.decode_join_group_protocol_metadata(
member.member_metadata)
assignments = self._round_robin_assignment(member_metadata,
topic_partitions)
log.debug("%s: generate_assignments %r", self, assignments)
encoded_assignments = []
for member in members:
encoded = KafkaCodec.encode_sync_group_member_assignment(
version=0,
assignments=assignments.get(member.member_id, {}),
user_data=b'',
)
encoded_assignments.append(
_SyncGroupRequestMember(member.member_id, encoded))
return encoded_assignments
def decode_assignment(self, assignment):
"""
Decode a topic partition assignment from the leader.
:returns: Map of topic name to partition IDs.
:rtype: Map[str, Tuple[int]]
"""
assignment = KafkaCodec.decode_sync_group_member_assignment(assignment)
log.debug("decode_assignment: assignment=%r", assignment)
return assignment.assignments
def test_encode_message_header(self):
expect = b"".join([
struct.pack(">h", 10), # API Key
struct.pack(">h", 0), # API Version
struct.pack(">i", 4), # Correlation Id
struct.pack(">h", len("client3")), # Length of clientId
b"client3", # ClientId
])
encoded = KafkaCodec._encode_message_header(b"client3", 4, 10)
self.assertEqual(encoded, expect)
def test_decode_consumermetadata_response(self):
expected = ConsumerMetadataResponse(0, 5, "theHost", 4242)
encoded = "".join([
struct.pack('>i', 9), # Correlation ID
struct.pack('>h', 0), # Error Code
struct.pack('>i', 5), # Coordinator ID
struct.pack('>h7s', 7, "theHost"), # Coordinator Host
struct.pack('>i', 4242), # Coordinator port
])
decoded = KafkaCodec.decode_consumermetadata_response(encoded)
self.assertEqual(decoded, expected)
def test_encode_message_header(self):
expect = b"".join([
struct.pack(">h", 10), # API Key
struct.pack(">h", 0), # API Version
struct.pack(">i", 4), # Correlation Id
struct.pack(">h", len("client3")), # Length of clientId
b"client3", # ClientId
])
encoded = KafkaCodec._encode_message_header(b"client3", 4, 10)
self.assertEqual(encoded, expect)
def test_decode_consumermetadata_response(self):
expected = ConsumerMetadataResponse(0, 5, "theHost", 4242)
encoded = b"".join([
struct.pack('>i', 9), # Correlation ID
struct.pack('>h', 0), # Error Code
struct.pack('>i', 5), # Coordinator ID
struct.pack('>h', len(b"theHost")), b"theHost", # Coordinator Host
struct.pack('>i', 4242), # Coordinator port
])
decoded = KafkaCodec.decode_consumermetadata_response(encoded)
self.assertEqual(decoded, expected)
def test_decode_produce_response(self):
t1 = "topic1"
t2 = u"topic2"
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)),
2, 2, len(t1), t1.encode(), 2, 0, 0, 10, 1, 1, 20,
len(t2), t2.encode(), 1, 0, 0, 30)
responses = list(KafkaCodec.decode_produce_response(encoded))
self.assertEqual(responses,
[ProduceResponse(t1, 0, 0, 10),
ProduceResponse(t1, 1, 1, 20),
ProduceResponse(t2, 0, 0, 30)])
def test_decode_produce_response(self):
t1 = b"topic1"
t2 = b"topic2"
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)), 2,
2, len(t1), t1, 2, 0, 0, 10, 1, 1, 20, len(t2),
t2, 1, 0, 0, 30)
responses = list(KafkaCodec.decode_produce_response(encoded))
self.assertEqual(responses, [
ProduceResponse(t1, 0, 0, 10),
ProduceResponse(t1, 1, 1, 20),
ProduceResponse(t2, 0, 0, 30)
])
def test_encode_metadata_request_no_topics(self):
expected = b"".join([
struct.pack('>h', 3), # API key metadata fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', 0), # No topics, give all the data!
])
encoded = KafkaCodec.encode_metadata_request(b"cid", 4)
self.assertEqual(encoded, expected)
def test_decode_produce_response(self):
t1 = "topic1"
t2 = u"topic2"
encoded = struct.pack('>iih%dsiihqlihqlh%dsiihql' % (len(t1), len(t2)),
2, 2, len(t1), t1.encode(), 2, 0, 0, 10, 4, 1, 1,
20, 5, len(t2), t2.encode(), 1, 0, 0, 30, 6)
responses = list(KafkaCodec.decode_produce_response(encoded))
self.assertEqual(responses, [
ProduceResponse(t1, 0, 0, 10, 4),
ProduceResponse(t1, 1, 1, 20, 5),
ProduceResponse(t2, 0, 0, 30, 6)
])
def test_encode_metadata_request_no_topics(self):
expected = b"".join([
struct.pack('>h', 3), # API key metadata fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', 0), # No topics, give all the data!
])
encoded = KafkaCodec.encode_metadata_request(b"cid", 4)
self.assertEqual(encoded, expected)
def test_encode_consumer_metadata_request(self):
expected = b"".join([
struct.pack('>h', 10), # API key ConsumerMetadataRequest
struct.pack('>h', 0), # API version
struct.pack('>i', 9), # Correlation ID
struct.pack('>h3s', 3, b"cID"), # The client ID
struct.pack('>h6s', 6, b"group1"), # Consumer group 'group1'
])
encoded = KafkaCodec.encode_consumermetadata_request(b"cID", 9, u"group1")
self.assertEqual(encoded, expected)
def test_encode_message(self):
message = create_message(b"test", b"key")
encoded = KafkaCodec._encode_message(message)
expect = b"".join([
struct.pack(">i", -1427009701), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 3), # Length of key
b"key", # key
struct.pack(">i", 4), # Length of value
b"test", # value
])
self.assertEqual(encoded, expect)
def test_encode_offset_request(self):
expected = b"".join([
struct.pack('>h', 2), # Message type = offset fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 0), # No topic/partitions
])
encoded = KafkaCodec.encode_offset_request(b"cid", 4)
self.assertEqual(encoded, expected)
def test_encode_consumer_metadata_request(self):
expected = b"".join([
struct.pack('>h', 10), # API key ConsumerMetadataRequest
struct.pack('>h', 0), # API version
struct.pack('>i', 9), # Correlation ID
struct.pack('>h3s', 3, b"cID"), # The client ID
struct.pack('>h6s', 6, b"group1"), # Consumer group 'group1'
])
encoded = KafkaCodec.encode_consumermetadata_request(
b"cID", 9, b"group1")
self.assertEqual(encoded, expected)
def test_encode_message(self):
message = create_message(b"test", b"key")
encoded = KafkaCodec._encode_message(message)
expect = b"".join([
struct.pack(">i", -1427009701), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 3), # Length of key
b"key", # key
struct.pack(">i", 4), # Length of value
b"test", # value
])
self.assertEqual(encoded, expect)
def test_encode_message(self):
message = create_message((47, b"test"), b"key", magic=1)
encoded = KafkaCodec._encode_message(message)
expect = b"".join([
struct.pack(">i", -1232077792), # CRC
struct.pack(">bbq", 1, 0, 47), # Magic, flags, timestamp
struct.pack(">i", 3), # Length of key
b"key", # key
struct.pack(">i", 4), # Length of value
b"test", # value
])
self.assertEqual(encoded, expect)
def test_encode_offset_request(self):
expected = b"".join([
struct.pack('>h', 2), # Message type = offset fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 0), # No topic/partitions
])
encoded = KafkaCodec.encode_offset_request(b"cid", 4)
self.assertEqual(encoded, expected)
def test_encode_metadata_request_with_topics(self):
expected = b"".join([
struct.pack('>h', 3), # API key metadata fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', 2), # Number of topics in the request
struct.pack('>h2s', 2, b"t1"), # Topic "t1"
struct.pack('>h2s', 2, b"t2"), # Topic "t2"
])
encoded = KafkaCodec.encode_metadata_request(b"cid", 4, [b"t1", b"t2"])
self.assertEqual(encoded, expected)
def test_encode_metadata_request_with_topics(self):
expected = b"".join([
struct.pack('>h', 3), # API key metadata fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', 2), # Number of topics in the request
struct.pack('>h2s', 2, b"t1"), # Topic "t1"
struct.pack('>h2s', 2, b"t2"), # Topic "t2"
])
encoded = KafkaCodec.encode_metadata_request(b"cid", 4, [u"t1", "t2"])
self.assertEqual(encoded, expected)
def test_decode_message(self):
encoded = b"".join([
struct.pack(">i", -1427009701), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 3), # Length of key
b"key", # key
struct.pack(">i", 4), # Length of value
b"test", # value
])
offset = 10
(returned_offset, decoded_message) = list(
KafkaCodec._decode_message(encoded, offset))[0]
self.assertEqual(returned_offset, offset)
self.assertEqual(decoded_message, create_message(b"test", b"key"))
def test_encode_offset_commit_request(self):
header = b"".join([
struct.pack('>h', 8), # Message type = offset commit
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 996), # Group generation ID
struct.pack('>h11s', 11, b'consumer_id'), # Consumer ID
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">q", 123), # Offset 123
struct.pack(">q", 1437585816816), # Timestamp in ms > epoch
struct.pack(">h", -1), # Null metadata
struct.pack(">i", 1), # Partition 1
struct.pack(">q", 234), # Offset 234
struct.pack(">q", 1436981054199), # Timestamp in ms > epoch
struct.pack(">h11s", 11, b'My_Metadata'), # Null metadata
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partition
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 345), # Offset 345
struct.pack(">q", -1), # Timestamp 'invalid-time'
struct.pack(">h", -1), # Null metadata
])
# A dict is used, so we can't predict the order of the topics...
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_commit_request(
b"client_id", 42, b"group_id", 996, b'consumer_id', [
OffsetCommitRequest(b"topic1", 0, 123, 1437585816816, None),
OffsetCommitRequest(b"topic1", 1, 234, 1436981054199,
b'My_Metadata'),
OffsetCommitRequest(b"topic2", 2, 345, -1, None),
])
self.assertIn(encoded, [expected1, expected2])
def test_encode_offset_commit_request(self):
header = b"".join([
struct.pack('>h', 8), # Message type = offset commit
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 996), # Group generation ID
struct.pack('>h11s', 11, b'consumer_id'), # Consumer ID
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">q", 123), # Offset 123
struct.pack(">q", 1437585816816), # Timestamp in ms > epoch
struct.pack(">h", -1), # Null metadata
struct.pack(">i", 1), # Partition 1
struct.pack(">q", 234), # Offset 234
struct.pack(">q", 1436981054199), # Timestamp in ms > epoch
struct.pack(">h11s", 11, b'My_Metadata'), # Null metadata
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partition
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 345), # Offset 345
struct.pack(">q", -1), # Timestamp 'invalid-time'
struct.pack(">h", -1), # Null metadata
])
# A dict is used, so we can't predict the order of the topics...
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_commit_request(
b"client_id", 42, u"group_id", 996, u'consumer_id', [
OffsetCommitRequest("topic1", 0, 123, 1437585816816, None),
OffsetCommitRequest(u"topic1", 1, 234, 1436981054199,
b'My_Metadata'),
OffsetCommitRequest(u"topic2", 2, 345, -1, None),
])
self.assertIn(encoded, [expected1, expected2])
def test_cancelRequestNoReply(self):
id2 = 87654
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'test_connect', 9092, 'clientId')
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testcancelRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# This one we cancel by ID. It should fail due to the
# expectResponse=False, since we don't keep the requestID after send
self.assertRaises(KeyError, c.cancelRequest, id2)
def test_decode_message(self):
encoded = b"".join([
struct.pack(">i", -1427009701), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 3), # Length of key
b"key", # key
struct.pack(">i", 4), # Length of value
b"test", # value
])
offset = 10
(returned_offset,
decoded_message) = list(KafkaCodec._decode_message(encoded,
offset))[0]
self.assertEqual(returned_offset, offset)
self.assertEqual(decoded_message, create_message(b"test", b"key"))
def test_cancelRequestNoReply(self):
id2 = 87654
reactor = MemoryReactorClock()
c = KafkaBrokerClient('test_connect', reactor=reactor)
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testcancelRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# This one we cancel by ID. It should fail due to the
# expectResponse=False, since we don't keep the requestID after send
self.assertRaises(KeyError, c.cancelRequest, id2)
def test_decode_message_gzip(self):
gzip_encoded = ('\xc0\x11\xb2\xf0\x00\x01\xff\xff\xff\xff\x00\x00\x000'
'\x1f\x8b\x08\x00\xa1\xc1\xc5R\x02\xffc`\x80\x03\x01'
'\x9f\xf9\xd1\x87\x18\x18\xfe\x03\x01\x90\xc7Tf\xc8'
'\x80$wu\x1aW\x05\x92\x9c\x11\x00z\xc0h\x888\x00\x00'
'\x00')
offset = 11
messages = list(KafkaCodec._decode_message(gzip_encoded, offset))
self.assertEqual(len(messages), 2)
msg1, msg2 = messages
returned_offset1, decoded_message1 = msg1
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message("v1"))
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset2, 0)
self.assertEqual(decoded_message2, create_message("v2"))
def test_decode_offset_commit_response(self):
encoded = "".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topic
struct.pack(">h6s", 6, "topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">h", 0), # No error
])
results = KafkaCodec.decode_offset_commit_response(encoded)
self.assertEqual(
set(results),
set([
OffsetCommitResponse(topic='topic1', partition=2, error=0),
OffsetCommitResponse(topic='topic1', partition=4, error=0),
]))
def test_decode_message_gzip(self):
gzip_encoded = (b'\xc0\x11\xb2\xf0\x00\x01\xff\xff\xff\xff\x00\x00\x000'
b'\x1f\x8b\x08\x00\xa1\xc1\xc5R\x02\xffc`\x80\x03\x01'
b'\x9f\xf9\xd1\x87\x18\x18\xfe\x03\x01\x90\xc7Tf\xc8'
b'\x80$wu\x1aW\x05\x92\x9c\x11\x00z\xc0h\x888\x00\x00'
b'\x00')
offset = 11
messages = list(KafkaCodec._decode_message(gzip_encoded, offset))
self.assertEqual(len(messages), 2)
msg1, msg2 = messages
returned_offset1, decoded_message1 = msg1
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message(b"v1"))
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset2, 0)
self.assertEqual(decoded_message2, create_message(b"v2"))
def test_decode_offset_commit_response(self):
encoded = b"".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topic
struct.pack(">h6s", 6, b"topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">h", 0), # No error
])
results = KafkaCodec.decode_offset_commit_response(encoded)
self.assertEqual(set(results), set([
OffsetCommitResponse(topic='topic1', partition=2, error=0),
OffsetCommitResponse(topic=u'topic1', partition=4, error=0),
]))
def test_decode_message_snappy(self):
if not has_snappy():
raise SkipTest("Snappy not available") # pragma: no cover
snappy_encoded = (b'\xec\x80\xa1\x95\x00\x02\xff\xff\xff\xff\x00\x00'
b'\x00,8\x00\x00\x19\x01@\x10L\x9f[\xc2\x00\x00\xff'
b'\xff\xff\xff\x00\x00\x00\x02v1\x19\x1bD\x00\x10\xd5'
b'\x96\nx\x00\x00\xff\xff\xff\xff\x00\x00\x00\x02v2')
offset = 11
messages = list(KafkaCodec._decode_message(snappy_encoded, offset))
self.assertEqual(len(messages), 2)
msg1, msg2 = messages
returned_offset1, decoded_message1 = msg1
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message(b"v1"))
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset2, 0)
self.assertEqual(decoded_message2, create_message(b"v2"))
def test_encode_fetch_request(self):
requests = [
FetchRequest("topic1", 0, 10, 1024),
FetchRequest(u"topic2", 1, 20, 100),
]
header = b"".join([
struct.pack('>h', 1), # Msg Header, Message type = Fetch
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 3), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 2), # Max wait time
struct.pack('>i', 100), # Min bytes
struct.pack('>i', 2), # Num requests
])
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 0), # Partition 0
struct.pack('>q', 10), # Offset
struct.pack('>i', 1024), # Max Bytes
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 1), # Partition 0
struct.pack('>q', 20), # Offset
struct.pack('>i', 100), # Max Bytes
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_fetch_request(
b"client1", 3, requests, 2, 100)
self.assertIn(encoded, [expected1, expected2])
def test_decode_metadata_response(self):
node_brokers = {
0: BrokerMetadata(0, "brokers1.afkak.rdio.com", 1000),
1: BrokerMetadata(1, "brokers1.afkak.rdio.com", 1001),
3: BrokerMetadata(3, "brokers2.afkak.rdio.com", 1000)
}
topic_partitions = {
"topic1": TopicMetadata(
'topic1', 0, {
0: PartitionMetadata("topic1", 0, 0, 1, (0, 2), (2,)),
1: PartitionMetadata("topic1", 1, 1, 3, (0, 1), (0, 1))
}),
"topic2": TopicMetadata(
'topic2', 1, {
0: PartitionMetadata("topic2", 0, 0, 0, (), ())
})
}
encoded = create_encoded_metadata_response(
node_brokers, topic_partitions)
decoded = KafkaCodec.decode_metadata_response(encoded)
self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_requestsRetried(self):
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testrequestsRetried',
reactor=reactor)
request = KafkaCodec.encode_fetch_request(
'testrequestsRetried', id1)
c.makeRequest(id1, request)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
# Before the reply 'comes back' drop the connection
from twisted.internet.main import CONNECTION_LOST
c.clientConnectionLost(c.connector, Failure(CONNECTION_LOST))
# Make sure the proto was reset
self.assertIs(c.proto, None)
# Advance the clock again
reactor.advance(0.1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
def test_cancelRequest(self):
errBackCalled = [False]
def _handleCancelErrback(reason):
log.debug("_handleCancelErrback: %r", reason)
reason.trap(CancelledError)
errBackCalled[0] = True
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('test_connect', reactor=reactor)
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testcancelRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
d.addErrback(_handleCancelErrback)
c.proto.sendString.assert_called_once_with(request)
# Now try to cancel the request
d.cancel()
self.assertTrue(errBackCalled[0])
def test_makeRequest_fails(self):
id1 = 15432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
c.proto.sendString.side_effect = StringTooLongError(
"Tried to send too many bytes")
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
