def test_makeRequest(self):
id1 = 54321
id2 = 76543
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testmakeRequest', 9092, 'clientId')
request = KafkaCodec.encode_fetch_request(b'testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testmakeRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
fail1 = eb1.call_args[0][0] # The actual failure sent to errback
self.assertTrue(fail1.check(CancelledError))
def test_makeRequest(self):
id1 = 54321
id2 = 76543
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testmakeRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
fail1 = eb1.call_args[0][0] # The actual failure sent to errback
self.assertTrue(fail1.check(CancelledError))
def test_makeUnconnectedRequest(self):
"""
Ensure that sending a request when not connected will attempt to bring
up a connection if one isn't already in the process of being brought up
"""
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testmakeUnconnectedRequest', 9092,
'clientId')
request = KafkaCodec.encode_fetch_request(
b'testmakeUnconnectedRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(1.0)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
def test_makeUnconnectedRequest(self):
""" test_makeUnconnectedRequest
Ensure that sending a request when not connected will attempt to bring
up a connection if one isn't already in the process of being brought up
"""
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeUnconnectedRequest',
reactor=reactor)
request = KafkaCodec.encode_fetch_request(
'testmakeUnconnectedRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(1.0)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
def test_cancelRequestNoReply(self):
id2 = 87654
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'test_connect', 9092, 'clientId')
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testcancelRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# This one we cancel by ID. It should fail due to the
# expectResponse=False, since we don't keep the requestID after send
self.assertRaises(KeyError, c.cancelRequest, id2)
def test_cancelRequestNoReply(self):
id2 = 87654
reactor = MemoryReactorClock()
c = KafkaBrokerClient('test_connect', reactor=reactor)
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testcancelRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# This one we cancel by ID. It should fail due to the
# expectResponse=False, since we don't keep the requestID after send
self.assertRaises(KeyError, c.cancelRequest, id2)
def test_encode_fetch_request(self):
requests = [
FetchRequest("topic1", 0, 10, 1024),
FetchRequest(u"topic2", 1, 20, 100),
]
header = b"".join([
struct.pack('>h', 1), # Msg Header, Message type = Fetch
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 3), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 2), # Max wait time
struct.pack('>i', 100), # Min bytes
struct.pack('>i', 2), # Num requests
])
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 0), # Partition 0
struct.pack('>q', 10), # Offset
struct.pack('>i', 1024), # Max Bytes
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 1), # Partition 0
struct.pack('>q', 20), # Offset
struct.pack('>i', 100), # Max Bytes
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_fetch_request(
b"client1", 3, requests, 2, 100)
self.assertIn(encoded, [expected1, expected2])
def test_encode_fetch_request(self):
requests = [
FetchRequest(b"topic1", 0, 10, 1024),
FetchRequest(b"topic2", 1, 20, 100),
]
header = b"".join([
struct.pack('>h', 1), # Msg Header, Message type = Fetch
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 3), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 2), # Max wait time
struct.pack('>i', 100), # Min bytes
struct.pack('>i', 2), # Num requests
])
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 0), # Partition 0
struct.pack('>q', 10), # Offset
struct.pack('>i', 1024), # Max Bytes
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # Topic
struct.pack('>i', 1), # Num Payloads
struct.pack('>i', 1), # Partition 0
struct.pack('>q', 20), # Offset
struct.pack('>i', 100), # Max Bytes
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_fetch_request(b"client1", 3, requests, 2,
100)
self.assertIn(encoded, [expected1, expected2])
def test_requestsRetried(self):
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testrequestsRetried',
reactor=reactor)
request = KafkaCodec.encode_fetch_request(
'testrequestsRetried', id1)
c.makeRequest(id1, request)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
# Before the reply 'comes back' drop the connection
from twisted.internet.main import CONNECTION_LOST
c.clientConnectionLost(c.connector, Failure(CONNECTION_LOST))
# Make sure the proto was reset
self.assertIs(c.proto, None)
# Advance the clock again
reactor.advance(0.1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
def test_cancelRequest(self):
errBackCalled = [False]
def _handleCancelErrback(reason):
log.debug("_handleCancelErrback: %r", reason)
reason.trap(CancelledError)
errBackCalled[0] = True
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'test_connect', 9092, 'clientId')
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testcancelRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
d.addErrback(_handleCancelErrback)
c.proto.sendString.assert_called_once_with(request)
# Now try to cancel the request
d.cancel()
self.assertTrue(errBackCalled[0])
def test_makeRequest_fails(self):
id1 = 15432
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testmakeRequest', 9092, 'clientId')
request = KafkaCodec.encode_fetch_request(b'testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
c.proto.sendString.side_effect = StringTooLongError(
"Tried to send too many bytes")
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
def test_cancelRequest(self):
errBackCalled = [False]
def _handleCancelErrback(reason):
log.debug("_handleCancelErrback: %r", reason)
reason.trap(CancelledError)
errBackCalled[0] = True
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('test_connect', reactor=reactor)
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Fake a protocol
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testcancelRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
d.addErrback(_handleCancelErrback)
c.proto.sendString.assert_called_once_with(request)
# Now try to cancel the request
d.cancel()
self.assertTrue(errBackCalled[0])
def test_makeRequest_fails(self):
id1 = 15432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
c.proto.sendString.side_effect = StringTooLongError(
"Tried to send too many bytes")
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
def test_requestsRetried(self):
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testrequestsRetried', 9092, 'clientId')
request = KafkaCodec.encode_fetch_request(b'testrequestsRetried', id1)
c.makeRequest(id1, request)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
# Before the reply 'comes back' drop the connection
from twisted.internet.main import CONNECTION_LOST
c.clientConnectionLost(c.connector, Failure(CONNECTION_LOST))
# Make sure the proto was reset
self.assertIs(c.proto, None)
# Advance the clock again
reactor.advance(0.1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(0.1)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
# And the request should be 'sent'
self.assertTrue(c.requests[id1].sent)
def test_handleResponse(self):
def make_fetch_response(id):
t1 = b"topic1"
t2 = b"topic2"
msgs = [
create_message(m)
for m in [b"message1", b"hi", b"boo", b"foo", b"so fun!"]
]
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
len(t1), len(ms1), len(ms2), len(t2), len(ms3))
return struct.pack(packFmt, id, 2, len(t1), t1, 2, 0, 0, 10,
len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2),
t2, 1, 0, 0, 30, len(ms3), ms3)
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'testhandleResponse', 9092, 'clientId')
logsave = brokerclient.log
try:
brokerclient.log = Mock()
badId = 98765
response = make_fetch_response(badId)
# First test that a response without first sending a request
# generates a log message
c.handleResponse(response)
brokerclient.log.warning.assert_called_once_with(
'Unexpected response:%r, %r', badId, response)
# Now try a request/response pair and ensure the deferred is called
goodId = 12345
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(b'testhandleResponse2',
goodId)
d = c.makeRequest(goodId, request)
self.assertIsInstance(d, Deferred)
c.proto.sendString.assert_called_once_with(request)
response = make_fetch_response(goodId)
self.assertFalse(d.called)
# Ensure trying to make another request with the same ID before
# we've got a response raises an exception
c.proto.sendString = Mock(side_effect=Exception(
'brokerclient sending duplicate request'))
self.assertRaises(DuplicateRequestError, c.makeRequest, goodId,
request)
c.handleResponse(response)
self.assertTrue(d.called)
# Now that we've got the response, try again with the goodId
c.proto = Mock() # Need a new mock...
d = c.makeRequest(goodId, request)
self.assertIsInstance(d, Deferred)
c.proto.sendString.assert_called_once_with(request)
response = make_fetch_response(goodId)
self.assertFalse(d.called)
c.handleResponse(response)
self.assertTrue(d.called)
# Now try with a real request, but with expectResponse=False
# We still get a deferred back, because the request can be
# cancelled before it's sent, and in that case, we errback()
# the deferred
d2 = c.makeRequest(goodId, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
# Send the (unexpected) response, and check for the log message
c.handleResponse(response)
brokerclient.log.warning.assert_called_with(
'Unexpected response:%r, %r', goodId, response)
finally:
brokerclient.log = logsave
def test_handleResponse(self):
def make_fetch_response(id):
t1 = "topic1"
t2 = "topic2"
msgs = map(
create_message, ["message1", "hi", "boo", "foo", "so fun!"])
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
len(t1), len(ms1), len(ms2), len(t2), len(ms3))
return struct.pack(
packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1,
1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
c = KafkaBrokerClient('testhandleResponse')
logsave = brokerclient.log
try:
brokerclient.log = Mock()
badId = 98765
response = make_fetch_response(badId)
# First test that a response without first sending a request
# generates a log message
c.handleResponse(response)
brokerclient.log.warning.assert_called_once_with(
'Unexpected response:%r, %r', badId, response)
# Now try a request/response pair and ensure the deferred is called
goodId = 12345
c.proto = Mock()
request = KafkaCodec.encode_fetch_request(
'testhandleResponse2', goodId)
d = c.makeRequest(goodId, request)
self.assertIsInstance(d, Deferred)
c.proto.sendString.assert_called_once_with(request)
response = make_fetch_response(goodId)
self.assertFalse(d.called)
# Ensure trying to make another request with the same ID before
# we've got a response raises an exception
c.proto.sendString = Mock(
side_effect=Exception(
'brokerclient sending duplicate request'))
self.assertRaises(
DuplicateRequestError, c.makeRequest, goodId, request)
c.handleResponse(response)
self.assertTrue(d.called)
# Now that we've got the response, try again with the goodId
c.proto = Mock() # Need a new mock...
d = c.makeRequest(goodId, request)
self.assertIsInstance(d, Deferred)
c.proto.sendString.assert_called_once_with(request)
response = make_fetch_response(goodId)
self.assertFalse(d.called)
c.handleResponse(response)
self.assertTrue(d.called)
# Now try with a real request, but with expectResponse=False
# We still get a deferred back, because the request can be
# cancelled before it's sent, and in that case, we errback()
# the deferred
d2 = c.makeRequest(goodId, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
# Send the (unexpected) response, and check for the log message
c.handleResponse(response)
brokerclient.log.warning.assert_called_with(
'Unexpected response:%r, %r', goodId, response)
finally:
brokerclient.log = logsave
