def test_abort_fails_all_pending_requests(self, read_message):
request1 = metadata.MetadataRequest()
request2 = metadata.MetadataRequest(topics=["example.foo"])
mock_responses = [
Mock(correlation_id=request1.correlation_id),
Mock(correlation_id=request2.correlation_id),
]
def get_next_response(*args):
return self.future_value(mock_responses.pop(0))
read_message.side_effect = get_next_response
conn = Connection("localhost", 1234)
conn.stream = Mock()
conn.stream.write.return_value = self.future_value(None)
responses = [conn.send(request1), conn.send(request2)]
conn.abort()
conn.abort() # second abort is a no-op
for response in responses:
error = response.exception()
self.assertEqual(error.host, "localhost")
self.assertEqual(error.port, 1234)
def get_metadata(self, topics=None):
"""
Retrieves metadata from a broker in the cluster, optionally limited
to a set of topics.
Each connection in the cluster is tried until one works. If no
connection in the cluster responds, a ``NoBrokersError`` is raised.
"""
log.debug("Gathering metadata (topics=%s)", topics)
if topics is None:
topics = []
response = None
for conn in self.conns.values():
try:
response = yield conn.send(
metadata.MetadataRequest(topics=topics))
break
except (iostream.StreamClosedError, BrokerConnectionError):
continue
if not response:
raise NoBrokersError
raise gen.Return(response)
def test_start_uses_metadata_api(self):
self.add_broker(
"kafka01", 9092,
responses=[
metadata.MetadataResponse(brokers=[], topics=[])
]
)
c = cluster.Cluster(["kafka01", "kafka02:9000"])
yield c.start()
self.assert_sent("kafka01", 9092, metadata.MetadataRequest(topics=[]))
def test_correlates_responses(self, read_message):
request1 = metadata.MetadataRequest()
request2 = metadata.MetadataRequest(topics=["example.foo"])
response1 = metadata.MetadataResponse(
brokers=[metadata.Broker(broker_id=1, host="broker01", port=333)],
topics=[
metadata.TopicMetadata(error_code=0, name="example.foo"),
metadata.TopicMetadata(error_code=0, name="example.bar"),
])
response1.correlation_id = request1.correlation_id
response2 = metadata.MetadataResponse(
brokers=[metadata.Broker(broker_id=1, host="broker01", port=333)],
topics=[
metadata.TopicMetadata(error_code=0, name="example.foo"),
])
response2.correlation_id = request2.correlation_id
# response2 comes over the wire before response1
responses = [response2, response1]
def get_next_response(*args):
return self.future_value(responses.pop(0))
read_message.side_effect = get_next_response
conn = Connection("localhost", 1234)
conn.stream = Mock()
conn.stream.write.return_value = self.future_value(None)
actual_responses = [conn.send(request1), conn.send(request2)]
yield conn.read_loop()
# first response is the one with two topics
self.assertEqual(len(actual_responses[0].result().topics), 2)
self.assertEqual(len(actual_responses[1].result().topics), 1)
def test_immediate_error_writing_to_stream_aborts(self):
class FakeException(Exception):
pass
conn = Connection("localhost", 1234)
conn.stream = Mock()
conn.stream.write.side_effect = FakeException("oh no!")
error = None
try:
yield conn.send(metadata.MetadataRequest())
except FakeException as e:
error = e
self.assertEqual(str(error), "oh no!")
self.assertEqual(conn.closing, True)
conn.stream.close.assert_called_once_with()
def start(self):
"""
Establishes connections to the brokers in a cluster as well as
gathers topic/partition metadata.
Cycles through each bootstrap host and attempts to send a metadata
request. Once a metadata request is successful the `heal()` method
is called.
"""
response = None
for host in self.bootstrap_hosts:
if ":" in host:
host, port = host.split(":")
else:
port = DEFAULT_KAFKA_PORT
conn = Connection(host, int(port))
log.info("Using bootstrap host '%s'", host)
try:
yield conn.connect()
except (iostream.StreamClosedError, BrokerConnectionError):
log.warn("Could not connect to bootstrap %s:%s", host, port)
continue
except Exception:
log.exception("Error connecting to bootstrap host '%s'", host)
continue
response = yield conn.send(metadata.MetadataRequest(topics=[]))
conn.close()
break
if not response:
raise NoBrokersError
log.info("Metadata gathered, setting up connections.")
yield self.heal(response)