def test_defaults(self):
c = cluster.Cluster(["kafka01", "kafka02"])
self.assertEqual(c.bootstrap_hosts, ["kafka01", "kafka02"])
self.assertEqual(c.conns, {})
self.assertEqual(c.topics, {})
self.assertEqual(c.leaders, {})
def test_heal_with_no_working_connections_raises_no_brokers(self):
c = cluster.Cluster(["kafka01", "kafka02:900"])
c.conns[1] = Mock()
c.conns[1].send.side_effect = iostream.StreamClosedError
with self.assertRaises(exc.NoBrokersError):
yield c.heal()
def test_iteration(self):
c = cluster.Cluster(["kafka01", "kafka02"])
conn1 = Mock()
conn2 = Mock()
conn3 = Mock()
c.conns = {1: conn1, 3: conn2, 8: conn3}
self.assertEqual(set(list(c)), set([1, 3, 8]))
def test_contains(self):
c = cluster.Cluster(["kafka01", "kafka02"])
conn1 = Mock()
conn2 = Mock()
c.conns = {1: conn1, 2: conn2}
self.assertEqual(1 in c, True)
self.assertEqual(2 in c, True)
self.assertEqual(3 in c, False)
def test_getitem(self):
c = cluster.Cluster(["kafka01", "kafka02"])
conn1 = Mock()
conn2 = Mock()
c.conns = {1: conn1, 2: conn2}
self.assertEqual(c[1], conn1)
self.assertEqual(c[2], conn2)
with self.assertRaises(KeyError):
c[3]
def test_stop_closes_all_connections(self):
c = cluster.Cluster(["kafka01", "kafka02"])
conn1 = Mock()
conn2 = Mock()
c.conns = {3: conn1, 8: conn2}
c.stop()
conn1.close.assert_called_once_with()
conn2.close.assert_called_once_with()
def test_start_uses_metadata_api(self):
self.add_broker(
"kafka01", 9092,
responses=[
metadata.MetadataResponse(brokers=[], topics=[])
]
)
c = cluster.Cluster(["kafka01", "kafka02:9000"])
yield c.start()
self.assert_sent("kafka01", 9092, metadata.MetadataRequest(topics=[]))
def test_get_leader(self):
c = cluster.Cluster(["kafka01", "kafka02"])
conn1 = Mock()
conn2 = Mock()
c.conns = {3: conn1, 8: conn2}
c.leaders = {
"test.topic": [8, 3],
"other.topic": [3],
}
self.assertEqual(c.get_leader("test.topic", 0), 8)
self.assertEqual(c.get_leader("test.topic", 1), 3)
self.assertEqual(c.get_leader("other.topic", 0), 3)
with self.assertRaises(KeyError):
c.get_leader("no.such.topic", 3)
def test_start_exceptions_in_all_connections(self):
self.add_broker(
"kafka01", 9092,
connect_error=Exception("oh no!")
)
self.add_broker(
"kafka02", 9000,
connect_error=iostream.StreamClosedError(),
)
self.add_broker(
"kafka03", 9092,
connect_error=exc.BrokerConnectionError("kafka03", 9092),
)
c = cluster.Cluster(["kafka01", "kafka02:9000", "kafka03"])
with self.assertRaises(exc.NoBrokersError):
yield c.start()
def test_heal_with_initial_broker_errors(self):
initial = metadata.MetadataResponse(
brokers=[
metadata.Broker(broker_id=2, host="kafka01", port=9092),
metadata.Broker(broker_id=7, host="kafka02", port=9000),
metadata.Broker(broker_id=8, host="kafka03", port=9092),
],
topics=[
metadata.TopicMetadata(
error_code=errors.no_error,
name="test.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=0,
leader=2,
replicas=[],
isrs=[],
),
]
),
],
)
fixed = metadata.MetadataResponse(
brokers=[
metadata.Broker(broker_id=2, host="kafka01", port=9092),
],
topics=[
metadata.TopicMetadata(
error_code=errors.no_error,
name="test.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=0,
leader=2,
replicas=[],
isrs=[],
),
]
),
],
)
self.add_broker("kafka01", 9092, responses=[initial, fixed])
self.add_broker(
"kafka02", 9000,
connect_error=iostream.StreamClosedError(),
)
self.add_broker(
"kafka03", 9092,
connect_error=exc.BrokerConnectionError("kafka03", 9092),
)
conn1 = cluster.Connection("kafka01", 9092)
c = cluster.Cluster(["kafka01", "kafka02:900"])
c.conns = {2: conn1}
yield c.heal()
self.assertEqual([2], list(c.conns.keys()))
def test_heal_with_intial_topic_errors(self):
initial = metadata.MetadataResponse(
brokers=[
metadata.Broker(broker_id=2, host="kafka01", port=9092),
],
topics=[
metadata.TopicMetadata(
error_code=errors.no_error,
name="test.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.leader_not_available,
partition_id=0,
leader=8,
replicas=[7, 2],
isrs=[7, 2],
),
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=1,
leader=7,
replicas=[2, 8],
isrs=[2, 8],
),
]
),
metadata.TopicMetadata(
error_code=errors.replica_not_available,
name="other.topic",
partitions=[],
),
metadata.TopicMetadata(
error_code=errors.unknown_topic_or_partition,
name="fake.topic",
partitions=[],
),
],
)
fixed = metadata.MetadataResponse(
brokers=[
metadata.Broker(broker_id=2, host="kafka01", port=9092),
],
topics=[
metadata.TopicMetadata(
error_code=errors.no_error,
name="test.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=0,
leader=2,
replicas=[],
isrs=[],
),
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=1,
leader=2,
replicas=[],
isrs=[],
),
]
),
],
)
self.add_broker("kafka01", 9092, responses=[initial, fixed])
conn1 = cluster.Connection("kafka01", 9092)
c = cluster.Cluster(["kafka01", "kafka02:900"])
c.conns = {2: conn1}
yield c.heal()
self.assertEqual(c.topics, {"test.topic": [0, 1]})
self.assertEqual(c.leaders, {"test.topic": {0: 2, 1: 2}})
def test_heal(self):
response = metadata.MetadataResponse(
brokers=[
metadata.Broker(broker_id=2, host="kafka01", port=9092),
metadata.Broker(broker_id=8, host="kafka02", port=9000),
metadata.Broker(broker_id=7, host="kafka03", port=9092),
],
topics=[
metadata.TopicMetadata(
error_code=errors.no_error,
name="test.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=0,
leader=8,
replicas=[7, 2],
isrs=[7, 2],
),
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=1,
leader=7,
replicas=[2, 8],
isrs=[2, 8],
),
]
),
metadata.TopicMetadata(
error_code=errors.no_error,
name="other.topic",
partitions=[
metadata.PartitionMetadata(
error_code=errors.no_error,
partition_id=3,
leader=2,
replicas=[8, 7],
isrs=[7],
),
]
),
],
)
self.add_broker("kafka01", 9092, responses=[response])
self.add_broker("kafka02", 9000, responses=[response])
self.add_broker("kafka04", 9092, responses=[response])
self.add_broker("kafka03", 9092, responses=[])
c = cluster.Cluster(["kafka01", "kafka02:900"])
# in the response, stays where it is
conn1 = cluster.Connection("kafka01", 9092)
# closing but in response, gets replaced
conn2 = cluster.Connection("kafka02", 9000)
conn2.closing = True
# not in response, gets aborted and dropped
conn3 = cluster.Connection("kafka04", 9092)
c.conns = {2: conn1, 8: conn2, 1: conn3}
def remove_connection():
del c.conns[1]
c.conns[1].abort.side_effect = remove_connection
yield c.heal()
self.assertEqual(c.topics, {"test.topic": [0, 1], "other.topic": [3]})
self.assertEqual(
c.leaders,
{"test.topic": {0: 8, 1: 7}, "other.topic": {3: 2}}
)
self.assertEqual(set(list(c)), set([2, 7, 8]))
self.assertEqual(c[7], self.broker_hosts[("kafka03", 9092)])
conn3.abort.assert_called_once_with()