def test_defaults(self): c = cluster.Cluster(["kafka01", "kafka02"]) self.assertEqual(c.bootstrap_hosts, ["kafka01", "kafka02"]) self.assertEqual(c.conns, {}) self.assertEqual(c.topics, {}) self.assertEqual(c.leaders, {})
def test_heal_with_no_working_connections_raises_no_brokers(self): c = cluster.Cluster(["kafka01", "kafka02:900"]) c.conns[1] = Mock() c.conns[1].send.side_effect = iostream.StreamClosedError with self.assertRaises(exc.NoBrokersError): yield c.heal()
def test_iteration(self): c = cluster.Cluster(["kafka01", "kafka02"]) conn1 = Mock() conn2 = Mock() conn3 = Mock() c.conns = {1: conn1, 3: conn2, 8: conn3} self.assertEqual(set(list(c)), set([1, 3, 8]))
def test_contains(self): c = cluster.Cluster(["kafka01", "kafka02"]) conn1 = Mock() conn2 = Mock() c.conns = {1: conn1, 2: conn2} self.assertEqual(1 in c, True) self.assertEqual(2 in c, True) self.assertEqual(3 in c, False)
def test_getitem(self): c = cluster.Cluster(["kafka01", "kafka02"]) conn1 = Mock() conn2 = Mock() c.conns = {1: conn1, 2: conn2} self.assertEqual(c[1], conn1) self.assertEqual(c[2], conn2) with self.assertRaises(KeyError): c[3]
def test_stop_closes_all_connections(self): c = cluster.Cluster(["kafka01", "kafka02"]) conn1 = Mock() conn2 = Mock() c.conns = {3: conn1, 8: conn2} c.stop() conn1.close.assert_called_once_with() conn2.close.assert_called_once_with()
def test_start_uses_metadata_api(self): self.add_broker( "kafka01", 9092, responses=[ metadata.MetadataResponse(brokers=[], topics=[]) ] ) c = cluster.Cluster(["kafka01", "kafka02:9000"]) yield c.start() self.assert_sent("kafka01", 9092, metadata.MetadataRequest(topics=[]))
def test_get_leader(self): c = cluster.Cluster(["kafka01", "kafka02"]) conn1 = Mock() conn2 = Mock() c.conns = {3: conn1, 8: conn2} c.leaders = { "test.topic": [8, 3], "other.topic": [3], } self.assertEqual(c.get_leader("test.topic", 0), 8) self.assertEqual(c.get_leader("test.topic", 1), 3) self.assertEqual(c.get_leader("other.topic", 0), 3) with self.assertRaises(KeyError): c.get_leader("no.such.topic", 3)
def test_start_exceptions_in_all_connections(self): self.add_broker( "kafka01", 9092, connect_error=Exception("oh no!") ) self.add_broker( "kafka02", 9000, connect_error=iostream.StreamClosedError(), ) self.add_broker( "kafka03", 9092, connect_error=exc.BrokerConnectionError("kafka03", 9092), ) c = cluster.Cluster(["kafka01", "kafka02:9000", "kafka03"]) with self.assertRaises(exc.NoBrokersError): yield c.start()
def test_heal_with_initial_broker_errors(self): initial = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), metadata.Broker(broker_id=7, host="kafka02", port=9000), metadata.Broker(broker_id=8, host="kafka03", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=2, replicas=[], isrs=[], ), ] ), ], ) fixed = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=2, replicas=[], isrs=[], ), ] ), ], ) self.add_broker("kafka01", 9092, responses=[initial, fixed]) self.add_broker( "kafka02", 9000, connect_error=iostream.StreamClosedError(), ) self.add_broker( "kafka03", 9092, connect_error=exc.BrokerConnectionError("kafka03", 9092), ) conn1 = cluster.Connection("kafka01", 9092) c = cluster.Cluster(["kafka01", "kafka02:900"]) c.conns = {2: conn1} yield c.heal() self.assertEqual([2], list(c.conns.keys()))
def test_heal_with_intial_topic_errors(self): initial = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.leader_not_available, partition_id=0, leader=8, replicas=[7, 2], isrs=[7, 2], ), metadata.PartitionMetadata( error_code=errors.no_error, partition_id=1, leader=7, replicas=[2, 8], isrs=[2, 8], ), ] ), metadata.TopicMetadata( error_code=errors.replica_not_available, name="other.topic", partitions=[], ), metadata.TopicMetadata( error_code=errors.unknown_topic_or_partition, name="fake.topic", partitions=[], ), ], ) fixed = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=2, replicas=[], isrs=[], ), metadata.PartitionMetadata( error_code=errors.no_error, partition_id=1, leader=2, replicas=[], isrs=[], ), ] ), ], ) self.add_broker("kafka01", 9092, responses=[initial, fixed]) conn1 = cluster.Connection("kafka01", 9092) c = cluster.Cluster(["kafka01", "kafka02:900"]) c.conns = {2: conn1} yield c.heal() self.assertEqual(c.topics, {"test.topic": [0, 1]}) self.assertEqual(c.leaders, {"test.topic": {0: 2, 1: 2}})
def test_heal(self): response = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), metadata.Broker(broker_id=8, host="kafka02", port=9000), metadata.Broker(broker_id=7, host="kafka03", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=8, replicas=[7, 2], isrs=[7, 2], ), metadata.PartitionMetadata( error_code=errors.no_error, partition_id=1, leader=7, replicas=[2, 8], isrs=[2, 8], ), ] ), metadata.TopicMetadata( error_code=errors.no_error, name="other.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=3, leader=2, replicas=[8, 7], isrs=[7], ), ] ), ], ) self.add_broker("kafka01", 9092, responses=[response]) self.add_broker("kafka02", 9000, responses=[response]) self.add_broker("kafka04", 9092, responses=[response]) self.add_broker("kafka03", 9092, responses=[]) c = cluster.Cluster(["kafka01", "kafka02:900"]) # in the response, stays where it is conn1 = cluster.Connection("kafka01", 9092) # closing but in response, gets replaced conn2 = cluster.Connection("kafka02", 9000) conn2.closing = True # not in response, gets aborted and dropped conn3 = cluster.Connection("kafka04", 9092) c.conns = {2: conn1, 8: conn2, 1: conn3} def remove_connection(): del c.conns[1] c.conns[1].abort.side_effect = remove_connection yield c.heal() self.assertEqual(c.topics, {"test.topic": [0, 1], "other.topic": [3]}) self.assertEqual( c.leaders, {"test.topic": {0: 8, 1: 7}, "other.topic": {3: 2}} ) self.assertEqual(set(list(c)), set([2, 7, 8])) self.assertEqual(c[7], self.broker_hosts[("kafka03", 9092)]) conn3.abort.assert_called_once_with()