def test_get_leader_for_unassigned_partitions(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): client._get_leader_for_partition('topic_no_partitions', 0) with self.assertRaises(UnknownTopicOrPartitionError): client._get_leader_for_partition('topic_unknown', 0)
def test_get_leader_returns_none_when_noleader(self, protocol, conn): "Getting leader for partitions returns None when the partiion has no leader" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) self.assertIsNone(client._get_leader_for_partition('topic_noleader', 0)) self.assertIsNone(client._get_leader_for_partition('topic_noleader', 1)) topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0]) } protocol.decode_metadata_response.return_value = (brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 2)) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics['topic_no_partitions'] = { 0: PartitionMetadata('topic_no_partitions', 0, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_no_partitions', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_no_partitions', 0): brokers[0]}, client.topics_to_brokers)
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_one_partition', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual({ TopicAndPartition('topic_one_partition', 0): brokers[0]}, client.topics_to_brokers)
def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = "response" # anything but None brokers = [BrokerMetadata(0, "broker_1", 4567), BrokerMetadata(1, "broker_2", 5678)] topics = [ TopicMetadata( "topic_noleader", NO_ERROR, [ PartitionMetadata("topic_noleader", 0, -1, [], [], NO_LEADER), PartitionMetadata("topic_noleader", 1, -1, [], [], NO_LEADER), ], ) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) client = KafkaClient(hosts=["broker_1:4567"]) self.assertDictEqual( {TopicAndPartition("topic_noleader", 0): None, TopicAndPartition("topic_noleader", 1): None}, client.topics_to_brokers, ) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone(client._get_leader_for_partition("topic_noleader", 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone(client._get_leader_for_partition("topic_noleader", 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone(client._get_leader_for_partition("topic_noleader", 2)) topics = [ TopicMetadata( "topic_noleader", NO_ERROR, [ PartitionMetadata("topic_noleader", 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata("topic_noleader", 1, 1, [1, 0], [1, 0], NO_ERROR), ], ) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition("topic_noleader", 0)) self.assertEqual(brokers[1], client._get_leader_for_partition("topic_noleader", 1))
def test_get_leader_for_unassigned_partitions(self, protocol, conn): "Get leader raises if no partitions is defined for a topic" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(PartitionUnavailableError): client._get_leader_for_partition('topic_no_partitions', 0)
def test_get_leader_for_unassigned_partitions(self, protocol, conn): "Get leader raises if no partitions is defined for a topic" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(PartitionUnavailableError): client._get_leader_for_partition('topic_no_partitions', 0)
def test_get_leader_for_unassigned_partitions(self, protocol, conn): conn.recv.return_value = "response" # anything but None brokers = [BrokerMetadata(0, "broker_1", 4567), BrokerMetadata(1, "broker_2", 5678)] topics = [ TopicMetadata("topic_no_partitions", NO_LEADER, []), TopicMetadata("topic_unknown", UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) client = KafkaClient(hosts=["broker_1:4567"]) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): client._get_leader_for_partition("topic_no_partitions", 0) with self.assertRaises(UnknownTopicOrPartitionError): client._get_leader_for_partition("topic_unknown", 0)
def test_get_leader_returns_none_when_noleader(self, protocol, conn): "Getting leader for partitions returns None when the partiion has no leader" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 0)) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 1)) topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0]) } protocol.decode_metadata_response.return_value = (brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))