def test_get_leader_for_unassigned_partitions(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual({}, client._topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client._get_leader_for_partition('topic_no_partitions', 0)) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client._get_leader_for_partition('topic_unknown', 0))
def test_get_leader_for_unassigned_partitions(self, protocol): @asyncio.coroutine def recv(request_id): return b"response" mocked_conns = {("broker_1", 4567): mock.MagicMock()} mocked_conns[("broker_1", 4567)].recv.side_effect = recv client = AIOKafkaClient(["broker_1:4567"], loop=self.loop) client._conns = mocked_conns brokers = [BrokerMetadata(0, "broker_1", 4567), BrokerMetadata(1, "broker_2", 5678)] topics = [ TopicMetadata("topic_no_partitions", NO_LEADER, []), TopicMetadata("topic_unknown", UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual({}, client._topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete(client._get_leader_for_partition("topic_no_partitions", 0)) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete(client._get_leader_for_partition("topic_unknown", 0))
def test_get_leader_exceptions_when_noleader(self, protocol): @asyncio.coroutine def recv(request_id): return b"response" mocked_conns = {("broker_1", 4567): mock.MagicMock()} mocked_conns[("broker_1", 4567)].recv.side_effect = recv client = AIOKafkaClient(["broker_1:4567"], loop=self.loop) client._conns = mocked_conns brokers = [BrokerMetadata(0, "broker_1", 4567), BrokerMetadata(1, "broker_2", 5678)] topics = [ TopicMetadata( "topic_noleader", NO_ERROR, [ PartitionMetadata("topic_noleader", 0, -1, [], [], NO_LEADER), PartitionMetadata("topic_noleader", 1, -1, [], [], NO_LEADER), ], ) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( {TopicAndPartition("topic_noleader", 0): None, TopicAndPartition("topic_noleader", 1): None}, client._topics_to_brokers, ) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone(self.loop.run_until_complete(client._get_leader_for_partition("topic_noleader", 0))) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone(self.loop.run_until_complete(client._get_leader_for_partition("topic_noleader", 1))) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone(self.loop.run_until_complete(client._get_leader_for_partition("topic_noleader", 2))) topics = [ TopicMetadata( "topic_noleader", NO_ERROR, [ PartitionMetadata("topic_noleader", 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata("topic_noleader", 1, 1, [1, 0], [1, 0], NO_ERROR), ], ) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) self.assertEqual( brokers[0], self.loop.run_until_complete(client._get_leader_for_partition("topic_noleader", 0)) ) self.assertEqual( brokers[1], self.loop.run_until_complete(client._get_leader_for_partition("topic_noleader", 1)) )
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete( client._get_leader_for_partition('topic_one_partition', 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client._topics_to_brokers)
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b"response" mocked_conns = {("broker_1", 4567): mock.MagicMock()} mocked_conns[("broker_1", 4567)].recv.side_effect = recv client = AIOKafkaClient(["broker_1:4567"], loop=self.loop) client._conns = mocked_conns brokers = [BrokerMetadata(0, "broker_1", 4567), BrokerMetadata(1, "broker_2", 5678)] topics = [TopicMetadata("topic_no_partitions", NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata( "topic_one_partition", NO_ERROR, [PartitionMetadata("topic_no_partition", 0, 0, [0, 1], [0, 1], NO_ERROR)], ) ] protocol.decode_metadata_response.return_value = MetadataResponse(brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete(client._get_leader_for_partition("topic_one_partition", 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual({TopicAndPartition("topic_one_partition", 0): brokers[0]}, client._topics_to_brokers)
def test_get_leader_exceptions_when_noleader(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client._topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1))) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 2))) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual( brokers[0], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) self.assertEqual( brokers[1], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1)))