Esempio n. 1
0
    def test_has_metadata_for_topic(self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_still_creating', NO_LEADER, []),
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
            TopicMetadata('topic_noleaders', NO_ERROR, [
                PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        # Topics with no partitions return False
        self.assertFalse(client.has_metadata_for_topic('topic_still_creating'))
        self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist'))

        # Topic with partition metadata, but no leaders return True
        self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
Esempio n. 2
0
    def test_ensure_topic_exists(self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_still_creating', NO_LEADER, []),
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
            TopicMetadata('topic_noleaders', NO_ERROR, [
                PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        with self.assertRaises(UnknownTopicOrPartitionError):
            client.ensure_topic_exists('topic_doesnt_exist', timeout=1)

        with self.assertRaises(KafkaTimeoutError):
            client.ensure_topic_exists('topic_still_creating', timeout=1)

        # This should not raise
        client.ensure_topic_exists('topic_noleaders', timeout=1)
Esempio n. 3
0
    def test_get_leader_for_unassigned_partitions(self, protocol):
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_no_partitions', NO_LEADER, []),
            TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        self.assertDictEqual({}, client._topics_to_brokers)

        with self.assertRaises(LeaderNotAvailableError):
            self.loop.run_until_complete(
                client._get_leader_for_partition('topic_no_partitions', 0))

        with self.assertRaises(UnknownTopicOrPartitionError):
            self.loop.run_until_complete(
                client._get_leader_for_partition('topic_unknown', 0))
Esempio n. 4
0
    def test_get_leader_for_unassigned_partitions(self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_no_partitions', NO_LEADER, []),
            TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        self.assertDictEqual({}, client.topics_to_brokers)

        with self.assertRaises(LeaderNotAvailableError):
            client._get_leader_for_partition('topic_no_partitions', 0)

        with self.assertRaises(UnknownTopicOrPartitionError):
            client._get_leader_for_partition('topic_unknown', 0)
Esempio n. 5
0
    def test_has_metadata_for_topic(self, protocol):
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_still_creating', NO_LEADER, []),
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
            TopicMetadata('topic_noleaders', NO_ERROR, [
                PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        # Topics with no partitions return False
        self.assertFalse(client.has_metadata_for_topic('topic_still_creating'))
        self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist'))

        # Topic with partition metadata, but no leaders return True
        self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
Esempio n. 6
0
    def test_get_leader_exceptions_when_noleader(self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])
        self.assertDictEqual(
            {
                TopicAndPartition('topic_noleader', 0): None,
                TopicAndPartition('topic_noleader', 1): None
            }, client.topics_to_brokers)

        # No leader partitions -- raise LeaderNotAvailableError
        with self.assertRaises(LeaderNotAvailableError):
            self.assertIsNone(
                client._get_leader_for_partition('topic_noleader', 0))
        with self.assertRaises(LeaderNotAvailableError):
            self.assertIsNone(
                client._get_leader_for_partition('topic_noleader', 1))

        # Unknown partitions -- raise UnknownTopicOrPartitionError
        with self.assertRaises(UnknownTopicOrPartitionError):
            self.assertIsNone(
                client._get_leader_for_partition('topic_noleader', 2))

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1],
                                  NO_ERROR),
                PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0],
                                  NO_ERROR)
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)
        self.assertEqual(brokers[0],
                         client._get_leader_for_partition('topic_noleader', 0))
        self.assertEqual(brokers[1],
                         client._get_leader_for_partition('topic_noleader', 1))
Esempio n. 7
0
    def test_load_metadata(self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata(
                'topic_1', NO_ERROR,
                [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR)
                 ]),
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
            TopicMetadata('topic_no_partitions', NO_LEADER, []),
            TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []),
            TopicMetadata('topic_3', NO_ERROR, [
                PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR),
                PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR),
                PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR)
            ])
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        # client loads metadata at init
        client = KafkaClient(hosts=['broker_1:4567'])
        self.assertDictEqual(
            {
                TopicAndPartition('topic_1', 0): brokers[1],
                TopicAndPartition('topic_noleader', 0): None,
                TopicAndPartition('topic_noleader', 1): None,
                TopicAndPartition('topic_3', 0): brokers[0],
                TopicAndPartition('topic_3', 1): brokers[1],
                TopicAndPartition('topic_3', 2): brokers[0]
            }, client.topics_to_brokers)

        # if we ask for metadata explicitly, it should raise errors
        with self.assertRaises(LeaderNotAvailableError):
            client.load_metadata_for_topics('topic_no_partitions')

        with self.assertRaises(UnknownTopicOrPartitionError):
            client.load_metadata_for_topics('topic_unknown')

        # This should not raise
        client.load_metadata_for_topics('topic_no_leader')
Esempio n. 8
0
    def test_send_produce_request_raises_when_topic_unknown(self, protocol):
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        requests = [
            ProduceRequest(
                "topic_doesnt_exist", 0,
                [create_message("a"), create_message("b")])
        ]

        with self.assertRaises(UnknownTopicOrPartitionError):
            self.loop.run_until_complete(client.send_produce_request(requests))
Esempio n. 9
0
    def test_send_produce_request_raises_when_topic_unknown(
            self, protocol, conn):

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        requests = [
            ProduceRequest(
                "topic_doesnt_exist", 0,
                [create_message("a"), create_message("b")])
        ]

        with self.assertRaises(UnknownTopicOrPartitionError):
            client.send_produce_request(requests)
Esempio n. 10
0
    def test_send_produce_request_raises_when_noleader(self, protocol, conn):
        "Send producer request raises LeaderNotAvailableError if leader is not available"

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        requests = [
            ProduceRequest(
                "topic_noleader", 0,
                [create_message("a"), create_message("b")])
        ]

        with self.assertRaises(LeaderNotAvailableError):
            client.send_produce_request(requests)
Esempio n. 11
0
    def test_get_leader_for_partitions_reloads_metadata(self, protocol):
        "Get leader for partitions reload metadata if it is not available"

        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        # topic metadata is loaded but empty
        self.assertDictEqual({}, client._topics_to_brokers)

        topics = [
            TopicMetadata('topic_one_partition', NO_ERROR, [
                PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1],
                                  NO_ERROR)
            ])
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        # calling _get_leader_for_partition (from any broker aware request)
        # will try loading metadata again for the same topic
        leader = self.loop.run_until_complete(
            client._get_leader_for_partition('topic_one_partition', 0))

        self.assertEqual(brokers[0], leader)
        self.assertDictEqual(
            {TopicAndPartition('topic_one_partition', 0): brokers[0]},
            client._topics_to_brokers)
Esempio n. 12
0
    def test_decode_metadata_response(self):
        node_brokers = [
            BrokerMetadata(0, b"brokers1.kafka.rdio.com", 1000),
            BrokerMetadata(1, b"brokers1.kafka.rdio.com", 1001),
            BrokerMetadata(3, b"brokers2.kafka.rdio.com", 1000)
        ]

        topic_partitions = [
            TopicMetadata(b"topic1", 0, [
                PartitionMetadata(b"topic1", 0, 1, (0, 2), (2, ), 0),
                PartitionMetadata(b"topic1", 1, 3, (0, 1), (0, 1), 1)
            ]),
            TopicMetadata(b"topic2", 1, [
                PartitionMetadata(b"topic2", 0, 0, (), (), 0),
            ]),
        ]
        encoded = self._create_encoded_metadata_response(
            node_brokers, topic_partitions)
        decoded = KafkaProtocol.decode_metadata_response(encoded)
        self.assertEqual(decoded, (node_brokers, topic_partitions))
Esempio n. 13
0
    def test_ensure_topic_exists(self, protocol):

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        fut = asyncio.Future(loop=self.loop)
        fut.set_result(b'response')
        mocked_conns[('broker_1', 4567)].send.return_value = fut
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_still_creating', NO_LEADER, []),
            TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
                          []),
            TopicMetadata('topic_noleaders', NO_ERROR, [
                PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        with self.assertRaises(UnknownTopicOrPartitionError):
            self.loop.run_until_complete(
                client.ensure_topic_exists('topic_doesnt_exist', timeout=1))

        with self.assertRaises(KafkaTimeoutError):
            self.loop.run_until_complete(
                client.ensure_topic_exists('topic_still_creating', timeout=1))

        # This should not raise
        self.loop.run_until_complete(
            client.ensure_topic_exists('topic_noleaders', timeout=1))
Esempio n. 14
0
    def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn):
        "Get leader for partitions reload metadata if it is not available"

        conn.recv.return_value = 'response'  # anything but None

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        client = KafkaClient(hosts=['broker_1:4567'])

        # topic metadata is loaded but empty
        self.assertDictEqual({}, client.topics_to_brokers)

        topics = [
            TopicMetadata('topic_one_partition', NO_ERROR, [
                PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1],
                                  NO_ERROR)
            ])
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        # calling _get_leader_for_partition (from any broker aware request)
        # will try loading metadata again for the same topic
        leader = client._get_leader_for_partition('topic_one_partition', 0)

        self.assertEqual(brokers[0], leader)
        self.assertDictEqual(
            {TopicAndPartition('topic_one_partition', 0): brokers[0]},
            client.topics_to_brokers)
Esempio n. 15
0
    def decode_metadata_response(cls, data):
        """
        Decode bytes to a MetadataResponse

        Params
        ======
        data: bytes to decode
        """
        ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0)

        # Broker info
        brokers = []
        for i in range(numbrokers):
            ((nodeId, ), cur) = relative_unpack('>i', data, cur)
            (host, cur) = read_short_string(data, cur)
            ((port,), cur) = relative_unpack('>i', data, cur)
            brokers.append(BrokerMetadata(nodeId, host, port))

        # Topic info
        ((num_topics,), cur) = relative_unpack('>i', data, cur)
        topic_metadata = []

        for i in range(num_topics):
            ((topic_error,), cur) = relative_unpack('>h', data, cur)
            (topic_name, cur) = read_short_string(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)
            partition_metadata = []

            for j in range(num_partitions):
                ((partition_error_code, partition, leader, numReplicas), cur) = \
                    relative_unpack('>hiii', data, cur)

                (replicas, cur) = relative_unpack(
                    '>%di' % numReplicas, data, cur)

                ((num_isr,), cur) = relative_unpack('>i', data, cur)
                (isr, cur) = relative_unpack('>%di' % num_isr, data, cur)

                partition_metadata.append(
                    PartitionMetadata(topic_name, partition, leader,
                                      replicas, isr, partition_error_code)
                )

            topic_metadata.append(
                TopicMetadata(topic_name, topic_error, partition_metadata)
            )

        return MetadataResponse(brokers, topic_metadata)
Esempio n. 16
0
    def test_send_produce_request_raises_when_noleader(self, protocol):
        """Send producer request raises LeaderNotAvailableError
           if leader is not available"""
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())

        requests = [
            ProduceRequest(
                "topic_noleader", 0,
                [create_message("a"), create_message("b")])
        ]

        with self.assertRaises(LeaderNotAvailableError):
            self.loop.run_until_complete(client.send_produce_request(requests))
Esempio n. 17
0
    def test_load_metadata(self, protocol):
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata(
                'topic_1', NO_ERROR,
                [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR)
                 ]),
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
            TopicMetadata('topic_no_partitions', NO_LEADER, []),
            TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []),
            TopicMetadata('topic_3', NO_ERROR, [
                PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR),
                PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR),
                PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR)
            ]),
            TopicMetadata('topic_4', NO_ERROR, [
                PartitionMetadata('topic_4', 0, 0, [0, 1], [0, 1], NO_ERROR),
                PartitionMetadata('topic_4', 1, 1, [1, 0], [1, 0],
                                  REPLICA_NOT_AVAILABLE),
            ])
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())
        self.assertDictEqual(
            {
                TopicAndPartition('topic_1', 0): brokers[1],
                TopicAndPartition('topic_noleader', 0): None,
                TopicAndPartition('topic_noleader', 1): None,
                TopicAndPartition('topic_3', 0): brokers[0],
                TopicAndPartition('topic_3', 1): brokers[1],
                TopicAndPartition('topic_3', 2): brokers[0],
                TopicAndPartition('topic_4', 0): brokers[0],
                TopicAndPartition('topic_4', 1): brokers[1]
            }, client._topics_to_brokers)

        # if we ask for metadata explicitly, it should raise errors
        with self.assertRaises(LeaderNotAvailableError):
            self.loop.run_until_complete(
                client.load_metadata_for_topics('topic_no_partitions'))

        with self.assertRaises(UnknownTopicOrPartitionError):
            self.loop.run_until_complete(
                client.load_metadata_for_topics('topic_unknown'))

        # This should not raise
        self.loop.run_until_complete(
            client.load_metadata_for_topics('topic_no_leader'))

        # This should not raise ReplicaNotAvailableError
        self.loop.run_until_complete(
            client.load_metadata_for_topics('topic_4'))
Esempio n. 18
0
    def test_get_leader_exceptions_when_noleader(self, protocol):
        @asyncio.coroutine
        def recv(request_id):
            return b'response'

        mocked_conns = {('broker_1', 4567): mock.MagicMock()}
        mocked_conns[('broker_1', 4567)].recv.side_effect = recv
        client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
        client._conns = mocked_conns

        brokers = [
            BrokerMetadata(0, 'broker_1', 4567),
            BrokerMetadata(1, 'broker_2', 5678)
        ]

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
                PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)

        self.loop.run_until_complete(client.load_metadata_for_topics())
        self.assertDictEqual(
            {
                TopicAndPartition('topic_noleader', 0): None,
                TopicAndPartition('topic_noleader', 1): None
            }, client._topics_to_brokers)

        # No leader partitions -- raise LeaderNotAvailableError
        with self.assertRaises(LeaderNotAvailableError):
            self.assertIsNone(
                self.loop.run_until_complete(
                    client._get_leader_for_partition('topic_noleader', 0)))
        with self.assertRaises(LeaderNotAvailableError):
            self.assertIsNone(
                self.loop.run_until_complete(
                    client._get_leader_for_partition('topic_noleader', 1)))

        # Unknown partitions -- raise UnknownTopicOrPartitionError
        with self.assertRaises(UnknownTopicOrPartitionError):
            self.assertIsNone(
                self.loop.run_until_complete(
                    client._get_leader_for_partition('topic_noleader', 2)))

        topics = [
            TopicMetadata('topic_noleader', NO_ERROR, [
                PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1],
                                  NO_ERROR),
                PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0],
                                  NO_ERROR)
            ]),
        ]
        protocol.decode_metadata_response.return_value = MetadataResponse(
            brokers, topics)
        self.assertEqual(
            brokers[0],
            self.loop.run_until_complete(
                client._get_leader_for_partition('topic_noleader', 0)))
        self.assertEqual(
            brokers[1],
            self.loop.run_until_complete(
                client._get_leader_for_partition('topic_noleader', 1)))