def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics['topic_no_partitions'] = { 0: PartitionMetadata('topic_no_partitions', 0, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_no_partitions', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_no_partitions', 0): brokers[0]}, client.topics_to_brokers)
def test_send_produce_request_raises_when_noleader(self, protocol, conn): mock_conn(conn) brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ (NO_ERROR, 'topic_noleader', [ (NO_LEADER, 0, -1, [], []), (NO_LEADER, 1, -1, [], []), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = SimpleClient(hosts=['broker_1:4567']) requests = [ ProduceRequestPayload( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): client.send_produce_request(requests)
def test_send_produce_request_raises_when_noleader(self, protocol, conn): "Send producer request raises LeaderUnavailableError if leader is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderUnavailableError): client.send_produce_request(requests)
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" mock_conn(conn) brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [(NO_LEADER, 'topic_no_partitions', [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = SimpleClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics = [(NO_ERROR, 'topic_one_partition', [(NO_ERROR, 0, 0, [0, 1], [0, 1])])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_one_partition', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicPartition('topic_one_partition', 0): brokers[0]}, client.topics_to_brokers)
def test_ensure_topic_exists(self, decode_metadata_response, conn): mock_conn(conn) brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ (NO_LEADER, 'topic_still_creating', []), (UNKNOWN_TOPIC_OR_PARTITION, 'topic_doesnt_exist', []), (NO_ERROR, 'topic_noleaders', [ (NO_LEADER, 0, -1, [], []), (NO_LEADER, 1, -1, [], []), ]), ] decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = SimpleClient(hosts=['broker_1:4567']) with self.assertRaises(UnknownTopicOrPartitionError): client.ensure_topic_exists('topic_doesnt_exist', timeout=1) with self.assertRaises(KafkaTimeoutError): client.ensure_topic_exists('topic_still_creating', timeout=1) # This should not raise client.ensure_topic_exists('topic_noleaders', timeout=1)
def test_has_metadata_for_topic(self, protocol, conn): mock_conn(conn) brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ (NO_LEADER, 'topic_still_creating', []), (UNKNOWN_TOPIC_OR_PARTITION, 'topic_doesnt_exist', []), (NO_ERROR, 'topic_noleaders', [ (NO_LEADER, 0, -1, [], []), (NO_LEADER, 1, -1, [], []), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = SimpleClient(hosts=['broker_1:4567']) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_has_metadata_for_topic(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_decode_metadata_response(self): node_brokers = { 0: BrokerMetadata(0, "brokers1.kafka.rdio.com", 1000), 1: BrokerMetadata(1, "brokers1.kafka.rdio.com", 1001), 3: BrokerMetadata(3, "brokers2.kafka.rdio.com", 1000) } topic_partitions = { "topic1": { 0: PartitionMetadata("topic1", 0, 1, (0, 2), (2, )), 1: PartitionMetadata("topic1", 1, 3, (0, 1), (0, 1)) }, "topic2": { 0: PartitionMetadata("topic2", 0, 0, (), ()) } } topic_errors = {"topic1": 0, "topic2": 1} partition_errors = { ("topic1", 0): 0, ("topic1", 1): 1, ("topic2", 0): 0 } encoded = self._create_encoded_metadata_response( node_brokers, topic_partitions, topic_errors, partition_errors) decoded = KafkaProtocol.decode_metadata_response(encoded) self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_get_leader_for_unassigned_partitions(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): client._get_leader_for_partition('topic_no_partitions', 0) with self.assertRaises(UnknownTopicOrPartitionError): client._get_leader_for_partition('topic_unknown', 0)
def test_send_produce_request_raises_when_topic_unknown( self, protocol, conn): mock_conn(conn) brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ (UNKNOWN_TOPIC_OR_PARTITION, 'topic_doesnt_exist', []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = SimpleClient(hosts=['broker_1:4567']) requests = [ ProduceRequestPayload( "topic_doesnt_exist", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(FailedPayloadsError): client.send_produce_request(requests)
def test_has_metadata_for_topic(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_ensure_topic_exists(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) with self.assertRaises(UnknownTopicOrPartitionError): client.ensure_topic_exists('topic_doesnt_exist', timeout=1) with self.assertRaises(KafkaTimeoutError): client.ensure_topic_exists('topic_still_creating', timeout=1) # This should not raise client.ensure_topic_exists('topic_noleaders', timeout=1)
def test_send_produce_request_raises_when_topic_unknown(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) requests = [ ProduceRequest( "topic_doesnt_exist", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete(client.send_produce_request(requests))
def test_send_produce_request_raises_when_noleader(self, protocol, conn): "Send producer request raises LeaderNotAvailableError if leader is not available" conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): client.send_produce_request(requests)
def test_get_leader_for_unassigned_partitions(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual({}, client._topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client._get_leader_for_partition('topic_no_partitions', 0)) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client._get_leader_for_partition('topic_unknown', 0))
def test_send_produce_request_raises_when_topic_unknown( self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_doesnt_exist", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(UnknownTopicOrPartitionError): client.send_produce_request(requests)
def test_decode_metadata_response(self): node_brokers = [ BrokerMetadata(0, b"brokers1.kafka.rdio.com", 1000), BrokerMetadata(1, b"brokers1.kafka.rdio.com", 1001), BrokerMetadata(3, b"brokers2.kafka.rdio.com", 1000) ] '''
def test_bootstrap_success(conn): conn.state = ConnectionStates.CONNECTED cli = KafkaClient() conn.assert_called_once_with('localhost', 9092, **cli.config) conn.connect.assert_called_with() conn.send.assert_called_once_with(MetadataRequest([])) assert cli._bootstrap_fails == 0 assert cli.cluster.brokers() == set([BrokerMetadata(0, 'foo', 12), BrokerMetadata(1, 'bar', 34)])
def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 2)) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_load_metadata(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): client.load_metadata_for_topics('topic_no_partitions') with self.assertRaises(UnknownTopicOrPartitionError): client.load_metadata_for_topics('topic_unknown') # This should not raise client.load_metadata_for_topics('topic_no_leader')
def test_get_leader_for_unassigned_partitions(self, protocol, conn): "Get leader raises if no partitions is defined for a topic" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(PartitionUnavailableError): client._get_leader_for_partition('topic_no_partitions', 0)
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete( client._get_leader_for_partition('topic_one_partition', 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client._topics_to_brokers)
def update_metadata(self, metadata): # In the common case where we ask for a single topic and get back an # error, we should fail the future if len(metadata.topics) == 1 and metadata.topics[0][0] != 0: error_code, topic, _ = metadata.topics[0] error = Errors.for_code(error_code)(topic) return self.failed_update(error) if not metadata.brokers: log.warning("No broker metadata found in MetadataResponse") for node_id, host, port in metadata.brokers: self._brokers.update({ node_id: BrokerMetadata(node_id, host, port) }) # Drop any UnknownTopic, InvalidTopic, and TopicAuthorizationFailed # but retain LeaderNotAvailable because it means topic is initializing self._partitions.clear() self._broker_partitions.clear() for error_code, topic, partitions in metadata.topics: error_type = Errors.for_code(error_code) if error_type is Errors.NoError: self._partitions[topic] = {} for _, partition, leader, _, _ in partitions: self._partitions[topic][partition] = leader if leader != -1: self._broker_partitions[leader].add(TopicPartition(topic, partition)) elif error_type is Errors.LeaderNotAvailableError: log.error("Topic %s is not available during auto-create" " initialization", topic) elif error_type is Errors.UnknownTopicOrPartitionError: log.error("Topic %s not found in cluster metadata", topic) elif error_type is Errors.TopicAuthorizationFailedError: log.error("Topic %s is not authorized for this client", topic) elif error_type is Errors.InvalidTopicError: log.error("'%s' is not a valid topic name", topic) else: log.error("Error fetching metadata for topic %s: %s", topic, error_type) if self._future: self._future.success(self) self._future = None self._need_update = False self._version += 1 now = time.time() * 1000 self._last_refresh_ms = now self._last_successful_refresh_ms = now log.debug("Updated cluster metadata version %d to %s", self._version, self) for listener in self._listeners: listener(self)
def test_decode_metadata_response(self): node_brokers = [ BrokerMetadata(0, b"brokers1.kafka.rdio.com", 1000), BrokerMetadata(1, b"brokers1.kafka.rdio.com", 1001), BrokerMetadata(3, b"brokers2.kafka.rdio.com", 1000) ] topic_partitions = [ TopicMetadata(b"topic1", 0, [ PartitionMetadata(b"topic1", 0, 1, (0, 2), (2, ), 0), PartitionMetadata(b"topic1", 1, 3, (0, 1), (0, 1), 1) ]), TopicMetadata(b"topic2", 1, [ PartitionMetadata(b"topic2", 0, 0, (), (), 0), ]), ] encoded = self._create_encoded_metadata_response( node_brokers, topic_partitions) decoded = KafkaProtocol.decode_metadata_response(encoded) self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_ensure_topic_exists(self, protocol): mocked_conns = {('broker_1', 4567): mock.MagicMock()} fut = asyncio.Future(loop=self.loop) fut.set_result(b'response') mocked_conns[('broker_1', 4567)].send.return_value = fut client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.ensure_topic_exists('topic_doesnt_exist', timeout=1)) with self.assertRaises(KafkaTimeoutError): self.loop.run_until_complete( client.ensure_topic_exists('topic_still_creating', timeout=1)) # This should not raise self.loop.run_until_complete( client.ensure_topic_exists('topic_noleaders', timeout=1))
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port, ), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics, ), cur) = relative_unpack('>i', data, cur) topic_metadata = {} for i in range(num_topics): # NOTE: topic_error is discarded. Should probably be returned with # the topic metadata. ((topic_error, ), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) partition_metadata = {} for j in range(num_partitions): # NOTE: partition_error_code is discarded. Should probably be # returned with the partition metadata. ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((num_isr, ), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata[partition] = \ PartitionMetadata( topic_name, partition, leader, replicas, isr) topic_metadata[topic_name] = partition_metadata return brokers, topic_metadata
def test_get_leader_returns_none_when_noleader(self, protocol, conn): "Getting leader for partitions returns None when the partiion has no leader" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 0)) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 1)) topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0]) } protocol.decode_metadata_response.return_value = (brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_load_metadata(self, protocol, conn): "Load metadata for all topics" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(1, 'broker_1', 4567) brokers[1] = BrokerMetadata(2, 'broker_2', 5678) topics = {} topics['topic_1'] = { 0: PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2]) } topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } topics['topic_no_partitions'] = {} topics['topic_3'] = { 0: PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0]), 2: PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers)
def test_send_produce_request_raises_when_noleader(self, protocol): """Send producer request raises LeaderNotAvailableError if leader is not available""" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete(client.send_produce_request(requests))
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = [] for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers.append(BrokerMetadata(nodeId, host, port)) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = [] for i in range(num_topics): ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = [] for j in range(num_partitions): ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata.append( PartitionMetadata(topic_name, partition, leader, replicas, isr, partition_error_code) ) topic_metadata.append( TopicMetadata(topic_name, topic_error, partition_metadata) ) return MetadataResponse(brokers, topic_metadata)