def test_has_metadata_for_topic(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_ensure_topic_exists(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) with self.assertRaises(UnknownTopicOrPartitionError): client.ensure_topic_exists('topic_doesnt_exist', timeout=1) with self.assertRaises(KafkaTimeoutError): client.ensure_topic_exists('topic_still_creating', timeout=1) # This should not raise client.ensure_topic_exists('topic_noleaders', timeout=1)
def test_get_leader_for_unassigned_partitions(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual({}, client._topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client._get_leader_for_partition('topic_no_partitions', 0)) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client._get_leader_for_partition('topic_unknown', 0))
def test_get_leader_for_unassigned_partitions(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual({}, client.topics_to_brokers) with self.assertRaises(LeaderNotAvailableError): client._get_leader_for_partition('topic_no_partitions', 0) with self.assertRaises(UnknownTopicOrPartitionError): client._get_leader_for_partition('topic_unknown', 0)
def test_has_metadata_for_topic(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 2)) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_load_metadata(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): client.load_metadata_for_topics('topic_no_partitions') with self.assertRaises(UnknownTopicOrPartitionError): client.load_metadata_for_topics('topic_unknown') # This should not raise client.load_metadata_for_topics('topic_no_leader')
def test_send_produce_request_raises_when_topic_unknown(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) requests = [ ProduceRequest( "topic_doesnt_exist", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete(client.send_produce_request(requests))
def test_send_produce_request_raises_when_topic_unknown( self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_doesnt_exist", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(UnknownTopicOrPartitionError): client.send_produce_request(requests)
def test_send_produce_request_raises_when_noleader(self, protocol, conn): "Send producer request raises LeaderNotAvailableError if leader is not available" conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): client.send_produce_request(requests)
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete( client._get_leader_for_partition('topic_one_partition', 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client._topics_to_brokers)
def test_decode_metadata_response(self): node_brokers = [ BrokerMetadata(0, b"brokers1.kafka.rdio.com", 1000), BrokerMetadata(1, b"brokers1.kafka.rdio.com", 1001), BrokerMetadata(3, b"brokers2.kafka.rdio.com", 1000) ] topic_partitions = [ TopicMetadata(b"topic1", 0, [ PartitionMetadata(b"topic1", 0, 1, (0, 2), (2, ), 0), PartitionMetadata(b"topic1", 1, 3, (0, 1), (0, 1), 1) ]), TopicMetadata(b"topic2", 1, [ PartitionMetadata(b"topic2", 0, 0, (), (), 0), ]), ] encoded = self._create_encoded_metadata_response( node_brokers, topic_partitions) decoded = KafkaProtocol.decode_metadata_response(encoded) self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_ensure_topic_exists(self, protocol): mocked_conns = {('broker_1', 4567): mock.MagicMock()} fut = asyncio.Future(loop=self.loop) fut.set_result(b'response') mocked_conns[('broker_1', 4567)].send.return_value = fut client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.ensure_topic_exists('topic_doesnt_exist', timeout=1)) with self.assertRaises(KafkaTimeoutError): self.loop.run_until_complete( client.ensure_topic_exists('topic_still_creating', timeout=1)) # This should not raise self.loop.run_until_complete( client.ensure_topic_exists('topic_noleaders', timeout=1))
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_one_partition', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client.topics_to_brokers)
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = [] for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers.append(BrokerMetadata(nodeId, host, port)) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = [] for i in range(num_topics): ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = [] for j in range(num_partitions): ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata.append( PartitionMetadata(topic_name, partition, leader, replicas, isr, partition_error_code) ) topic_metadata.append( TopicMetadata(topic_name, topic_error, partition_metadata) ) return MetadataResponse(brokers, topic_metadata)
def test_send_produce_request_raises_when_noleader(self, protocol): """Send producer request raises LeaderNotAvailableError if leader is not available""" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete(client.send_produce_request(requests))
def test_load_metadata(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]), TopicMetadata('topic_4', NO_ERROR, [ PartitionMetadata('topic_4', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_4', 1, 1, [1, 0], [1, 0], REPLICA_NOT_AVAILABLE), ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0], TopicAndPartition('topic_4', 0): brokers[0], TopicAndPartition('topic_4', 1): brokers[1] }, client._topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_partitions')) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_unknown')) # This should not raise self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_leader')) # This should not raise ReplicaNotAvailableError self.loop.run_until_complete( client.load_metadata_for_topics('topic_4'))
def test_get_leader_exceptions_when_noleader(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client._topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1))) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 2))) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual( brokers[0], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) self.assertEqual( brokers[1], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1)))