def test_has_metadata_for_topic(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_has_metadata_for_topic(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # Topics with no partitions return False self.assertFalse(client.has_metadata_for_topic('topic_still_creating')) self.assertFalse(client.has_metadata_for_topic('topic_doesnt_exist')) # Topic with partition metadata, but no leaders return True self.assertTrue(client.has_metadata_for_topic('topic_noleaders'))
def test_ensure_topic_exists(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) with self.assertRaises(UnknownTopicOrPartitionError): client.ensure_topic_exists('topic_doesnt_exist', timeout=1) with self.assertRaises(KafkaTimeoutError): client.ensure_topic_exists('topic_still_creating', timeout=1) # This should not raise client.ensure_topic_exists('topic_noleaders', timeout=1)
def test_decode_metadata_response(self): node_brokers = { 0: BrokerMetadata(0, "brokers1.kafka.rdio.com", 1000), 1: BrokerMetadata(1, "brokers1.kafka.rdio.com", 1001), 3: BrokerMetadata(3, "brokers2.kafka.rdio.com", 1000) } topic_partitions = { "topic1": { 0: PartitionMetadata("topic1", 0, 1, (0, 2), (2, )), 1: PartitionMetadata("topic1", 1, 3, (0, 1), (0, 1)) }, "topic2": { 0: PartitionMetadata("topic2", 0, 0, (), ()) } } topic_errors = {"topic1": 0, "topic2": 1} partition_errors = { ("topic1", 0): 0, ("topic1", 1): 1, ("topic2", 0): 0 } encoded = self._create_encoded_metadata_response( node_brokers, topic_partitions, topic_errors, partition_errors) decoded = KafkaProtocol.decode_metadata_response(encoded) self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_send_produce_request_raises_when_noleader(self, protocol, conn): "Send producer request raises LeaderNotAvailableError if leader is not available" conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): client.send_produce_request(requests)
def test_send_produce_request_raises_when_noleader(self, protocol, conn): "Send producer request raises LeaderUnavailableError if leader is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderUnavailableError): client.send_produce_request(requests)
def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 2)) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics['topic_no_partitions'] = { 0: PartitionMetadata('topic_no_partitions', 0, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_no_partitions', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_no_partitions', 0): brokers[0]}, client.topics_to_brokers)
def test__process_topic_partition_metadata(): METADATA_RESPONSE = { 'topic0': { 0: PartitionMetadata( topic='topic0', partition=0, leader=13, replicas=(13, 100), isr=(13, 100), error=0, ), 1: PartitionMetadata( topic='topic0', partition=1, leader=100, replicas=(13, 100), isr=(13, 100), error=0, ), }, 'topic1': { 0: PartitionMetadata( topic='topic1', partition=0, leader=666, replicas=(300, 500, 666), # one more replica isr=(500, 666), error=9, ), 1: PartitionMetadata( topic='topic1', partition=1, leader=300, replicas=(300, 500, 666), isr=(500, 666), error=0, ), }, } result = _process_topic_partition_metadata(METADATA_RESPONSE) assert result == set([('topic1', 0)])
def test__process_topic_partition_metadata_few_for_one_broker(): METADATA_RESPONSE = { 'topic0': { 0: PartitionMetadata( topic='topic0', partition=0, leader=100, replicas=(100), isr=(100,), error=9, ), 1: PartitionMetadata( topic='topic0', partition=1, leader=100, replicas=(13, 100), isr=(13, 100), error=0, ), }, 'topic1': { 0: PartitionMetadata( topic='topic1', partition=0, leader=666, replicas=(13, 500, 666), isr=(13, 500, 666), error=0, ), 1: PartitionMetadata( topic='topic1', partition=1, leader=666, replicas=(666), isr=(666,), error=9, ), }, } result = _process_topic_partition_metadata(METADATA_RESPONSE) assert result == set([('topic0', 0), ('topic1', 1)])
def test_decode_metadata_response(self): node_brokers = [ BrokerMetadata(0, b"brokers1.kafka.rdio.com", 1000), BrokerMetadata(1, b"brokers1.kafka.rdio.com", 1001), BrokerMetadata(3, b"brokers2.kafka.rdio.com", 1000) ] topic_partitions = [ TopicMetadata(b"topic1", 0, [ PartitionMetadata(b"topic1", 0, 1, (0, 2), (2, ), 0), PartitionMetadata(b"topic1", 1, 3, (0, 1), (0, 1), 1) ]), TopicMetadata(b"topic2", 1, [ PartitionMetadata(b"topic2", 0, 0, (), (), 0), ]), ] encoded = self._create_encoded_metadata_response( node_brokers, topic_partitions) decoded = KafkaProtocol.decode_metadata_response(encoded) self.assertEqual(decoded, (node_brokers, topic_partitions))
def test_ensure_topic_exists(self, protocol): mocked_conns = {('broker_1', 4567): mock.MagicMock()} fut = asyncio.Future(loop=self.loop) fut.set_result(b'response') mocked_conns[('broker_1', 4567)].send.return_value = fut client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_still_creating', NO_LEADER, []), TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_noleaders', NO_ERROR, [ PartitionMetadata('topic_noleaders', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleaders', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.ensure_topic_exists('topic_doesnt_exist', timeout=1)) with self.assertRaises(KafkaTimeoutError): self.loop.run_until_complete( client.ensure_topic_exists('topic_still_creating', timeout=1)) # This should not raise self.loop.run_until_complete( client.ensure_topic_exists('topic_noleaders', timeout=1))
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port, ), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics, ), cur) = relative_unpack('>i', data, cur) topic_metadata = {} for i in range(num_topics): # NOTE: topic_error is discarded. Should probably be returned with # the topic metadata. ((topic_error, ), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) partition_metadata = {} for j in range(num_partitions): # NOTE: partition_error_code is discarded. Should probably be # returned with the partition metadata. ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((num_isr, ), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata[partition] = \ PartitionMetadata( topic_name, partition, leader, replicas, isr) topic_metadata[topic_name] = partition_metadata return brokers, topic_metadata
def test_get_leader_returns_none_when_noleader(self, protocol, conn): "Getting leader for partitions returns None when the partiion has no leader" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 0)) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 1)) topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0]) } protocol.decode_metadata_response.return_value = (brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_send_produce_request_raises_when_noleader(self, protocol): """Send producer request raises LeaderNotAvailableError if leader is not available""" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) requests = [ ProduceRequest( "topic_noleader", 0, [create_message("a"), create_message("b")]) ] with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete(client.send_produce_request(requests))
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numbrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = [] for i in range(numbrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port,), cur) = relative_unpack('>i', data, cur) brokers.append(BrokerMetadata(nodeId, host, port)) # Topic info ((num_topics,), cur) = relative_unpack('>i', data, cur) topic_metadata = [] for i in range(num_topics): ((topic_error,), cur) = relative_unpack('>h', data, cur) (topic_name, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) partition_metadata = [] for j in range(num_partitions): ((partition_error_code, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack( '>%di' % numReplicas, data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur) partition_metadata.append( PartitionMetadata(topic_name, partition, leader, replicas, isr, partition_error_code) ) topic_metadata.append( TopicMetadata(topic_name, topic_error, partition_metadata) ) return MetadataResponse(brokers, topic_metadata)
def decode_metadata_response(cls, data): """ Decode bytes to a MetadataResponse Params ====== data: bytes to decode """ ((correlation_id, numBrokers), cur) = relative_unpack('>ii', data, 0) # Broker info brokers = {} for i in range(numBrokers): ((nodeId, ), cur) = relative_unpack('>i', data, cur) (host, cur) = read_short_string(data, cur) ((port, ), cur) = relative_unpack('>i', data, cur) brokers[nodeId] = BrokerMetadata(nodeId, host, port) # Topic info ((num_topics, ), cur) = relative_unpack('>i', data, cur) topicMetadata = {} for i in range(num_topics): ((topicError, ), cur) = relative_unpack('>h', data, cur) (topicName, cur) = read_short_string(data, cur) ((num_partitions, ), cur) = relative_unpack('>i', data, cur) partitionMetadata = {} for j in range(num_partitions): ((partitionErrorCode, partition, leader, numReplicas), cur) = \ relative_unpack('>hiii', data, cur) (replicas, cur) = relative_unpack('>%di' % numReplicas, data, cur) ((numIsr, ), cur) = relative_unpack('>i', data, cur) (isr, cur) = relative_unpack('>%di' % numIsr, data, cur) partitionMetadata[partition] = \ PartitionMetadata(topicName, partition, leader, replicas, isr) topicMetadata[topicName] = partitionMetadata return (brokers, topicMetadata)
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete( client._get_leader_for_partition('topic_one_partition', 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client._topics_to_brokers)
def test_load_metadata(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): client.load_metadata_for_topics('topic_no_partitions') with self.assertRaises(UnknownTopicOrPartitionError): client.load_metadata_for_topics('topic_unknown') # This should not raise client.load_metadata_for_topics('topic_no_leader')
def test_load_metadata(self, protocol, conn): "Load metadata for all topics" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(1, 'broker_1', 4567) brokers[1] = BrokerMetadata(2, 'broker_2', 5678) topics = {} topics['topic_1'] = { 0: PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2]) } topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } topics['topic_no_partitions'] = {} topics['topic_3'] = { 0: PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0]), 2: PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers)
import mock from kafka.common import PartitionMetadata from kafka_utils.util.config import ClusterConfig from kafka_utils.util.metadata import get_topic_partition_with_error from kafka_utils.util.metadata import LEADER_NOT_AVAILABLE_ERROR from kafka_utils.util.metadata import REPLICA_NOT_AVAILABLE_ERROR METADATA_RESPONSE_ALL_FINE = { 'topic0': { 0: PartitionMetadata( topic='topic0', partition=0, leader=13, replicas=(13, 100), isr=(13, 100), error=0, ), 1: PartitionMetadata( topic='topic0', partition=1, leader=100, replicas=(13, 100), isr=(13, 100), error=0, ), }, 'topic1': { 0:
from __future__ import absolute_import import mock from kafka.common import PartitionMetadata from pytest import fixture from kafka_utils.kafka_check.commands.replication_factor import _find_topics_with_wrong_rp from kafka_utils.kafka_check.commands.replication_factor import _prepare_output TOPICS_STATE = { 'topic_0': { 0: PartitionMetadata( topic='topic_0', partition=0, leader=170396635, replicas=(170396635, 170398981, 170396665), isr=(170398981, 170396635), error=0, ), }, 'topic_1': { 0: PartitionMetadata( topic='topic_1', partition=0, leader=170396635, replicas=(170396635, 170398981), isr=(170396635, 170398981), error=0, ), },
def update_metadata(self, metadata): # In the common case where we ask for a single topic and get back an # error, we should fail the future if len(metadata.topics) == 1 and metadata.topics[0][0] != 0: error_code, topic, _ = metadata.topics[0] error = Errors.for_code(error_code)(topic) return self.failed_update(error) if not metadata.brokers: log.warning("No broker metadata found in MetadataResponse") for node_id, host, port in metadata.brokers: self._brokers.update( {node_id: BrokerMetadata(node_id, host, port)}) _new_partitions = {} _new_broker_partitions = collections.defaultdict(set) _new_unauthorized_topics = set() for error_code, topic, partitions in metadata.topics: error_type = Errors.for_code(error_code) if error_type is Errors.NoError: _new_partitions[topic] = {} for p_error, partition, leader, replicas, isr in partitions: _new_partitions[topic][partition] = PartitionMetadata( topic=topic, partition=partition, leader=leader, replicas=replicas, isr=isr, error=p_error) if leader != -1: _new_broker_partitions[leader].add( TopicPartition(topic, partition)) elif error_type is Errors.LeaderNotAvailableError: log.warning( "Topic %s is not available during auto-create" " initialization", topic) elif error_type is Errors.UnknownTopicOrPartitionError: log.error("Topic %s not found in cluster metadata", topic) elif error_type is Errors.TopicAuthorizationFailedError: log.error("Topic %s is not authorized for this client", topic) _new_unauthorized_topics.add(topic) elif error_type is Errors.InvalidTopicError: log.error("'%s' is not a valid topic name", topic) else: log.error("Error fetching metadata for topic %s: %s", topic, error_type) with self._lock: self._partitions = _new_partitions self._broker_partitions = _new_broker_partitions self.unauthorized_topics = _new_unauthorized_topics f = None if self._future: f = self._future self._future = None self._need_update = False now = time.time() * 1000 self._last_refresh_ms = now self._last_successful_refresh_ms = now if f: f.success(self) log.debug("Updated cluster metadata to %s", self) for listener in self._listeners: listener(self)
def test_load_metadata(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]), TopicMetadata('topic_4', NO_ERROR, [ PartitionMetadata('topic_4', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_4', 1, 1, [1, 0], [1, 0], REPLICA_NOT_AVAILABLE), ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0], TopicAndPartition('topic_4', 0): brokers[0], TopicAndPartition('topic_4', 1): brokers[1] }, client._topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_partitions')) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_unknown')) # This should not raise self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_leader')) # This should not raise ReplicaNotAvailableError self.loop.run_until_complete( client.load_metadata_for_topics('topic_4'))
def test_get_leader_exceptions_when_noleader(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client._topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1))) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 2))) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual( brokers[0], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) self.assertEqual( brokers[1], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1)))