def test_get_leader_exceptions_when_noleader(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 0)) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 1)) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( client._get_leader_for_partition('topic_noleader', 2)) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def test_get_leader_for_partitions_reloads_metadata(self, protocol, conn): "Get leader for partitions reload metadata if it is not available" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {'topic_no_partitions': {}} protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) # topic metadata is loaded but empty self.assertDictEqual({}, client.topics_to_brokers) topics['topic_no_partitions'] = { 0: PartitionMetadata('topic_no_partitions', 0, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = client._get_leader_for_partition('topic_no_partitions', 0) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_no_partitions', 0): brokers[0]}, client.topics_to_brokers)
def test_first_send_failed(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicAndPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [FailedPayloadsError(req) for req in reqs] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponse(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # plus 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 5)
def load_metadata_for_topics(self, *topics): """ Discover brokers and metadata for a set of topics. This function is called lazily whenever metadata is unavailable. """ request_id = self._next_id() request = KafkaProtocol.encode_metadata_request( self.client_id, request_id, topics) response = self._send_broker_unaware_request(request_id, request) (brokers, topics) = KafkaProtocol.decode_metadata_response(response) log.debug("Broker metadata: %s", brokers) log.debug("Topic metadata: %s", topics) self.brokers = brokers for topic, partitions in topics.items(): self.reset_topic_metadata(topic) if not partitions: log.warning('No partitions for %s', topic) continue self.topic_partitions[topic] = [] for partition, meta in partitions.items(): self.topic_partitions[topic].append(partition) topic_part = TopicAndPartition(topic, partition) if meta.leader == -1: log.warning('No leader for topic %s partition %s', topic, partition) self.topics_to_brokers[topic_part] = None else: self.topics_to_brokers[topic_part] = brokers[meta.leader]
def send_fetch_request(self, payloads=[], fail_on_error=True, callback=None, max_wait_time=100, min_bytes=4096): """ Encode and send a FetchRequest Payloads are grouped by topic and partition so they can be pipelined to the same brokers. """ encoder = partial(KafkaProtocol.encode_fetch_request, max_wait_time=max_wait_time, min_bytes=min_bytes) resps = self._send_broker_aware_request( payloads, encoder, KafkaProtocol.decode_fetch_response) out = [] for resp in resps: # Check for errors if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR: raise Exception( "FetchRequest for %s failed with errorcode=%d" % (TopicAndPartition(resp.topic, resp.partition), resp.error)) # Run the callback if callback is not None: out.append(callback(resp)) else: out.append(resp) return out
def str_to_kafka_seq(seq): seq = json.loads(seq) # deconstruct tuple keys marshaled_seq = {} for key, val in seq.items(): topic, partition = key.split(',') marshaled_seq[TopicAndPartition(topic, int(partition))] = val return marshaled_seq
def _get_leader_for_partition(self, topic, partition): key = TopicAndPartition(topic, partition) if key not in self.topics_to_brokers: self.load_metadata_for_topics(topic) if key not in self.topics_to_brokers: raise KafkaRequestError("Partition does not exist: %s" % str(key)) return self.topics_to_brokers[key]
def _raise_on_response_error(self, resp): if resp.error == ErrorMapping.NO_ERROR: return if resp.error in (ErrorMapping.UNKNOWN_TOPIC_OR_PARTITON, ErrorMapping.NOT_LEADER_FOR_PARTITION): self.reset_topic_metadata(resp.topic) raise BrokerResponseError( "Request for %s failed with errorcode=%d" % (TopicAndPartition(resp.topic, resp.partition), resp.error))
def reset_topic_metadata(self, *topics): for topic in topics: try: partitions = self.topic_partitions[topic] except KeyError: continue for partition in partitions: self.topics_to_brokers.pop(TopicAndPartition(topic, partition), None) del self.topic_partitions[topic]
def get_or_create_wrapped(self, verify_unchanged=None): checkpoints = self._get_checkpoints() ret = {} if checkpoints: timestamp = checkpoints[0].last_modified for checkpoint in checkpoints: ret[TopicAndPartition(checkpoint.topic, checkpoint.partition)] = checkpoint.offset if checkpoint.last_modified > timestamp: timestamp = checkpoint.last_modified else: timestamp = datetime.fromtimestamp(0) return WrappedCheckpoint(ret, timestamp)
def test_wo_retries(self): # lets create a queue and add 10 messages for 1 partition for i in range(10): self.queue.put((TopicAndPartition("test", 0), "msg %i", "key %i")) self._run_process() # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 4 non-void cals: # 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 4)
def test_clear_metadata(self): # make sure that mapping topic/partition exists for partition in self.client._topic_partitions[self.topic].values(): self.assertIsInstance(partition, PartitionMetadata) # make sure that mapping TopicAndPartition to broker exists key = TopicAndPartition(self.topic, 0) self.assertIsInstance(self.client._topics_to_brokers[key], BrokerMetadata) self.client.reset_all_metadata() self.assertEqual(self.client._topics_to_brokers, {}) self.assertEqual(self.client._topic_partitions, {})
def send_produce_request(self, payloads=[], acks=1, timeout=1000, fail_on_error=True, callback=None): """ Encode and send some ProduceRequests ProduceRequests will be grouped by (topic, partition) and then sent to a specific broker. Output is a list of responses in the same order as the list of payloads specified Params ====== payloads: list of ProduceRequest fail_on_error: boolean, should we raise an Exception if we encounter an API error? callback: function, instead of returning the ProduceResponse, first pass it through this function Return ====== list of ProduceResponse or callback(ProduceResponse), in the order of input payloads """ encoder = partial( KafkaProtocol.encode_produce_request, acks=acks, timeout=timeout) if acks == 0: decoder = None else: decoder = KafkaProtocol.decode_produce_response resps = self._send_broker_aware_request(payloads, encoder, decoder) out = [] for resp in resps: # Check for errors if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR: raise Exception( "ProduceRequest for %s failed with errorcode=%d" % (TopicAndPartition(resp.topic, resp.partition), resp.error)) # Run the callback if callback is not None: out.append(callback(resp)) else: out.append(resp) return out
def test_get_leader_returns_none_when_noleader(self, protocol, conn): "Getting leader for partitions returns None when the partiion has no leader" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(0, 'broker_1', 4567) brokers[1] = BrokerMetadata(1, 'broker_2', 5678) topics = {} topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } protocol.decode_metadata_response.return_value = (brokers, topics) client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client.topics_to_brokers) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 0)) self.assertIsNone(client._get_leader_for_partition( 'topic_noleader', 1)) topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0]) } protocol.decode_metadata_response.return_value = (brokers, topics) self.assertEqual(brokers[0], client._get_leader_for_partition('topic_noleader', 0)) self.assertEqual(brokers[1], client._get_leader_for_partition('topic_noleader', 1))
def handle(self, topic, num_partitions, **options): stop_pillows = raw_input("did you stop pillows? [y/n]") if stop_pillows not in ['y', 'yes']: print("then stop them") kafka_command = ( "./kafka-topics.sh --alter --zookeeper <zk IP>:2181 --partitions={} --topic={}" .format(num_partitions, topic)) added_partition = raw_input( "have you run {} ? [y/n]".format(kafka_command)) if added_partition not in ['y', 'yes']: print("then run it") for checkpoint in DjangoPillowCheckpoint.objects.filter( sequence_format='json'): try: kafka_seq = str_to_kafka_seq(checkpoint.sequence) except ValueError: print("unable to parse {}", checkpoint.checkpoint_id) continue topics = [tp.topic for tp in kafka_seq] if topic not in topics: print("topic does not exist in {}", checkpoint.checkpoint_id) continue changed = False for partition in range(num_partitions): tp = TopicAndPartition(topic, partition) if tp in kafka_seq: continue else: changed = True kafka_seq[tp] = 0 if changed: checkpoint.old_sequence = checkpoint.sequence checkpoint.sequence = kafka_seq_to_str(kafka_seq) checkpoint.save() for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=checkpoint.checkpoint_id, topic=topic_partition.topic, partition=topic_partition.partition, defaults={'offset': offset}) print("please restart the pillows")
def test_get_leader_for_partitions_reloads_metadata(self, protocol): "Get leader for partitions reload metadata if it is not available" @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [TopicMetadata('topic_no_partitions', NO_LEADER, [])] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) # topic metadata is loaded but empty self.assertDictEqual({}, client._topics_to_brokers) topics = [ TopicMetadata('topic_one_partition', NO_ERROR, [ PartitionMetadata('topic_no_partition', 0, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # calling _get_leader_for_partition (from any broker aware request) # will try loading metadata again for the same topic leader = self.loop.run_until_complete( client._get_leader_for_partition('topic_one_partition', 0)) self.assertEqual(brokers[0], leader) self.assertDictEqual( {TopicAndPartition('topic_one_partition', 0): brokers[0]}, client._topics_to_brokers)
def _get_leader_for_partition(self, topic, partition): """ Returns the leader for a partition or None if the partition exists but has no leader. PartitionUnavailableError will be raised if the topic or partition is not part of the metadata. """ key = TopicAndPartition(topic, partition) # reload metadata whether the partition is not available # or has no leader (broker is None) if self.topics_to_brokers.get(key) is None: self.load_metadata_for_topics(topic) if key not in self.topics_to_brokers: raise PartitionUnavailableError("%s not available" % str(key)) return self.topics_to_brokers[key]
def test_load_metadata(self, protocol, conn): conn.recv.return_value = 'response' # anything but None brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): client.load_metadata_for_topics('topic_no_partitions') with self.assertRaises(UnknownTopicOrPartitionError): client.load_metadata_for_topics('topic_unknown') # This should not raise client.load_metadata_for_topics('topic_no_leader')
def _load_metadata_for_topics(self, *topics): """ Discover brokers and metadata for a set of topics. This method will recurse in the event of a retry. """ request_id = self._next_id() request = KafkaProtocol.encode_metadata_request(self.client_id, request_id, topics) response = self._send_broker_unaware_request(request_id, request) if response is None: raise Exception("All servers failed to process request") (brokers, topics) = KafkaProtocol.decode_metadata_response(response) log.debug("Broker metadata: %s", brokers) log.debug("Topic metadata: %s", topics) self.brokers = brokers self.topics_to_brokers = {} for topic, partitions in topics.items(): # Clear the list once before we add it. This removes stale entries # and avoids duplicates self.topic_partitions.pop(topic, None) if not partitions: log.info("Partition is unassigned, delay for 1s and retry") time.sleep(1) self._load_metadata_for_topics(topic) break for partition, meta in partitions.items(): if meta.leader == -1: log.info("Partition is unassigned, delay for 1s and retry") time.sleep(1) self._load_metadata_for_topics(topic) else: topic_part = TopicAndPartition(topic, partition) self.topics_to_brokers[topic_part] = brokers[meta.leader] self.topic_partitions[topic].append(partition)
def test_with_limited_retries(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicAndPartition("test", i), "msg %i" % i, "key %i" % i)) def send_side_effect(reqs, *args, **kwargs): return [FailedPayloadsError(req) for req in reqs] self.client.send_produce_request.side_effect = send_side_effect self._run_process(3, 3) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 16 non-void calls: # 3 initial batches of 3 msgs each + 1 initial batch of 1 msg + # 3 retries of the batches above = (1 + 3 retries) * 4 batches = 16 self.assertEqual(self.client.send_produce_request.call_count, 16)
def test_async_producer_not_leader(self): for i in range(10): self.queue.put((TopicAndPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [ ProduceResponse(req.topic, req.partition, NotLeaderForPartitionError.errno, -1) for req in reqs ] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponse(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # + 3 batches of 3 msgs each + 1 batch of 1 msg = 1 + 3 + 1 = 5 self.assertEqual(self.client.send_produce_request.call_count, 5)
def _get_leader_for_partition(self, topic, partition): """ Returns the leader for a partition or None if the partition exists but has no leader. UnknownTopicOrPartitionError will be raised if the topic or partition is not part of the metadata. LeaderNotAvailableError is raised if server has metadata, but there is no current leader """ key = TopicAndPartition(topic, partition) # Use cached metadata if it is there if self.topics_to_brokers.get(key) is not None: return self.topics_to_brokers[key] # Otherwise refresh metadata # If topic does not already exist, this will raise # UnknownTopicOrPartitionError if not auto-creating # LeaderNotAvailableError otherwise until partitions are created self.load_metadata_for_topics(topic) # If the partition doesn't actually exist, raise if partition not in self.topic_partitions[topic]: raise UnknownTopicOrPartitionError(key) # If there's no leader for the partition, raise meta = self.topic_partitions[topic][partition] if meta.leader == -1: raise LeaderNotAvailableError(meta) # Otherwise return the BrokerMetadata return self.brokers[meta.leader]
def test_load_metadata(self, protocol, conn): "Load metadata for all topics" conn.recv.return_value = 'response' # anything but None brokers = {} brokers[0] = BrokerMetadata(1, 'broker_1', 4567) brokers[1] = BrokerMetadata(2, 'broker_2', 5678) topics = {} topics['topic_1'] = { 0: PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2]) } topics['topic_noleader'] = { 0: PartitionMetadata('topic_noleader', 0, -1, [], []), 1: PartitionMetadata('topic_noleader', 1, -1, [], []) } topics['topic_no_partitions'] = {} topics['topic_3'] = { 0: PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1]), 1: PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0]), 2: PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1]) } protocol.decode_metadata_response.return_value = (brokers, topics) # client loads metadata at init client = KafkaClient(hosts=['broker_1:4567']) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0] }, client.topics_to_brokers)
def test_load_metadata(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata( 'topic_1', NO_ERROR, [PartitionMetadata('topic_1', 0, 1, [1, 2], [1, 2], NO_ERROR) ]), TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), TopicMetadata('topic_no_partitions', NO_LEADER, []), TopicMetadata('topic_unknown', UNKNOWN_TOPIC_OR_PARTITION, []), TopicMetadata('topic_3', NO_ERROR, [ PartitionMetadata('topic_3', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_3', 1, 1, [1, 0], [1, 0], NO_ERROR), PartitionMetadata('topic_3', 2, 0, [0, 1], [0, 1], NO_ERROR) ]), TopicMetadata('topic_4', NO_ERROR, [ PartitionMetadata('topic_4', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_4', 1, 1, [1, 0], [1, 0], REPLICA_NOT_AVAILABLE), ]) ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_1', 0): brokers[1], TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None, TopicAndPartition('topic_3', 0): brokers[0], TopicAndPartition('topic_3', 1): brokers[1], TopicAndPartition('topic_3', 2): brokers[0], TopicAndPartition('topic_4', 0): brokers[0], TopicAndPartition('topic_4', 1): brokers[1] }, client._topics_to_brokers) # if we ask for metadata explicitly, it should raise errors with self.assertRaises(LeaderNotAvailableError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_partitions')) with self.assertRaises(UnknownTopicOrPartitionError): self.loop.run_until_complete( client.load_metadata_for_topics('topic_unknown')) # This should not raise self.loop.run_until_complete( client.load_metadata_for_topics('topic_no_leader')) # This should not raise ReplicaNotAvailableError self.loop.run_until_complete( client.load_metadata_for_topics('topic_4'))
def test_get_leader_exceptions_when_noleader(self, protocol): @asyncio.coroutine def recv(request_id): return b'response' mocked_conns = {('broker_1', 4567): mock.MagicMock()} mocked_conns[('broker_1', 4567)].recv.side_effect = recv client = AIOKafkaClient(['broker_1:4567'], loop=self.loop) client._conns = mocked_conns brokers = [ BrokerMetadata(0, 'broker_1', 4567), BrokerMetadata(1, 'broker_2', 5678) ] topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER), PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER), ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.loop.run_until_complete(client.load_metadata_for_topics()) self.assertDictEqual( { TopicAndPartition('topic_noleader', 0): None, TopicAndPartition('topic_noleader', 1): None }, client._topics_to_brokers) # No leader partitions -- raise LeaderNotAvailableError with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) with self.assertRaises(LeaderNotAvailableError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1))) # Unknown partitions -- raise UnknownTopicOrPartitionError with self.assertRaises(UnknownTopicOrPartitionError): self.assertIsNone( self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 2))) topics = [ TopicMetadata('topic_noleader', NO_ERROR, [ PartitionMetadata('topic_noleader', 0, 0, [0, 1], [0, 1], NO_ERROR), PartitionMetadata('topic_noleader', 1, 1, [1, 0], [1, 0], NO_ERROR) ]), ] protocol.decode_metadata_response.return_value = MetadataResponse( brokers, topics) self.assertEqual( brokers[0], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 0))) self.assertEqual( brokers[1], self.loop.run_until_complete( client._get_leader_for_partition('topic_noleader', 1)))
def _kill_leader(self, topic, partition): leader = self.client.topics_to_brokers[TopicAndPartition( kafka_bytestring(topic), partition)] broker = self.brokers[leader.nodeId] broker.close() return broker
def load_metadata_for_topics(self, *topics): """ Fetch broker and topic-partition metadata from the server, and update internal data: broker list, topic/partition list, and topic/parition -> broker map This method should be called after receiving any error @param: *topics (optional) If a list of topics is provided, the metadata refresh will be limited to the specified topics only. Exceptions: ---------- If the broker is configured to not auto-create topics, expect UnknownTopicOrPartitionError for topics that don't exist If the broker is configured to auto-create topics, expect LeaderNotAvailableError for new topics until partitions have been initialized. Exceptions *will not* be raised in a full refresh (i.e. no topic list) In this case, error codes will be logged as errors Partition-level errors will also not be raised here (a single partition w/o a leader, for example) """ resp = self.send_metadata_request(topics) log.debug("Broker metadata: %s", resp.brokers) log.debug("Topic metadata: %s", resp.topics) self.brokers = dict([(broker.nodeId, broker) for broker in resp.brokers]) for topic_metadata in resp.topics: topic = topic_metadata.topic partitions = topic_metadata.partitions self.reset_topic_metadata(topic) # Errors expected for new topics try: kafka.common.check_error(topic_metadata) except (UnknownTopicOrPartitionError, LeaderNotAvailableError) as e: # Raise if the topic was passed in explicitly if topic in topics: raise # Otherwise, just log a warning log.error("Error loading topic metadata for %s: %s", topic, type(e)) continue self.topic_partitions[topic] = {} for partition_metadata in partitions: partition = partition_metadata.partition leader = partition_metadata.leader self.topic_partitions[topic][partition] = partition_metadata # Populate topics_to_brokers dict topic_part = TopicAndPartition(topic, partition) # Check for partition errors try: kafka.common.check_error(partition_metadata) # If No Leader, topics_to_brokers topic_partition -> None except LeaderNotAvailableError: log.error('No leader for topic %s partition %d', topic, partition) self.topics_to_brokers[topic_part] = None continue # If Known Broker, topic_partition -> BrokerMetadata if leader in self.brokers: self.topics_to_brokers[topic_part] = self.brokers[leader] # If Unknown Broker, fake BrokerMetadata so we dont lose the id # (not sure how this could happen. server could be in bad state) else: self.topics_to_brokers[topic_part] = BrokerMetadata( leader, None, None )