def _consume_topic_partition(self, topic, partition): topic = kafka_bytestring(topic) if not isinstance(partition, int): raise KafkaConfigurationError('Unknown partition type (%s) ' '-- expected int' % type(partition)) if topic not in self._client.topic_partitions: raise UnknownTopicOrPartitionError("Topic %s not found in broker metadata" % topic) if partition not in self._client.get_partition_ids_for_topic(topic): raise UnknownTopicOrPartitionError("Partition %d not found in Topic %s " "in broker metadata" % (partition, topic)) logger.info("Configuring consumer to fetch topic '%s', partition %d", topic, partition) self._topics.append((topic, partition))
def assign(self, partitions): """Manually assign a list of TopicPartitions to this consumer. Arguments: partitions (list of TopicPartition): assignment for this instance. Raises: IllegalStateError: if consumer has already called subscribe() Warning: It is not possible to use both manual partition assignment with assign() and group assignment with subscribe(). Note: This interface does not support incremental assignment and will replace the previous assignment (if there was one). Note: Manual topic assignment through this method does not use the consumer's group management functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic metadata change. """ for tp in partitions: p_ids = self.partitions_for_topic(tp.topic) if not p_ids or tp.partition not in p_ids: raise UnknownTopicOrPartitionError(tp) self._subscription.assign_from_user(partitions) self._on_change_subscription() self._client.set_topics([tp.topic for tp in partitions])
def _wait_on_metadata(self, topic): """ Wait for cluster metadata including partitions for the given topic to be available. Arguments: topic (str): topic we want metadata for Returns: set: partition ids for the topic Raises: UnknownTopicOrPartitionError: if no topic or partitions found in cluster metadata """ if topic in self.client.cluster.topics(): return self._metadata.partitions_for_topic(topic) # add topic to metadata topic list if it is not there already. self.client.add_topic(topic) yield from self.client.force_metadata_update() if topic not in self.client.cluster.topics(): raise UnknownTopicOrPartitionError() return self._metadata.partitions_for_topic(topic)
def _wait_on_metadata(self, topic): """ Wait for cluster metadata including partitions for the given topic to be available. Arguments: topic (str): topic we want metadata for Returns: set: partition ids for the topic Raises: UnknownTopicOrPartitionError: if no topic or partitions found in cluster metadata """ if topic in self.cluster.topics(): return self.cluster.partitions_for_topic(topic) # add topic to metadata topic list if it is not there already. self.add_topic(topic) t0 = self._loop.time() while True: yield from self.force_metadata_update() if topic in self.cluster.topics(): break if (self._loop.time() - t0) > (self._request_timeout_ms / 1000): raise UnknownTopicOrPartitionError() yield from asyncio.sleep(self._retry_backoff, loop=self._loop) return self.cluster.partitions_for_topic(topic)
def start(self): yield from self._client.bootstrap() # Check Broker Version if not set explicitly if self._api_version == 'auto': self._api_version = yield from self._client.check_version() # Convert api_version config to tuple for easy comparisons self._api_version = tuple(map(int, self._api_version.split('.'))) if self._api_version < (0, 9): raise ValueError("Unsupported Kafka version: {}".format( self._api_version)) self._fetcher = Fetcher( self._client, self._subscription, loop=self._loop, key_deserializer=self._key_deserializer, value_deserializer=self._value_deserializer, fetch_min_bytes=self._fetch_min_bytes, fetch_max_wait_ms=self._fetch_max_wait_ms, max_partition_fetch_bytes=self._max_partition_fetch_bytes, check_crcs=self._check_crcs, fetcher_timeout=self._consumer_timeout) if self._group_id is not None: # using group coordinator for automatic partitions assignment self._coordinator = GroupCoordinator( self._client, self._subscription, loop=self._loop, group_id=self._group_id, heartbeat_interval_ms=self._heartbeat_interval_ms, retry_backoff_ms=self._retry_backoff_ms, enable_auto_commit=self._enable_auto_commit, auto_commit_interval_ms=self._auto_commit_interval_ms, assignors=self._partition_assignment_strategy) self._coordinator.on_group_rebalanced(self._on_change_subscription) yield from self._coordinator.ensure_active_group() elif self._subscription.needs_partition_assignment: # using manual partitions assignment by topic(s) yield from self._client.force_metadata_update() partitions = [] for topic in self._subscription.subscription: p_ids = self.partitions_for_topic(topic) if not p_ids: raise UnknownTopicOrPartitionError() for p_id in p_ids: partitions.append(TopicPartition(topic, p_id)) self._subscription.unsubscribe() self._subscription.assign_from_user(partitions) yield from self._update_fetch_positions( self._subscription.missing_fetch_positions())
def _get_leader_for_partition(self, topic, partition): """ Returns the leader for a partition or None if the partition exists but has no leader. UnknownTopicOrPartitionError will be raised if the topic or partition is not part of the metadata. LeaderNotAvailableError is raised if server has metadata, but there is no current leader """ key = TopicAndPartition(topic, partition) # Use cached metadata if it is there if self.topics_to_brokers.get(key) is not None: return self.topics_to_brokers[key] # Otherwise refresh metadata # If topic does not already exist, this will raise # UnknownTopicOrPartitionError if not auto-creating # LeaderNotAvailableError otherwise until partitions are created self.load_metadata_for_topics(topic) # If the partition doesn't actually exist, raise if partition not in self.topic_partitions[topic]: raise UnknownTopicOrPartitionError(key) # If there's no leader for the partition, raise meta = self.topic_partitions[topic][partition] if meta.leader == -1: raise LeaderNotAvailableError(meta) # Otherwise return the BrokerMetadata return self.brokers[meta.leader]
def mocked_func(node_id, request): if not send_fut.done(): send_fut.set_result(None) raise UnknownTopicOrPartitionError()