def get_group(self, group_name, cache=True): """ Get information on a group in the cluster. If cache is True, used cached group information if it has not expired. Otherwise, send a request to refresh information on the group first. Args: group_name (string): The name of the group to return cache (boolean): If False, ignore cached information and always fetch from the cluster Returns: Group: The Group object holding the detail for the group Raises: ConnectionError: If there is a failure to send the request to all brokers in the cluster GroupError: If the group does not exist or there is a problem fetching information for it """ self._raise_if_not_connected() try: group = self.cluster.groups[group_name] force_update = (not cache) or (not group.updated_since(time.time() - self.configuration.metadata_refresh)) except KeyError: force_update = True if force_update: # Group detail must come from the coordinator broker group_info = self._send_group_aware_request(group_name, DescribeGroupsV0Request({'group_ids': [group_name]})) raise_if_error(GroupError, group_info['groups'][0]['error']) self._update_groups_from_describe(group_info) return self.cluster.groups[group_name]
def get_topic(self, topic_name, cache=True): """ Get information on a topic in the cluster. If cache is True, used cached topic metadata if it has not expired. Otherwise, perform a metadata request to refresh information on the topic first. Args: topic_name (string): The name of the topic to return cache (boolean): If False, ignore cached metadata and always fetch from the cluster Returns: Topic: The Topic object holding the detail for the topic Raises: ConnectionError: If there is a failure to send the request to all brokers in the cluster TopicError: If the topic does not exist, or there is a problem getting metadata for it """ self._raise_if_not_connected() try: topic = self.cluster.topics[topic_name] force_update = (not cache) or (not topic.updated_since(time.time() - self.configuration.metadata_refresh)) except KeyError: force_update = True if force_update: # It doesn't matter what broker we fetch topic metadata from metadata = self._send_any_broker(TopicMetadataV1Request({'topics': [topic_name]})) raise_if_error(TopicError, metadata['topics'][0]['error']) self._update_from_metadata(metadata) return self.cluster.topics[topic_name]
def get_group(self, group_name, cache=True): """ Get information on a group in the cluster. If cache is True, used cached group information if it has not expired. Otherwise, send a request to refresh information on the group first. Args: group_name (string): The name of the group to return cache (boolean): If False, ignore cached information and always fetch from the cluster Returns: Group: The Group object holding the detail for the group Raises: ConnectionError: If there is a failure to send the request to all brokers in the cluster GroupError: If the group does not exist or there is a problem fetching information for it """ self._raise_if_not_connected() try: group = self.cluster.groups[group_name] force_update = (not cache) or ( not group.updated_since(time.time() - self.configuration.metadata_refresh)) except KeyError: force_update = True if force_update: # Group detail must come from the coordinator broker group_info = self._send_group_aware_request( group_name, DescribeGroupsV0Request({'group_ids': [group_name]})) raise_if_error(GroupError, group_info['groups'][0]['error']) self._update_groups_from_describe(group_info) return self.cluster.groups[group_name]
def get_topic(self, topic_name, cache=True): """ Get information on a topic in the cluster. If cache is True, used cached topic metadata if it has not expired. Otherwise, perform a metadata request to refresh information on the topic first. Args: topic_name (string): The name of the topic to return cache (boolean): If False, ignore cached metadata and always fetch from the cluster Returns: Topic: The Topic object holding the detail for the topic Raises: ConnectionError: If there is a failure to send the request to all brokers in the cluster TopicError: If the topic does not exist, or there is a problem getting metadata for it """ self._raise_if_not_connected() try: topic = self.cluster.topics[topic_name] force_update = (not cache) or ( not topic.updated_since(time.time() - self.configuration.metadata_refresh)) except KeyError: force_update = True if force_update: # It doesn't matter what broker we fetch topic metadata from metadata = self._send_any_broker( TopicMetadataV1Request({'topics': [topic_name]})) raise_if_error(TopicError, metadata['topics'][0]['error']) self._update_from_metadata(metadata) return self.cluster.topics[topic_name]
def set_offsets_from_fetch(self, partitions): """ Given a partition_responses object from a OffsetFetch response, update the offsets with the values in the response Args: partitions (Array): the partition_response object from a OffsetFetch response Raises: OffsetError: If there was a failure retrieving any of the offsets """ for partition in partitions: raise_if_error(OffsetError, partition['error']) self.partitions[partition['partition']] = partition['offset']
def set_offsets_from_list(self, partitions): """ Given a partition_responses object from a ListOffsets response, update the offsets with the values in the response Args: partitions (Array): the partition_response object from a ListOffsets response Raises: OffsetError: If there was a failure retrieving any of the offsets """ for partition in partitions: raise_if_error(OffsetError, partition['error']) if len(partition['offsets']) > 0: self.partitions[partition['partition']] = partition['offsets'][0] else: # We received no offsets back, so we'll just return -1 to indicate that self.partitions[partition['partition']] = -1
def _send_group_aware_request(self, group_name, request): """ Sends a request to the broker currently serving as the group coordinator for the specified group name. The request is not introspected for whether or not this is the correct broker, and the response is not checked as to whether or not there was an error. As a side effect of this call, the group object is created if it exists, and the coordinator attribute is set to the broker that is currently the coordinator for that group. Args: group_name (string): The name of the group to find the coordinator for request (BaseRequest): A request instance, inherited from BaseRequest, to send to the coordinator broker Returns: BaseResponse: The response instance, appropriate for the request, that is returned from the broker Raises: ConnectionError: If there is a failure to send the request to the coordinator broker, or a failure to retrieve the coordinator information GroupError: If an error is returned when fetching coordinator information """ response = self._send_any_broker( GroupCoordinatorV0Request({'group_id': group_name})) raise_if_error(GroupError, response['error']) if group_name not in self.cluster.groups: self.cluster.add_group(Group(group_name)) try: self.cluster.groups[group_name].coordinator = self.cluster.brokers[ response['node_id']] except KeyError: broker = self._make_broker(response) self.cluster.add_broker(broker) self.cluster.groups[group_name].coordinator = broker response = self._send_to_broker( self.cluster.groups[group_name].coordinator.id, request) return response
def _send_group_aware_request(self, group_name, request): """ Sends a request to the broker currently serving as the group coordinator for the specified group name. The request is not introspected for whether or not this is the correct broker, and the response is not checked as to whether or not there was an error. As a side effect of this call, the group object is created if it exists, and the coordinator attribute is set to the broker that is currently the coordinator for that group. Args: group_name (string): The name of the group to find the coordinator for request (BaseRequest): A request instance, inherited from BaseRequest, to send to the coordinator broker Returns: BaseResponse: The response instance, appropriate for the request, that is returned from the broker Raises: ConnectionError: If there is a failure to send the request to the coordinator broker, or a failure to retrieve the coordinator information GroupError: If an error is returned when fetching coordinator information """ response = self._send_any_broker(GroupCoordinatorV0Request({'group_id': group_name})) raise_if_error(GroupError, response['error']) if group_name not in self.cluster.groups: self.cluster.add_group(Group(group_name)) try: self.cluster.groups[group_name].coordinator = self.cluster.brokers[response['node_id']] except KeyError: broker = self._make_broker(response) self.cluster.add_broker(broker) self.cluster.groups[group_name].coordinator = broker response = self._send_to_broker(self.cluster.groups[group_name].coordinator.id, request) return response