def get_group(self, group_name, cache=True):
"""
Get information on a group in the cluster. If cache is True, used cached group information
if it has not expired. Otherwise, send a request to refresh information on the group first.
Args:
group_name (string): The name of the group to return
cache (boolean): If False, ignore cached information and always fetch from the cluster
Returns:
Group: The Group object holding the detail for the group
Raises:
ConnectionError: If there is a failure to send the request to all brokers in the cluster
GroupError: If the group does not exist or there is a problem fetching information for it
"""
self._raise_if_not_connected()
try:
group = self.cluster.groups[group_name]
force_update = (not cache) or (not group.updated_since(time.time() - self.configuration.metadata_refresh))
except KeyError:
force_update = True
if force_update:
# Group detail must come from the coordinator broker
group_info = self._send_group_aware_request(group_name, DescribeGroupsV0Request({'group_ids': [group_name]}))
raise_if_error(GroupError, group_info['groups'][0]['error'])
self._update_groups_from_describe(group_info)
return self.cluster.groups[group_name]
def get_topic(self, topic_name, cache=True):
"""
Get information on a topic in the cluster. If cache is True, used cached topic metadata
if it has not expired. Otherwise, perform a metadata request to refresh information on
the topic first.
Args:
topic_name (string): The name of the topic to return
cache (boolean): If False, ignore cached metadata and always fetch from the cluster
Returns:
Topic: The Topic object holding the detail for the topic
Raises:
ConnectionError: If there is a failure to send the request to all brokers in the cluster
TopicError: If the topic does not exist, or there is a problem getting metadata for it
"""
self._raise_if_not_connected()
try:
topic = self.cluster.topics[topic_name]
force_update = (not cache) or (not topic.updated_since(time.time() - self.configuration.metadata_refresh))
except KeyError:
force_update = True
if force_update:
# It doesn't matter what broker we fetch topic metadata from
metadata = self._send_any_broker(TopicMetadataV1Request({'topics': [topic_name]}))
raise_if_error(TopicError, metadata['topics'][0]['error'])
self._update_from_metadata(metadata)
return self.cluster.topics[topic_name]
def get_group(self, group_name, cache=True):
"""
Get information on a group in the cluster. If cache is True, used cached group information
if it has not expired. Otherwise, send a request to refresh information on the group first.
Args:
group_name (string): The name of the group to return
cache (boolean): If False, ignore cached information and always fetch from the cluster
Returns:
Group: The Group object holding the detail for the group
Raises:
ConnectionError: If there is a failure to send the request to all brokers in the cluster
GroupError: If the group does not exist or there is a problem fetching information for it
"""
self._raise_if_not_connected()
try:
group = self.cluster.groups[group_name]
force_update = (not cache) or (
not group.updated_since(time.time() -
self.configuration.metadata_refresh))
except KeyError:
force_update = True
if force_update:
# Group detail must come from the coordinator broker
group_info = self._send_group_aware_request(
group_name,
DescribeGroupsV0Request({'group_ids': [group_name]}))
raise_if_error(GroupError, group_info['groups'][0]['error'])
self._update_groups_from_describe(group_info)
return self.cluster.groups[group_name]
def get_topic(self, topic_name, cache=True):
"""
Get information on a topic in the cluster. If cache is True, used cached topic metadata
if it has not expired. Otherwise, perform a metadata request to refresh information on
the topic first.
Args:
topic_name (string): The name of the topic to return
cache (boolean): If False, ignore cached metadata and always fetch from the cluster
Returns:
Topic: The Topic object holding the detail for the topic
Raises:
ConnectionError: If there is a failure to send the request to all brokers in the cluster
TopicError: If the topic does not exist, or there is a problem getting metadata for it
"""
self._raise_if_not_connected()
try:
topic = self.cluster.topics[topic_name]
force_update = (not cache) or (
not topic.updated_since(time.time() -
self.configuration.metadata_refresh))
except KeyError:
force_update = True
if force_update:
# It doesn't matter what broker we fetch topic metadata from
metadata = self._send_any_broker(
TopicMetadataV1Request({'topics': [topic_name]}))
raise_if_error(TopicError, metadata['topics'][0]['error'])
self._update_from_metadata(metadata)
return self.cluster.topics[topic_name]
def set_offsets_from_fetch(self, partitions):
"""
Given a partition_responses object from a OffsetFetch response, update the offsets
with the values in the response
Args:
partitions (Array): the partition_response object from a OffsetFetch response
Raises:
OffsetError: If there was a failure retrieving any of the offsets
"""
for partition in partitions:
raise_if_error(OffsetError, partition['error'])
self.partitions[partition['partition']] = partition['offset']
def set_offsets_from_list(self, partitions):
"""
Given a partition_responses object from a ListOffsets response, update the offsets
with the values in the response
Args:
partitions (Array): the partition_response object from a ListOffsets response
Raises:
OffsetError: If there was a failure retrieving any of the offsets
"""
for partition in partitions:
raise_if_error(OffsetError, partition['error'])
if len(partition['offsets']) > 0:
self.partitions[partition['partition']] = partition['offsets'][0]
else:
# We received no offsets back, so we'll just return -1 to indicate that
self.partitions[partition['partition']] = -1
def _send_group_aware_request(self, group_name, request):
"""
Sends a request to the broker currently serving as the group coordinator for the specified
group name. The request is not introspected for whether or not this is the correct broker,
and the response is not checked as to whether or not there was an error.
As a side effect of this call, the group object is created if it exists, and the coordinator
attribute is set to the broker that is currently the coordinator for that group.
Args:
group_name (string): The name of the group to find the coordinator for
request (BaseRequest): A request instance, inherited from BaseRequest, to send to the
coordinator broker
Returns:
BaseResponse: The response instance, appropriate for the request, that is returned from
the broker
Raises:
ConnectionError: If there is a failure to send the request to the coordinator broker,
or a failure to retrieve the coordinator information
GroupError: If an error is returned when fetching coordinator information
"""
response = self._send_any_broker(
GroupCoordinatorV0Request({'group_id': group_name}))
raise_if_error(GroupError, response['error'])
if group_name not in self.cluster.groups:
self.cluster.add_group(Group(group_name))
try:
self.cluster.groups[group_name].coordinator = self.cluster.brokers[
response['node_id']]
except KeyError:
broker = self._make_broker(response)
self.cluster.add_broker(broker)
self.cluster.groups[group_name].coordinator = broker
response = self._send_to_broker(
self.cluster.groups[group_name].coordinator.id, request)
return response
def _send_group_aware_request(self, group_name, request):
"""
Sends a request to the broker currently serving as the group coordinator for the specified
group name. The request is not introspected for whether or not this is the correct broker,
and the response is not checked as to whether or not there was an error.
As a side effect of this call, the group object is created if it exists, and the coordinator
attribute is set to the broker that is currently the coordinator for that group.
Args:
group_name (string): The name of the group to find the coordinator for
request (BaseRequest): A request instance, inherited from BaseRequest, to send to the
coordinator broker
Returns:
BaseResponse: The response instance, appropriate for the request, that is returned from
the broker
Raises:
ConnectionError: If there is a failure to send the request to the coordinator broker,
or a failure to retrieve the coordinator information
GroupError: If an error is returned when fetching coordinator information
"""
response = self._send_any_broker(GroupCoordinatorV0Request({'group_id': group_name}))
raise_if_error(GroupError, response['error'])
if group_name not in self.cluster.groups:
self.cluster.add_group(Group(group_name))
try:
self.cluster.groups[group_name].coordinator = self.cluster.brokers[response['node_id']]
except KeyError:
broker = self._make_broker(response)
self.cluster.add_broker(broker)
self.cluster.groups[group_name].coordinator = broker
response = self._send_to_broker(self.cluster.groups[group_name].coordinator.id, request)
return response