def discovery_info_from_group(connect, group): """discovery topics and parts from group """ lags = [] for member in connect.describe_consumer_groups([group])[0][5]: for data in MemberAssignment.decode(member[4]).assignment: topic = data[0] parts = data[1] for part in parts: lags.append({"{#CGRP}": group, "{#PRT}": part, "{#TOP}": topic}) print(json.dumps(lags))
def get_lag_from_group(connect, group): """get lag from group """ consumer = KafkaConsumer(**CONNECT_PARAMS, request_timeout_ms=10001, group_id=group) for member in connect.describe_consumer_groups([group])[0][5]: for data in MemberAssignment.decode(member[4]).assignment: topic = data[0] parts = data[1] for part in parts: topart = TopicPartition(topic, part) last_offsets = consumer.end_offsets([topart]) committed = consumer.committed(topart) if last_offsets != None and committed != None: lag = last_offsets.get(topart) - committed print("group: {} topic: {} partition: {} lag: {}".\ format(group, topic, part, lag))
def generate_consumer_groups_for_broker(broker, response): """ From a `broker` and `response` generate a list of consumer groups """ consumer_groups = {} for err, gid, gstate, prot_type, prot, _members in response.groups: members = {} for mid, cid, chost, mdata, assign in _members: mdata = ProtocolMetadata.decode(mdata) assign = MemberAssignment.decode(assign) assignment = {} for t, p in assign.assignment: assignment[t] = p members[mid] = { 'client_id': cid, 'client_host': chost, 'member_metadata': { 'version': mdata.version, 'subscription': mdata.subscription, 'user_data': mdata.user_data.decode('utf-8') }, 'member_assignment': { 'version': assign.version, 'assignment': assignment, 'user_data': assign.user_data.decode('utf-8') } } group = { 'error_code': err, 'group_state': gstate, 'members': members, 'protocol_type': prot_type, 'protocol': prot, 'coordinator': { 'host': broker.host, 'nodeId': broker.nodeId, 'port': broker.port, 'rack': broker.rack } } consumer_groups[gid] = group return consumer_groups
def check(self, group_topics=None, discovery=None): """ { "<group>": { "state": <str>, "topics": { "<topic>": { "consumer_lag": <int>, "partitions": { "<partition>": { "offset_first": <int>, "offset_consumed": <int>, "offset_last": <int>, "lag": <int> } } } } } } :param persist_groups: :return: consumer statistics """ cluster = self.client.cluster brokers = cluster.brokers() # Consumer group ID -> list(topics) if group_topics is None: group_topics = {} if discovery is None: discovery = True else: group_topics = copy.deepcopy(group_topics) # Set of consumer group IDs consumer_groups = set(group_topics.iterkeys()) # Set of all known topics topics = set(itertools.chain(*group_topics.itervalues())) # Consumer group ID -> coordinating broker consumer_coordinator = {} # Coordinating broker - > list(consumer group IDs) coordinator_consumers = {} results = {} for consumer_group in group_topics.iterkeys(): results[consumer_group] = {'state': None, 'topics': {}} # Ensure connections to all brokers for broker in brokers: while not self.client.is_ready(broker.nodeId): self.client.ready(broker.nodeId) # Collect all active consumer groups if discovery: for broker in brokers: response = self._send(broker.nodeId, _ListGroupsRequest(), _ListGroupsResponse) if response: for group in response.groups: consumer_groups.add(group[0]) # Identify which broker is coordinating each consumer group for group in consumer_groups: response = self._send(next(iter(brokers)).nodeId, _GroupCoordinatorRequest(group), _GroupCoordinatorResponse) if response: consumer_coordinator[group] = response.coordinator_id if response.coordinator_id not in coordinator_consumers: coordinator_consumers[response.coordinator_id] = [] coordinator_consumers[response.coordinator_id].append(group) # Populate consumer groups into dict for group in consumer_groups: if group not in group_topics: group_topics[group] = [] # Add groups to results dict for group, topic_list in group_topics.iteritems(): results[group] = {'state': None, 'topics': {}} # Identify group information and topics read by each consumer group for coordinator, consumers in coordinator_consumers.iteritems(): response = self._send(coordinator, _DescribeGroupsRequest(consumers), _DescribeGroupsResponse) for group in response.groups: if group[1] in results: results[group[1]]['state'] = group[2] # TODO Also include member data? if discovery: members = group[5] for member in members: try: assignment = MemberAssignment.decode(member[4]) if assignment: for partition in assignment.partition_assignment: topic = partition[0] # Add topic to topic set topics.add(topic) # Add topic to group group_topics[group[1]].append(topic) except: pass # Add topics to groups in results dict for group, topic_list in group_topics.iteritems(): for topic in topic_list: results[group]['topics'][topic] = {'consumer_lag': 0, 'partitions': {}} # For storing the latest offset for all partitions of all topics # topic -> partition -> offset start_offsets = {} end_offsets = {} # Identify all the topic partitions that each broker is leader for # and request next new offset for each partition for broker, partitions in cluster._broker_partitions.iteritems(): # topic -> List(partition, time, max_offsets) request_partitions = {} for tp in partitions: if tp.topic in topics: if tp.topic not in request_partitions: request_partitions[tp.topic] = [] # Time value '-2' is to get the offset for first available message request_partitions[tp.topic].append((tp.partition, -2, 1)) # List(topic, List(partition, time, max_offsets)) topic_partitions = [] for tp in request_partitions.iteritems(): topic_partitions.append(tp) # Request partition start offsets response = self._send(broker, _OffsetRequest(-1, topic_partitions), _OffsetResponse) if response: for offset in response.topics: topic = offset[0] if topic not in start_offsets: start_offsets[topic] = {} for p in offset[1]: start_offsets[topic][p[0]] = p[2][0] for tp in topic_partitions: for i, ptm in enumerate(tp[1]): # Time value '-1' is to get the offset for next new message tp[1][i] = (ptm[0], -1, 1) # Request partition end offsets response = self._send(broker, _OffsetRequest(-1, topic_partitions), _OffsetResponse) if response: for offset in response.topics: topic = offset[0] if topic not in end_offsets: end_offsets[topic] = {} for p in offset[1]: end_offsets[topic][p[0]] = p[2][0] # Populate with offset values for group, topics in group_topics.iteritems(): coordinator = consumer_coordinator[group] # topic -> list(partition) request_partitions = {} for topic in topics: results[group]['topics'][topic]['consumer_lag'] = 0 results[group]['topics'][topic]['partitions'] = {} if topic in start_offsets: for p in start_offsets[topic]: results[group]['topics'][topic]['partitions'][p] = { 'offset_first': start_offsets[topic][p], 'offset_last': end_offsets[topic][p], 'offset_consumed': 0, 'lag' : 0} if topic not in request_partitions: request_partitions[topic] = [] request_partitions[topic].append(p) # List(topic -> list(partition)) topic_partitions = [] for tp in request_partitions.iteritems(): topic_partitions.append(tp) response = self._send(coordinator, _OffsetFetchRequest(group, topic_partitions), _OffsetFetchResponse) if response: for offset in response.topics: topic = offset[0] offsets = offset[1] if topic not in results[group]['topics']: continue for p_offset in offsets: partition = p_offset[0] offset_consumed = p_offset[1] p_results = results[group]['topics'][topic]['partitions'][partition] if offset_consumed != -1: p_results['offset_consumed'] = offset_consumed p_results['lag'] = p_results['offset_last'] - offset_consumed else: p_results['offset_consumed'] = 0 p_results['lag'] = p_results['offset_last'] - p_results['offset_first'] results[group]['topics'][topic]['consumer_lag'] += p_results['lag'] return results
def check(self, group_topics=None, discovery=None): """ { "<group>": { "state": <str>, "topics": { "<topic>": { "consumer_lag": <int>, "partitions": { "<partition>": { "offset_first": <int>, "offset_consumed": <int>, "offset_last": <int>, "lag": <int> } } } } } } :param persist_groups: :return: consumer statistics """ cluster = self.client.cluster brokers = cluster.brokers() # Consumer group ID -> list(topics) if group_topics is None: group_topics = {} if discovery is None: discovery = True else: group_topics = copy.deepcopy(group_topics) # Set of consumer group IDs consumer_groups = set(group_topics.iterkeys()) # Set of all known topics topics = set(itertools.chain(*group_topics.itervalues())) # Consumer group ID -> coordinating broker consumer_coordinator = {} # Coordinating broker - > list(consumer group IDs) coordinator_consumers = {} results = {} for consumer_group in group_topics.iterkeys(): results[consumer_group] = {'state': None, 'topics': {}} # Ensure connections to all brokers for broker in brokers: while not self.client.is_ready(broker.nodeId): self.client.ready(broker.nodeId) # Collect all active consumer groups if discovery: for broker in brokers: response = self._send(broker.nodeId, _ListGroupsRequest(), _ListGroupsResponse) if response: for group in response.groups: consumer_groups.add(group[0]) # Identify which broker is coordinating each consumer group for group in consumer_groups: response = self._send( next(iter(brokers)).nodeId, _GroupCoordinatorRequest(group), _GroupCoordinatorResponse) if response: consumer_coordinator[group] = response.coordinator_id if response.coordinator_id not in coordinator_consumers: coordinator_consumers[response.coordinator_id] = [] coordinator_consumers[response.coordinator_id].append(group) # Populate consumer groups into dict for group in consumer_groups: if group not in group_topics: group_topics[group] = [] # Add groups to results dict for group, topic_list in group_topics.iteritems(): results[group] = {'state': None, 'topics': {}} # Identify group information and topics read by each consumer group for coordinator, consumers in coordinator_consumers.iteritems(): response = self._send(coordinator, _DescribeGroupsRequest(consumers), _DescribeGroupsResponse) for group in response.groups: if group[1] in results: results[group[1]]['state'] = group[2] # TODO Also include member data? if discovery: members = group[5] for member in members: try: assignment = MemberAssignment.decode(member[4]) if assignment: for partition in assignment.partition_assignment: topic = partition[0] # Add topic to topic set topics.add(topic) # Add topic to group group_topics[group[1]].append(topic) except: pass # Add topics to groups in results dict for group, topic_list in group_topics.iteritems(): for topic in topic_list: results[group]['topics'][topic] = { 'consumer_lag': 0, 'partitions': {} } # For storing the latest offset for all partitions of all topics # topic -> partition -> offset start_offsets = {} end_offsets = {} # Identify all the topic partitions that each broker is leader for # and request next new offset for each partition for broker, partitions in cluster._broker_partitions.iteritems(): # topic -> List(partition, time, max_offsets) request_partitions = {} for tp in partitions: if tp.topic in topics: if tp.topic not in request_partitions: request_partitions[tp.topic] = [] # Time value '-2' is to get the offset for first available message request_partitions[tp.topic].append((tp.partition, -2, 1)) # List(topic, List(partition, time, max_offsets)) topic_partitions = [] for tp in request_partitions.iteritems(): topic_partitions.append(tp) # Request partition start offsets response = self._send(broker, _OffsetRequest(-1, topic_partitions), _OffsetResponse) if response: for offset in response.topics: topic = offset[0] if topic not in start_offsets: start_offsets[topic] = {} for p in offset[1]: start_offsets[topic][p[0]] = p[2][0] for tp in topic_partitions: for i, ptm in enumerate(tp[1]): # Time value '-1' is to get the offset for next new message tp[1][i] = (ptm[0], -1, 1) # Request partition end offsets response = self._send(broker, _OffsetRequest(-1, topic_partitions), _OffsetResponse) if response: for offset in response.topics: topic = offset[0] if topic not in end_offsets: end_offsets[topic] = {} for p in offset[1]: end_offsets[topic][p[0]] = p[2][0] # Populate with offset values for group, topics in group_topics.iteritems(): coordinator = consumer_coordinator[group] # topic -> list(partition) request_partitions = {} for topic in topics: results[group]['topics'][topic]['consumer_lag'] = 0 results[group]['topics'][topic]['partitions'] = {} if topic in start_offsets: for p in start_offsets[topic]: results[group]['topics'][topic]['partitions'][p] = { 'offset_first': start_offsets[topic][p], 'offset_last': end_offsets[topic][p], 'offset_consumed': 0, 'lag': 0 } if topic not in request_partitions: request_partitions[topic] = [] request_partitions[topic].append(p) # List(topic -> list(partition)) topic_partitions = [] for tp in request_partitions.iteritems(): topic_partitions.append(tp) response = self._send(coordinator, _OffsetFetchRequest(group, topic_partitions), _OffsetFetchResponse) if response: for offset in response.topics: topic = offset[0] offsets = offset[1] if topic not in results[group]['topics']: continue for p_offset in offsets: partition = p_offset[0] offset_consumed = p_offset[1] p_results = results[group]['topics'][topic][ 'partitions'][partition] if offset_consumed != -1: p_results['offset_consumed'] = offset_consumed p_results['lag'] = p_results[ 'offset_last'] - offset_consumed else: p_results['offset_consumed'] = 0 p_results['lag'] = p_results[ 'offset_last'] - p_results['offset_first'] results[group]['topics'][topic][ 'consumer_lag'] += p_results['lag'] return results
def describe(self, node_id, group_name): describe_groups_request = admin.DescribeGroupsRequest_v0( groups=[(group_name)]) future = self.client.send(node_id, describe_groups_request) self.client.poll(timeout_ms=self.timeout, future=future) (error_code, group_id, state, protocol_type, protocol, members) = future.value.groups[0] if error_code != 0: print( "Kafka API - RET admin.DescribeGroupsRequest, error_code={}, group_id={}, state={}, protocol_type={}, protocol={}, members_count={}" .format(error_code, group_id, state, protocol_type, protocol, len(members))) exit(1) metadata_consumer_group = { 'id': group_name, 'state': state, 'topics': [], 'lag': 0, 'members': [] } if len(members) != 0: for member in members: (member_id, client_id, client_host, member_metadata, member_assignment) = member member_topics_assignment = [] for (topic, partitions) in MemberAssignment.decode( member_assignment).assignment: member_topics_assignment.append(topic) metadata_consumer_group['members'].append({ 'member_id': member_id, 'client_id': client_id, 'client_host': client_host, 'topic': member_topics_assignment }) metadata_consumer_group['topics'] += member_topics_assignment (lag_total, topics_found) = self.get_lag_by_topic_list( group_name, topics=metadata_consumer_group['topics']) metadata_consumer_group[ 'lag'] = metadata_consumer_group['lag'] + lag_total else: all_topics = self.client.cluster.topics() while '__consumer_offsets' in all_topics: all_topics.remove('__consumer_offsets') (lag_total, topics_found) = self.get_lag_by_topic_list(group_name, topics=all_topics) metadata_consumer_group[ 'lag'] = metadata_consumer_group['lag'] + lag_total metadata_consumer_group['topics'] = topics_found return metadata_consumer_group