def _get_kafka_offsets(self, kafka_conn, consumer_groups):
# Query Kafka for consumer offsets
consumer_offsets = {}
topics = collections.defaultdict(set)
for consumer_group, topic_partitions in consumer_groups.iteritems():
for topic, partitions in topic_partitions.iteritems():
if len(
partitions
) == 0: # Use all found partitions if none are specified.
try:
kafka_consumer = consumer.SimpleConsumer(
kafka_conn, consumer_group, topic)
partitions = kafka_consumer.offsets.keys()
except Exception:
self.log.error(
'Error fetching partition list for topic {0}'.
format(topic))
continue
finally:
kafka_consumer.stop()
# Remember the topic partitions encountered so that we can look up their broker offsets later
topics[topic].update(set(partitions))
consumer_offsets[(consumer_group, topic)] = {}
for partition in partitions:
try:
kafka_consumer = consumer.SimpleConsumer(
kafka_conn, consumer_group, topic)
consumer_offsets[(
consumer_group, topic
)][partition] = kafka_consumer.offsets[partition]
except KeyError:
self.log.error(
'Error fetching consumer offset for {0} partition {1}'
.format(topic, partition))
finally:
kafka_consumer.stop()
# Query Kafka for the broker offsets, done in a separate loop so only one query is done
# per topic/partition even if multiple consumer groups watch the same topic
broker_offsets = {}
for topic, partitions in topics.iteritems():
offset_responses = []
for p in partitions:
try:
response = kafka_conn.send_offset_request(
[common.OffsetRequest(topic, p, -1, 1)])
offset_responses.append(response[0])
except common.KafkaError as e:
self.log.error(
"Error fetching broker offset: {0}".format(e))
for resp in offset_responses:
broker_offsets[(resp.topic, resp.partition)] = resp.offsets[0]
return consumer_offsets, broker_offsets
def _init_consumer(self):
try:
if not self._client:
self._init_client()
self._consumer = consumer.SimpleConsumer(
self._client, self.group, self.topic,
auto_commit=self.auto_commit,
partitions=self.partitions)
LOG.debug('Consumer was created successfully.')
except Exception:
self._consumer = None
LOG.exception('Kafka (%s) consumer can not be created.' %
self.uri)
def check(self, instance):
consumer_groups = self.read_config(instance,
'consumer_groups',
cast=self._validate_consumer_groups)
kafka_host_ports = self.read_config(instance, 'kafka_connect_str')
full_output = self.read_config(instance, 'full_output', cast=bool)
dimensions = {'component': 'kafka', 'service': 'kafka'}
try:
# Connect to Kafka
kafka_conn = client.KafkaClient(kafka_host_ports)
# Query Kafka for consumer offsets
consumer_offsets = {}
topics = collections.defaultdict(set)
for consumer_group, topic_partitions in consumer_groups.iteritems(
):
for topic, partitions in topic_partitions.iteritems():
kafka_consumer = consumer.SimpleConsumer(
kafka_conn, consumer_group, topic)
# Remember the topic partitions that we've see so that we can
# look up their broker offsets later
topics[topic].update(set(partitions))
for partition in partitions:
try:
consumer_offsets[(
consumer_group, topic,
partition)] = kafka_consumer.offsets[partition]
except KeyError:
kafka_consumer.stop()
self.log.error(
'Error fetching consumer offset for {0} partition {1}'
.format(topic, partition))
kafka_consumer.stop()
# Query Kafka for the broker offsets, done in a separate loop so only one query is done
# per topic even if multiple consumer groups watch the same topic
broker_offsets = {}
for topic, partitions in topics.items():
offset_responses = []
for p in partitions:
try:
response = kafka_conn.send_offset_request(
[common.OffsetRequest(topic, p, -1, 1)])
offset_responses.append(response[0])
except common.KafkaError as e:
self.log.error(
"Error fetching broker offset: {0}".format(e))
for resp in offset_responses:
broker_offsets[(resp.topic,
resp.partition)] = resp.offsets[0]
finally:
try:
kafka_conn.close()
except Exception:
self.log.exception('Error cleaning up Kafka connection')
# Report the broker data
if full_output:
broker_dimensions = dimensions.copy()
for (topic, partition), broker_offset in broker_offsets.items():
broker_dimensions.update({
'topic': topic,
'partition': partition
})
broker_offset = broker_offsets.get((topic, partition))
self.gauge('kafka.broker_offset',
broker_offset,
dimensions=self._set_dimensions(
broker_dimensions, instance))
# Report the consumer data
consumer_dimensions = dimensions.copy()
for (consumer_group, topic,
partition), consumer_offset in consumer_offsets.items():
# Get the broker offset
broker_offset = broker_offsets.get((topic, partition))
# Report the consumer offset and lag
consumer_dimensions.update({
'topic': topic,
'partition': partition,
'consumer_group': consumer_group
})
if full_output:
self.gauge('kafka.consumer_offset',
consumer_offset,
dimensions=self._set_dimensions(
consumer_dimensions, instance))
self.gauge('kafka.consumer_lag',
broker_offset - consumer_offset,
dimensions=self._set_dimensions(consumer_dimensions,
instance))