def _get_commit_offsets(self):
logger.info("Consumer fetching stored offsets")
for topic_partition in self._topics:
resps = []
if self._config['offset_storage'] in ('zookeeper', 'dual'):
resps += self._client.send_offset_fetch_request(
self._config['group_id'], [
OffsetFetchRequestPayload(topic_partition[0],
topic_partition[1])
],
fail_on_error=False)
if self._config['offset_storage'] in ('kafka', 'dual'):
resps += self._client.send_offset_fetch_request_kafka(
self._config['group_id'], [
OffsetFetchRequestPayload(topic_partition[0],
topic_partition[1])
],
fail_on_error=False)
try:
for r in resps:
check_error(r)
# API spec says server wont set an error here
# but 0.8.1.1 does actually...
except UnknownTopicOrPartitionError:
pass
# -1 offset signals no commit is currently stored
max_offset = max(r.offset for r in resps)
if max_offset == -1:
self._offsets.commit[topic_partition] = None
# Otherwise we committed the stored offset
# and need to fetch the next one
else:
self._offsets.commit[topic_partition] = max_offset
def fetch_last_known_offsets(self, partitions=None):
if self.group is None:
raise ValueError('SimpleClient.group must not be None')
if partitions is None:
partitions = self.client.get_partition_ids_for_topic(self.topic)
responses = self.client.send_offset_fetch_request(
self.group,
[OffsetFetchRequestPayload(self.topic, p) for p in partitions],
fail_on_error=False)
for resp in responses:
try:
check_error(resp)
# API spec says server wont set an error here
# but 0.8.1.1 does actually...
except UnknownTopicOrPartitionError:
pass
# -1 offset signals no commit is currently stored
if resp.offset == -1:
self.offsets[resp.partition] = 0
# Otherwise we committed the stored offset
# and need to fetch the next one
else:
self.offsets[resp.partition] = resp.offset
def get_values_kafak(self, groupName, topicName):
kafka_values = dict()
broker = SimpleClient(kafka_conn)
zk = KazooClient(hosts=zookeepers_conn, read_only=True)
zk.start()
logsize = 0
if topicName:
logsize = 0
partitions = broker.get_partition_ids_for_topic(topicName)
responses = broker.send_offset_fetch_request(groupName,
[OffsetFetchRequestPayload(topicName, p) for p in partitions],
fail_on_error=True)
latest_offset = 0
for res in responses:
if topicName != "test":
latest_offset += res[2]
for partition in partitions:
log = "/consumers/%s/offsets/%s/%s" % (groupName, topicName, partition)
if zk.exists(log):
data, stat = zk.get(log)
logsize += int(data)
lag = latest_offset - logsize
broker.close()
zk.stop()
zk.close()
kafka_values['offset'] = latest_offset
kafka_values['logsize'] = logsize
kafka_values['lag'] = lag
return kafka_values
def get_current_consumer_offsets(
kafka_client,
group,
topics,
raise_on_error=True,
):
""" Get current consumer offsets.
NOTE: This method does not refresh client metadata. It is up to the caller
to avoid using stale metadata.
If any partition leader is not available, the request fails for all the
other topics. This is the tradeoff of sending all topic requests in batch
and save both in performance and Kafka load.
:param kafka_client: a connected KafkaToolClient
:param group: kafka group_id
:param topics: topic list or dict {<topic>: [partitions]}
:param raise_on_error: if False the method ignores missing topics and
missing partitions. It still may fail on the request send.
:returns: a dict topic: partition: offset
:raises:
:py:class:`kafka_utils.util.error.UnknownTopic`: upon missing
topics and raise_on_error=True
:py:class:`kafka_utils.util.error.UnknownPartition`: upon missing
partitions and raise_on_error=True
FailedPayloadsError: upon send request error.
"""
topics = _verify_topics_and_partitions(kafka_client, topics, raise_on_error)
group_offset_reqs = [
OffsetFetchRequestPayload(topic, partition)
for topic, partitions in six.iteritems(topics)
for partition in partitions
]
group_offsets = {}
send_api = kafka_client.send_offset_fetch_request_kafka
if group_offset_reqs:
# fail_on_error = False does not prevent network errors
group_resps = send_api(
group=group,
payloads=group_offset_reqs,
fail_on_error=False,
callback=pluck_topic_offset_or_zero_on_unknown,
)
for resp in group_resps:
group_offsets.setdefault(
resp.topic,
{},
)[resp.partition] = resp.offset
return group_offsets
def test_commit_fetch_offsets(self):
req = OffsetCommitRequestPayload(self.topic, 0, 42, 'metadata')
(resp, ) = self.client.send_offset_commit_request('group', [req])
self.assertEqual(resp.error, 0)
req = OffsetFetchRequestPayload(self.topic, 0)
(resp, ) = self.client.send_offset_fetch_request('group', [req])
self.assertEqual(resp.error, 0)
self.assertEqual(resp.offset, 42)
self.assertEqual(resp.metadata, '') # Metadata isn't stored for now
lags = {}
zk = KazooClient(hosts=zookeepers, read_only=True)
zk.start()
logsize = 0
# topics=zk.get_children("/consumers/%s/owners" % (group) )
topic = sys.argv[1]
data_need = sys.argv[2]
# for topic in topics:
if topic:
logsize = 0
# print topic
partitions = broker.get_partition_ids_for_topic(topic)
# print partitions
consumer = KafkaConsumer(broker, group, str(topic))
responses = broker.send_offset_fetch_request(
group, [OffsetFetchRequestPayload(topic, p) for p in partitions],
fail_on_error=True)
# print responses
latest_offset = 0
for res in responses:
if topic != "test":
latest_offset += res[2]
# print latest_offset
for partition in partitions:
log = "/consumers/%s/offsets/%s/%s" % (group, topic, partition)
if zk.exists(log):
data, stat = zk.get(log)
logsize += int(data)
# print logsize
lag = latest_offset - logsize
# print lag
