def checker( ):
client = KafkaClient(kafka2)
topic = "twittercontent"
partitions = client.topic_partitions[topic]
offset_requests = [OffsetRequestPayload(topic, p, -1, 1) for p in partitions.keys()]
offsets_responses = client.send_offset_fetch_request("cpp_service_customers_50000", offset_requests)
for r in offsets_responses:
print "partition = %s, offset = %s"%(r.partition, r.offsets[0])
class Kafka(object):
executor = ThreadPoolExecutor(20)
def __init__(self, broker):
self.broker = broker
self.client = KafkaClient(broker, timeout=3)
@run_on_executor
def getPartition(self, topic):
""" 指定topic返回partition列表 """
return self.client.get_partition_ids_for_topic(topic)
@run_on_executor
def getLogsize(self, topic, partitions):
""" 指定topic与partition列表, 返回logsize数据 """
tp = self.client.send_offset_request(
[OffsetRequestPayload(topic, p, -1, 1) for p in partitions])
return {p.partition: p.offsets[0] for p in tp}
@run_on_executor
def getOffsets(self, topic, partitions, group):
""" 指定topic、partition和group, 返回offsets数据 """
try:
# 尝试使用zookeeper-storage api获取offsets数据
# 未获得指定group的offsets数据将抛出UnknownTopicOrPartitionError异常
tp = self.client.send_offset_fetch_request(
group,
[OffsetRequestPayload(topic, p, -1, 1) for p in partitions])
offsets = {p.partition: p.offset for p in tp}
except UnknownTopicOrPartitionError:
# 收到异常后使用kafka-storage api获取offsets数据
consumer = KafkaConsumer(group_id=group,
bootstrap_servers=self.broker,
enable_auto_commit=False)
tp = [TopicPartition(topic, p) for p in partitions]
consumer.assign(tp)
offsets = {p.partition: consumer.position(p) for p in tp}
return offsets
