def checker( ): client = KafkaClient(kafka2) topic = "twittercontent" partitions = client.topic_partitions[topic] offset_requests = [OffsetRequestPayload(topic, p, -1, 1) for p in partitions.keys()] offsets_responses = client.send_offset_fetch_request("cpp_service_customers_50000", offset_requests) for r in offsets_responses: print "partition = %s, offset = %s"%(r.partition, r.offsets[0])
class Kafka(object): executor = ThreadPoolExecutor(20) def __init__(self, broker): self.broker = broker self.client = KafkaClient(broker, timeout=3) @run_on_executor def getPartition(self, topic): """ 指定topic返回partition列表 """ return self.client.get_partition_ids_for_topic(topic) @run_on_executor def getLogsize(self, topic, partitions): """ 指定topic与partition列表, 返回logsize数据 """ tp = self.client.send_offset_request( [OffsetRequestPayload(topic, p, -1, 1) for p in partitions]) return {p.partition: p.offsets[0] for p in tp} @run_on_executor def getOffsets(self, topic, partitions, group): """ 指定topic、partition和group, 返回offsets数据 """ try: # 尝试使用zookeeper-storage api获取offsets数据 # 未获得指定group的offsets数据将抛出UnknownTopicOrPartitionError异常 tp = self.client.send_offset_fetch_request( group, [OffsetRequestPayload(topic, p, -1, 1) for p in partitions]) offsets = {p.partition: p.offset for p in tp} except UnknownTopicOrPartitionError: # 收到异常后使用kafka-storage api获取offsets数据 consumer = KafkaConsumer(group_id=group, bootstrap_servers=self.broker, enable_auto_commit=False) tp = [TopicPartition(topic, p) for p in partitions] consumer.assign(tp) offsets = {p.partition: consumer.position(p) for p in tp} return offsets