def get_topic_offset(self, brokers, topic, all=False):
"""
获取一个topic的offset值的和
"""
from kafkao import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
partitions = client.topic_partitions[topic]
offset_requests = [
OffsetRequestPayload(topic, p, -1, 1)
for p in partitions.keys()
]
offsets_responses = client.send_offset_request(offset_requests)
if len(partitions.keys()) > 1 and not all:
# for r in offsets_responses:
# print(r.offsets, r.partition)
return [{
"{}".format(r.partition): r.offsets[0] - 3
} for r in offsets_responses]
else:
return sum([r.offsets[0] for r in offsets_responses]) - 1
except Exception as e:
logger.error("{} {} : {}".format(brokers, topic, e))
# if self.err_var is not None:
# self.err_var.set(e)
setup_err_msg(self, e)
return None
def get_topic_offset(self, brokers, topic, all=False):
"""
获取一个topic的offset值的和
"""
from kafkao import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
partitions = client.topic_partitions[topic]
if len(partitions.keys()) > 1 and not all:
data = []
for p in partitions.keys():
result = dict()
q = [OffsetRequestPayload(topic, p, -1, 1)]
try:
r = client.send_offset_request(q)
result[r[0].partition] = r[0].offsets[0] - 3
except Exception as e:
print(e)
result[str(p)] = None
# for r in offsets_responses:
# print(r.offsets, r.partition)
data.append(result)
return data
else:
offset_requests = [OffsetRequestPayload(topic, p, -1, 1) for p in partitions.keys()]
offsets_responses = client.send_offset_request(offset_requests)
return sum([r.offsets[0] for r in offsets_responses]) - 1
except Exception as e:
logger.error("{} {} : {}".format(brokers, topic, e))
return None
def create_partiton(self):
from kafka.admin import KafkaAdminClient, NewPartitions
try:
adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker)
partitions = dict()
pn = self.newpn_input.get()
if pn and pn != "":
pn = int(pn)
newp = NewPartitions(total_count=pn)
print(self.cur_topic)
partitions[self.cur_topic] = newp
adminClient.create_partitions(partitions)
except Exception as e:
logger.error("{}".format(e))
def get_topics(self, brokers):
"""
获取一个topic的offset值的和
"""
from kafkao.client import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
topics = client.topic_partitions.keys()
topics = sorted(topics)
except Exception as e:
logger.error("{}: {}".format(brokers, e))
print("{} connect fail".format(brokers))
return []
return list(topics)
def get_topic_single_partiton_offset(self, brokers, topic, p):
"""
获取一个topic的offset值的和
"""
from kafkao import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
offset_requests = [OffsetRequestPayload(topic, p, -1, 1)]
offsets_responses = client.send_offset_request(offset_requests)
return sum([r.offsets[0] for r in offsets_responses])
except Exception as e:
logger.error("{} {} : {}".format(brokers, topic, e))
# if self.err_var is not None:
# self.err_var.set(e)
print(e)
return None
def get_topics(self, brokers):
"""
获取一个topic的offset值的和
"""
from kafkao.client import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
topics = client.topic_partitions.keys()
except Exception as e:
logger.error("{}: {}".format(brokers, e))
setup_err_msg(self, "{} connect fail".format(brokers))
return []
# setup_err_msg(self, "")
if self.err_var is not None:
self.err_var.set("")
return list(topics)
def check_filter(self, message):
bmsg = message.value
if self.strfilter is not None:
strmsg = bmsg.decode('utf-8')
# print(strmsg)
if self.strfilter in strmsg:
try:
# msg = json.loads(strmsg)
infos = '{"offset":%s, "partition": %s}' % (str(message.offset), str(message.partition))
# print(infos)
self.write_file(infos)
self.write_file(strmsg)
logger.info("offset: {} filter:{}".format(message.offset, self.strfilter))
return 0
except Exception as e:
logger.error("{}".format(e))
return 1
def create_topics(self):
from kafka.admin import KafkaAdminClient, NewTopic
try:
adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker)
newt = self.newt_input.get()
newpn = int(self.newp_input.get())
newrf = 1
topics = []
newtopic = NewTopic(name=newt,
num_partitions=newpn,
replication_factor=newrf)
topics.append(newtopic)
adminClient.create_topics(topics)
# self.get_connect()
# setup_admin(self)
self.all_topic = sorted(self.get_topics(self.cur_broker))
self.combox_topic_listp["values"] = self.all_topic
self.combox_topic_listp.update()
self.combox_topic_listp.set(newt)
self.combox_topic_list["values"] = self.all_topic
self.combox_topic_list.update()
self.combox_topic_list.set(newt)
except Exception as e:
logger.error("{}".format(e))
def get_filter_data(self, kp, begin_time=None, end_time=None, filter_str=None, p=0):
from kafka import KafkaConsumer, TopicPartition
try:
if begin_time is None or end_time is None:
logger.error("begin time or end time is None, please reset")
msg = "begin time or end time is None, please reset"
print(msg)
return None
# consumer = KafkaConsumer(group_id=config.get("db", "main_group_id"),
# bootstrap_servers=config.get("db", "bootstrap_servers"))
consumer = KafkaConsumer(group_id="test",
# sasl_plain_username='******', sasl_plain_password='******',
# security_protocol='SASL_PLAINTEXT', sasl_mechanism='PLAIN',
bootstrap_servers=self.cur_broker,
max_poll_records=100,
)
tps = []
# for p in consumer.partitions_for_topic(self.cur_topic):
# tp = TopicPartition(self.cur_topic, p)
# tps.append(tp)
tp = TopicPartition(self.cur_topic, p)
tps.append(tp)
try:
consumer, end_offset = get_offset_time_window(consumer, tps, begin_time, end_time)
if end_offset == "null":
msg = "no msg filter"
logger.info(msg)
print(msg)
return 1
except Exception as e:
print(e)
end_offset = 0
return 1
start_time = end_time = int(time.time())
# if True:
while True:
try:
value_ans = consumer.poll(max_records=10).values()
message_sets = []
# print("get msg:", len(value_ans))
if len(value_ans) > 0:
for par in value_ans:
if isinstance(par, list):
for record in par:
# print(record)
message_sets.append(record)
try:
msg_offset = int(record.offset)
if msg_offset >= end_offset:
print("end:", msg_offset, end_offset)
# exit(0)
# break
return 0
if end_offset == 0:
print("end:", end_offset)
# exit(0)
# break
return 0
except Exception as e:
print("offset:", e)
# break
return 1
else:
pass
data = kp.msg_process(message_sets, filter_str)
message_sets = []
except Exception as e:
print(2, ":================", e)
# break
return 1
except Exception as e:
print(1, ":", e)
return 1
def workline1(self, kp, begin_time=None, end_time=None, filter_str=None):
try:
if begin_time is None or end_time is None:
logger.error("begin time or end time is None, please reset")
return None
# consumer = KafkaConsumer(group_id=config.get("db", "main_group_id"),
# bootstrap_servers=config.get("db", "bootstrap_servers"))
consumer = KafkaConsumer(group_id="test",
# sasl_plain_username='******', sasl_plain_password='******',
# security_protocol='SASL_PLAINTEXT', sasl_mechanism='PLAIN',
bootstrap_servers=broker
)
tps = []
for p in consumer.partitions_for_topic(topic):
tp = TopicPartition(topic, p)
tps.append(tp)
print("version:", consumer.config['api_version'], tps)
consumer.assign(tps)
try:
consumer, end_offset = get_offset_time_window(consumer, tps, begin_time, end_time)
except Exception as e:
# print(e)
logger.error("{}".format(e))
exit(0)
start_time = end_time = int(time.time())
while True:
try:
value_ans = consumer.poll(max_records=200).values()
message_sets = []
# print("get msg:", len(value_ans))
if len(value_ans) > 0:
for par in value_ans:
if isinstance(par, list):
for record in par:
# print(record)
message_sets.append(record)
try:
msg_offset = int(record.offset)
if msg_offset >= end_offset:
logger.info("end:", msg_offset, end_offset)
exit(0)
break
except Exception as e:
# print("offset:", e)
logger.error("{}".format(e))
break
else:
msg_offset = int(par.offset)
# msg_partition = int(par.partition)
# msg_topic = str(par.topic)
# if (对应分片的截止时间戳的end_offset到达)：
# 停止
# message_sets += par
print("to msg process...")
kp.msg_process(message_sets, filter_str)
# del message_sets[:]
except Exception as e:
logger.error("{}".format(e))
print(2, ":================", e)
break
except Exception as e:
print(1, ":", e)
logger.error("{}".format(e))
return None