def get_admin(self):
if self.cur_broker is None:
if self.err_var is None:
setup_err_msg(self, "")
self.err_var.set("broker not connect please connect firset.")
self.ui.update()
return
if self.admin_ui is None:
""" new ui """
logger.info("new admin ui")
self.admin_ui = Tk()
# from tkinter import Widget
# self.admin_ui = Widget(self.ui, 'a')
self.admin_ui.title('Kafka Client Admin')
self.admin_ui.geometry(
'600x400+572+264') # 宽x高+偏移量(相对于屏幕)=width*height+xpos+ypos
# print(self.admin_ui.winfo_x(), self.admin_ui.winfo_y())
setup_admin(self)
else:
""" if exist then show """
# print(self.admin_ui)
self.admin_ui.update()
self.admin_ui.deiconify()
def get_topic_offset(self, brokers, topic, all=False):
"""
获取一个topic的offset值的和
"""
from kafkao import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
partitions = client.topic_partitions[topic]
offset_requests = [
OffsetRequestPayload(topic, p, -1, 1)
for p in partitions.keys()
]
offsets_responses = client.send_offset_request(offset_requests)
if len(partitions.keys()) > 1 and not all:
# for r in offsets_responses:
# print(r.offsets, r.partition)
return [{
"{}".format(r.partition): r.offsets[0] - 3
} for r in offsets_responses]
else:
return sum([r.offsets[0] for r in offsets_responses]) - 1
except Exception as e:
logger.error("{} {} : {}".format(brokers, topic, e))
# if self.err_var is not None:
# self.err_var.set(e)
setup_err_msg(self, e)
return None
def get_topic_single_partiton_offset(self, brokers, topic, p):
"""
获取一个topic的offset值的和
"""
from kafkao import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
offset_requests = [OffsetRequestPayload(topic, p, -1, 1)]
offsets_responses = client.send_offset_request(offset_requests)
return sum([r.offsets[0] for r in offsets_responses])
except Exception as e:
logger.error("{} {} : {}".format(brokers, topic, e))
# if self.err_var is not None:
# self.err_var.set(e)
setup_err_msg(self, e)
return None
def get_topics(self, brokers):
"""
获取一个topic的offset值的和
"""
from kafkao.client import SimpleClient
from kafkao.common import OffsetRequestPayload
try:
client = SimpleClient(brokers, timeout=2)
topics = client.topic_partitions.keys()
except Exception as e:
logger.error("{}: {}".format(brokers, e))
setup_err_msg(self, "{} connect fail".format(brokers))
return []
# setup_err_msg(self, "")
if self.err_var is not None:
self.err_var.set("")
return list(topics)
def get_filter_data(self,
kp,
begin_time=None,
end_time=None,
filter_str=None,
p=0):
from kafka import KafkaConsumer, TopicPartition
try:
if begin_time is None or end_time is None:
logger.error("begin time or end time is None, please reset")
msg = "begin time or end time is None, please reset"
if self.err_var is not None:
self.err_var.set(msg)
else:
setup_err_msg(self, msg)
return None
# consumer = KafkaConsumer(group_id=config.get("db", "main_group_id"),
# bootstrap_servers=config.get("db", "bootstrap_servers"))
consumer = KafkaConsumer(
group_id="test",
# sasl_plain_username='******', sasl_plain_password='******',
# security_protocol='SASL_PLAINTEXT', sasl_mechanism='PLAIN',
bootstrap_servers=self.cur_broker,
max_poll_records=100,
)
tps = []
# for p in consumer.partitions_for_topic(self.cur_topic):
# tp = TopicPartition(self.cur_topic, p)
# tps.append(tp)
tp = TopicPartition(self.cur_topic, p)
tps.append(tp)
# print("version:", consumer.config['api_version'], tps)
consumer.assign(tps)
try:
consumer, end_offset = get_offset_time_window(
consumer, tps, begin_time, end_time)
if end_offset == "null":
msg = "no msg filter"
logger.info(msg)
if self.err_var is not None:
self.err_var.set(msg)
else:
setup_err_msg(self, msg)
return 1
except Exception as e:
print(e)
end_offset = 0
return 1
start_time = end_time = int(time.time())
# if True:
while True:
try:
value_ans = consumer.poll(max_records=10).values()
message_sets = []
# print("get msg:", len(value_ans))
if len(value_ans) > 0:
for par in value_ans:
if isinstance(par, list):
for record in par:
# print(record)
message_sets.append(record)
try:
msg_offset = int(record.offset)
if msg_offset >= end_offset:
print("end:", msg_offset,
end_offset)
# exit(0)
# break
return 0
if end_offset == 0:
print("end:", end_offset)
# exit(0)
# break
return 0
except Exception as e:
print("offset:", e)
# break
return 1
else:
pass
data = kp.msg_process(message_sets, filter_str, p)
message_sets = []
except Exception as e:
print(2, ":================", e)
# break
return 1
except Exception as e:
print(1, ":", e)
return 1