def __init__(self):
self.ui = Tk()
self.ui.title('Kafka Client Get Newest Offset')
self.ui.geometry(
'880x680+532+244') # 宽x高+偏移量(相对于屏幕)=width*height+xpos+ypos
if os.path.exists('./kafka.ico'):
self.ui.iconbitmap('./kafka.ico')
self.def_ip = "127.0.0.1"
self.def_port = "9092"
# self.bk_status = False
# self.error_msg = None
self.admin_ui = None
self.err_var = None
self.combox_topic_list = None # topic list
self.combox_consumer_list = None # consumer list
self.comvalue = None
self.comsumervalue = None
self.begin_input = None
self.end_input = None
self.filter_input = None
self.service_input = ""
self.port_input = ""
self.cur_topic = "ods.h323" # default topic
self.cur_consumer = "H323-Etl" # default consumer
self.cur_broker = None
self.all_topic = []
self.content_text = None
self.status = None
self.setup_ui()
# ui exit
# self.ui.protocol("WM_DELETE_WINDOW", self.ui.iconify())
self.ui.bind('<Escape>', lambda e: self.on_closing())
# self.ui.event_add()
logger.info("kafka client start")
def region_set_inner(self, *args):
# global return_val
return_val = self.combox_topic_list.get()
self.cur_topic = return_val
# print(self.cur_topic)
self.change_status()
logger.info("select topic:{}".format(return_val))
def get_admin(self):
if self.cur_broker is None:
if self.err_var is None:
setup_err_msg(self, "")
self.err_var.set("broker not connect please connect firset.")
self.ui.update()
return
if self.admin_ui is None:
""" new ui """
logger.info("new admin ui")
self.admin_ui = Tk()
# from tkinter import Widget
# self.admin_ui = Widget(self.ui, 'a')
self.admin_ui.title('Kafka Client Admin')
self.admin_ui.geometry(
'600x400+572+264') # 宽x高+偏移量(相对于屏幕)=width*height+xpos+ypos
# print(self.admin_ui.winfo_x(), self.admin_ui.winfo_y())
setup_admin(self)
else:
""" if exist then show """
# print(self.admin_ui)
self.admin_ui.update()
self.admin_ui.deiconify()
def get_offset_time_window(consumer, partitions_structs, begin_time, end_time):
begin_search = {}
for partition in partitions_structs:
begin_search[partition] = begin_time if isinstance(begin_time, int) else __str_to_timestamp(begin_time)
begin_offset = consumer.offsets_for_times(begin_search)
# print("begin timestamp:", begin_search)
# print("begin OffsetAndTimestamp:", begin_offset)
end_search = {}
for partition in partitions_structs:
end_search[partition] = end_time if isinstance(end_time, int) else __str_to_timestamp(end_time)
end_offset = consumer.offsets_for_times(end_search)
e_offset = 'null'
for topic_partition, offset_and_timestamp in begin_offset.items():
b_offset = 'null' if offset_and_timestamp is None else offset_and_timestamp[0]
e_offset = 'null' if end_offset[topic_partition] is None else end_offset[topic_partition][0]
print('Between {0} and {1}, {2} offset range = [{3}, {4}]'.format(begin_time, end_time, topic_partition, b_offset,
e_offset))
logger.info(
'Between {0} and {1}, {2} offset range = [{3}, {4}]'.format(begin_time, end_time, topic_partition, b_offset,
e_offset))
if b_offset != 'null':
print("begin offset:", topic_partition, b_offset)
consumer.seek(topic_partition, b_offset)
return consumer, e_offset
def consumer_set_inner(self, *args):
# global consumer_val
consumer_val = self.combox_consumer_list.get()
self.cur_consumer = consumer_val
# print("callback:", self.comsumervalue.get())
# self.comsumervalue.set(self.comsumervalue.get())
self.change_status()
logger.info("select consumer:{}".format(consumer_val))
def check_filter(self, message):
bmsg = message.value
if self.strfilter is not None:
strmsg = bmsg.decode('utf-8')
# print(strmsg)
if self.strfilter in strmsg:
try:
# msg = json.loads(strmsg)
infos = '{"offset":%s, "partition": %s}' % (str(message.offset), str(message.partition))
# print(infos)
self.write_file(infos)
self.write_file(strmsg)
logger.info("offset: {} filter:{}".format(message.offset, self.strfilter))
return 0
except Exception as e:
logger.error("{}".format(e))
return 1
def __init__(self):
self.def_ip = "10.67.18.100"
self.def_port = "9092"
# self.bk_status = False
# self.error_msg = None
self.err_var = None
self.combox_topic_list = None # topic list
self.combox_consumer_list = None # consumer list
self.comvalue = None
self.comsumervalue = None
self.begin_input = None
self.end_input = None
self.filter_input = None
self.service_input = ""
self.port_input = ""
self.cur_topic = "ods.h323" # default topic
self.cur_consumer = "H323-Etl" # default consumer
self.cur_broker = None
self.all_topic = []
self.all_consumer = []
self.content_text = None
self.status = None
logger.info("kafka client start")
def get_filter_data(self, kp, begin_time=None, end_time=None, filter_str=None, p=0):
from kafka import KafkaConsumer, TopicPartition
try:
if begin_time is None or end_time is None:
logger.error("begin time or end time is None, please reset")
msg = "begin time or end time is None, please reset"
print(msg)
return None
# consumer = KafkaConsumer(group_id=config.get("db", "main_group_id"),
# bootstrap_servers=config.get("db", "bootstrap_servers"))
consumer = KafkaConsumer(group_id="test",
# sasl_plain_username='******', sasl_plain_password='******',
# security_protocol='SASL_PLAINTEXT', sasl_mechanism='PLAIN',
bootstrap_servers=self.cur_broker,
max_poll_records=100,
)
tps = []
# for p in consumer.partitions_for_topic(self.cur_topic):
# tp = TopicPartition(self.cur_topic, p)
# tps.append(tp)
tp = TopicPartition(self.cur_topic, p)
tps.append(tp)
try:
consumer, end_offset = get_offset_time_window(consumer, tps, begin_time, end_time)
if end_offset == "null":
msg = "no msg filter"
logger.info(msg)
print(msg)
return 1
except Exception as e:
print(e)
end_offset = 0
return 1
start_time = end_time = int(time.time())
# if True:
while True:
try:
value_ans = consumer.poll(max_records=10).values()
message_sets = []
# print("get msg:", len(value_ans))
if len(value_ans) > 0:
for par in value_ans:
if isinstance(par, list):
for record in par:
# print(record)
message_sets.append(record)
try:
msg_offset = int(record.offset)
if msg_offset >= end_offset:
print("end:", msg_offset, end_offset)
# exit(0)
# break
return 0
if end_offset == 0:
print("end:", end_offset)
# exit(0)
# break
return 0
except Exception as e:
print("offset:", e)
# break
return 1
else:
pass
data = kp.msg_process(message_sets, filter_str)
message_sets = []
except Exception as e:
print(2, ":================", e)
# break
return 1
except Exception as e:
print(1, ":", e)
return 1
def workline1(self, kp, begin_time=None, end_time=None, filter_str=None):
try:
if begin_time is None or end_time is None:
logger.error("begin time or end time is None, please reset")
return None
# consumer = KafkaConsumer(group_id=config.get("db", "main_group_id"),
# bootstrap_servers=config.get("db", "bootstrap_servers"))
consumer = KafkaConsumer(group_id="test",
# sasl_plain_username='******', sasl_plain_password='******',
# security_protocol='SASL_PLAINTEXT', sasl_mechanism='PLAIN',
bootstrap_servers=broker
)
tps = []
for p in consumer.partitions_for_topic(topic):
tp = TopicPartition(topic, p)
tps.append(tp)
print("version:", consumer.config['api_version'], tps)
consumer.assign(tps)
try:
consumer, end_offset = get_offset_time_window(consumer, tps, begin_time, end_time)
except Exception as e:
# print(e)
logger.error("{}".format(e))
exit(0)
start_time = end_time = int(time.time())
while True:
try:
value_ans = consumer.poll(max_records=200).values()
message_sets = []
# print("get msg:", len(value_ans))
if len(value_ans) > 0:
for par in value_ans:
if isinstance(par, list):
for record in par:
# print(record)
message_sets.append(record)
try:
msg_offset = int(record.offset)
if msg_offset >= end_offset:
logger.info("end:", msg_offset, end_offset)
exit(0)
break
except Exception as e:
# print("offset:", e)
logger.error("{}".format(e))
break
else:
msg_offset = int(par.offset)
# msg_partition = int(par.partition)
# msg_topic = str(par.topic)
# if (对应分片的截止时间戳的end_offset到达)：
# 停止
# message_sets += par
print("to msg process...")
kp.msg_process(message_sets, filter_str)
# del message_sets[:]
except Exception as e:
logger.error("{}".format(e))
print(2, ":================", e)
break
except Exception as e:
print(1, ":", e)
logger.error("{}".format(e))
return None