def kafka_get_time(host, topic):
"""
读取kafka中消费者的消息
:param host:
:param topic:
:return:
"""
consumer = kafka.KafkaConsumer(topic, bootstrap_servers=host)
# 连接postgresql数据库
host = '127.0.0.1'
db_password = '******'
connet = postgres.connect(host, db_password)
# 写入本地文件
# f = open("test_time.csv", "w")
# n = 0
for msg in consumer:
# print(msg.value)
# f.write(msg.value)
# 解决json loads报错'utf-32-be'
# print(msg.value)
str_1 = bytes.decode(msg.value)
# 解决json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
msg_str = json.loads(str_1)
#
# (msg_str)
# print(msg_str, print(type(msg_str)))
# 转换处理时间
# 云侧
time_end = float(msg_str["it"])
# 边缘侧
# time_end = float(msg_str[0]["tt"])
# 事件发生时间
# 云侧
time_start = float(msg_str["t"])
# 边缘侧
# time_start = float(msg_str[0]["Timestamp"])
time_use = time_end - time_start
insert_sql = "INSERT INTO time(time_end, time_start, time_use) VALUES " \
"(" + str(time_end) + "," + str(time_start) + "," + str(time_use) + ")"
# insert_sql = "INSERT INTO time_kafka(time_end) VALUES " \
# "(" + str(time_end) + ")"
postgres.exec_table(connet, insert_sql)
connet.commit()
# author:丑牛
# datetime:2020/5/25 9:03
from PostgresqlTest import postgres
if __name__ == "__main__":
host = '192.168.175.198'
db_password = '******'
connet = postgres.connect(host, db_password)
create_sql = 'CREATE TABLE use_kafa ( cpu float8,mem float8)'
drop_sql = 'drop table if exists time_tsts;'
insert_sql = 'insert into ts_event(topic, label, value, timestamp, message, state, rule, starttime, endtime) ' \
'values (?,?,?,?,?,?,?,?,?) '
select_sql = 'select * from ts_event'
postgres.exec_table(connet, select_sql)
connet.commit()
connet.close()
from PostgresqlTest import postgres
def inner_get_avg(url):
response = requests.get(url)
# print(response.text)
cpu = response.json()[1]["totalCores"]
mem = response.json()[1]["memoryMetrics"]["totalOnHeapStorageMemory"]
response.close()
return cpu, mem
if __name__ == '__main__':
spark_url = 'http://192.168.175.230:18088/api/v1/applications/application_1594110503511_0098/executors'
host = '127.0.0.1'
db_password = '******'
connect = postgres.connect(host, db_password)
inner_get_avg(spark_url)
n = 0
while True:
cpu_use, mem_use = inner_get_avg(spark_url)
insert_sql = "INSERT INTO use(cpu, mem) VALUES (" + str(
cpu_use) + "," + str(mem_use) + ")"
postgres.exec_table(connect, insert_sql)
connect.commit()
time.sleep(1)
n = n + 1
if n == 60 * 5:
break
connect.close()