def __init__(self, host_name):
self.c = paramiko.SSHClient()
self.c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.c.connect(host_name,
CONFIG_MQTT_UAT.get('port'),
CONFIG_MQTT_UAT.get('username'),
CONFIG_MQTT_UAT.get('password'),
timeout=20)
def __init__(self, host):
self._trans = paramiko.Transport(
sock=(host, sftp_conf.port)
)
self._trans.connect(
username=CONFIG_MQTT_UAT.get('username'),
password=CONFIG_MQTT_UAT.get("password")
)
self._sftp = paramiko.SFTPClient.from_transport(self._trans)
def _execl_name(self):
if CONFIG_MQTT_UAT.get("log_date") == "":
# 获得当前时间时间戳
now = int(time.time())
# 转换为其他日期格式,如:"%Y-%m-%d %H:%M:%S"
timeStruct = time.localtime(now)
return time.strftime("%Y-%m-%d", timeStruct)
else:
return CONFIG_MQTT_UAT.get("log_date")
def files2list(self):
k_v_list = []
remote_date_name = "" if CONFIG_MQTT_UAT.get("log_date", "") == "" else ".{}".format(
CONFIG_MQTT_UAT.get("log_date"))
for h in hosts:
file = os.path.join(os.path.join(IN_DATA, f'{h}{remote_date_name}_in_data'))
with open(file, 'r') as f:
for line in f:
if "IC auth consumed time" in line:
log_time = line[:23]
values = [log_time, h]
values.extend(re.compile(r'[lock_id|serial|card_id|card_type|time]:(\d*)', re.I).findall(line))
values[-1] = int(values[-1])
k_v_list.append(dict(zip(self._headers.keys(), values)))
return sorted(k_v_list, key=lambda x: int(x['time']), reverse=True)
def ssh_service(host_name):
c = SSH(host_name)
remote_date_name = "" if CONFIG_MQTT_UAT.get(
"log_date", "") == "" else ".{}".format(
CONFIG_MQTT_UAT.get("log_date"))
ip_in_data = '{}{}_in_data'.format(host_name, remote_date_name)
server_ret = c.exec_cmd(f'ls ~/{ip_in_data}')
if server_ret:
c.exec_cmd('')
else:
c.exec_cmd(
f'grep "\'message\': \'IC auth consumed time" /data/logs/backend.iot-mqtt-dispatch-service/app.log{remote_date_name} > ~/{ip_in_data}'
)
sftp_service(ip_in_data, host_name)
c.exec_cmd(f'rm -rf /home/develop/{ip_in_data}')
c.__del__()
def source_parse():
date = CONFIG_MQTT_UAT.get("log_date")
# for host_name in hosts:
# remote_date_name = "" if CONFIG_MQTT_UAT.get("log_date", "") == "" else ".{}".format(CONFIG_MQTT_UAT.get("log_date"))
# ip_in_data = '{}{}_in_data'.format(host_name, remote_date_name)
# source_path = os.path.join(BASE_DIR, "data/source", f"{host_name}/app.log{remote_date_name}")
# os.system(f'grep "\'message\': \'IC auth consumed time" {source_path} > ../{ip_in_data}')
# print(f"{host_name} 日志过滤完成。")
# time.sleep(2)
# try:
# cmd = "bash " + os.path.join(BASE_DIR, "data/source", f'source_parse.sh {date}')
# os.system(cmd)
# except Exception as e:
# print("目前还没有从服务器下载相应的log文件！请手动下载")
#
# print("原数据整理完成!")
pass
class SftpConf:
host_name = CONFIG_MQTT_UAT.get('host_name')
port = int(CONFIG_MQTT_UAT.get('port'))
#! coding=utf-8
from mqtt.ssh_ssh import ssh_service
from conf.config_mqtt_uat import CONFIG_MQTT_UAT
from tqdm import tqdm
import time
import os
import re
import xlwt
import json
import operator
from mqtt.service import IN_DATA, BASE_DIR
hosts = CONFIG_MQTT_UAT.get('hostname').split(',')
env = "prod" # "uat"
def ssh_sftp_service():
for host in hosts:
ssh_service(host)
time.sleep(5)
def source_parse():
date = CONFIG_MQTT_UAT.get("log_date")
# for host_name in hosts:
# remote_date_name = "" if CONFIG_MQTT_UAT.get("log_date", "") == "" else ".{}".format(CONFIG_MQTT_UAT.get("log_date"))
# ip_in_data = '{}{}_in_data'.format(host_name, remote_date_name)
# source_path = os.path.join(BASE_DIR, "data/source", f"{host_name}/app.log{remote_date_name}")
# os.system(f'grep "\'message\': \'IC auth consumed time" {source_path} > ../{ip_in_data}')