def check_dir(path):
logger.debug("检查下载路径是否存在")
if not os.path.exists(path):
logger.info("下载路径不存在,创建下载路径")
os.makedirs(path)
logger.debug("下载路径检查完成")
def servers_close(server_list):
logger.info("====关闭后台服务器连接====")
num = int(config.remote_host_num)
for server_index in range(0, num):
server_list[server_index].download_nmon_files(config)
server_list[server_index].close()
logger.info("====后台服务器完全关闭====")
def exe_command(command):
logger.debug("正在执行命令:"+command)
result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if not result.returncode:
logger.info(result.stdout.decode("gbk"))
else:
# 调用 lr 时,会抛出一个 log4cxx 的异常, 但是脚本正常跑完,结果保存成功,此异常暂时忽略
err_msg = result.stderr.decode('gbk')
if not err_msg.find("log4cxx") >= 0:
raise CustomError(err_msg)
def analyse_nmon(server_list, nmon_result_list):
logger.info("开始解析nmon文件")
for server in server_list:
for filepath in server.file_list:
nmon = NmonAnalyse()
nmon.set_ip(server.server_name)
nmon.file_analyse(filepath)
nmon_result_list.append(nmon)
logger.info("解析nmon文件结束")
def analyse_loadrunner(loadrunner_file_list, loadrunner_result_list):
logger.info("开始解析loadrunner文件")
for loadrunner_file in loadrunner_file_list:
loadrunner = LoadRunnerAnalyse()
try:
loadrunner.file_analyse(loadrunner_file)
except Exception as e:
logger.error(e)
continue
loadrunner_result_list.append(loadrunner)
logger.info("解析loadrunner文件结束")
def analyse_jmeter(jmeter_file_list, jmeter_result_list):
logger.info("开始解析jmeter文件")
for jmeter_file in jmeter_file_list:
jmeter = JmeterAnalyse()
try:
jmeter.file_analyse(jmeter_file)
except Exception as e:
logger.error(e)
continue
jmeter_result_list.append(jmeter)
logger.info("解析jmeter文件结束")
def file_analyse(self, file):
"""
解析jmeter报告
:param file: jmeter报告所在目录
"""
try:
logger.info("开始解析%s jmeter结果文件" % os.path.basename(file))
super().file_analyse(file)
file_all_path = file + r"\content\js\dashboard.js"
with open(file_all_path, "r", encoding="utf8") as jmeterfile:
text = jmeterfile.read()
static_data_match_result = re.match(
r'[\s\S]*statisticsTable"\),(.*?), function', text)
if static_data_match_result is not None:
static_json_data = static_data_match_result.group(
1).strip()
logger.debug("取到 %s 的压测结果数据为: %s" %
(os.path.basename(file), static_json_data))
static_data = json.loads(static_json_data)
logger.debug("转化成json格式:%s" % static_data)
if "items" not in static_data.keys():
raise CustomError("%s获取压测结果失败,提取到的数据中未找到item标签" %
os.path.basename(file))
static_items_data = static_data["items"]
logger.debug("提取到的数据为: %s" % static_items_data)
for static_item_data in static_items_data:
tmp_data = static_item_data['data']
# list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)]
tmp_list = [
tmp_data[1],
round(float(tmp_data[10]), 2), tmp_data[3],
round(float(tmp_data[4]), 2),
round(float(tmp_data[5]), 2),
round(float(tmp_data[6]), 2)
]
# dict: {name:list}
self.result_dict[tmp_data[0]] = tmp_list
logger.debug("%s 提取结果 %s" %
(os.path.basename(file), self.result_dict))
else:
raise CustomError("%s获取压测结果失败,未找到匹配数据" %
os.path.basename(file))
finally:
logger.info("%s jmeter 结果文件解析结束" % os.path.basename(file))
def get_report(self, result_list, nmon_list, file_name="report.html"):
"""
:param result_list: 压测结果 list
:param nmon_list : nmon 结果 List
:param file_name: html 全路径,不填默认当前路径
生成报告
"""
logger.info("开始生成报告")
load_result = self._change_to_load_table(result_list)
nmon_result = self._change_to_nmon_table(nmon_list)
with open(file_name, "w") as report_file:
report_file.write(load_result)
report_file.write(nmon_result)
logger.info("生成报告结束")
def servers_connect(server_list):
logger.info("====开始连接后台服务器====")
num = int(config.remote_host_num)
for server_index in range(0, num):
if config.__getattribute__("nmon_path" + str(server_index)) == "":
server = Server(config.__getattribute__("ip" + str(server_index)), taskid="nmon" + time_stamp)
else:
server = Server(config.__getattribute__("ip" + str(server_index)), taskid="nmon" + time_stamp,
path=config.__getattribute__("nmon_path" + str(server_index)))
server.connect(config.__getattribute__("user" + str(server_index)),
config.__getattribute__("passwd" + str(server_index)))
server_list.append(server)
logger.info("====连接服务器结束====")
def get_all_script_path(file_path, file_extension):
"""
获取当前文件夹下所有制定后缀文件
:param file_path: 文件夹路径
:param file_extension: 文件类型
:return:返回脚本文件全路径列表
"""
files_path = []
for root, dirs, files in os.walk(file_path):
for file in files:
logger.debug("文件名："+file)
if os.path.splitext(file)[1] == file_extension:
file_path = os.path.join(root, file)
logger.info("含有"+file_extension+"后缀的文件:"+file+",全路径为:"+file_path)
files_path.append(os.path.splitext(file_path)[0])
return files_path
def get_report(self, result_list, nmon_list, file_name, file_path=""):
"""
:param result_list: 压测结果 list
:param nmon_list : nmon 结果 List
:param file_name: html 报告名称
:param file_path html 报告路径, 不填默认当前路径
生成报告
"""
logger.info("开始生成报告")
load_result = self._change_to_load_table(result_list)
nmon_result = self._change_to_nmon_table(nmon_list)
file = file_path + os.path.sep + file_name
with open(file, "w") as report_file:
report_file.write(load_result)
report_file.write(nmon_result)
logger.info("生成报告结束")
def download_nmon_files(self, config):
if not hasattr(self, "ssh"):
raise CustomError("未与服务端进行连接")
download_local_path = config.download_local_path + os.path.sep + self.server_name + os.path.sep + self.taskid
if not os.path.exists(download_local_path):
logger.info("正在创建文件夹" + self.server_name)
os.mkdir(download_local_path)
trans = self.ssh.get_transport()
sftp = paramiko.SFTPClient.from_transport(trans)
files = sftp.listdir_attr(self.path + "/" + self.server_name + "/" +
self.taskid)
logger.info("开始下载" + self.server_name + "监控文件")
for file in files:
logger.debug("正在下载:" + file.filename)
sftp.get(
self.path + "/" + self.server_name + "/" + self.taskid + "/" +
file.filename, download_local_path + "\\" + file.filename)
self.file_list.append(download_local_path + "\\" + file.filename)
trans.close()
# --add 20200515 报告显示顺序存在乱序的情况, 在保存完所有的文件路径后, 进行排序
self.file_list.sort()
logger.info("%s 监控文件下载完成, 文件保存在 %s" %
(self.server_name, download_local_path))
def servers_connect(self):
logger.info("====开始连接后台服务器====")
num = int(self.config.remote_host_num)
for server_index in range(0, num):
# ip 判空
if self.config.__getattribute__("ip" + str(server_index)) == "":
continue
if self.config.__getattribute__("nmon_path" +
str(server_index)) == "":
server = Server(
self.config.__getattribute__("ip" + str(server_index)))
else:
server = Server(
self.config.__getattribute__("ip" + str(server_index)),
self.config.__getattribute__("nmon_path" +
str(server_index)))
server.connect(
self.config.__getattribute__("user" + str(server_index)),
self.config.__getattribute__("passwd" + str(server_index)))
self.servers.append(server)
logger.info("====连接服务器结束====")
def file_analyse(self, file):
"""
Nmon 文件解析入口
:param file nmon 文件全路径
"""
logger.info("%s 文件数据解析开始" % os.path.basename(file))
super().file_analyse(file)
cpu_line = []
mem_line = []
disk_line = []
net_line = []
# 打开文件, 提取存有关键数据的行
with open(file, "r", encoding='utf8') as nmonfile:
text = nmonfile.readlines()
for line in text:
# cpu
if "CPU_ALL,T" in line:
cpu_line.append(line)
# mem
elif "MEM,T" in line:
mem_line.append(line)
# disk
elif "DISK" in line:
disk_line.append(line)
# net
elif "NET," in line:
net_line.append(line)
# 分别对关键数据进行处理
logger.info("开始提取cpu数据")
self.fetch_cpu(cpu_line)
logger.info("开始提取内存数据")
self.fetch_mem(mem_line)
logger.info("开始提取磁盘数据")
self.fetch_disk(disk_line)
logger.info("开始提取网络数据")
self.fetch_net(net_line)
logger.info("%s 文件数据解析结束" % os.path.basename(file))
def file_analyse(self, file):
"""
解析 Loadrunner 报告
:param file: loadrunner 报告所在路径
"""
try:
logger.info("开始解析 %s loadrunner 报告" % os.path.basename(file))
super().file_analyse(file)
tps_list = []
resp_avg_list = []
resp_min_list = []
resp_max_list = []
summary_html_path = file + r'\An_Report1\summary.html'
content_html_path = file + r'\An_Report1\contents.html'
with open(summary_html_path, "r",
encoding='utf8') as summary_html_file:
summary_str = summary_html_file.read()
transaction_name_list = re.findall(
r'headers="LraTransaction Name".*?8">(.*?)</td>',
summary_str)
logger.debug(
"trasaction_name_list is None: %s" %
str(False if
(transaction_name_list is not None) else True))
pass_list = re.findall(r'headers="LraPass".*?8">(.*?)</td>',
summary_str)
logger.debug("pass_list is None: %s" %
str(False if (pass_list is not None) else True))
fail_list = re.findall(r'headers="LraFail".*?8">(.*?)</td>',
summary_str)
logger.debug("fail_list is None: %s" %
str(False if (fail_list is not None) else True))
if not pass_list or not fail_list or not transaction_name_list:
raise CustomError("%s 有未匹配到的数据" % self.name)
# TPS 从 TPS html 页面中获取, 先从 contents.html 获取到 TPS html 名称
# Respnse Time 从 Response Time html 页面中获取,先从 contents.html 获取到 Response Time html 名称
with open(content_html_path, "r",
encoding='utf8') as content_html_file:
content_str = content_html_file.read()
tps_html_name_match = re.match(
r'[\s\S]*href="(.*?)" Target.*?>Transactions per Second',
content_str)
response_time_html_name_match = re.match(
r'[\s\S]*href="(.*?)" Target.*?>Average Transaction Response Time',
content_str)
if tps_html_name_match is None:
raise CustomError("%s 未找到 tps html 报告" % self.name)
elif response_time_html_name_match is None:
raise CustomError("%s 未找到 Respnse Time html 报告" %
self.name)
tps_html_name = tps_html_name_match.group(1)
logger.debug("%s tps html name %s " %
(os.path.basename(file), tps_html_name))
tps_html_path = file + r'\An_Report1' + os.path.sep + tps_html_name
logger.debug("%s tps html path %s " %
(os.path.basename(file), tps_html_path))
response_time_html_name = response_time_html_name_match.group(
1)
logger.debug("%s response time html name %s" %
(os.path.basename(file), response_time_html_name))
response_time_html_path = file + r'\An_Report1' + os.path.sep + response_time_html_name
logger.debug("%s response time html path %s" %
(os.path.basename(file), response_time_html_path))
self.fetch_tps(tps_html_path, tps_list)
self.fetch_resp_time(response_time_html_path, resp_avg_list,
resp_min_list, resp_max_list)
# 长整数取到的数字带有逗号,例如1024是1,024,在取数字时,先将逗号去掉
for index in range(0, len(transaction_name_list)):
transaction_name = transaction_name_list[index]
logger.debug("transaction name %s" % transaction_name)
tps = tps_list[index]
logger.debug("tps %s" % tps)
pass_tsc = pass_list[index].replace(",", "")
logger.debug("pass transaction: %s" % pass_tsc)
fail_tsc = fail_list[index].replace(",", "")
logger.debug("fail transaction: %s" % fail_tsc)
# 时间转化成 ms 单位
resp_avg = resp_avg_list[index]
logger.debug("resp average time : %sms" % resp_avg)
resp_max = resp_max_list[index]
logger.debug("resp max time: %sms" % resp_max)
resp_min = resp_min_list[index]
logger.debug("resp min time: %sms" % resp_min)
all_tsc = str(int(fail_tsc) + int(pass_tsc))
error = round(int(fail_tsc) / int(all_tsc) * 100, 2)
# list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)]
data_list = [all_tsc, tps, error, resp_avg, resp_min, resp_max]
# dict:{transaction name:list}
self.result_dict[transaction_name] = data_list
finally:
logger.info("%s loadrunner 报告解析结束" % os.path.basename(file))
def _change_to_load_table(self, result_list):
"""
将压测结果转化成 html 中的 table 返回
:param result_list:需要转化的压测list
:return: str table str
"""
logger.info("开始将压测报告数据转化成 table")
html_str = """
<h1>summary</h1>
<table border="1">
<tr>
<th>script name</th>
<th>trasaction name</th>
<th>trasaction number</th>
<th>tps</th>
<th>error%</th>
<th>response time(average) ms</th>
<th>response time(min) ms</th>
<th>response time(max) ms</th>
</tr>
"""
for result in result_list:
keys_dict = result.result_dict.keys()
keys = list(keys_dict)
if len(keys) == 0:
raise CustomError("%s 脚本提取数据异常,无法获取到取样器" % result.name)
logger.debug('%s 含有 transaction %s' % (result.name, keys))
result_value_one = result.result_dict[keys[0]]
summary_html_one = """
<tr>
<td rowspan= '%d'>%s</td>
<td>%s</td>
<td>%s</td>
<td>%s</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
<td>%s</td>
</tr>
""" % (len(keys), result.name, keys[0], result_value_one[0],
result_value_one[1], result_value_one[2],
result_value_one[3], result_value_one[4],
result_value_one[5])
if len(keys) == 1:
html_str += summary_html_one
continue
for key_index in range(1, len(keys)):
result_value = result.result_dict[keys[key_index]]
summary_html = """
<tr>
<td>%s</td>
<td>%s</td>
<td>%s</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
<td>%s</td>
</tr>
""" % (keys[key_index], result_value[0], result_value[1],
result_value[2], result_value[3], result_value[4],
result_value[5])
summary_html_one += summary_html
html_str += summary_html_one
return html_str + "</table>"
def _change_to_nmon_table(self, nmon_list):
"""
将nmon 结果转化成 html 中的 table 返回
:param nmon_list:
:return: str table str
"""
logger.info("开始将nmon结果转化为 table")
nmon_table_dict = {}
for nmon in nmon_list:
if nmon.ip in nmon_table_dict.keys():
nmon_table_dict[nmon.ip].append(nmon)
else:
nmon_table_dict[nmon.ip] = []
nmon_table_dict[nmon.ip].append(nmon)
html_str = """
<h1>Server resource(nmon)</h1>
<table border="1">
<tr>
<th>server ip</th>
<th>trasaction name</th>
<th>cpu</th>
<th>mem</th>
<th>mem(contain virtual)</th>
<th>diskread (kb/s)</th>
<th>diskwrite (kb/s)</th>
<th>diskio</th>
<th>diskbusy</th>
<th>netread (kb/s)</th>
<th>netwrite (kb/s)</th>
</tr>
"""
for key in nmon_table_dict:
nmon_table_value_one = nmon_table_dict[key][0]
summary_html_one = """
<tr>
<td rowspan= '%d'>%s</td>
<td>%s</td>
<td>%s%%</td>
<td>%s%%</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
<td>%s</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
</tr>
""" % (
len(nmon_table_dict[key]), key, nmon_table_value_one.name,
nmon_table_value_one.cpu, nmon_table_value_one.mem[0],
nmon_table_value_one.mem[1], nmon_table_value_one.disk[0],
nmon_table_value_one.disk[1], nmon_table_value_one.disk[2],
nmon_table_value_one.disk[3], nmon_table_value_one.net[0],
nmon_table_value_one.net[1])
if len(nmon_table_dict[key]) == 1:
html_str += summary_html_one
continue
for index in range(1, len(nmon_table_dict[key])):
nmon_table_value = nmon_table_dict[key][index]
summary_html = """
<tr>
<td>%s</td>
<td>%s%%</td>
<td>%s%%</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
<td>%s</td>
<td>%s%%</td>
<td>%s</td>
<td>%s</td>
</tr>
""" % (nmon_table_value.name, nmon_table_value.cpu,
nmon_table_value.mem[0], nmon_table_value.mem[1],
nmon_table_value.disk[0], nmon_table_value.disk[1],
nmon_table_value.disk[2], nmon_table_value.disk[3],
nmon_table_value.net[0], nmon_table_value.net[1])
summary_html_one += summary_html
html_str += summary_html_one
return html_str + "</table>"
def servers_start_nmon_control(script_file_name):
logger.info("====开启后台监控====")
num = int(config.remote_host_num)
for server_index in range(0, num):
servers[server_index].start_nmon_control(config, script_file_name)
