def check_dir(path): logger.debug("检查下载路径是否存在") if not os.path.exists(path): logger.info("下载路径不存在,创建下载路径") os.makedirs(path) logger.debug("下载路径检查完成")
def servers_close(server_list): logger.info("====关闭后台服务器连接====") num = int(config.remote_host_num) for server_index in range(0, num): server_list[server_index].download_nmon_files(config) server_list[server_index].close() logger.info("====后台服务器完全关闭====")
def exe_command(command): logger.debug("正在执行命令:"+command) result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not result.returncode: logger.info(result.stdout.decode("gbk")) else: # 调用 lr 时,会抛出一个 log4cxx 的异常, 但是脚本正常跑完,结果保存成功,此异常暂时忽略 err_msg = result.stderr.decode('gbk') if not err_msg.find("log4cxx") >= 0: raise CustomError(err_msg)
def analyse_nmon(server_list, nmon_result_list): logger.info("开始解析nmon文件") for server in server_list: for filepath in server.file_list: nmon = NmonAnalyse() nmon.set_ip(server.server_name) nmon.file_analyse(filepath) nmon_result_list.append(nmon) logger.info("解析nmon文件结束")
def analyse_loadrunner(loadrunner_file_list, loadrunner_result_list): logger.info("开始解析loadrunner文件") for loadrunner_file in loadrunner_file_list: loadrunner = LoadRunnerAnalyse() try: loadrunner.file_analyse(loadrunner_file) except Exception as e: logger.error(e) continue loadrunner_result_list.append(loadrunner) logger.info("解析loadrunner文件结束")
def analyse_jmeter(jmeter_file_list, jmeter_result_list): logger.info("开始解析jmeter文件") for jmeter_file in jmeter_file_list: jmeter = JmeterAnalyse() try: jmeter.file_analyse(jmeter_file) except Exception as e: logger.error(e) continue jmeter_result_list.append(jmeter) logger.info("解析jmeter文件结束")
def file_analyse(self, file): """ 解析jmeter报告 :param file: jmeter报告所在目录 """ try: logger.info("开始解析%s jmeter结果文件" % os.path.basename(file)) super().file_analyse(file) file_all_path = file + r"\content\js\dashboard.js" with open(file_all_path, "r", encoding="utf8") as jmeterfile: text = jmeterfile.read() static_data_match_result = re.match( r'[\s\S]*statisticsTable"\),(.*?), function', text) if static_data_match_result is not None: static_json_data = static_data_match_result.group( 1).strip() logger.debug("取到 %s 的压测结果数据为: %s" % (os.path.basename(file), static_json_data)) static_data = json.loads(static_json_data) logger.debug("转化成json格式:%s" % static_data) if "items" not in static_data.keys(): raise CustomError("%s获取压测结果失败,提取到的数据中未找到item标签" % os.path.basename(file)) static_items_data = static_data["items"] logger.debug("提取到的数据为: %s" % static_items_data) for static_item_data in static_items_data: tmp_data = static_item_data['data'] # list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)] tmp_list = [ tmp_data[1], round(float(tmp_data[10]), 2), tmp_data[3], round(float(tmp_data[4]), 2), round(float(tmp_data[5]), 2), round(float(tmp_data[6]), 2) ] # dict: {name:list} self.result_dict[tmp_data[0]] = tmp_list logger.debug("%s 提取结果 %s" % (os.path.basename(file), self.result_dict)) else: raise CustomError("%s获取压测结果失败,未找到匹配数据" % os.path.basename(file)) finally: logger.info("%s jmeter 结果文件解析结束" % os.path.basename(file))
def get_report(self, result_list, nmon_list, file_name="report.html"): """ :param result_list: 压测结果 list :param nmon_list : nmon 结果 List :param file_name: html 全路径,不填默认当前路径 生成报告 """ logger.info("开始生成报告") load_result = self._change_to_load_table(result_list) nmon_result = self._change_to_nmon_table(nmon_list) with open(file_name, "w") as report_file: report_file.write(load_result) report_file.write(nmon_result) logger.info("生成报告结束")
def servers_connect(server_list): logger.info("====开始连接后台服务器====") num = int(config.remote_host_num) for server_index in range(0, num): if config.__getattribute__("nmon_path" + str(server_index)) == "": server = Server(config.__getattribute__("ip" + str(server_index)), taskid="nmon" + time_stamp) else: server = Server(config.__getattribute__("ip" + str(server_index)), taskid="nmon" + time_stamp, path=config.__getattribute__("nmon_path" + str(server_index))) server.connect(config.__getattribute__("user" + str(server_index)), config.__getattribute__("passwd" + str(server_index))) server_list.append(server) logger.info("====连接服务器结束====")
def get_all_script_path(file_path, file_extension): """ 获取当前文件夹下所有制定后缀文件 :param file_path: 文件夹路径 :param file_extension: 文件类型 :return:返回脚本文件全路径列表 """ files_path = [] for root, dirs, files in os.walk(file_path): for file in files: logger.debug("文件名:"+file) if os.path.splitext(file)[1] == file_extension: file_path = os.path.join(root, file) logger.info("含有"+file_extension+"后缀的文件:"+file+",全路径为:"+file_path) files_path.append(os.path.splitext(file_path)[0]) return files_path
def get_report(self, result_list, nmon_list, file_name, file_path=""): """ :param result_list: 压测结果 list :param nmon_list : nmon 结果 List :param file_name: html 报告名称 :param file_path html 报告路径, 不填默认当前路径 生成报告 """ logger.info("开始生成报告") load_result = self._change_to_load_table(result_list) nmon_result = self._change_to_nmon_table(nmon_list) file = file_path + os.path.sep + file_name with open(file, "w") as report_file: report_file.write(load_result) report_file.write(nmon_result) logger.info("生成报告结束")
def download_nmon_files(self, config): if not hasattr(self, "ssh"): raise CustomError("未与服务端进行连接") download_local_path = config.download_local_path + os.path.sep + self.server_name + os.path.sep + self.taskid if not os.path.exists(download_local_path): logger.info("正在创建文件夹" + self.server_name) os.mkdir(download_local_path) trans = self.ssh.get_transport() sftp = paramiko.SFTPClient.from_transport(trans) files = sftp.listdir_attr(self.path + "/" + self.server_name + "/" + self.taskid) logger.info("开始下载" + self.server_name + "监控文件") for file in files: logger.debug("正在下载:" + file.filename) sftp.get( self.path + "/" + self.server_name + "/" + self.taskid + "/" + file.filename, download_local_path + "\\" + file.filename) self.file_list.append(download_local_path + "\\" + file.filename) trans.close() # --add 20200515 报告显示顺序存在乱序的情况, 在保存完所有的文件路径后, 进行排序 self.file_list.sort() logger.info("%s 监控文件下载完成, 文件保存在 %s" % (self.server_name, download_local_path))
def servers_connect(self): logger.info("====开始连接后台服务器====") num = int(self.config.remote_host_num) for server_index in range(0, num): # ip 判空 if self.config.__getattribute__("ip" + str(server_index)) == "": continue if self.config.__getattribute__("nmon_path" + str(server_index)) == "": server = Server( self.config.__getattribute__("ip" + str(server_index))) else: server = Server( self.config.__getattribute__("ip" + str(server_index)), self.config.__getattribute__("nmon_path" + str(server_index))) server.connect( self.config.__getattribute__("user" + str(server_index)), self.config.__getattribute__("passwd" + str(server_index))) self.servers.append(server) logger.info("====连接服务器结束====")
def file_analyse(self, file): """ Nmon 文件解析入口 :param file nmon 文件全路径 """ logger.info("%s 文件数据解析开始" % os.path.basename(file)) super().file_analyse(file) cpu_line = [] mem_line = [] disk_line = [] net_line = [] # 打开文件, 提取存有关键数据的行 with open(file, "r", encoding='utf8') as nmonfile: text = nmonfile.readlines() for line in text: # cpu if "CPU_ALL,T" in line: cpu_line.append(line) # mem elif "MEM,T" in line: mem_line.append(line) # disk elif "DISK" in line: disk_line.append(line) # net elif "NET," in line: net_line.append(line) # 分别对关键数据进行处理 logger.info("开始提取cpu数据") self.fetch_cpu(cpu_line) logger.info("开始提取内存数据") self.fetch_mem(mem_line) logger.info("开始提取磁盘数据") self.fetch_disk(disk_line) logger.info("开始提取网络数据") self.fetch_net(net_line) logger.info("%s 文件数据解析结束" % os.path.basename(file))
def file_analyse(self, file): """ 解析 Loadrunner 报告 :param file: loadrunner 报告所在路径 """ try: logger.info("开始解析 %s loadrunner 报告" % os.path.basename(file)) super().file_analyse(file) tps_list = [] resp_avg_list = [] resp_min_list = [] resp_max_list = [] summary_html_path = file + r'\An_Report1\summary.html' content_html_path = file + r'\An_Report1\contents.html' with open(summary_html_path, "r", encoding='utf8') as summary_html_file: summary_str = summary_html_file.read() transaction_name_list = re.findall( r'headers="LraTransaction Name".*?8">(.*?)</td>', summary_str) logger.debug( "trasaction_name_list is None: %s" % str(False if (transaction_name_list is not None) else True)) pass_list = re.findall(r'headers="LraPass".*?8">(.*?)</td>', summary_str) logger.debug("pass_list is None: %s" % str(False if (pass_list is not None) else True)) fail_list = re.findall(r'headers="LraFail".*?8">(.*?)</td>', summary_str) logger.debug("fail_list is None: %s" % str(False if (fail_list is not None) else True)) if not pass_list or not fail_list or not transaction_name_list: raise CustomError("%s 有未匹配到的数据" % self.name) # TPS 从 TPS html 页面中获取, 先从 contents.html 获取到 TPS html 名称 # Respnse Time 从 Response Time html 页面中获取,先从 contents.html 获取到 Response Time html 名称 with open(content_html_path, "r", encoding='utf8') as content_html_file: content_str = content_html_file.read() tps_html_name_match = re.match( r'[\s\S]*href="(.*?)" Target.*?>Transactions per Second', content_str) response_time_html_name_match = re.match( r'[\s\S]*href="(.*?)" Target.*?>Average Transaction Response Time', content_str) if tps_html_name_match is None: raise CustomError("%s 未找到 tps html 报告" % self.name) elif response_time_html_name_match is None: raise CustomError("%s 未找到 Respnse Time html 报告" % self.name) tps_html_name = tps_html_name_match.group(1) logger.debug("%s tps html name %s " % (os.path.basename(file), tps_html_name)) tps_html_path = file + r'\An_Report1' + os.path.sep + tps_html_name logger.debug("%s tps html path %s " % (os.path.basename(file), tps_html_path)) response_time_html_name = response_time_html_name_match.group( 1) logger.debug("%s response time html name %s" % (os.path.basename(file), response_time_html_name)) response_time_html_path = file + r'\An_Report1' + os.path.sep + response_time_html_name logger.debug("%s response time html path %s" % (os.path.basename(file), response_time_html_path)) self.fetch_tps(tps_html_path, tps_list) self.fetch_resp_time(response_time_html_path, resp_avg_list, resp_min_list, resp_max_list) # 长整数取到的数字带有逗号,例如1024是1,024,在取数字时,先将逗号去掉 for index in range(0, len(transaction_name_list)): transaction_name = transaction_name_list[index] logger.debug("transaction name %s" % transaction_name) tps = tps_list[index] logger.debug("tps %s" % tps) pass_tsc = pass_list[index].replace(",", "") logger.debug("pass transaction: %s" % pass_tsc) fail_tsc = fail_list[index].replace(",", "") logger.debug("fail transaction: %s" % fail_tsc) # 时间转化成 ms 单位 resp_avg = resp_avg_list[index] logger.debug("resp average time : %sms" % resp_avg) resp_max = resp_max_list[index] logger.debug("resp max time: %sms" % resp_max) resp_min = resp_min_list[index] logger.debug("resp min time: %sms" % resp_min) all_tsc = str(int(fail_tsc) + int(pass_tsc)) error = round(int(fail_tsc) / int(all_tsc) * 100, 2) # list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)] data_list = [all_tsc, tps, error, resp_avg, resp_min, resp_max] # dict:{transaction name:list} self.result_dict[transaction_name] = data_list finally: logger.info("%s loadrunner 报告解析结束" % os.path.basename(file))
def _change_to_load_table(self, result_list): """ 将压测结果转化成 html 中的 table 返回 :param result_list:需要转化的压测list :return: str table str """ logger.info("开始将压测报告数据转化成 table") html_str = """ <h1>summary</h1> <table border="1"> <tr> <th>script name</th> <th>trasaction name</th> <th>trasaction number</th> <th>tps</th> <th>error%</th> <th>response time(average) ms</th> <th>response time(min) ms</th> <th>response time(max) ms</th> </tr> """ for result in result_list: keys_dict = result.result_dict.keys() keys = list(keys_dict) if len(keys) == 0: raise CustomError("%s 脚本提取数据异常,无法获取到取样器" % result.name) logger.debug('%s 含有 transaction %s' % (result.name, keys)) result_value_one = result.result_dict[keys[0]] summary_html_one = """ <tr> <td rowspan= '%d'>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr> """ % (len(keys), result.name, keys[0], result_value_one[0], result_value_one[1], result_value_one[2], result_value_one[3], result_value_one[4], result_value_one[5]) if len(keys) == 1: html_str += summary_html_one continue for key_index in range(1, len(keys)): result_value = result.result_dict[keys[key_index]] summary_html = """ <tr> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr> """ % (keys[key_index], result_value[0], result_value[1], result_value[2], result_value[3], result_value[4], result_value[5]) summary_html_one += summary_html html_str += summary_html_one return html_str + "</table>"
def _change_to_nmon_table(self, nmon_list): """ 将nmon 结果转化成 html 中的 table 返回 :param nmon_list: :return: str table str """ logger.info("开始将nmon结果转化为 table") nmon_table_dict = {} for nmon in nmon_list: if nmon.ip in nmon_table_dict.keys(): nmon_table_dict[nmon.ip].append(nmon) else: nmon_table_dict[nmon.ip] = [] nmon_table_dict[nmon.ip].append(nmon) html_str = """ <h1>Server resource(nmon)</h1> <table border="1"> <tr> <th>server ip</th> <th>trasaction name</th> <th>cpu</th> <th>mem</th> <th>mem(contain virtual)</th> <th>diskread (kb/s)</th> <th>diskwrite (kb/s)</th> <th>diskio</th> <th>diskbusy</th> <th>netread (kb/s)</th> <th>netwrite (kb/s)</th> </tr> """ for key in nmon_table_dict: nmon_table_value_one = nmon_table_dict[key][0] summary_html_one = """ <tr> <td rowspan= '%d'>%s</td> <td>%s</td> <td>%s%%</td> <td>%s%%</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> </tr> """ % ( len(nmon_table_dict[key]), key, nmon_table_value_one.name, nmon_table_value_one.cpu, nmon_table_value_one.mem[0], nmon_table_value_one.mem[1], nmon_table_value_one.disk[0], nmon_table_value_one.disk[1], nmon_table_value_one.disk[2], nmon_table_value_one.disk[3], nmon_table_value_one.net[0], nmon_table_value_one.net[1]) if len(nmon_table_dict[key]) == 1: html_str += summary_html_one continue for index in range(1, len(nmon_table_dict[key])): nmon_table_value = nmon_table_dict[key][index] summary_html = """ <tr> <td>%s</td> <td>%s%%</td> <td>%s%%</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> </tr> """ % (nmon_table_value.name, nmon_table_value.cpu, nmon_table_value.mem[0], nmon_table_value.mem[1], nmon_table_value.disk[0], nmon_table_value.disk[1], nmon_table_value.disk[2], nmon_table_value.disk[3], nmon_table_value.net[0], nmon_table_value.net[1]) summary_html_one += summary_html html_str += summary_html_one return html_str + "</table>"
def servers_start_nmon_control(script_file_name): logger.info("====开启后台监控====") num = int(config.remote_host_num) for server_index in range(0, num): servers[server_index].start_nmon_control(config, script_file_name)