def get_all_script(script_path, file_extension): """ 获取当前文件夹下所有指定后缀文件 :param script_path: 文件夹路径 :param file_extension: 文件类型 :return:返回脚本文件列表 """ script_files = [] if not os.path.exists(script_path): raise CustomError("路径错误,文件夹不存在") files = os.listdir(script_path) logger.debug("当前路径" + script_path + "下所有文件与文件夹") logger.debug(files) for file in files: if not os.path.isfile(script_path + "\\" + file): continue if os.path.splitext(file)[1] == file_extension: script_files.append(os.path.splitext(file)[0]) if not script_files.__len__(): raise CustomError("路径下无后缀为%s的脚本文件" % file_extension) logger.debug("所有脚本文件") logger.debug(script_files) return script_files
def start(self): logger.error("*****************************************************") self.config = Config.get_instance() self.check_dir(self.config.download_local_path) # 生成脚本运行命令 if self.config.type == "1": # 判断参数是否填写 self.check_jmeter_config_param() # jmeter 生成脚本命令前, 检查 jmeter 程序是否存在 self.check_exe() script_path = self.config.jmeter_script_dir script_file, path = get_all_script(script_path, ".jmx") script_command, result_jmeter_file_list = jmeter_cmd( script_file, path) elif self.config.type == "2": self.check_loadrunner_config_param() script_path = self.config.loadrunner_script_dir script_file, path = get_all_script(script_path, ".lrs") script_command, result_analyse_command, result_loadrunner_file_list = lr_cmd( script_file, path) else: raise CustomError("参数type值只能为1或者2,目前值 %s 非法" % self.config.type) # 连接后台服务器,运行脚本,开启监控 self.servers_connect() for command in script_command: index = script_command.index(command) self.servers_start_nmon_control(script_file[index]) exe_command(command) # 下载nmon文件,关闭后台连接 self.servers_close() # 如果是loadrunner需要额外调用命令,解析文件 if not self.config.loadrunner_script_dir == "" and self.config.jmeter_script_dir == "": if len(result_analyse_command) == 0: raise CustomError("无法获取 loadrunner 解析命令") for command in result_analyse_command: exe_command(command) self.analyse_nmon(self.servers, self.result_nmon_variable_list) if not self.config.jmeter_script_dir == "": if len(result_jmeter_file_list) == 0: raise CustomError("jmeter 解析时出现异常,找不到结果文件所在路径") self.analyse_jmeter(result_jmeter_file_list, self.result_file_analyse_variable_list) elif not self.config.loadrunner_script_dir == "": if len(result_loadrunner_file_list) == 0: raise CustomError("loadrunner 解析时出现异常,找不到结果文件所在路径") self.analyse_loadrunner(result_loadrunner_file_list, self.result_file_analyse_variable_list) else: raise CustomError("脚本路径不能全为空,解析结果失败") report = Report() report.get_report(self.result_file_analyse_variable_list, self.result_nmon_variable_list, file_name=self.config.report_name, file_path=self.config.report_path)
def start_nmon_control(self, config, filename): """ 开启后台监控 :param config:config 对象 :param filename: nmon 文件名 :return: """ if not hasattr(self, "ssh"): raise CustomError("未与服务端进行连接") stdin, stdout, stderr = self.ssh.exec_command("ls -dl " + self.server_name) if stdout.channel.recv_exit_status(): stdin, stdout, stderr = self.ssh.exec_command("mkdir " + self.server_name) if stdout.channel.recv_exit_status(): raise CustomError(stderr.read().decode('utf-8')) nmon_filename = filename + ".nmon" nmon_cmd = (self.path + "/nmon -F ./" + self.server_name + "/" + nmon_filename + " -t -s " + config.nmon_acquisition_interval + " -c " + config.nmon_all_time) logger.debug("正在开启" + self.server_name + "监控,监控结果文件名为:" + nmon_filename) logger.debug("监控命令 %s" % nmon_cmd) stdin, stdout, stderr = self.ssh.exec_command(nmon_cmd) if stdout.channel.recv_exit_status(): err_msg = stderr.read().decode("utf-8") raise CustomError(err_msg)
def check_exe(): ''' 检查jmeter是否在运行, 正在运行则退出 :return: ''' command = "tasklist | findstr java.exe" result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode == 1: if result.stderr: raise CustomError(result.stderr.decode('gbk')) elif result.returncode == 0: command_result_str = result.stdout.decode('gbk') logger.debug("命令 %s 执行结果 %a" % (command, command_result_str)) command_result_list = command_result_str.split(os.linesep) logger.debug(command_result_list) for command_result in command_result_list: if command_result != '': pid = command_result.split()[1] find_jemeter = "jstack %s" % pid result_jm = subprocess.run(find_jemeter, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result_jm.returncode == 0: if "jmeter" in result_jm.stdout.decode('gbk'): raise CustomError("jmeter 程序正在运行, 请关闭 jmeter 再开启脚本") else: logger.debug("jmeter不在运行运行") else: if result_jm.stderr: raise CustomError(result_jm.stderr.decode('gbk'))
def check_jmeter_config_param(self): # 检查jmeter参数会否填写 if self.config.jmeter_script_dir == "": raise CustomError("JMETER_SCRIPT_DIR 参数不允许为空") if self.config.run_in_win == "0": if self.config.jmeter_ip == "": raise CustomError("参数 jmeter ip 不允许为空") elif self.config.jmeter_user == "": raise CustomError("参数 jmeter user 不允许为空") elif self.config.jmeter_passwd == "": raise CustomError("参数 jmeter passwd 不允许为空")
def file_analyse(self, file): """ 解析jmeter报告 :param file: jmeter报告所在目录 """ try: logger.info("开始解析%s jmeter结果文件" % os.path.basename(file)) super().file_analyse(file) file_all_path = file + r"\content\js\dashboard.js" with open(file_all_path, "r", encoding="utf8") as jmeterfile: text = jmeterfile.read() static_data_match_result = re.match( r'[\s\S]*statisticsTable"\),(.*?), function', text) if static_data_match_result is not None: static_json_data = static_data_match_result.group( 1).strip() logger.debug("取到 %s 的压测结果数据为: %s" % (os.path.basename(file), static_json_data)) static_data = json.loads(static_json_data) logger.debug("转化成json格式:%s" % static_data) if "items" not in static_data.keys(): raise CustomError("%s获取压测结果失败,提取到的数据中未找到item标签" % os.path.basename(file)) static_items_data = static_data["items"] logger.debug("提取到的数据为: %s" % static_items_data) for static_item_data in static_items_data: tmp_data = static_item_data['data'] # list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)] tmp_list = [ tmp_data[1], round(float(tmp_data[10]), 2), tmp_data[3], round(float(tmp_data[4]), 2), round(float(tmp_data[5]), 2), round(float(tmp_data[6]), 2) ] # dict: {name:list} self.result_dict[tmp_data[0]] = tmp_list logger.debug("%s 提取结果 %s" % (os.path.basename(file), self.result_dict)) else: raise CustomError("%s获取压测结果失败,未找到匹配数据" % os.path.basename(file)) finally: logger.info("%s jmeter 结果文件解析结束" % os.path.basename(file))
def set_default_value(self, section_item): if not section_item[1] == "": return False if section_item[0] == "nmon_path": self.__setattr__(section_item[0], ".") elif section_item[0] == "nmon_acquisition_interval": self.__setattr__(section_item[0], "1") elif section_item[0] == "download_local_path": raise CustomError("存放监控文件路径不能为空") elif section_item[0] == "remote_host_num": raise CustomError("后台服务器数量不能为空") else: self.__setattr__(section_item[0], "") return True
def __new__(cls, *args, **kwargs): if cls.__instance is None: cls.__instance = super().__new__(cls) else: raise CustomError("Config 类只能含有一个实例, " "使用类方法 get_instance 获取实例") return cls.__instance
def fetch_tps(self, file_path, tps_list): """ 提取 tps html 中 tps 的值 :param file_path: tps html 绝对路径 :param tps_list: 保存 tps 值的 list """ logger.debug("%s 开始提取 tps 数据" % self.name) with open(file_path, "r", encoding='utf8') as tps_html_file: tps_str = tps_html_file.read() tps_table_list = re.findall( r'<tr class="legendRow">([\s\S]*?)</tr>', tps_str) if not tps_table_list: raise CustomError("%s 未匹配到 tps 数据" % self.name) logger.debug("%s 共匹配到 %d 条tps记录" % (self.name, len(tps_table_list))) for index in range(0, len(tps_table_list)): tps_table_str = tps_table_list[index].replace("\n", "") tps_data_list = tps_table_str.split("<td>", 5) # 判断是否为成功记录,成功记录提取数据, 失败记录跳过 if tps_data_list[2][:-5].split(":")[1] != "Pass": continue logger.debug("%s 交易 transaction %s tps %s" % (self.name, tps_data_list[2][:-5].split(":")[0], tps_data_list[4][:-5])) tps_list.append(tps_data_list[4][:-5])
def __new__(cls, *args, **kwargs): if cls.__instance is None: cls.__instance = super().__new__(cls) else: raise CustomError("Log 只能存在一个实例") return cls.__instance
def reload_all_value(self, config_file): self.conf = configparser.ConfigParser() # 「优化」自行传入配置文件, if not config_file: if getattr(sys, 'frozen', False): config_file = os.path.dirname( sys.executable) + "\\conf\\config.ini" elif __file__: config_file = os.path.dirname(__file__) + "\\conf\\config.ini" print(config_file) if os.path.exists(config_file): self.conf.read(config_file) else: print(config_file) raise CustomError("配置文件不存在") sections = self.conf.sections() for section in sections: items = self.conf.items(section) for item in items: if not self.set_default_value(item): self.__setattr__(item[0], item[1].strip()) # 替换路径 if item[0] in [ 'download_local_path', 'jmeter_script_dir', 'report_path' ]: self.replace_path(item[1])
def __init__(self): self.conf = configparser.ConfigParser() config_path = os.path.dirname(__file__) if os.path.exists(config_path+"\\conf\\config.ini"): self.conf.read(config_path+"\\conf\\config.ini", encoding="GBK") else: raise CustomError("配置文件不存在")
def download_nmon_files(self, config): if not hasattr(self, "ssh"): raise CustomError("未与服务端进行连接") download_local_path = config.download_local_path + os.path.sep + self.server_name + os.path.sep + self.taskid if not os.path.exists(download_local_path): logger.info("正在创建文件夹" + self.server_name) os.mkdir(download_local_path) trans = self.ssh.get_transport() sftp = paramiko.SFTPClient.from_transport(trans) files = sftp.listdir_attr(self.path + "/" + self.server_name + "/" + self.taskid) logger.info("开始下载" + self.server_name + "监控文件") for file in files: logger.debug("正在下载:" + file.filename) sftp.get( self.path + "/" + self.server_name + "/" + self.taskid + "/" + file.filename, download_local_path + "\\" + file.filename) self.file_list.append(download_local_path + "\\" + file.filename) trans.close() # --add 20200515 报告显示顺序存在乱序的情况, 在保存完所有的文件路径后, 进行排序 self.file_list.sort() logger.info("%s 监控文件下载完成, 文件保存在 %s" % (self.server_name, download_local_path))
def close(self): """ 关闭后台连接 """ if not hasattr(self, "ssh"): raise CustomError("未与服务端进行连接") logger.debug("正在关闭" + self.server_name + "的连接") self.ssh.close()
def __init__(self, ip, path="."): if not isinstance(ip, (str, bytes)): raise CustomError("IP 需要是字符串类型或者bytes类型") self.server_name = ip self.path = path # 保存下载 nmon 文件全路径 self.file_list = []
def exe_command(command): logger.debug("正在执行命令:"+command) result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not result.returncode: logger.info(result.stdout.decode("gbk")) else: # 调用 lr 时,会抛出一个 log4cxx 的异常, 但是脚本正常跑完,结果保存成功,此异常暂时忽略 err_msg = result.stderr.decode('gbk') if not err_msg.find("log4cxx") >= 0: raise CustomError(err_msg)
def get_all_script(script_path, file_extension): """ 获取当前文件夹下所有指定后缀文件 :param script_path: 文件夹路径 :param file_extension: 文件类型 :return:返回脚本文件列表 :return 脚本路径 """ # 如果是一个文件,判断后缀是否合法后,返回 if os.path.isfile(script_path): file_and_path = os.path.split(script_path) file = file_and_path[1] if os.path.splitext(file)[1] == file_extension: return [os.path.splitext(file)[0]], file_and_path[0] else: raise CustomError("检查到文件后缀与脚本类型不符, 预期脚本类型为: %s" % file_extension) script_files = [] if not os.path.exists(script_path): raise CustomError("路径错误,文件夹或者文件不存在: %s" % script_path) files = os.listdir(script_path) logger.debug("当前路径" + script_path + "下所有文件与文件夹") logger.debug(files) for file in files: if not os.path.isfile(script_path + "\\" + file): continue if os.path.splitext(file)[1] == file_extension: script_files.append(os.path.splitext(file)[0]) if not script_files.__len__(): raise CustomError("路径下无后缀为%s的脚本文件" % file_extension) logger.debug("所有脚本文件") logger.debug(script_files) return script_files, script_path
def fetch_resp_time(self, file_path, resp_avg_list, resp_min_list, resp_max_list): """ 提取 response time html 中 各 response time 的值 :param file_path: response time html 绝对路径 :param resp_avg_list: 保存 response time average 值 :param resp_min_list: 保存 response time min 值 :param resp_max_list: 保存 response time max 值 """ logger.debug("%s 开始提取 response time 数据" % self.name) with open(file_path, "r", encoding='utf8') as response_time_html_file: response_time_str = response_time_html_file.read() response_time_table_list = re.findall( r'<tr class="legendRow">([\s\S]*?)</tr>', response_time_str) if not response_time_table_list: raise CustomError("%s 未匹配到 response time 数据" % self.name) logger.debug("%s 共匹配到 %d 条 response time 记录" % (self.name, len(response_time_table_list))) for index in range(0, len(response_time_table_list)): response_time_table_str = response_time_table_list[ index].replace("\n", "") response_time_data_list = response_time_table_str.split( "<td>", 6) trasaction_name = response_time_data_list[2][:-5] # 单位转化为 ms response_time_average = round( float(response_time_data_list[4][:-5]) * 1000, 2) logger.debug( "%s 交易 transcation %s response time average: %.2fms" % (self.name, trasaction_name, response_time_average)) resp_avg_list.append(response_time_average) response_time_min = round( float(response_time_data_list[3][:-5]) * 1000, 2) logger.debug("%s 交易 transcation %s response time min: %.2fms" % (self.name, trasaction_name, response_time_min)) resp_min_list.append(response_time_min) response_time_max = round( float(response_time_data_list[5][:-5]) * 1000, 2) logger.debug("%s 交易 transcation %s response time max: %.2fms" % (self.name, trasaction_name, response_time_max)) resp_max_list.append(response_time_max)
def fetch_mem(self, lines): """ 获取 mem 的关键数据包括: 纯物理内存使用率, 包含虚拟内存的内存使用率(无则为0) :param lines: 带 mem 关键数据行 """ mem_sum = float(0) mem_virtual_sum = float(0) for line in lines: mems = line.split(",") if len(mems) == 17: # (Memtotal - Memfree - cached - buffers)/Memtotal * 100 mem_sum += ((float(mems[2]) - float(mems[6]) - float(mems[11]) - float(mems[14])) / float( mems[2]) * 100) elif len(mems) == 8: # (Real total - Real free)/Real total * 100 mem_sum += ((float(mems[6]) - float(mems[4])) / float(mems[6]) * 100) # (Real total - Real free + Virtual total - Virtual free) /(Real total + Virtual total) * 100 mem_virtual_sum += ((float(mems[6]) - float(mems[4]) + float(mems[7]) - float(mems[5])) / ( float(mems[6]) + float(mems[7])) * 100) else: raise CustomError("暂不支持此内存页面数据读取") self.mem = (round(mem_sum / len(lines), 2), round(mem_virtual_sum / len(lines), 2)) logger.debug("mem: 不含虚拟内存的使用率 %.2f%%, 包含虚拟内存的使用率 %.2f%%" % (self.mem[0], self.mem[1]))
def exec_command(self, command) -> str: stdin, stdout, stderr = self.ssh.exec_command(command) if stdout.channel.recv_exit_status(): err_msg = stderr.read().decode('utf-8') raise CustomError(err_msg) return stdout.read().decode('utf-8')
def file_analyse(self, file): """ 解析 Loadrunner 报告 :param file: loadrunner 报告所在路径 """ try: logger.info("开始解析 %s loadrunner 报告" % os.path.basename(file)) super().file_analyse(file) tps_list = [] resp_avg_list = [] resp_min_list = [] resp_max_list = [] summary_html_path = file + r'\An_Report1\summary.html' content_html_path = file + r'\An_Report1\contents.html' with open(summary_html_path, "r", encoding='utf8') as summary_html_file: summary_str = summary_html_file.read() transaction_name_list = re.findall( r'headers="LraTransaction Name".*?8">(.*?)</td>', summary_str) logger.debug( "trasaction_name_list is None: %s" % str(False if (transaction_name_list is not None) else True)) pass_list = re.findall(r'headers="LraPass".*?8">(.*?)</td>', summary_str) logger.debug("pass_list is None: %s" % str(False if (pass_list is not None) else True)) fail_list = re.findall(r'headers="LraFail".*?8">(.*?)</td>', summary_str) logger.debug("fail_list is None: %s" % str(False if (fail_list is not None) else True)) if not pass_list or not fail_list or not transaction_name_list: raise CustomError("%s 有未匹配到的数据" % self.name) # TPS 从 TPS html 页面中获取, 先从 contents.html 获取到 TPS html 名称 # Respnse Time 从 Response Time html 页面中获取,先从 contents.html 获取到 Response Time html 名称 with open(content_html_path, "r", encoding='utf8') as content_html_file: content_str = content_html_file.read() tps_html_name_match = re.match( r'[\s\S]*href="(.*?)" Target.*?>Transactions per Second', content_str) response_time_html_name_match = re.match( r'[\s\S]*href="(.*?)" Target.*?>Average Transaction Response Time', content_str) if tps_html_name_match is None: raise CustomError("%s 未找到 tps html 报告" % self.name) elif response_time_html_name_match is None: raise CustomError("%s 未找到 Respnse Time html 报告" % self.name) tps_html_name = tps_html_name_match.group(1) logger.debug("%s tps html name %s " % (os.path.basename(file), tps_html_name)) tps_html_path = file + r'\An_Report1' + os.path.sep + tps_html_name logger.debug("%s tps html path %s " % (os.path.basename(file), tps_html_path)) response_time_html_name = response_time_html_name_match.group( 1) logger.debug("%s response time html name %s" % (os.path.basename(file), response_time_html_name)) response_time_html_path = file + r'\An_Report1' + os.path.sep + response_time_html_name logger.debug("%s response time html path %s" % (os.path.basename(file), response_time_html_path)) self.fetch_tps(tps_html_path, tps_list) self.fetch_resp_time(response_time_html_path, resp_avg_list, resp_min_list, resp_max_list) # 长整数取到的数字带有逗号,例如1024是1,024,在取数字时,先将逗号去掉 for index in range(0, len(transaction_name_list)): transaction_name = transaction_name_list[index] logger.debug("transaction name %s" % transaction_name) tps = tps_list[index] logger.debug("tps %s" % tps) pass_tsc = pass_list[index].replace(",", "") logger.debug("pass transaction: %s" % pass_tsc) fail_tsc = fail_list[index].replace(",", "") logger.debug("fail transaction: %s" % fail_tsc) # 时间转化成 ms 单位 resp_avg = resp_avg_list[index] logger.debug("resp average time : %sms" % resp_avg) resp_max = resp_max_list[index] logger.debug("resp max time: %sms" % resp_max) resp_min = resp_min_list[index] logger.debug("resp min time: %sms" % resp_min) all_tsc = str(int(fail_tsc) + int(pass_tsc)) error = round(int(fail_tsc) / int(all_tsc) * 100, 2) # list: [Transaction, TPS, Error%, Response Time(average), Response Time(min), Response Time(max)] data_list = [all_tsc, tps, error, resp_avg, resp_min, resp_max] # dict:{transaction name:list} self.result_dict[transaction_name] = data_list finally: logger.info("%s loadrunner 报告解析结束" % os.path.basename(file))
def check_loadrunner_config_param(self): if self.config.loadrunner_script_dir == "": raise CustomError("LOADRUNNER_SCRIPT_DIR 参数不允许为空") if self.config.run_in_win == "0": raise CustomError("Loadrunner 压测方式暂不支持脚本在非 WINDOWS 机器上运行")
def __init__(self): self.conf = configparser.ConfigParser() if os.path.exists(".\\conf\\config_test.ini"): self.conf.read(".\\conf\\config_test.ini", encoding="GBK") else: raise CustomError("配置文件不存在")
def _change_to_load_table(self, result_list): """ 将压测结果转化成 html 中的 table 返回 :param result_list:需要转化的压测list :return: str table str """ logger.info("开始将压测报告数据转化成 table") html_str = """ <h1>summary</h1> <table border="1"> <tr> <th>script name</th> <th>trasaction name</th> <th>trasaction number</th> <th>tps</th> <th>error%</th> <th>response time(average) ms</th> <th>response time(min) ms</th> <th>response time(max) ms</th> </tr> """ for result in result_list: keys_dict = result.result_dict.keys() keys = list(keys_dict) if len(keys) == 0: raise CustomError("%s 脚本提取数据异常,无法获取到取样器" % result.name) logger.debug('%s 含有 transaction %s' % (result.name, keys)) result_value_one = result.result_dict[keys[0]] summary_html_one = """ <tr> <td rowspan= '%d'>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr> """ % (len(keys), result.name, keys[0], result_value_one[0], result_value_one[1], result_value_one[2], result_value_one[3], result_value_one[4], result_value_one[5]) if len(keys) == 1: html_str += summary_html_one continue for key_index in range(1, len(keys)): result_value = result.result_dict[keys[key_index]] summary_html = """ <tr> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s%%</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr> """ % (keys[key_index], result_value[0], result_value[1], result_value[2], result_value[3], result_value[4], result_value[5]) summary_html_one += summary_html html_str += summary_html_one return html_str + "</table>"
# 保存解析文件结果 result_nmon_variable_list = [] result_file_analyse_variable_list = [] check_dir(config.download_local_path) # 生成脚本运行命令 if not config.jmeter_script_dir == "": # jmeter 生成脚本命令前, 检查 jmeter 程序是否存在 check_exe() script_path = config.jmeter_script_dir script_file, path = get_all_script(script_path, ".jmx") script_command, result_jmeter_file_list = jmeter_cmd(script_file, path, time_stamp) elif not config.loadrunner_script_dir == "": if config.run_in_win == "0": raise CustomError("Loadrunner 压测方式暂不支持脚本在非 WINDOWS 机器上运行") script_path = config.loadrunner_script_dir script_file, path = get_all_script(script_path, ".lrs") script_command, result_analyse_command, result_loadrunner_file_list = lr_cmd(script_file, path) else: raise CustomError("脚本路径不能全为空") # 连接后台服务器,运行脚本,开启监控 servers = [] servers_connect(servers) for command in script_command: index = script_command.index(command) servers_start_nmon_control(script_file[index]) exe_command(command) # 下载nmon文件,关闭后台连接