def run(self, path_or_testsets, mapping=None): """ start to run test with varaibles mapping @param path_or_testsets: YAML/JSON testset file path or testset list path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) testsets: testset or list of testset - (dict) testset_dict - (list) list of testset_dict [ testset_dict_1, testset_dict_2 ] @param (dict) mapping: if mapping specified, it will override variables in config block """ try: task_suite = init_task_suite(path_or_testsets, mapping) except exception.TestcaseNotFound: logger.log_error( "Testcases not found in {}".format(path_or_testsets)) sys.exit(1) result = self.runner.run(task_suite) self.summary = get_summary(result) output = [] for task in task_suite.tasks: output.extend(task.output) self.summary["output"] = output return self
def aggregate(self, tests_results): """ aggregate results Args: tests_results (list): list of (testcase, result) """ self.exception_stage = "aggregate results" self.summary = { "success": True, "stat": {}, "time": {}, "platform": report.get_platform(), "details": [] } for tests_result in tests_results: testcase, result = tests_result testcase_summary = report.get_summary(result) self.summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") testcase_summary["base_url"] = testcase.config.get( "request", {}).get("base_url", "") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) testcase_summary["in_out"] = in_out report.aggregate_stat(self.summary["stat"], testcase_summary["stat"]) report.aggregate_stat(self.summary["time"], testcase_summary["time"]) self.summary["details"].append(testcase_summary)
def run(self, path_or_testsets, mapping=None): """ start to run test with varaibles mapping @param path_or_testsets: YAML/JSON testset file path or testset list path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) testsets: testset or list of testset - (dict) testset_dict - (list) list of testset_dict [ testset_dict_1, testset_dict_2 ] @param (dict) mapping: if mapping specified, it will override variables in config block """ try: task_suite = init_task_suite(path_or_testsets, mapping) except exception.TestcaseNotFound: logger.log_error("Testcases not found in {}".format(path_or_testsets)) sys.exit(1) result = self.runner.run(task_suite) self.summary = get_summary(result) output = [] for task in task_suite.tasks: output.extend(task.output) self.summary["output"] = output return self
def run(self, mapping=None, html_report_name=None, html_report_template=None): """ start to run suite @param (dict) mapping: if mapping specified, it will override variables in config block @param (str) html_report_name: output html report file name @param (str) html_report_template: report template file path, template should be in Jinja2 format """ try: mapping = mapping or {} task_suite = TaskSuite(self.path, mapping) except exception.TestcaseNotFound: sys.exit(1) result = self.runner.run(task_suite) output = {} for task in task_suite.tasks: output.update(task.output) if self.gen_html_report: summary = result.summary summary["report_path"] = result.render_html_report( html_report_name, html_report_template) else: summary = get_summary(result) summary["output"] = output return summary
def run(self, path_or_testsets, mapping=None): """ start to run test with varaibles mapping @param path_or_testsets: YAML/JSON testset file path or testset list path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) testsets: testset or list of testset - (dict) testset_dict - (list) list of testset_dict [ testset_dict_1, testset_dict_2 ] @param (dict) mapping: if mapping specified, it will override variables in config block """ try: test_suite_list = init_test_suites(path_or_testsets, mapping) except exceptions.TestcaseNotFound: logger.log_error("Testcases not found in {}".format(path_or_testsets)) sys.exit(1) self.summary = { "success": True, "stat": {}, "time": {}, "platform": get_platform(), "details": [] } def accumulate_stat(origin_stat, new_stat): """ accumulate new_stat to origin_stat """ for key in new_stat: if key not in origin_stat: origin_stat[key] = new_stat[key] elif key == "start_at": # start datetime origin_stat[key] = min(origin_stat[key], new_stat[key]) else: origin_stat[key] += new_stat[key] for test_suite in test_suite_list: result = self.runner.run(test_suite) test_suite_summary = get_summary(result) self.summary["success"] &= test_suite_summary["success"] test_suite_summary["name"] = test_suite.config.get("name") test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "") test_suite_summary["output"] = test_suite.output print_output(test_suite_summary["output"]) accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) accumulate_stat(self.summary["time"], test_suite_summary["time"]) self.summary["details"].append(test_suite_summary) return self
def _aggregate(self, tests_results, failed_testcase_list): """ aggregate results Args: tests_results (list): list of (testcase, result) """ summary = { "success": True, "stat": { "testcases": { "total": len(tests_results) + len(failed_testcase_list), "success": 0, "fail": 0 }, "teststeps": {} }, "time": {}, "platform": report.get_platform(), "details": [] } for testcase in failed_testcase_list: summary["stat"]["testcases"]["fail"] += 1 summary["success"] = False testcase_summary = report.get_summary_ex(testcase) testcase_summary["name"] = testcase report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) summary["details"].append(testcase_summary) for tests_result in tests_results: testcase, result = tests_result testcase_summary = report.get_summary(result) if testcase_summary["success"]: summary["stat"]["testcases"]["success"] += 1 else: summary["stat"]["testcases"]["fail"] += 1 summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") testcase_summary["in_out"] = utils.get_testcase_io(testcase) #add by zlinghu testcase_summary["base_url"] = testcase.config.get("base_url") #add end report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) report.aggregate_stat(summary["time"], testcase_summary["time"]) summary["details"].append(testcase_summary) return summary
def _aggregate(self, tests_results): """ aggregate results Args: tests_results (list): list of (testcases, result) """ summary = { "success": True, "stat": { "testcases": { "total": len(tests_results), "success": 0, "fail": 0 }, "teststeps": {} }, "time": {}, "platform": report.get_platform(), "details": [] } for index, tests_result in enumerate(tests_results): testcase, result = tests_result testcase_summary = report.get_summary(result) if testcase_summary["success"]: summary["stat"]["testcases"]["success"] += 1 else: summary["stat"]["testcases"]["fail"] += 1 summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") testcase_summary["in_out"] = utils.get_testcase_io(testcase) report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) report.aggregate_stat(summary["time"], testcase_summary["time"]) if self.save_tests: logs_file_abs_path = utils.prepare_log_file_abs_path( self.test_path, f"testcase_{index+1}.log") testcase_summary["log"] = logs_file_abs_path testcase_summary[ "HRUN-Request-ID"] = testcase.runner.hrun_request_id summary["details"].append(testcase_summary) return summary
def _aggregate(self, tests_results): """ aggregate results Args: tests_results (list): list of (testcase, result) """ summary = { "success": True, "stat": { "testcases": { "total": len(tests_results), "success": 0, "fail": 0 }, "teststeps": {} }, "time": {}, "platform": report.get_platform(), "details": [] } for tests_result in tests_results: testcase, result = tests_result testcase_summary = report.get_summary(result) if testcase_summary["success"]: summary["stat"]["testcases"]["success"] += 1 else: summary["stat"]["testcases"]["fail"] += 1 summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) testcase_summary["in_out"] = in_out report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) report.aggregate_stat(summary["time"], testcase_summary["time"]) summary["details"].append(testcase_summary) return summary
def run(self, path_or_testcases, mapping=None): """ start to run test with variables mapping. Args: path_or_testcases (str/list/dict): YAML/JSON testcase file path or testcase list path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) testcases: testcase dict or list of testcases - (dict) testset_dict - (list) list of testset_dict [ testset_dict_1, testset_dict_2 ] mapping (dict): if mapping specified, it will override variables in config block. Returns: instance: HttpRunner() instance """ # loader testcases_list = self.load_tests(path_or_testcases) # parser parsed_testcases_list = self.parse_tests(testcases_list) # initialize unittest_runner, test_suite = self.__initialize(parsed_testcases_list) # aggregate self.summary = { "success": True, "stat": {}, "time": {}, "platform": report.get_platform(), "details": [] } # execution for testcase in test_suite: testcase_name = testcase.config.get("name") logger.log_info("Start to run testcase: {}".format(testcase_name)) result = unittest_runner.run(testcase) testcase_summary = report.get_summary(result) self.summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase_name testcase_summary["base_url"] = testcase.config.get( "request", {}).get("base_url", "") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) testcase_summary["in_out"] = in_out report.aggregate_stat(self.summary["stat"], testcase_summary["stat"]) report.aggregate_stat(self.summary["time"], testcase_summary["time"]) self.summary["details"].append(testcase_summary) return self
def _aggregate(self, testcases_results, testcase_details=None): """ aggregate results Args: testcases_results (list): list of (test_case, result) """ summary = { "success": True, "stat": { "testcases": { "total": len(testcases_results), "success": 0, "fail": 0 }, "teststeps": {} }, "time": {}, "platform": report.get_platform(), "details": [] } # custom for index_case, tests_result in enumerate(testcases_results): testcase, result = tests_result testcase_summary = report.get_summary(result) if testcase_summary["success"]: summary["stat"]["testcases"]["success"] += 1 else: summary["stat"]["testcases"]["fail"] += 1 summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") testcase_summary["in_out"] = utils.get_testcase_io(testcase) report.aggregate_stat( summary["stat"]["teststeps"], testcase_summary["stat"]) report.aggregate_stat(summary["time"], testcase_summary["time"]) # ================================================== # custom: add step detail to summary, by zheng.zhang # step detail path in summary: summary.details[i].records[j].step_detail teststeps = testcase.teststeps for index_step, teststep in enumerate(teststeps): try: step_detail = parser.parse_variables_mapping(teststep["variables"]) testcase_summary["records"][index_step]["step_detail"] = step_detail except exceptions.VariableNotFound as e: # TODO: deal with various built-in Exception testcase_summary["records"][index_step]["step_detail"] = { "internel error message": \ "error during adding step details: VariableNotFound", "Exception message": e } # print("error during adding step detail to summary, \ # in {}".format(__file__)) # ================================================== summary["details"].append(testcase_summary) return summary
def run3(self, path_or_testsets, mapping=None): try: test_suite_list = init_test_suites2(path_or_testsets) except exceptions.TestcaseNotFound: logger.log_error( "Testcases not found in {}".format(path_or_testsets)) sys.exit(1) self.summary = { "success": True, "stat": {}, "time": {}, "platform": get_platform(), "details": [] } mapping = mapping or {} def accumulate_stat(origin_stat, new_stat): """ accumulate new_stat to origin_stat """ for key in new_stat: if key not in origin_stat: origin_stat[key] = new_stat[key] elif key == "start_at": # start datetime origin_stat[key] = min(origin_stat[key], new_stat[key]) else: origin_stat[key] += new_stat[key] # 各用例提取的变量 extract_parameter = {} for test_suite in test_suite_list: extract_list = test_suite.get('testcases', {})[0].get('extract') # # 合并各用例提取的变量 # variables = test_suite.testcase_parser.variables # test_suite.testcase_parser.update_binded_variables(dict(variables, **extract_parameter)) try: test_suite = TestSuite_ext(test_suite, dict(extract_parameter, **mapping)) except exceptions.ParamsError as e: raise Exception("出现异常,参数错误: {0}".format(e)) except exceptions.VariableNotFound as e: raise Exception("出现异常,变量不存在: {0}".format(e)) except BaseException as e: raise Exception("出现异常: {0}".format(e)) result = self.runner.run(test_suite) test_suite_summary = get_summary(result) name = test_suite.config.get("name") test_infos = TestCaseInfo.objects.filter(name=name).all() test_infos = list(test_infos) if test_infos: # 清除get_cache_case的缓存 del_case_cache(test_infos[0].id) self.summary["success"] &= test_suite_summary["success"] test_suite_summary["name"] = name test_suite_summary["base_url"] = test_suite.config.get( "request", {}).get("base_url", "") test_suite_summary["output"] = test_suite.output print_output(test_suite_summary["output"]) accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) accumulate_stat(self.summary["time"], test_suite_summary["time"]) # 根据返回结果提取变量值 if extract_list is not None and isinstance(extract_list, list): records = test_suite_summary.get('records') data_result = records[0].get('meta_data').get('response').get( 'json') if data_result is not None: for extract in extract_list: print(extract) for key, value in extract.items(): try: extract_value = {} extract_value['content'] = data_result fields = str(value).split('.') extract_success = True for field in fields: if isinstance(extract_value, dict): extract_value = extract_value.get( field) elif isinstance(extract_value, list): if extract_value: extract_value = extract_value[int( field)] else: extract_success = False # raise Exception('提取变量失败,', '路径:', value, '下的结果不存在,请检查') if extract_success: extract_parameter[key] = extract_value except AttributeError as e: print('run运行错误:未提取到变量') except Exception as e: logger.log_error("出现错误.{0}".format(e)) self.summary["details"].append(test_suite_summary) return self
def render_html_report(self, html_report_name=None, html_report_template=None): return render_html_report(get_summary(self.test_result), html_report_name, html_report_template)