def run(self, path_or_testsets, mapping=None):
""" start to run test with varaibles mapping
@param path_or_testsets: YAML/JSON testset file path or testset list
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
testsets: testset or list of testset
- (dict) testset_dict
- (list) list of testset_dict
[
testset_dict_1,
testset_dict_2
]
@param (dict) mapping:
if mapping specified, it will override variables in config block
"""
try:
task_suite = init_task_suite(path_or_testsets, mapping)
except exception.TestcaseNotFound:
logger.log_error(
"Testcases not found in {}".format(path_or_testsets))
sys.exit(1)
result = self.runner.run(task_suite)
self.summary = get_summary(result)
output = []
for task in task_suite.tasks:
output.extend(task.output)
self.summary["output"] = output
return self
def aggregate(self, tests_results):
""" aggregate results
Args:
tests_results (list): list of (testcase, result)
"""
self.exception_stage = "aggregate results"
self.summary = {
"success": True,
"stat": {},
"time": {},
"platform": report.get_platform(),
"details": []
}
for tests_result in tests_results:
testcase, result = tests_result
testcase_summary = report.get_summary(result)
self.summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
testcase_summary["base_url"] = testcase.config.get(
"request", {}).get("base_url", "")
in_out = utils.get_testcase_io(testcase)
utils.print_io(in_out)
testcase_summary["in_out"] = in_out
report.aggregate_stat(self.summary["stat"],
testcase_summary["stat"])
report.aggregate_stat(self.summary["time"],
testcase_summary["time"])
self.summary["details"].append(testcase_summary)
def run(self, path_or_testsets, mapping=None):
""" start to run test with varaibles mapping
@param path_or_testsets: YAML/JSON testset file path or testset list
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
testsets: testset or list of testset
- (dict) testset_dict
- (list) list of testset_dict
[
testset_dict_1,
testset_dict_2
]
@param (dict) mapping:
if mapping specified, it will override variables in config block
"""
try:
task_suite = init_task_suite(path_or_testsets, mapping)
except exception.TestcaseNotFound:
logger.log_error("Testcases not found in {}".format(path_or_testsets))
sys.exit(1)
result = self.runner.run(task_suite)
self.summary = get_summary(result)
output = []
for task in task_suite.tasks:
output.extend(task.output)
self.summary["output"] = output
return self
def run(self,
mapping=None,
html_report_name=None,
html_report_template=None):
""" start to run suite
@param (dict) mapping:
if mapping specified, it will override variables in config block
@param (str) html_report_name:
output html report file name
@param (str) html_report_template:
report template file path, template should be in Jinja2 format
"""
try:
mapping = mapping or {}
task_suite = TaskSuite(self.path, mapping)
except exception.TestcaseNotFound:
sys.exit(1)
result = self.runner.run(task_suite)
output = {}
for task in task_suite.tasks:
output.update(task.output)
if self.gen_html_report:
summary = result.summary
summary["report_path"] = result.render_html_report(
html_report_name, html_report_template)
else:
summary = get_summary(result)
summary["output"] = output
return summary
def run(self, path_or_testsets, mapping=None):
""" start to run test with varaibles mapping
@param path_or_testsets: YAML/JSON testset file path or testset list
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
testsets: testset or list of testset
- (dict) testset_dict
- (list) list of testset_dict
[
testset_dict_1,
testset_dict_2
]
@param (dict) mapping:
if mapping specified, it will override variables in config block
"""
try:
test_suite_list = init_test_suites(path_or_testsets, mapping)
except exceptions.TestcaseNotFound:
logger.log_error("Testcases not found in {}".format(path_or_testsets))
sys.exit(1)
self.summary = {
"success": True,
"stat": {},
"time": {},
"platform": get_platform(),
"details": []
}
def accumulate_stat(origin_stat, new_stat):
""" accumulate new_stat to origin_stat
"""
for key in new_stat:
if key not in origin_stat:
origin_stat[key] = new_stat[key]
elif key == "start_at":
# start datetime
origin_stat[key] = min(origin_stat[key], new_stat[key])
else:
origin_stat[key] += new_stat[key]
for test_suite in test_suite_list:
result = self.runner.run(test_suite)
test_suite_summary = get_summary(result)
self.summary["success"] &= test_suite_summary["success"]
test_suite_summary["name"] = test_suite.config.get("name")
test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "")
test_suite_summary["output"] = test_suite.output
print_output(test_suite_summary["output"])
accumulate_stat(self.summary["stat"], test_suite_summary["stat"])
accumulate_stat(self.summary["time"], test_suite_summary["time"])
self.summary["details"].append(test_suite_summary)
return self
def _aggregate(self, tests_results, failed_testcase_list):
""" aggregate results
Args:
tests_results (list): list of (testcase, result)
"""
summary = {
"success": True,
"stat": {
"testcases": {
"total": len(tests_results) + len(failed_testcase_list),
"success": 0,
"fail": 0
},
"teststeps": {}
},
"time": {},
"platform": report.get_platform(),
"details": []
}
for testcase in failed_testcase_list:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] = False
testcase_summary = report.get_summary_ex(testcase)
testcase_summary["name"] = testcase
report.aggregate_stat(summary["stat"]["teststeps"],
testcase_summary["stat"])
summary["details"].append(testcase_summary)
for tests_result in tests_results:
testcase, result = tests_result
testcase_summary = report.get_summary(result)
if testcase_summary["success"]:
summary["stat"]["testcases"]["success"] += 1
else:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
testcase_summary["in_out"] = utils.get_testcase_io(testcase)
#add by zlinghu
testcase_summary["base_url"] = testcase.config.get("base_url")
#add end
report.aggregate_stat(summary["stat"]["teststeps"],
testcase_summary["stat"])
report.aggregate_stat(summary["time"], testcase_summary["time"])
summary["details"].append(testcase_summary)
return summary
def _aggregate(self, tests_results):
""" aggregate results
Args:
tests_results (list): list of (testcases, result)
"""
summary = {
"success": True,
"stat": {
"testcases": {
"total": len(tests_results),
"success": 0,
"fail": 0
},
"teststeps": {}
},
"time": {},
"platform": report.get_platform(),
"details": []
}
for index, tests_result in enumerate(tests_results):
testcase, result = tests_result
testcase_summary = report.get_summary(result)
if testcase_summary["success"]:
summary["stat"]["testcases"]["success"] += 1
else:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
testcase_summary["in_out"] = utils.get_testcase_io(testcase)
report.aggregate_stat(summary["stat"]["teststeps"],
testcase_summary["stat"])
report.aggregate_stat(summary["time"], testcase_summary["time"])
if self.save_tests:
logs_file_abs_path = utils.prepare_log_file_abs_path(
self.test_path, f"testcase_{index+1}.log")
testcase_summary["log"] = logs_file_abs_path
testcase_summary[
"HRUN-Request-ID"] = testcase.runner.hrun_request_id
summary["details"].append(testcase_summary)
return summary
def _aggregate(self, tests_results):
""" aggregate results
Args:
tests_results (list): list of (testcase, result)
"""
summary = {
"success": True,
"stat": {
"testcases": {
"total": len(tests_results),
"success": 0,
"fail": 0
},
"teststeps": {}
},
"time": {},
"platform": report.get_platform(),
"details": []
}
for tests_result in tests_results:
testcase, result = tests_result
testcase_summary = report.get_summary(result)
if testcase_summary["success"]:
summary["stat"]["testcases"]["success"] += 1
else:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
in_out = utils.get_testcase_io(testcase)
utils.print_io(in_out)
testcase_summary["in_out"] = in_out
report.aggregate_stat(summary["stat"]["teststeps"],
testcase_summary["stat"])
report.aggregate_stat(summary["time"], testcase_summary["time"])
summary["details"].append(testcase_summary)
return summary
def run(self, path_or_testcases, mapping=None):
""" start to run test with variables mapping.
Args:
path_or_testcases (str/list/dict): YAML/JSON testcase file path or testcase list
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
testcases: testcase dict or list of testcases
- (dict) testset_dict
- (list) list of testset_dict
[
testset_dict_1,
testset_dict_2
]
mapping (dict): if mapping specified, it will override variables in config block.
Returns:
instance: HttpRunner() instance
"""
# loader
testcases_list = self.load_tests(path_or_testcases)
# parser
parsed_testcases_list = self.parse_tests(testcases_list)
# initialize
unittest_runner, test_suite = self.__initialize(parsed_testcases_list)
# aggregate
self.summary = {
"success": True,
"stat": {},
"time": {},
"platform": report.get_platform(),
"details": []
}
# execution
for testcase in test_suite:
testcase_name = testcase.config.get("name")
logger.log_info("Start to run testcase: {}".format(testcase_name))
result = unittest_runner.run(testcase)
testcase_summary = report.get_summary(result)
self.summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase_name
testcase_summary["base_url"] = testcase.config.get(
"request", {}).get("base_url", "")
in_out = utils.get_testcase_io(testcase)
utils.print_io(in_out)
testcase_summary["in_out"] = in_out
report.aggregate_stat(self.summary["stat"],
testcase_summary["stat"])
report.aggregate_stat(self.summary["time"],
testcase_summary["time"])
self.summary["details"].append(testcase_summary)
return self
def _aggregate(self, testcases_results, testcase_details=None):
""" aggregate results
Args:
testcases_results (list): list of (test_case, result)
"""
summary = {
"success": True,
"stat": {
"testcases": {
"total": len(testcases_results),
"success": 0,
"fail": 0
},
"teststeps": {}
},
"time": {},
"platform": report.get_platform(),
"details": []
}
# custom
for index_case, tests_result in enumerate(testcases_results):
testcase, result = tests_result
testcase_summary = report.get_summary(result)
if testcase_summary["success"]:
summary["stat"]["testcases"]["success"] += 1
else:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
testcase_summary["in_out"] = utils.get_testcase_io(testcase)
report.aggregate_stat(
summary["stat"]["teststeps"], testcase_summary["stat"])
report.aggregate_stat(summary["time"], testcase_summary["time"])
# ==================================================
# custom: add step detail to summary, by zheng.zhang
# step detail path in summary: summary.details[i].records[j].step_detail
teststeps = testcase.teststeps
for index_step, teststep in enumerate(teststeps):
try:
step_detail = parser.parse_variables_mapping(teststep["variables"])
testcase_summary["records"][index_step]["step_detail"] = step_detail
except exceptions.VariableNotFound as e:
# TODO: deal with various built-in Exception
testcase_summary["records"][index_step]["step_detail"] = {
"internel error message": \
"error during adding step details: VariableNotFound",
"Exception message": e
}
# print("error during adding step detail to summary, \
# in {}".format(__file__))
# ==================================================
summary["details"].append(testcase_summary)
return summary
def run3(self, path_or_testsets, mapping=None):
try:
test_suite_list = init_test_suites2(path_or_testsets)
except exceptions.TestcaseNotFound:
logger.log_error(
"Testcases not found in {}".format(path_or_testsets))
sys.exit(1)
self.summary = {
"success": True,
"stat": {},
"time": {},
"platform": get_platform(),
"details": []
}
mapping = mapping or {}
def accumulate_stat(origin_stat, new_stat):
""" accumulate new_stat to origin_stat
"""
for key in new_stat:
if key not in origin_stat:
origin_stat[key] = new_stat[key]
elif key == "start_at":
# start datetime
origin_stat[key] = min(origin_stat[key], new_stat[key])
else:
origin_stat[key] += new_stat[key]
# 各用例提取的变量
extract_parameter = {}
for test_suite in test_suite_list:
extract_list = test_suite.get('testcases', {})[0].get('extract')
# # 合并各用例提取的变量
# variables = test_suite.testcase_parser.variables
# test_suite.testcase_parser.update_binded_variables(dict(variables, **extract_parameter))
try:
test_suite = TestSuite_ext(test_suite,
dict(extract_parameter, **mapping))
except exceptions.ParamsError as e:
raise Exception("出现异常，参数错误: {0}".format(e))
except exceptions.VariableNotFound as e:
raise Exception("出现异常，变量不存在: {0}".format(e))
except BaseException as e:
raise Exception("出现异常: {0}".format(e))
result = self.runner.run(test_suite)
test_suite_summary = get_summary(result)
name = test_suite.config.get("name")
test_infos = TestCaseInfo.objects.filter(name=name).all()
test_infos = list(test_infos)
if test_infos:
# 清除get_cache_case的缓存
del_case_cache(test_infos[0].id)
self.summary["success"] &= test_suite_summary["success"]
test_suite_summary["name"] = name
test_suite_summary["base_url"] = test_suite.config.get(
"request", {}).get("base_url", "")
test_suite_summary["output"] = test_suite.output
print_output(test_suite_summary["output"])
accumulate_stat(self.summary["stat"], test_suite_summary["stat"])
accumulate_stat(self.summary["time"], test_suite_summary["time"])
# 根据返回结果提取变量值
if extract_list is not None and isinstance(extract_list, list):
records = test_suite_summary.get('records')
data_result = records[0].get('meta_data').get('response').get(
'json')
if data_result is not None:
for extract in extract_list:
print(extract)
for key, value in extract.items():
try:
extract_value = {}
extract_value['content'] = data_result
fields = str(value).split('.')
extract_success = True
for field in fields:
if isinstance(extract_value, dict):
extract_value = extract_value.get(
field)
elif isinstance(extract_value, list):
if extract_value:
extract_value = extract_value[int(
field)]
else:
extract_success = False
# raise Exception('提取变量失败，', '路径：', value, '下的结果不存在,请检查')
if extract_success:
extract_parameter[key] = extract_value
except AttributeError as e:
print('run运行错误：未提取到变量')
except Exception as e:
logger.log_error("出现错误.{0}".format(e))
self.summary["details"].append(test_suite_summary)
return self
def render_html_report(self,
html_report_name=None,
html_report_template=None):
return render_html_report(get_summary(self.test_result),
html_report_name, html_report_template)
