def run_tests(self, tests_mapping):
""" run testcases/testsuites data
"""
capture_message("start to run tests")
self.test_path = tests_mapping.get("project_mapping",
{}).get("test_path", "")
if self.save_tests:
utils.dump_json_file(
tests_mapping,
utils.prepare_log_file_abs_path(self.test_path, "loaded.json"))
# parse tests
self.exception_stage = "parse tests"
parsed_testcases = parser.parse_tests(tests_mapping)
parse_failed_testfiles = parser.get_parse_failed_testfiles()
if parse_failed_testfiles:
logger.warning("parse failures occurred ...")
utils.dump_json_file(
parse_failed_testfiles,
utils.prepare_log_file_abs_path(self.test_path,
"parse_failed.json"))
if len(parsed_testcases) == 0:
logger.error("failed to parse all cases, abort.")
raise exceptions.ParseTestsFailure
if self.save_tests:
utils.dump_json_file(
parsed_testcases,
utils.prepare_log_file_abs_path(self.test_path, "parsed.json"))
# add tests to test suite
self.exception_stage = "add tests to test suite"
test_suite = self._add_tests(parsed_testcases)
# run test suite
self.exception_stage = "run test suite"
results = self._run_suite(test_suite)
# aggregate results
self.exception_stage = "aggregate results"
self._summary = self._aggregate(results)
# generate html report
self.exception_stage = "generate html report"
report.stringify_summary(self._summary)
if self.save_tests:
utils.dump_json_file(
self._summary,
utils.prepare_log_file_abs_path(self.test_path,
"summary.json"))
# save variables and export data
vars_out = self.get_vars_out()
utils.dump_json_file(
vars_out,
utils.prepare_log_file_abs_path(self.test_path, "io.json"))
return self._summary
def _run_suite(self, test_suite):
""" run tests in test_suite
Args:
test_suite: unittest.TestSuite()
Returns:
list: tests_results
"""
tests_results = []
for index, testcase in enumerate(test_suite):
log_handler = None
if self.save_tests:
logs_file_abs_path = utils.prepare_log_file_abs_path(
self.test_path, f"testcase_{index+1}.log")
log_handler = logger.add(logs_file_abs_path, level="DEBUG")
testcase_name = testcase.config.get("name")
logger.info(f"Start to run testcases: {testcase_name}")
result = self.unittest_runner.run(testcase)
if result.wasSuccessful():
tests_results.append((testcase, result))
else:
tests_results.insert(0, (testcase, result))
if self.save_tests and log_handler:
logger.remove(log_handler)
return tests_results
def test_prepare_dump_json_file_path_for_file(self):
# hrun tests/httpbin/a.b.c/rpc.yml --save-tests
test_path = os.path.join("tests", "httpbin", "a.b.c", "rpc.yml")
self.assertEqual(
utils.prepare_log_file_abs_path(test_path, "loaded.json"),
os.path.join(os.getcwd(), "logs",
"tests/httpbin/a.b.c/rpc.loaded.json"))
def _aggregate(self, tests_results):
""" aggregate results
Args:
tests_results (list): list of (testcases, result)
"""
summary = {
"success": True,
"stat": {
"testcases": {
"total": len(tests_results),
"success": 0,
"fail": 0
},
"teststeps": {}
},
"time": {},
"platform": report.get_platform(),
"details": []
}
for index, tests_result in enumerate(tests_results):
testcase, result = tests_result
testcase_summary = report.get_summary(result)
if testcase_summary["success"]:
summary["stat"]["testcases"]["success"] += 1
else:
summary["stat"]["testcases"]["fail"] += 1
summary["success"] &= testcase_summary["success"]
testcase_summary["name"] = testcase.config.get("name")
testcase_summary["in_out"] = utils.get_testcase_io(testcase)
report.aggregate_stat(summary["stat"]["teststeps"],
testcase_summary["stat"])
report.aggregate_stat(summary["time"], testcase_summary["time"])
if self.save_tests:
logs_file_abs_path = utils.prepare_log_file_abs_path(
self.test_path, f"testcase_{index+1}.log")
testcase_summary["log"] = logs_file_abs_path
testcase_summary[
"HRUN-Request-ID"] = testcase.runner.hrun_request_id
summary["details"].append(testcase_summary)
return summary
def test_prepare_dump_json_file_path_for_passed_testcase(self):
test_path = ""
self.assertEqual(
utils.prepare_log_file_abs_path(test_path, "loaded.json"),
os.path.join(os.getcwd(), "logs", "tests_mapping.loaded.json"))