def test_json_report(self):
def _clean_json_report(json_report_file, results=None):
if results is None:
with codecs_open(json_report_file, "r", encoding="utf-8") as fd:
content = fd.read()
results = json.loads(content)["results"]
json_files = results[:]
json_files.append(json_report_file)
for json_file in json_files:
os.remove(json_file)
test_pairs = [("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),]
for test_name, reason in test_pairs:
time_str = get_time_str()
test_report_name = "%s_%s.json" % (time_str, test_name)
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report.report_types["json"](fd=fd)
test_runner = runner.runner_types["basic"](test_report)
test_name = "test.sampletest.hellotest.%s" % test_name
print("json report test for test: " + test_name)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
self.assertEqual(report_json["loaded_testcases"][0]["name"], test_name)
test_results = report_json["results"]
self.addCleanup(_clean_json_report, test_report_name, test_results)
self.assertEqual(len(test_results), 1)
with codecs_open(test_results[0], "r", encoding="utf-8") as fd2:
content = fd2.read()
result_json = json.loads(content)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
test_name = "test.sampletest.hellotest.HelloTest test.sampletest.hellotest.TimeoutTest"
time_str = get_time_str()
test_report_name = "test_json_report_%s.json" % time_str
retry_count = 2
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report.report_types["json"](fd=fd, title="test_json_report")
test_runner = runner.runner_types["multithread"](test_report, retries=retry_count)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
summary = report_json["summary"]
self.assertEqual(summary["testcase_total_run"], (retry_count + 1) * 2)
self.assertEqual(summary["testcase_total_count"], 2)
self.assertTrue("hostname" in summary)
self.assertTrue("os" in summary)
self.addCleanup(_clean_json_report, test_report_name)
def test_runtest_return_code(self):
from testbase.management import RunTest
test_cases = list(
map(lambda x: "tests.sampletest.hellotest." + x, [
"PassedCase",
"FailedCase",
"PassedCase",
]))
report_pairs = [("stream", ""), ("json", "-o xxxx.json"), ("html", ""),
("xml", "")]
context = modify_attributes(os, {"system": lambda _: None})
for report_type, report_args in report_pairs:
args = argparse.Namespace()
args.tests = test_cases
args.working_dir = "runtest_%s" % get_time_str()
self.addCleanup(shutil.rmtree, args.working_dir, True)
args.report_type = report_type
args.report_args = report_args
args.report_args_help = None
args.runner_type = "basic"
args.runner_args = ""
args.runner_args_help = None
args.priorities = None
args.status = None
args.owners = None
args.excluded_names = None
args.tags = None
args.excluded_tags = None
args.resmgr_backend_type = "local"
try:
with context:
RunTest().execute(args)
except SystemExit as e:
self.assertEqual(e.args[0], 1)
else:
self.fail(
"runtest didn't failed with code 1 for report type: %s" %
report_type)
args.working_dir = "runtest_%s" % get_time_str()
self.addCleanup(shutil.rmtree, args.working_dir, True)
args.tests = test_cases[:1]
try:
with context:
RunTest().execute(args)
except SystemExit as e:
self.fail("online test report return non-zero: \n%s" %
traceback.format_exc())
def test_discover(self):
file_name = "discovertest_%s.txt" % get_time_str()
cmdline = "--excluded-tag test --priority High --status Ready --owner xxx "
cmdline += "--output-file %s tests.sampletest.hellotest " % file_name
cmdline += "tests.sampletest.tagtest tests.sampletest.loaderr"
args = DiscoverTests.parser.parse_args(cmdline.split())
self.assertEqual(args.priorities, [TestCase.EnumPriority.High])
self.assertEqual(args.status, [TestCase.EnumStatus.Ready])
self.assertEqual(args.excluded_tags, ["test"])
self.assertEqual(args.owners, ["xxx"])
self.assertEqual(args.output_type, "stream")
self.assertEqual(args.output_file, file_name)
DiscoverTests().execute(args)
self.assertTrue(os.path.exists(file_name))
self.addCleanup(os.remove, file_name)
with codecs.open(file_name, "r", encoding="utf-8") as fd:
content = fd.read()
self.assertTrue(
content.find("tests.sampletest.tagtest.TagTest2, reason") >= 0)
self.assertTrue(
content.find("tests.sampletest.hellotest.PassedCase") >= 0)
self.assertTrue(
content.find("cannot load test \"tests.sampletest.loaderr\"") >= 0)
def test_xml_report(self):
test_pairs = [("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),]
old_cwd = os.getcwd()
for test_name, reason in test_pairs:
try:
test_report = report.report_types["xml"]()
test_runner = runner.runner_types["basic"](test_report)
test_name = "test.sampletest.hellotest.%s" % test_name
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
print("xml report test for test: %s, wokring dir=%s" % (test_name, working_dir))
test_runner.run(test_name)
report_path = os.path.join(os.getcwd(), "TestReport.xml")
xml_report = minidom.parse(report_path)
test_results = xml_report.getElementsByTagName("TestResult")
self.assertEqual(len(test_results), 1)
attrs = test_results[0].attributes
self.assertEqual(attrs["name"].value, test_name)
self.assertEqual(attrs["result"].value, "False")
result_path = os.path.join(os.getcwd(), attrs["log"].value)
result_xml = minidom.parse(result_path)
error_nodes = result_xml.getElementsByTagName("ERROR")
self.assertEqual(len(error_nodes), 1)
failed_reason = smart_text(error_nodes[0].childNodes[0].nodeValue)
self.assertRegexpMatches(failed_reason, reason)
finally:
os.chdir(old_cwd)
def get_file(self):
file_name = '%s_%s.json' % (self._translated_name, get_time_str())
if not path_exists(file_name):
content = json.dumps(self._data)
with codecs_open(file_name, mode="w", encoding="utf-8") as fd:
fd.write(content)
return file_name
def test_json_report_retry(self):
old_cwd = os.getcwd()
try:
test_name = "tests.sampletest.hellotest.HelloTest tests.sampletest.hellotest.TimeoutTest"
time_str = get_time_str()
working_dir = test_name + "_" + time_str
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report_name = "test_json_report_%s.json" % time_str
retry_count = 2
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report_types["json"](fd=fd,
title="test_json_report")
test_runner = runner_types["multithread"](test_report,
retries=retry_count)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
summary = report_json["summary"]
self.assertEqual(summary["testcase_total_run"],
(retry_count + 1) * 2)
self.assertEqual(summary["testcase_total_count"], 2)
self.assertTrue("hostname" in summary["environment"])
self.assertTrue("os" in summary["environment"])
self.assertTrue("qtaf_version" in summary["environment"])
self.assertTrue("python_version" in summary["environment"])
finally:
os.chdir(old_cwd)
def test_failed_returncode(self):
working_dir = "test_online_report_%s" % get_time_str()
cmdline = '--report-type html tests.sampletest.hellotest.FailedCase'
cmdline += " -w " + working_dir
self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True)
args = RunTest.parser.parse_args(cmdline.split())
return_code = RunTest().execute(args)
self.assertEqual(return_code, 1)
def get_file(self):
file_name = '%s_%s.json' % (self._translated_name, get_time_str())
file_path = os.path.join(os.getcwd(), file_name)
if not os.path.exists(file_path):
content = json.dumps(self._data)
with codecs_open(file_path, mode="w", encoding="utf-8") as fd:
fd.write(content)
return file_path
def test_config_file_success(self):
working_dir = "test_online_report_%s" % get_time_str()
cmdline = 'runtest --report-type html tests.sampletest.hellotest.FailedCase --config-file tests/sampletest/test.json'
cmdline += " -w " + working_dir
self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True)
sys.argv = ["qtaf"]
sys.argv.extend(cmdline.split())
exitcode = ManagementTools().run()
self.assertEqual(exitcode, 0)
def test_config_file_with_global_parameters(self):
with modify_settings(QTAF_PARAM_MODE=True):
working_dir = "test_online_report_%s" % get_time_str()
cmdline = 'runtest --config-file tests/sampletest/test_global_parameters.json'
cmdline += " -w " + working_dir
self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True)
sys.argv = ["qtaf"]
sys.argv.extend(cmdline.split())
exitcode = ManagementTools().run()
self.assertEqual(exitcode, 0)
def get_file(self):
file_name = '%s_%s.js' % (self._translated_name, get_time_str())
if not path_exists(file_name):
var_name = os.path.basename(file_name)
var_name = os.path.splitext(file_name)[0].replace(".", "_")
content = "var %s = %s" % (var_name, json.dumps(self._data))
content = smart_binary(content)
with codecs_open(file_name, mode="wb") as fd:
fd.write(content)
return file_name
def test_success_returncode(self):
working_dir = "test_online_report_%s" % get_time_str()
cmdline = '--report-type html tests.sampletest.hellotest.PassedCase'
cmdline += " -w " + working_dir
self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True)
args = RunTest.parser.parse_args(cmdline.split())
proc = multiprocessing.Process(target=RunTest().execute, args=(args,))
proc.start()
proc.join()
self.assertEqual(proc.exitcode, 0)
def __init__(self, testcase):
'''构造函数
:param file_path: XML文件路径
:type file_path: string
'''
super(XmlResult, self).__init__()
self._xmldoc = dom.Document()
translated_name = translate_bad_char(testcase.test_name)
self._file_path = '%s_%s.xml' % (translated_name, get_time_str())
def test_runtest_return_code(self):
from testbase.management import RunTest
test_cases = list(
map(lambda x: "tests.sampletest.hellotest." + x, [
"PassedCase",
"FailedCase",
"PassedCase",
]))
report_pairs = [("stream", ""), ("json", "-o xxxx.json"), ("html", ""),
("xml", "")]
context = modify_attributes(os, {"system": lambda _: None})
for report_type, report_args in report_pairs:
args = argparse.Namespace()
args.tests = test_cases
args.working_dir = "runtest_%s" % get_time_str()
self.addCleanup(shutil.rmtree, args.working_dir, True)
args.report_type = report_type
args.report_args = report_args
args.report_args_help = None
args.runner_type = "basic"
args.runner_args = ""
args.runner_args_help = None
args.priorities = None
args.status = None
args.owners = None
args.excluded_names = None
args.tags = None
args.excluded_tags = None
args.resmgr_backend_type = "local"
args.global_parameters = None
args.config_file = None
with context:
self.assertEqual(RunTest().execute(args), 1)
args.working_dir = "runtest_%s" % get_time_str()
self.addCleanup(shutil.rmtree, args.working_dir, True)
args.tests = test_cases[:1]
with context:
self.assertEqual(RunTest().execute(args), None)
def test_discover_show(self):
file_name = "discovertest_%s.txt" % get_time_str()
cmdline = "--excluded-tag test --owner xxx --output-file %s --show error " % file_name
cmdline += "tests.sampletest.hellotest tests.sampletest.tagtest tests.sampletest.loaderr"
args = DiscoverTests.parser.parse_args(cmdline.split())
DiscoverTests().execute(args)
self.assertTrue(os.path.exists(file_name))
self.addCleanup(os.remove, file_name)
with open(file_name, "r") as fd:
content = fd.read()
self.assertTrue(content.find("filtered test") == -1)
self.assertTrue(content.find("normal test") == -1)
self.assertTrue(content.find("error test") >= 0)
self.assertTrue(content.find("cannot load test \"tests.sampletest.loaderr\"") >= 0)
def test_html_report_content(self):
test_pairs = [
("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),
]
old_cwd = os.getcwd()
for test_name, reason in test_pairs:
try:
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report = report_types["html"](title="test html report")
test_runner = runner_types["basic"](test_report)
test_name = "tests.sampletest.hellotest.%s" % test_name
print("html report test for test: " + test_name)
test_runner.run(test_name)
html_report_file = os.path.join(os.getcwd(), "qta-report.js")
with codecs_open(html_report_file, encoding="utf-8") as fd:
content = fd.read()
index = content.find("{")
qta_report_data = content[index:]
qta_report = json.loads(qta_report_data)
failed_test_names = list(qta_report["failed_tests"].keys())
self.assertEqual(failed_test_names[0], test_name)
failed_tests = qta_report["failed_tests"]
self.assertEqual(len(failed_tests), 1)
with codecs_open(failed_tests[test_name]["records"][0],
"r",
encoding="utf-8") as fd2:
content = fd2.read()
index = content.find("{")
result_json_data = content[index:]
result_json = json.loads(result_json_data)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(
failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
finally:
os.chdir(old_cwd)
def test_json_report_content(self):
test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败")]
old_cwd = os.getcwd()
for test_name, reason in test_pairs:
try:
time_str = get_time_str()
working_dir = test_name + "_" + time_str
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report_name = "%s_%s.json" % (time_str, test_name)
with codecs_open(test_report_name, "w",
encoding="utf-8") as fd:
test_report = report_types["json"](fd=fd)
test_runner = runner_types["basic"](test_report)
test_name = "tests.sampletest.hellotest.%s" % test_name
print("json report test for test: " + test_name)
test_runner.run(test_name)
with codecs_open(test_report_name, "r",
encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
failed_test_names = list(
report_json["failed_tests"].keys())
self.assertEqual(failed_test_names[0], test_name)
failed_tests = report_json["failed_tests"]
self.assertEqual(len(failed_tests), 1)
with codecs_open(failed_tests[test_name]["records"][0],
"r",
encoding="utf-8") as fd2:
content = fd2.read()
result_json = json.loads(content)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(
failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
finally:
os.chdir(old_cwd)
def test_xml_report_with_extra_properties(self):
test_pairs = [
("HelloTest", "断言失败"),
("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"),
("QT4iTest", "run_test执行失败"),
]
old_cwd = os.getcwd()
try:
test_report = report_types["xml"]()
test_runner = runner_types["basic"](test_report)
test_name = "tests.sampletest.hellotest.ExtraPropertyTest"
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
print("xml report test for test: %s" % test_name)
test_runner.run(test_name)
report_path = os.path.join(os.getcwd(), "TestReport.xml")
xml_report = minidom.parse(report_path)
test_results = xml_report.getElementsByTagName("TestResult")
self.assertEqual(len(test_results), 1)
attrs = test_results[0].attributes
self.assertEqual(attrs["name"].value, test_name)
self.assertEqual(attrs["result"].value, "True")
result_path = os.path.join(os.getcwd(), attrs["log"].value)
result_xml = minidom.parse(result_path)
test_nodes = result_xml.getElementsByTagName("TEST")
self.assertEqual(len(test_nodes), 1)
attrs = test_nodes[0].attributes
self.assertEqual(attrs['base_property_str'].value, '123')
self.assertEqual(attrs['base_property_int'].value, '123')
self.assertEqual(attrs['base_property_func_value'].value, 'True')
self.assertEqual(attrs['base_property_bool'].value, 'True')
self.assertEqual(attrs['property_str'].value, '123')
self.assertEqual(attrs['property_int'].value, '123')
self.assertEqual(attrs['property_func_value'].value, 'True')
self.assertEqual(attrs['property_bool'].value, 'True')
self.assertEqual(attrs['property_variable2'].value, 'b1_test')
finally:
os.chdir(old_cwd)
def test_html_report_datadrive(self):
old_cwd = os.getcwd()
try:
test_name = "DataDriveCase"
working_dir = test_name + "_" + get_time_str()
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report = report_types["html"](title="test html report")
test_runner = runner_types["basic"](test_report)
test_name = "tests.sampletest.hellotest." + test_name
test_runner.run(test_name)
with codecs_open("qta-report.js", encoding="utf-8") as fd:
content = fd.read()
index = content.find("{")
qta_report_data = content[index:]
qta_report = json.loads(qta_report_data)
self.assertEqual(qta_report["summary"]["testcase_total_run"],
2)
passed_tests = qta_report["passed_tests"]
for item in ["中国", "xxx"]:
is_ok = any((lambda x: item in x, passed_tests.keys()))
self.assertTrue(is_ok)
self.assertEqual(len(passed_tests), 2)
for test_name in passed_tests:
with codecs_open(passed_tests[test_name]["records"][0],
"r",
encoding="utf-8") as fd2:
content = fd2.read()
index = content.find("{")
result_json_data = content[index:]
result_json = json.loads(result_json_data)
self.assertEqual(result_json["succeed"], True)
finally:
os.chdir(old_cwd)
def test_json_report(self):
test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"),
("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败")]
old_cwd = os.getcwd()
try:
for test_name, reason in test_pairs:
time_str = get_time_str()
working_dir = test_name + "_" + time_str
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report_name = "%s_%s.json" % (time_str, test_name)
with codecs_open(test_report_name, "w",
encoding="utf-8") as fd:
test_report = report.report_types["json"](fd=fd)
test_runner = runner.runner_types["basic"](test_report)
test_name = "test.sampletest.hellotest.%s" % test_name
print("json report test for test: " + test_name)
test_runner.run(test_name)
with codecs_open(test_report_name, "r",
encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
self.assertEqual(
report_json["loaded_testcases"][0]["name"], test_name)
test_results = report_json["results"]
self.assertEqual(len(test_results), 1)
with codecs_open(test_results[test_name][0],
"r",
encoding="utf-8") as fd2:
content = fd2.read()
result_json = json.loads(content)
self.assertEqual(result_json["succeed"], False)
failed_step = result_json["steps"][-1]
self.assertEqual(failed_step["succeed"], False)
actual_reson = smart_text(
failed_step["logs"][0]["message"])
self.assertRegexpMatches(actual_reson, reason)
finally:
os.chdir(old_cwd)
try:
test_name = "test.sampletest.hellotest.HelloTest test.sampletest.hellotest.TimeoutTest"
time_str = get_time_str()
working_dir = test_name + "_" + time_str
os.makedirs(working_dir)
os.chdir(working_dir)
self.addCleanup(shutil.rmtree, working_dir, True)
test_report_name = "test_json_report_%s.json" % time_str
retry_count = 2
with codecs_open(test_report_name, "w", encoding="utf-8") as fd:
test_report = report.report_types["json"](
fd=fd, title="test_json_report")
test_runner = runner.runner_types["multithread"](
test_report, retries=retry_count)
test_runner.run(test_name)
with codecs_open(test_report_name, "r", encoding="utf-8") as fd:
content = fd.read()
report_json = json.loads(content)
summary = report_json["summary"]
self.assertEqual(summary["testcase_total_run"],
(retry_count + 1) * 2)
self.assertEqual(summary["testcase_total_count"], 2)
self.assertTrue("hostname" in summary["environment"])
self.assertTrue("os" in summary["environment"])
self.assertTrue("qtaf_version" in summary["environment"])
self.assertTrue("python_version" in summary["environment"])
finally:
os.chdir(old_cwd)
