def test_json_report(self): def _clean_json_report(json_report_file, results=None): if results is None: with codecs_open(json_report_file, "r", encoding="utf-8") as fd: content = fd.read() results = json.loads(content)["results"] json_files = results[:] json_files.append(json_report_file) for json_file in json_files: os.remove(json_file) test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败"),] for test_name, reason in test_pairs: time_str = get_time_str() test_report_name = "%s_%s.json" % (time_str, test_name) with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report.report_types["json"](fd=fd) test_runner = runner.runner_types["basic"](test_report) test_name = "test.sampletest.hellotest.%s" % test_name print("json report test for test: " + test_name) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) self.assertEqual(report_json["loaded_testcases"][0]["name"], test_name) test_results = report_json["results"] self.addCleanup(_clean_json_report, test_report_name, test_results) self.assertEqual(len(test_results), 1) with codecs_open(test_results[0], "r", encoding="utf-8") as fd2: content = fd2.read() result_json = json.loads(content) self.assertEqual(result_json["succeed"], False) failed_step = result_json["steps"][-1] self.assertEqual(failed_step["succeed"], False) actual_reson = smart_text(failed_step["logs"][0]["message"]) self.assertRegexpMatches(actual_reson, reason) test_name = "test.sampletest.hellotest.HelloTest test.sampletest.hellotest.TimeoutTest" time_str = get_time_str() test_report_name = "test_json_report_%s.json" % time_str retry_count = 2 with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report.report_types["json"](fd=fd, title="test_json_report") test_runner = runner.runner_types["multithread"](test_report, retries=retry_count) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) summary = report_json["summary"] self.assertEqual(summary["testcase_total_run"], (retry_count + 1) * 2) self.assertEqual(summary["testcase_total_count"], 2) self.assertTrue("hostname" in summary) self.assertTrue("os" in summary) self.addCleanup(_clean_json_report, test_report_name)
def test_runtest_return_code(self): from testbase.management import RunTest test_cases = list( map(lambda x: "tests.sampletest.hellotest." + x, [ "PassedCase", "FailedCase", "PassedCase", ])) report_pairs = [("stream", ""), ("json", "-o xxxx.json"), ("html", ""), ("xml", "")] context = modify_attributes(os, {"system": lambda _: None}) for report_type, report_args in report_pairs: args = argparse.Namespace() args.tests = test_cases args.working_dir = "runtest_%s" % get_time_str() self.addCleanup(shutil.rmtree, args.working_dir, True) args.report_type = report_type args.report_args = report_args args.report_args_help = None args.runner_type = "basic" args.runner_args = "" args.runner_args_help = None args.priorities = None args.status = None args.owners = None args.excluded_names = None args.tags = None args.excluded_tags = None args.resmgr_backend_type = "local" try: with context: RunTest().execute(args) except SystemExit as e: self.assertEqual(e.args[0], 1) else: self.fail( "runtest didn't failed with code 1 for report type: %s" % report_type) args.working_dir = "runtest_%s" % get_time_str() self.addCleanup(shutil.rmtree, args.working_dir, True) args.tests = test_cases[:1] try: with context: RunTest().execute(args) except SystemExit as e: self.fail("online test report return non-zero: \n%s" % traceback.format_exc())
def test_discover(self): file_name = "discovertest_%s.txt" % get_time_str() cmdline = "--excluded-tag test --priority High --status Ready --owner xxx " cmdline += "--output-file %s tests.sampletest.hellotest " % file_name cmdline += "tests.sampletest.tagtest tests.sampletest.loaderr" args = DiscoverTests.parser.parse_args(cmdline.split()) self.assertEqual(args.priorities, [TestCase.EnumPriority.High]) self.assertEqual(args.status, [TestCase.EnumStatus.Ready]) self.assertEqual(args.excluded_tags, ["test"]) self.assertEqual(args.owners, ["xxx"]) self.assertEqual(args.output_type, "stream") self.assertEqual(args.output_file, file_name) DiscoverTests().execute(args) self.assertTrue(os.path.exists(file_name)) self.addCleanup(os.remove, file_name) with codecs.open(file_name, "r", encoding="utf-8") as fd: content = fd.read() self.assertTrue( content.find("tests.sampletest.tagtest.TagTest2, reason") >= 0) self.assertTrue( content.find("tests.sampletest.hellotest.PassedCase") >= 0) self.assertTrue( content.find("cannot load test \"tests.sampletest.loaderr\"") >= 0)
def test_xml_report(self): test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败"),] old_cwd = os.getcwd() for test_name, reason in test_pairs: try: test_report = report.report_types["xml"]() test_runner = runner.runner_types["basic"](test_report) test_name = "test.sampletest.hellotest.%s" % test_name working_dir = test_name + "_" + get_time_str() os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) print("xml report test for test: %s, wokring dir=%s" % (test_name, working_dir)) test_runner.run(test_name) report_path = os.path.join(os.getcwd(), "TestReport.xml") xml_report = minidom.parse(report_path) test_results = xml_report.getElementsByTagName("TestResult") self.assertEqual(len(test_results), 1) attrs = test_results[0].attributes self.assertEqual(attrs["name"].value, test_name) self.assertEqual(attrs["result"].value, "False") result_path = os.path.join(os.getcwd(), attrs["log"].value) result_xml = minidom.parse(result_path) error_nodes = result_xml.getElementsByTagName("ERROR") self.assertEqual(len(error_nodes), 1) failed_reason = smart_text(error_nodes[0].childNodes[0].nodeValue) self.assertRegexpMatches(failed_reason, reason) finally: os.chdir(old_cwd)
def get_file(self): file_name = '%s_%s.json' % (self._translated_name, get_time_str()) if not path_exists(file_name): content = json.dumps(self._data) with codecs_open(file_name, mode="w", encoding="utf-8") as fd: fd.write(content) return file_name
def test_json_report_retry(self): old_cwd = os.getcwd() try: test_name = "tests.sampletest.hellotest.HelloTest tests.sampletest.hellotest.TimeoutTest" time_str = get_time_str() working_dir = test_name + "_" + time_str os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report_name = "test_json_report_%s.json" % time_str retry_count = 2 with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report_types["json"](fd=fd, title="test_json_report") test_runner = runner_types["multithread"](test_report, retries=retry_count) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) summary = report_json["summary"] self.assertEqual(summary["testcase_total_run"], (retry_count + 1) * 2) self.assertEqual(summary["testcase_total_count"], 2) self.assertTrue("hostname" in summary["environment"]) self.assertTrue("os" in summary["environment"]) self.assertTrue("qtaf_version" in summary["environment"]) self.assertTrue("python_version" in summary["environment"]) finally: os.chdir(old_cwd)
def test_failed_returncode(self): working_dir = "test_online_report_%s" % get_time_str() cmdline = '--report-type html tests.sampletest.hellotest.FailedCase' cmdline += " -w " + working_dir self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True) args = RunTest.parser.parse_args(cmdline.split()) return_code = RunTest().execute(args) self.assertEqual(return_code, 1)
def get_file(self): file_name = '%s_%s.json' % (self._translated_name, get_time_str()) file_path = os.path.join(os.getcwd(), file_name) if not os.path.exists(file_path): content = json.dumps(self._data) with codecs_open(file_path, mode="w", encoding="utf-8") as fd: fd.write(content) return file_path
def test_config_file_success(self): working_dir = "test_online_report_%s" % get_time_str() cmdline = 'runtest --report-type html tests.sampletest.hellotest.FailedCase --config-file tests/sampletest/test.json' cmdline += " -w " + working_dir self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True) sys.argv = ["qtaf"] sys.argv.extend(cmdline.split()) exitcode = ManagementTools().run() self.assertEqual(exitcode, 0)
def test_config_file_with_global_parameters(self): with modify_settings(QTAF_PARAM_MODE=True): working_dir = "test_online_report_%s" % get_time_str() cmdline = 'runtest --config-file tests/sampletest/test_global_parameters.json' cmdline += " -w " + working_dir self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True) sys.argv = ["qtaf"] sys.argv.extend(cmdline.split()) exitcode = ManagementTools().run() self.assertEqual(exitcode, 0)
def get_file(self): file_name = '%s_%s.js' % (self._translated_name, get_time_str()) if not path_exists(file_name): var_name = os.path.basename(file_name) var_name = os.path.splitext(file_name)[0].replace(".", "_") content = "var %s = %s" % (var_name, json.dumps(self._data)) content = smart_binary(content) with codecs_open(file_name, mode="wb") as fd: fd.write(content) return file_name
def test_success_returncode(self): working_dir = "test_online_report_%s" % get_time_str() cmdline = '--report-type html tests.sampletest.hellotest.PassedCase' cmdline += " -w " + working_dir self.addCleanup(shutil.rmtree, working_dir, ignore_errors=True) args = RunTest.parser.parse_args(cmdline.split()) proc = multiprocessing.Process(target=RunTest().execute, args=(args,)) proc.start() proc.join() self.assertEqual(proc.exitcode, 0)
def __init__(self, testcase): '''构造函数 :param file_path: XML文件路径 :type file_path: string ''' super(XmlResult, self).__init__() self._xmldoc = dom.Document() translated_name = translate_bad_char(testcase.test_name) self._file_path = '%s_%s.xml' % (translated_name, get_time_str())
def test_runtest_return_code(self): from testbase.management import RunTest test_cases = list( map(lambda x: "tests.sampletest.hellotest." + x, [ "PassedCase", "FailedCase", "PassedCase", ])) report_pairs = [("stream", ""), ("json", "-o xxxx.json"), ("html", ""), ("xml", "")] context = modify_attributes(os, {"system": lambda _: None}) for report_type, report_args in report_pairs: args = argparse.Namespace() args.tests = test_cases args.working_dir = "runtest_%s" % get_time_str() self.addCleanup(shutil.rmtree, args.working_dir, True) args.report_type = report_type args.report_args = report_args args.report_args_help = None args.runner_type = "basic" args.runner_args = "" args.runner_args_help = None args.priorities = None args.status = None args.owners = None args.excluded_names = None args.tags = None args.excluded_tags = None args.resmgr_backend_type = "local" args.global_parameters = None args.config_file = None with context: self.assertEqual(RunTest().execute(args), 1) args.working_dir = "runtest_%s" % get_time_str() self.addCleanup(shutil.rmtree, args.working_dir, True) args.tests = test_cases[:1] with context: self.assertEqual(RunTest().execute(args), None)
def test_discover_show(self): file_name = "discovertest_%s.txt" % get_time_str() cmdline = "--excluded-tag test --owner xxx --output-file %s --show error " % file_name cmdline += "tests.sampletest.hellotest tests.sampletest.tagtest tests.sampletest.loaderr" args = DiscoverTests.parser.parse_args(cmdline.split()) DiscoverTests().execute(args) self.assertTrue(os.path.exists(file_name)) self.addCleanup(os.remove, file_name) with open(file_name, "r") as fd: content = fd.read() self.assertTrue(content.find("filtered test") == -1) self.assertTrue(content.find("normal test") == -1) self.assertTrue(content.find("error test") >= 0) self.assertTrue(content.find("cannot load test \"tests.sampletest.loaderr\"") >= 0)
def test_html_report_content(self): test_pairs = [ ("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败"), ] old_cwd = os.getcwd() for test_name, reason in test_pairs: try: working_dir = test_name + "_" + get_time_str() os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report = report_types["html"](title="test html report") test_runner = runner_types["basic"](test_report) test_name = "tests.sampletest.hellotest.%s" % test_name print("html report test for test: " + test_name) test_runner.run(test_name) html_report_file = os.path.join(os.getcwd(), "qta-report.js") with codecs_open(html_report_file, encoding="utf-8") as fd: content = fd.read() index = content.find("{") qta_report_data = content[index:] qta_report = json.loads(qta_report_data) failed_test_names = list(qta_report["failed_tests"].keys()) self.assertEqual(failed_test_names[0], test_name) failed_tests = qta_report["failed_tests"] self.assertEqual(len(failed_tests), 1) with codecs_open(failed_tests[test_name]["records"][0], "r", encoding="utf-8") as fd2: content = fd2.read() index = content.find("{") result_json_data = content[index:] result_json = json.loads(result_json_data) self.assertEqual(result_json["succeed"], False) failed_step = result_json["steps"][-1] self.assertEqual(failed_step["succeed"], False) actual_reson = smart_text( failed_step["logs"][0]["message"]) self.assertRegexpMatches(actual_reson, reason) finally: os.chdir(old_cwd)
def test_json_report_content(self): test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败")] old_cwd = os.getcwd() for test_name, reason in test_pairs: try: time_str = get_time_str() working_dir = test_name + "_" + time_str os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report_name = "%s_%s.json" % (time_str, test_name) with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report_types["json"](fd=fd) test_runner = runner_types["basic"](test_report) test_name = "tests.sampletest.hellotest.%s" % test_name print("json report test for test: " + test_name) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) failed_test_names = list( report_json["failed_tests"].keys()) self.assertEqual(failed_test_names[0], test_name) failed_tests = report_json["failed_tests"] self.assertEqual(len(failed_tests), 1) with codecs_open(failed_tests[test_name]["records"][0], "r", encoding="utf-8") as fd2: content = fd2.read() result_json = json.loads(content) self.assertEqual(result_json["succeed"], False) failed_step = result_json["steps"][-1] self.assertEqual(failed_step["succeed"], False) actual_reson = smart_text( failed_step["logs"][0]["message"]) self.assertRegexpMatches(actual_reson, reason) finally: os.chdir(old_cwd)
def test_xml_report_with_extra_properties(self): test_pairs = [ ("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败"), ] old_cwd = os.getcwd() try: test_report = report_types["xml"]() test_runner = runner_types["basic"](test_report) test_name = "tests.sampletest.hellotest.ExtraPropertyTest" working_dir = test_name + "_" + get_time_str() os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) print("xml report test for test: %s" % test_name) test_runner.run(test_name) report_path = os.path.join(os.getcwd(), "TestReport.xml") xml_report = minidom.parse(report_path) test_results = xml_report.getElementsByTagName("TestResult") self.assertEqual(len(test_results), 1) attrs = test_results[0].attributes self.assertEqual(attrs["name"].value, test_name) self.assertEqual(attrs["result"].value, "True") result_path = os.path.join(os.getcwd(), attrs["log"].value) result_xml = minidom.parse(result_path) test_nodes = result_xml.getElementsByTagName("TEST") self.assertEqual(len(test_nodes), 1) attrs = test_nodes[0].attributes self.assertEqual(attrs['base_property_str'].value, '123') self.assertEqual(attrs['base_property_int'].value, '123') self.assertEqual(attrs['base_property_func_value'].value, 'True') self.assertEqual(attrs['base_property_bool'].value, 'True') self.assertEqual(attrs['property_str'].value, '123') self.assertEqual(attrs['property_int'].value, '123') self.assertEqual(attrs['property_func_value'].value, 'True') self.assertEqual(attrs['property_bool'].value, 'True') self.assertEqual(attrs['property_variable2'].value, 'b1_test') finally: os.chdir(old_cwd)
def test_html_report_datadrive(self): old_cwd = os.getcwd() try: test_name = "DataDriveCase" working_dir = test_name + "_" + get_time_str() os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report = report_types["html"](title="test html report") test_runner = runner_types["basic"](test_report) test_name = "tests.sampletest.hellotest." + test_name test_runner.run(test_name) with codecs_open("qta-report.js", encoding="utf-8") as fd: content = fd.read() index = content.find("{") qta_report_data = content[index:] qta_report = json.loads(qta_report_data) self.assertEqual(qta_report["summary"]["testcase_total_run"], 2) passed_tests = qta_report["passed_tests"] for item in ["中国", "xxx"]: is_ok = any((lambda x: item in x, passed_tests.keys())) self.assertTrue(is_ok) self.assertEqual(len(passed_tests), 2) for test_name in passed_tests: with codecs_open(passed_tests[test_name]["records"][0], "r", encoding="utf-8") as fd2: content = fd2.read() index = content.find("{") result_json_data = content[index:] result_json = json.loads(result_json_data) self.assertEqual(result_json["succeed"], True) finally: os.chdir(old_cwd)
def test_json_report(self): test_pairs = [("HelloTest", "断言失败"), ("TimeoutTest", "用例执行超时"), ("CrashTest", "App Crash"), ("QT4iTest", "run_test执行失败")] old_cwd = os.getcwd() try: for test_name, reason in test_pairs: time_str = get_time_str() working_dir = test_name + "_" + time_str os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report_name = "%s_%s.json" % (time_str, test_name) with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report.report_types["json"](fd=fd) test_runner = runner.runner_types["basic"](test_report) test_name = "test.sampletest.hellotest.%s" % test_name print("json report test for test: " + test_name) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) self.assertEqual( report_json["loaded_testcases"][0]["name"], test_name) test_results = report_json["results"] self.assertEqual(len(test_results), 1) with codecs_open(test_results[test_name][0], "r", encoding="utf-8") as fd2: content = fd2.read() result_json = json.loads(content) self.assertEqual(result_json["succeed"], False) failed_step = result_json["steps"][-1] self.assertEqual(failed_step["succeed"], False) actual_reson = smart_text( failed_step["logs"][0]["message"]) self.assertRegexpMatches(actual_reson, reason) finally: os.chdir(old_cwd) try: test_name = "test.sampletest.hellotest.HelloTest test.sampletest.hellotest.TimeoutTest" time_str = get_time_str() working_dir = test_name + "_" + time_str os.makedirs(working_dir) os.chdir(working_dir) self.addCleanup(shutil.rmtree, working_dir, True) test_report_name = "test_json_report_%s.json" % time_str retry_count = 2 with codecs_open(test_report_name, "w", encoding="utf-8") as fd: test_report = report.report_types["json"]( fd=fd, title="test_json_report") test_runner = runner.runner_types["multithread"]( test_report, retries=retry_count) test_runner.run(test_name) with codecs_open(test_report_name, "r", encoding="utf-8") as fd: content = fd.read() report_json = json.loads(content) summary = report_json["summary"] self.assertEqual(summary["testcase_total_run"], (retry_count + 1) * 2) self.assertEqual(summary["testcase_total_count"], 2) self.assertTrue("hostname" in summary["environment"]) self.assertTrue("os" in summary["environment"]) self.assertTrue("qtaf_version" in summary["environment"]) self.assertTrue("python_version" in summary["environment"]) finally: os.chdir(old_cwd)