def test_init_skipped(): tc = Case("Skipped-Message-and-Output") tc.add_skipped_info("skipped message", "I skipped!") ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Message-and-Output"}, skipped_message="skipped message", skipped_output="I skipped!")
def get_test_suites(self, use_bc_ids=False) -> List[TestSuite]: test_cases = defaultdict(list) test_suites = [] records = self.passed_checks + self.failed_checks + self.skipped_checks for record in records: check_name = f"{record.get_output_id(use_bc_ids)}/{record.check_name}" test_name = f"{self.check_type} {check_name} {record.resource}" test_case = TestCase(name=test_name, file=record.file_path, classname=record.check_class) if record.check_result["result"] == CheckResult.FAILED: if record.file_path and record.file_line_range: test_case.add_failure_info( f"Resource {record.resource} failed in check {check_name} - {record.file_path}:{record.file_line_range} - Guideline: {record.guideline}" ) else: test_case.add_failure_info( f"Resource {record.resource} failed in check {check_name}" ) if record.check_result["result"] == CheckResult.SKIPPED: test_case.add_skipped_info( f'Resource {record.resource} skipped in check {check_name} \n Suppress comment: {record.check_result["suppress_comment"]} - Guideline: {record.guideline}' ) test_cases[check_name].append(test_case) for key in test_cases.keys(): test_suites.append( TestSuite( name=key, test_cases=test_cases[key], package=test_cases[key][0].classname, )) return test_suites
def test_init_skipped_output(): tc = Case("Skipped-Output") tc.add_skipped_info(output="I skipped!") #pylint: disable=unused-variable ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Output"}, skipped_output="I skipped!")
def test_init_utf8(): tc = Case( name="Test äöü", classname="some.class.name.äöü", elapsed_sec=123.345, stdout="I am stdöüt!", stderr="I am stdärr!", ) tc.add_skipped_info(message="Skipped äöü", output="I skippäd!") tc.add_error_info(message="Skipped error äöü", output="I skippäd with an error!") test_suite = Suite("Test UTF-8", [tc]) #pylint: disable=unused-variable ts, tcs = serialize_and_read(test_suite, encoding="utf-8")[0] verify_test_case( tcs[0], { "name": decode("Test äöü", "utf-8"), "classname": decode("some.class.name.äöü", "utf-8"), "time": ("%f" % 123.345), }, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), skipped_message=decode("Skipped äöü", "utf-8"), skipped_output=decode("I skippäd!", "utf-8"), error_message=decode("Skipped error äöü", "utf-8"), error_output=decode("I skippäd with an error!", "utf-8"), )
def test_init_skipped_message(): tc = Case("Skipped-Message") tc.add_skipped_info("skipped message") #pylint: disable=unused-variable ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Message"}, skipped_message="skipped message")
def write_test_results(results, args): exit_code = ExitCodes.OK test_cases = [] for test_result in results["result"]: test_case = TestCase(test_result.name, elapsed_sec=test_result.elapsed_time, timestamp=test_result.timestamp) if test_result.name in args.ignore or test_result.state in [ TestStates.DISABLED, TestStates.UNCLEAR, TestStates.MANUAL, TestStates.NA, TestStates.OPTIONAL ]: test_case.add_skipped_info(test_result.detail) elif test_result.state in [TestStates.WARNING, TestStates.FAIL]: test_case.add_failure_info(test_result.detail, failure_type=str(test_result.state)) if test_result.state == TestStates.FAIL: exit_code = max(exit_code, ExitCodes.FAIL) elif test_result.state == TestStates.WARNING: exit_code = max(exit_code, ExitCodes.WARNING) elif test_result.state != TestStates.PASS: test_case.add_error_info(test_result.detail, error_type=str(test_result.state)) test_cases.append(test_case) ts = TestSuite(results["name"] + ": " + results["base_url"], test_cases) with open(args.output, "w") as f: TestSuite.to_file(f, [ts], prettyprint=False) print(" * Test results written to file: {}".format(args.output)) return exit_code
def run_tests(self): test_cases = [] for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info(message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output t_start = time.time() result = self.run_test(index) t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = test_output.getvalue() print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) test_output.close() test_cases += [test_case]; return TestSuite(self.name, test_cases)
def get_test_suites(self): test_cases = {} test_suites = [] records = self.passed_checks + self.failed_checks + self.skipped_checks for record in records: check_name = record.check_name if check_name not in test_cases: test_cases[check_name] = [] test_name = "{} {} {}".format(self.check_type, check_name, record.resource) test_case = TestCase(name=test_name, file=record.file_path, classname=record.check_class) if record.check_result['result'] == CheckResult.FAILED: test_case.add_failure_info( "Resource \"{}\" failed in check \"{}\"".format( record.resource, check_name)) if record.check_result['result'] == CheckResult.SKIPPED: test_case.add_skipped_info( "Resource \"{}\" skipped in check \"{}\"\n Suppress comment: {}" .format(record.resource, check_name, record.check_result['suppress_comment'])) test_cases[check_name].append(test_case) for key in test_cases.keys(): test_suites.append( TestSuite(name=key, test_cases=test_cases[key], package=test_cases[key][0].classname)) return test_suites
def simple_report(self): """empty test report""" test_case_1 = TestCase("testcase1", elapsed_sec=1.5) test_case_2 = TestCase("testcase2", elapsed_sec=0.5) test_case_2.add_skipped_info("was skipped") test_case_3 = TestCase("testcase3", elapsed_sec=1.0) test_case_3.add_failure_info("failed") test_case_4 = TestCase("testcase4", elapsed_sec=0.25) test_case_4.add_error_info("errored") test_case_5 = TestCase("testcase5", elapsed_sec=0.1) test_cases = [ test_case_1, test_case_2, test_case_3, test_case_4, test_case_5 ] test_suites = [ TestSuite('testsuite1', test_cases, timestamp=datetime.datetime.utcnow()) ] return TestReport(NAME, {"module": test_suites}, BUILD_NUMBER, True)
def test_init_unicode(): tc = Case( name=decode("Test äöü", "utf-8"), classname=decode("some.class.name.äöü", "utf-8"), elapsed_sec=123.345, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), ) tc.add_skipped_info(message=decode("Skipped äöü", "utf-8"), output=decode("I skippäd!", "utf-8")) tc.add_error_info(message=decode("Skipped error äöü", "utf-8"), output=decode("I skippäd with an error!", "utf-8")) ts, tcs = serialize_and_read(Suite("Test Unicode", [tc]))[0] verify_test_case( tcs[0], { "name": decode("Test äöü", "utf-8"), "classname": decode("some.class.name.äöü", "utf-8"), "time": ("%f" % 123.345), }, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), skipped_message=decode("Skipped äöü", "utf-8"), skipped_output=decode("I skippäd!", "utf-8"), error_message=decode("Skipped error äöü", "utf-8"), error_output=decode("I skippäd with an error!", "utf-8"), )
def test_multiple_skipped(): """Tests multiple skipped messages in one test case""" tc = Case("Multiple skipped", allow_multiple_subelements=True) tc.add_skipped_info("First skipped", "First skipped message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Multiple skipped"}, skipped=[{ "message": "First skipped", "output": "First skipped message" }]) tc.add_skipped_info("Second skipped", "Second skipped message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple skipped"}, skipped=[ { "message": "First skipped", "output": "First skipped message" }, { "message": "Second skipped", "output": "Second skipped message" }, ], )
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] for platform in sorted(test_result_ext.keys()): # {platform : ['Platform', 'Result', 'Scope', 'Description']) test_cases = [] for tr_result in test_result_ext[platform]: result, name, scope, description = tr_result classname = "test.ioper.%s.%s.%s" % (platform, name, scope) elapsed_sec = 0 _stdout = description _stderr = "" # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if result == "FAIL": tc.add_failure_info(description, _stdout) elif result == "ERROR": tc.add_error_info(description, _stdout) elif result == "SKIP" or result == "NOT_SUPPORTED": tc.add_skipped_info(description, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def print_matches(self, matches, rules=None, filenames=None): """Output all the matches""" if not rules: return None test_cases = [] for rule in rules.all_rules: if not rules.is_rule_enabled(rule): if not rule.id: continue test_case = TestCase( name='{0} {1}'.format(rule.id, rule.shortdesc)) if rule.experimental: test_case.add_skipped_info( message='Experimental rule - not enabled') else: test_case.add_skipped_info(message='Ignored rule') test_cases.append(test_case) else: test_case = TestCase(name='{0} {1}'.format( rule.id, rule.shortdesc), allow_multiple_subelements=True, url=rule.source_url) for match in matches: if match.rule.id == rule.id: test_case.add_failure_info( message=self._failure_format(match), failure_type=match.message) test_cases.append(test_case) test_suite = TestSuite('CloudFormation Lint', test_cases) return to_xml_report_string([test_suite], prettyprint=True)
def test_init_skipped_output(self): tc = TestCase('Skipped-Output') tc.add_skipped_info(output="I skipped!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Skipped-Output'}, skipped_output="I skipped!")
def test_init_skipped_message(self): tc = TestCase('Skipped-Message') tc.add_skipped_info("skipped message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Skipped-Message'}, skipped_message="skipped message")
def test_init_unicode(self): tc = TestCase(name=decode('Test äöü', 'utf-8'), classname=decode('some.class.name.äöü', 'utf-8'), elapsed_sec=123.345, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8')) tc.add_skipped_info(message=decode('Skipped äöü', 'utf-8'), output=decode('I skippäd!', 'utf-8')) tc.add_error_info(message=decode('Skipped error äöü', 'utf-8'), output=decode('I skippäd with an error!', 'utf-8')) (ts, tcs) = serialize_and_read(TestSuite('Test Unicode', [tc]))[0] verify_test_case( self, tcs[0], { 'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345) }, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def test_init_skipped(self): tc = TestCase('Skipped-Message-and-Output') tc.add_skipped_info("skipped message", "I skipped!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Skipped-Message-and-Output'}, skipped_message="skipped message", skipped_output="I skipped!")
def main(): parser = argparse.ArgumentParser(description='dummy test') parser.add_argument('-classes', type=int, default=5, help='number of classes') parser.add_argument('-testcases', type=int, default=10, help='number of testcases') parser.add_argument('-pass_rate', type=int, default=75, help='pass rate') parser.add_argument('-error_rate', type=int, default=20, help='error rate') parser.add_argument('-failure_rate', type=int, default=10, help='failure rate') parser.add_argument('-skip_rate', type=int, default=10, help='skip rate') parser.add_argument('-outputfile', type=str, default='test_results.xml', help='output file') parser.add_argument('-print', action='store_true', help='print the test results') args = parser.parse_args() ts = TestSuite(name='my test suite', hostname=platform.node(), timestamp=datetime.now()) for i in range(args.classes): for j in range(args.testcases): tc = TestCase(classname=f"myclass{i}", name=f"mytest{j}", elapsed_sec=random.randint(100, 1000), stdout = "stdout output", stderr = "stderr output") if random.randint(0, 100) < args.pass_rate: if random.randint(0, 100) < args.error_rate: tc.add_error_info(message=f"error {i} {j}", output="error output message", error_type="ERR1") elif random.randint(0, 100) < args.failure_rate: tc.add_failure_info(message=f"failure {i} {j}", output="failure output message", failure_type="FAIL1") elif random.randint(0, 100) < args.skip_rate: tc.add_skipped_info(message=f"skipped {i} {j}", output="skipped output message") ts.test_cases.append(tc) # pretty printing is on by default but can be disabled using prettyprint=False if args.print: print(TestSuite.to_xml_string([ts])) with open(args.outputfile, 'w') as f: TestSuite.to_file(f, [ts], prettyprint=True)
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] for platform in sorted(test_result_ext.keys()): # {platform : ['Platform', 'Result', 'Scope', 'Description']) test_cases = [] for tr_result in test_result_ext[platform]: result, name, scope, description = tr_result classname = 'test.ioper.%s.%s.%s' % (platform, name, scope) elapsed_sec = 0 _stdout = description _stderr = '' # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if result == 'FAIL': tc.add_failure_info(description, _stdout) elif result == 'ERROR': tc.add_error_info(description, _stdout) elif result == 'SKIP': tc.add_skipped_info(description, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def _gen_cases(n_passes, n_fails, n_skips, n_errors): result = [] for i in range(n_passes): case = TestCase(name='TestPassed%s' % i, classname='generated.xml.test.case.passes', elapsed_sec=rand_duration()) result.append(case) for i in range(n_skips): case = TestCase(name='TestSkipped%s' % i, classname='generated.xml.test.case.skips', elapsed_sec=rand_duration()) case.add_skipped_info(message=rand_string('skipped!')) result.append(case) for i in range(n_fails): case = TestCase(name='TestFailed%s' % i, classname='generated.xml.test.case.fails', elapsed_sec=rand_duration()) case.add_failure_info(message=rand_string('failure!')) result.append(case) for i in range(n_errors): case = TestCase(name='TestErrored%s' % i, classname='generated.xml.test.case.errors', elapsed_sec=rand_duration()) case.add_error_info(message=rand_string('error!')) result.append(case) return result
def run_tests(self): test_cases = [] should_update_env = True for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info( message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output print('running test "{}"'.format(name)) if should_update_env: res = self.update_env(self.env_vars) if res != BSTestRunner.SUCCESS: print('failed to set environment variables') break res = self.pretest() if res != BSTestRunner.SUCCESS: print('failed to run pretest init') break should_update_env = False if name in self.mocks: debug_print('setting up mocks') self.mocks[name]['request_env'] = self.request_env self.mocks[name]['setup']() extra_env = mock_decorators.get_all_envs(name) if extra_env is not None: self.update_env(extra_env) t_start = time.time() result = self.run_test(index) if name in self.mocks: debug_print('tearing down mocks') try: self.mocks[name]['teardown']() except AssertionError: debug_print('teardown assert failure') result = BSTestRunner.FAIL t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = filter(lambda c: ord(c) < 128, test_output.getvalue()) print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) should_update_env = True test_output.close() test_cases += [test_case] return TestSuite(self.name, test_cases)
def test_init_skipped_err_output(): tc = Case("Skipped-Output") tc.add_skipped_info(output="I skipped!") tc.add_error_info(output="I skipped with an error!") ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Output"}, skipped_output="I skipped!", error_output="I skipped with an error!")
def format_test_results(results, endpoints, format, args): formatted = None total_time = 0 max_name_len = 0 ignored_tests = [] if "suite" in vars(args): ignored_tests = args.ignore for test_result in results["result"]: _check_test_result(test_result, results) total_time += test_result.elapsed_time max_name_len = max(max_name_len, len(test_result.name)) if format == "json": formatted = { "suite": results["suite"], "timestamp": time.time(), "duration": total_time, "results": [], "config": _export_config(), "endpoints": endpoints } for test_result in results["result"]: formatted["results"].append({ "name": test_result.name, "state": str(TestStates.DISABLED if test_result.name in ignored_tests else test_result.state), "detail": test_result.detail, "duration": test_result.elapsed_time }) formatted = json.dumps(formatted, sort_keys=True, indent=4) elif format == "junit": test_cases = [] for test_result in results["result"]: test_case = TestCase(test_result.name, classname=results["suite"], elapsed_sec=test_result.elapsed_time, timestamp=test_result.timestamp) if test_result.name in ignored_tests or test_result.state in [ TestStates.DISABLED, TestStates.UNCLEAR, TestStates.MANUAL, TestStates.NA, TestStates.OPTIONAL ]: test_case.add_skipped_info(test_result.detail) elif test_result.state in [TestStates.WARNING, TestStates.FAIL]: test_case.add_failure_info(test_result.detail, failure_type=str(test_result.state)) elif test_result.state != TestStates.PASS: test_case.add_error_info(test_result.detail, error_type=str(test_result.state)) test_cases.append(test_case) formatted = TestSuite(results["def"]["name"] + ": " + ", ".join(results["urls"]), test_cases) elif format == "console": formatted = "\r\nPrinting test results for suite '{}' using API(s) '{}'\r\n" \ .format(results["suite"], ", ".join(results["urls"])) formatted += "----------------------------\r\n" for test_result in results["result"]: num_extra_dots = max_name_len - len(test_result.name) test_state = str(TestStates.DISABLED if test_result.name in ignored_tests else test_result.state) formatted += "{} ...{} {}\r\n".format(test_result.name, ("." * num_extra_dots), test_state) formatted += "----------------------------\r\n" formatted += "Ran {} tests in ".format(len(results["result"])) + "{0:.3f}s".format(total_time) + "\r\n" return formatted
def test_init_skipped_err_output(self): tc = TestCase('Skipped-Output') tc.add_skipped_info(output="I skipped!") tc.add_error_info(output="I skipped with an error!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Skipped-Output'}, skipped_output="I skipped!", error_output="I skipped with an error!")
def generate_generic_test_case(name, classname, time, message, result): default_pass_message = 'Test passed, check build log for additional details' default_skip_message = 'Test skipped, check build log for additional details' tc = TestCase(name, classname, time, default_pass_message if result == 'pass' else '') if result == 'fail': tc.add_failure_info(message=message) if result == 'skip': tc.add_skipped_info(message=default_skip_message) return tc
def take_action(self, args): test_cases = [] if args.playbook is not None: playbooks = args.playbook results = (models.TaskResult().query .join(models.Task) .filter(models.TaskResult.task_id == models.Task.id) .filter(models.Task.playbook_id.in_(playbooks))) else: results = models.TaskResult().query.all() for result in results: task_name = result.task.name if not task_name: task_name = result.task.action additional_results = { 'host': result.host.name, 'playbook_path': result.task.playbook.path } result_str = jsonutils.dumps(additional_results) test_path = \ u'{playbook_file}.{play_name}'.format( playbook_file=os.path.basename(result.task.playbook.path), play_name=result.task.play.name) test_case = TestCase( name=task_name, classname=test_path, elapsed_sec=result.duration.seconds, stdout=result_str) if result.status == 'skipped': test_case.add_skipped_info(message=result.result) elif ((result.status in ('failed', 'unreachable') and result.ignore_errors is False and 'EXPECTED FAILURE' not in task_name and 'TOGGLE RESULT' not in task_name) or (result.status == 'ok' and 'TOGGLE RESULT' in task_name)): test_case.add_failure_info(message=result.result) test_cases.append(test_case) test_suite = TestSuite('Ansible Tasks', test_cases) # TODO: junit_xml doesn't order the TestCase parameters. # This makes it so the order of the parameters for the same exact # TestCase is not guaranteed to be the same and thus results in a # different stdout (or file). This is easily reproducible on Py3. xml_string = six.text_type(test_suite.to_xml_string([test_suite])) if args.output_file == '-': if six.PY2: sys.stdout.write(encodeutils.safe_encode(xml_string)) else: sys.stdout.buffer.write(encodeutils.safe_encode(xml_string)) else: with open(args.output_file, 'wb') as f: f.write(encodeutils.safe_encode(xml_string))
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: toolchains = sorted(test_result_ext[target].keys()) for toolchain in toolchains: test_cases = [] tests = sorted(test_result_ext[target][toolchain].keys()) for test in tests: test_results = test_result_ext[target][toolchain][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result['description'] classname = '%s.%s.%s.%s' % (self.package, target, toolchain, test_result['id']) elapsed_sec = test_result['elapsed_time'] _stdout = test_result['output'] if 'target_name_unique' in test_result: _stderr = test_result['target_name_unique'] else: _stderr = test_result['target_name'] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info message = test_result['result'] if test_result['result'] == 'FAIL': tc.add_failure_info(message, _stdout) elif test_result['result'] == 'SKIP' or test_result[ "result"] == 'NOT_SUPPORTED': tc.add_skipped_info(message, _stdout) elif test_result['result'] != 'OK': tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite( "test.suite.%s.%s" % (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def run_tests(self): test_cases = [] should_update_env = True for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info(message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output print('running test "{}"'.format(name)) if should_update_env: res = self.update_env(self.env_vars) if res != BSTestRunner.SUCCESS: print('failed to set environment variables') break; should_update_env = False if name in self.mocks: debug_print('setting up mocks') self.mocks[name]['request_env'] = self.request_env self.mocks[name]['setup']() extra_env = mock_decorators.get_all_envs(name) if extra_env is not None: self.update_env(extra_env) t_start = time.time() result = self.run_test(index) if name in self.mocks: debug_print('tearing down mocks') try: self.mocks[name]['teardown']() except AssertionError: debug_print('teardown assert failure') result = BSTestRunner.FAIL t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = test_output.getvalue() print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) should_update_env = True test_output.close() test_cases += [test_case]; return TestSuite(self.name, test_cases)
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: toolchains = sorted(test_result_ext[target].keys()) for toolchain in toolchains: test_cases = [] tests = sorted(test_result_ext[target][toolchain].keys()) for test in tests: test_results = test_result_ext[target][toolchain][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result["description"] classname = "%s.%s.%s.%s" % (self.package, target, toolchain, test_result["id"]) elapsed_sec = test_result["elapsed_time"] _stdout = test_result["output"] if "target_name_unique" in test_result: _stderr = test_result["target_name_unique"] else: _stderr = test_result["target_name"] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info message = test_result["result"] if test_result["result"] == "FAIL": tc.add_failure_info(message, _stdout) elif test_result["result"] == "SKIP" or test_result["result"] == "NOT_SUPPORTED": tc.add_skipped_info(message, _stdout) elif test_result["result"] != "OK": tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite( "test.suite.%s.%s" % (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain], ) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def take_action(self, args): test_cases = [] if args.playbook is not None: playbooks = args.playbook results = (models.TaskResult().query.join(models.Task).filter( models.TaskResult.task_id == models.Task.id).filter( models.Task.playbook_id.in_(playbooks))) else: results = models.TaskResult().query.all() for result in results: task_name = result.task.name if not task_name: task_name = result.task.action additional_results = { 'host': result.host.name, 'playbook_path': result.task.playbook.path } result_str = jsonutils.dumps(additional_results) test_path = \ u'{playbook_file}.{play_name}'.format( playbook_file=os.path.basename(result.task.playbook.path), play_name=result.task.play.name) test_case = TestCase(name=task_name, classname=test_path, elapsed_sec=result.duration.seconds, stdout=result_str) if result.status == 'skipped': test_case.add_skipped_info(message=result.result) elif ((result.status in ('failed', 'unreachable') and result.ignore_errors is False and 'EXPECTED FAILURE' not in task_name and 'TOGGLE RESULT' not in task_name) or (result.status == 'ok' and 'TOGGLE RESULT' in task_name)): test_case.add_failure_info(message=result.result) test_cases.append(test_case) test_suite = TestSuite('Ansible Tasks', test_cases) # TODO: junit_xml doesn't order the TestCase parameters. # This makes it so the order of the parameters for the same exact # TestCase is not guaranteed to be the same and thus results in a # different stdout (or file). This is easily reproducible on Py3. xml_string = six.text_type(test_suite.to_xml_string([test_suite])) if args.output_file == '-': if six.PY2: sys.stdout.write(encodeutils.safe_encode(xml_string)) else: sys.stdout.buffer.write(encodeutils.safe_encode(xml_string)) else: with open(args.output_file, 'wb') as f: f.write(encodeutils.safe_encode(xml_string))
def format_test_results(results, format): formatted = None if format == "json": formatted = { "suite": results["suite"], "url": results["base_url"], "timestamp": time.time(), "results": [] } for test_result in results["result"]: formatted["results"].append({ "name": test_result.name, "state": str(test_result.state), "detail": test_result.detail }) formatted = json.dumps(formatted, sort_keys=True, indent=4) elif format == "junit": test_cases = [] for test_result in results["result"]: test_case = TestCase(test_result.name, classname=results["suite"], elapsed_sec=test_result.elapsed_time, timestamp=test_result.timestamp) if test_result.name in args.ignore or test_result.state in [ TestStates.DISABLED, TestStates.UNCLEAR, TestStates.MANUAL, TestStates.NA, TestStates.OPTIONAL ]: test_case.add_skipped_info(test_result.detail) elif test_result.state in [TestStates.WARNING, TestStates.FAIL]: test_case.add_failure_info(test_result.detail, failure_type=str(test_result.state)) elif test_result.state != TestStates.PASS: test_case.add_error_info(test_result.detail, error_type=str(test_result.state)) test_cases.append(test_case) formatted = TestSuite( results["def"]["name"] + ": " + results["base_url"], test_cases) elif format == "console": formatted = "\r\nPrinting test results for suite '{}' using API '{}'\r\n" \ .format(results["suite"], results["base_url"]) formatted += "----------------------------\r\n" total_time = 0 for test_result in results["result"]: formatted += "{} ... {}\r\n".format(test_result.name, str(test_result.state)) total_time += test_result.elapsed_time formatted += "----------------------------\r\n" formatted += "Ran {} tests in ".format(len( results["result"])) + "{0:.3f}s".format(total_time) + "\r\n" return formatted
def test_init_utf8(self): tc = TestCase('Test äöü', 'some.class.name.äöü', 123.345, 'I am stdöüt!', 'I am stdärr!') tc.add_skipped_info(message='Skipped äöü', output="I skippäd!") tc.add_error_info(message='Skipped error äöü', output="I skippäd with an error!") test_suite = TestSuite('Test UTF-8', [tc]) (ts, tcs) = serialize_and_read(test_suite, encoding='utf-8')[0] verify_test_case(self, tcs[0], {'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345)}, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def create_junit_results(data, output=None, **kwargs): """ Creates a Junit result, can write to a file if desired, or return xml string. (used by Jenkins) input either dict(dict(dict())) or dict(list(dict())) dict = {suite: {test: {stderr,stdout,time,class,err,fail,skip}}} list = {suite: [(test, {stderr,stdout,time,class,err,fail,skip})]} :param data: A dictionary with dict or list hierarchy :param output: A filename to write results to /path/to/file/*.junit.xml :return: Returns an XML string if no output, else nothing. """ log.debug('creating junit results: output={}'.format(output)) stdout_format = kwargs.pop('stdout_format', None) test_class = kwargs.pop('test_class', None) package = kwargs.pop('package', None) from junit_xml import TestSuite, TestCase test_suites = [] for suite, tests in data.items(): test_cases = [] for test, result in (tests if isinstance(tests, list) else tests.items()): tc = TestCase(test) stdout = result.get('stdout') if stdout_format is not None and callable(stdout_format): if hasattr(stdout_format, 'func_code') and 'kwargs' in stdout_format.func_code.co_varnames: stdout = stdout_format(stdout, suite_name=suite, test_name=test, **kwargs) else: stdout = stdout_format(stdout) tc.stdout = stdout tc.stderr = result.get('stderr') tc.elapsed_sec = result.get('time') tc.classname = result.get('class', test_class) err = result.get('err') if err: tc.add_error_info(*err if isinstance(err, (list, tuple)) else [err]) fail = result.get('fail') if fail: tc.add_failure_info(*fail if isinstance(fail, (list, tuple)) else [fail]) skip = result.get('skip') if skip: tc.add_skipped_info(*skip if isinstance(skip, (list, tuple)) else [skip]) test_cases.append(tc) ts = TestSuite(suite, test_cases, package=package) test_suites.append(ts) if output: check_makedir(os.path.dirname(output)) with open(output, 'w') as out: TestSuite.to_file(out, test_suites) return output else: return TestSuite.to_xml_string(test_suites)
def _generate_junit_xml(self, config_name): testcases = [] tests=self.data_source.get_test_results(config_name) for test in tests: test_time = 0 if test.func_end_time != None and test.func_start_time != None: test_time = test.func_end_time - test.func_start_time tc = TestCase(test.name,config_name,test_time, test.description, test.message) if 'failed' in test.result: tc.add_failure_info(test.result) elif 'skipped' in test.result: tc.add_skipped_info(test.result) testcases.append(tc) testsuite = TestSuite(config_name+'_'+self.name, testcases) return testsuite
def _generate_junit_xml(self, config_name): testcases = [] tests = self.data_source.get_test_results(config_name) for test in tests: test_time = 0 if test.func_end_time != None and test.func_start_time != None: test_time = test.func_end_time - test.func_start_time tc = TestCase(test.name, config_name, test_time, test.description, test.message) if 'failed' in test.result: tc.add_failure_info(test.result) elif 'skipped' in test.result: tc.add_skipped_info(test.result) testcases.append(tc) testsuite = TestSuite(config_name + '_' + self.name, testcases) return testsuite
def _build_test_case(self, task_data, host_data): """ build a TestCase from the given TaskData and HostData """ name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name) duration = host_data.finish - task_data.start if self._task_relative_path: junit_classname = os.path.relpath(task_data.path, self._task_relative_path) else: junit_classname = task_data.path if self._task_class == 'true': junit_classname = re.sub(r'\.yml:[0-9]+$', '', junit_classname) if host_data.status == 'included': return TestCase(name, junit_classname, duration, host_data.result) res = host_data.result._result rc = res.get('rc', 0) dump = self._dump_results(res, indent=0) dump = self._cleanse_string(dump) if host_data.status == 'ok': return TestCase(name, junit_classname, duration, dump) test_case = TestCase(name, junit_classname, duration) if host_data.status == 'failed': if 'exception' in res: message = res['exception'].strip().split('\n')[-1] output = res['exception'] test_case.add_error_info(message, output) elif 'msg' in res: message = res['msg'] test_case.add_failure_info(message, dump) else: test_case.add_failure_info('rc=%s' % rc, dump) elif host_data.status == 'skipped': if 'skip_reason' in res: message = res['skip_reason'] else: message = 'skipped' test_case.add_skipped_info(message) return test_case
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: toolchains = sorted(test_result_ext[target].keys()) for toolchain in toolchains: test_cases = [] tests = sorted(test_result_ext[target][toolchain].keys()) for test in tests: test_results = test_result_ext[target][toolchain][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result['description'] classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id']) elapsed_sec = test_result['elapsed_time'] _stdout = test_result['output'] if 'target_name_unique' in test_result: _stderr = test_result['target_name_unique'] else: _stderr = test_result['target_name'] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info message = test_result['result'] if test_result['result'] == 'FAIL': tc.add_failure_info(message, _stdout) elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED': tc.add_skipped_info(message, _stdout) elif test_result['result'] != 'OK': tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def test_init_unicode(self): tc = TestCase(decode('Test äöü', 'utf-8'), decode('some.class.name.äöü', 'utf-8'), 123.345, decode('I am stdöüt!', 'utf-8'), decode('I am stdärr!', 'utf-8')) tc.add_skipped_info(message=decode('Skipped äöü', 'utf-8'), output=decode('I skippäd!', 'utf-8')) tc.add_error_info(message=decode('Skipped error äöü', 'utf-8'), output=decode('I skippäd with an error!', 'utf-8')) (ts, tcs) = serialize_and_read(TestSuite('Test Unicode', [tc]))[0] verify_test_case(self, tcs[0], {'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345)}, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def handle(line, inpipe): global logged_in global test_cases print("> {}".format(line), end='', flush=True) if not logged_in: if line.startswith('Welcome to Buildroot'): time.sleep(1) print("we got the prompt!") send("root\n", inpipe) logged_in = True time.sleep(5) send("cd /usr/lib/uclibc-ng-test/test\n", inpipe) send("sh uclibcng-testrunner.sh\n", inpipe) else: if 'PASS ' in line: r = re.match("PASS (.*)", line) if r: test_name = r.group(1) test = TestCase(test_name, '', time.time()) test_cases.append(test) if 'FAIL ' in line: r = re.match("FAIL (.*)", line) if r: test_name = r.group(1) test = TestCase(test_name, '', time.time()) test.add_failure_info(message="FAIL") test_cases.append(test) if 'SKIP' in line: r = re.match("SKIP (.*)", line) if r: test_name = r.group(1) test = TestCase(test_name, '', time.time()) test.add_skipped_info(message="SKIP") test_cases.append(test) if 'Total passed:' in line: print( "uClibc-ng testsuite run is over, writing test results and exiting." ) return False return True
def readTest(f, expectedFailures): cl = ".".join(f.split(".")[:-1]) name = f.split(".")[-2] with open(f) as fin: try: res = simplejson.load(fin) except simplejson.errors.JSONDecodeError: print("Error loading file %s" % f) raise expectFail = cl in expectedFailures if "killed" in res: tc1 = TestCase(name, cl, 0, '', '') tc2 = TestCase(name, cl, 0, '', '') if expectFail: tc1.add_skipped_info('Killed or crashed; expected failure') else: tc1.add_error_info('Killed or crashed') tc2.add_error_info('Killed or crashed') return (tc1, tc2, cl) tc1 = TestCase(name, cl, res["time"], res["messages"], '') tc2 = TestCase(name, cl, res["time"], res["messages"], '') success = res["success"] shouldPass = res["shouldPass"] if expectFail: if success: tc1.add_error_info( 'This testcase started working (failure was expected)') else: tc1.add_skipped_info('This testcase still fails (as expected)') elif not success: if shouldPass: tc1.add_error_info('failed') else: tc1.add_error_info('expected failure, but passed') if not success: if shouldPass: tc2.add_error_info('failed') else: tc2.add_error_info('expected failure, but passed') return (tc1, tc2, None if success else cl)
def take_action(self, args): test_cases = [] if args.playbook is not None: playbooks = args.playbook results = (models.TaskResult().query .join(models.Task) .filter(models.TaskResult.task_id == models.Task.id) .filter(models.Task.playbook_id.in_(playbooks))) else: results = models.TaskResult().query.all() for result in results: task_name = result.task.name if not task_name: task_name = result.task.action additional_results = { 'host': result.host.name, 'playbook_path': result.task.playbook.path } result_str = json.dumps(additional_results) test_path = \ "{playbook_file}.{play_name}".format( playbook_file=os.path.basename(result.task.playbook.path), play_name=result.task.play.name) test_case = TestCase( name=task_name, classname=test_path, elapsed_sec=result.duration.seconds, stdout=result_str) if result.status == "skipped": test_case.add_skipped_info(message=result.result) elif (result.status in ("failed", "unreachable") and result.ignore_errors is False): test_case.add_failure_info(message=result.result) test_cases.append(test_case) test_suite = TestSuite("Ansible Tasks", test_cases) xml_string = test_suite.to_xml_string([test_suite]) if args.output_file == "-": sys.stdout.write(xml_string) else: with open(args.output_file, "w") as f: f.write(xml_string)
def readTest(f, expectedFailures): cl = ".".join(f.split(".")[:-1]) name = f.split(".")[-2] with open(f) as fin: try: res = simplejson.load(fin) except simplejson.errors.JSONDecodeError: print("Error loading file %s" % f) raise expectFail = cl in expectedFailures if "killed" in res: tc1 = TestCase(name, cl, 0, '', '') tc2 = TestCase(name, cl, 0, '', '') if expectFail: tc1.add_skipped_info('Killed or crashed; expected failure') else: tc1.add_error_info('Killed or crashed') tc2.add_error_info('Killed or crashed') return (tc1, tc2, cl) tc1 = TestCase(name, cl, res["time"], res["messages"], '') tc2 = TestCase(name, cl, res["time"], res["messages"], '') success = res["success"] shouldPass = res["shouldPass"] if expectFail: if success: tc1.add_error_info('This testcase started working (failure was expected)') else: tc1.add_skipped_info('This testcase still fails (as expected)') elif not success: if shouldPass: tc1.add_error_info('failed') else: tc1.add_error_info('expected failure, but passed') if not success: if shouldPass: tc2.add_error_info('failed') else: tc2.add_error_info('expected failure, but passed') return (tc1, tc2, None if success else cl)
def _build_test_case(self, task_data, host_data): """ build a TestCase from the given TaskData and HostData """ name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name) duration = host_data.finish - task_data.start if self._task_class == 'true': junit_classname = re.sub('\.yml:[0-9]+$', '', task_data.path) else: junit_classname = task_data.path if host_data.status == 'included': return TestCase(name, junit_classname, duration, host_data.result) res = host_data.result._result rc = res.get('rc', 0) dump = self._dump_results(res, indent=0) dump = self._cleanse_string(dump) if host_data.status == 'ok': return TestCase(name, junit_classname, duration, dump) test_case = TestCase(name, junit_classname, duration) if host_data.status == 'failed': if 'exception' in res: message = res['exception'].strip().split('\n')[-1] output = res['exception'] test_case.add_error_info(message, output) elif 'msg' in res: message = res['msg'] test_case.add_failure_info(message, dump) else: test_case.add_failure_info('rc=%s' % rc, dump) elif host_data.status == 'skipped': if 'skip_reason' in res: message = res['skip_reason'] else: message = 'skipped' test_case.add_skipped_info(message) return test_case
def test_init_utf8(self): tc = TestCase('Test äöü', 'some.class.name.äöü', 123.345, 'I am stdöüt!', 'I am stdärr!') tc.add_skipped_info(message='Skipped äöü', output="I skippäd!") tc.add_error_info(message='Skipped error äöü', output="I skippäd with an error!") test_suite = TestSuite('Test UTF-8', [tc]) (ts, tcs) = serialize_and_read(test_suite, encoding='utf-8')[0] verify_test_case( self, tcs[0], { 'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345) }, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def test_deploy_openstack_run_tempest(self, underlay, config, ccpcluster, k8s_actions, rally): """Deploy base environment Scenario: 1. Revert snapshot 2. Install ccp 3. Deploy environment 4. Run tempest Duration 35 min """ remote = underlay.remote(host=config.k8s.kube_host) if settings.REGISTRY == "127.0.0.1:31500": k8s_actions.create_registry() ccpcluster.build() ccpcluster.deploy() post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=4500) post_os_deploy_checks.check_pods_status(k8s_actions.api, timeout=4500) # prepare rally rally.prepare() rally.pull_image() rally.run() # run tempest rally.run_tempest() LOG.info('Storing tests results...') res_file_name = 'result.json' file_prefix = 'results_' + datetime.datetime.now().strftime( '%Y%m%d_%H%M%S') + '_' file_dst = '{0}/logs/{1}{2}'.format( settings.LOGS_DIR, file_prefix, res_file_name) remote.download( '/home/{0}/rally/{1}'.format(settings.SSH_LOGIN, res_file_name), file_dst) res = json.load(remote.open('/home/{}/rally/result.json'.format( settings.SSH_LOGIN))) formatted_tc = [] failed_cases = [res['test_cases'][case] for case in res['test_cases'] if res['test_cases'][case]['status'] in 'fail'] for case in failed_cases: if case: tc = TestCase(case['name']) tc.add_failure_info(case['traceback']) formatted_tc.append(tc) skipped_cases = [res['test_cases'][case] for case in res['test_cases'] if res['test_cases'][case]['status'] in 'skip'] for case in skipped_cases: if case: tc = TestCase(case['name']) tc.add_skipped_info(case['reason']) formatted_tc.append(tc) error_cases = [res['test_cases'][case] for case in res['test_cases'] if res['test_cases'][case]['status'] in 'error'] for case in error_cases: if case: tc = TestCase(case['name']) tc.add_error_info(case['traceback']) formatted_tc.append(tc) success = [res['test_cases'][case] for case in res['test_cases'] if res['test_cases'][case]['status'] in 'success'] for case in success: if case: tc = TestCase(case['name']) formatted_tc.append(tc) ts = TestSuite("tempest", formatted_tc) with open('tempest.xml', 'w') as f: ts.to_file(f, [ts], prettyprint=False) fail_msg = 'Tempest verification fails {}'.format(res) assert res['failures'] == 0, fail_msg
def add_skipped(self, behavior): super().add_skipped(behavior) test_case = TestCase(self.get_behaviors_description(behavior)) test_case.add_skipped_info(message="Skipped") self.junit_cases.append(test_case)
def _log_skip(self, text, package, version): logger.debug(text, package, version) log_entry = TestCase('{} {}'.format(package, version)) log_entry.add_skipped_info(text % (package, version)) self._results.append(log_entry)
def exporter_testcase_junit(test_result_ext, test_suite_properties=None): """! Export test results in JUnit XML compliant format @param test_result_ext Extended report from Greentea @param test_spec Dictionary of test build names to test suite properties @details This function will import junit_xml library to perform report conversion @return String containing Junit XML formatted test result output """ from junit_xml import TestSuite, TestCase test_suites = [] for target_name in test_result_ext: test_results = test_result_ext[target_name] for test_suite_name in test_results: test = test_results[test_suite_name] # tc_elapsed_sec = test['elapsed_time'] tc_stdout = str() #test['single_test_output'] try: tc_stdout = test['single_test_output'].decode('unicode_escape').encode('ascii','ignore') except UnicodeDecodeError as e: err_mgs = "(UnicodeDecodeError) exporter_testcase_junit:", str(e) tc_stdout = err_mgs print err_mgs # testcase_result stores info about test case results testcase_result = test['testcase_result'] # "testcase_result": { # "STRINGS004": { # "duration": 0.009999990463256836, # "time_start": 1453073018.275, # "time_end": 1453073018.285, # "result": 1 # }, test_cases = [] for tc_name in sorted(testcase_result.keys()): duration = testcase_result[tc_name].get('duration', 0.0) utest_log = testcase_result[tc_name].get('utest_log', '') result_text = testcase_result[tc_name].get('result_text', "UNDEF") try: tc_stderr = '\n'.join(utest_log).decode('unicode_escape').encode('ascii','ignore') except UnicodeDecodeError as e: err_mgs = "(UnicodeDecodeError) exporter_testcase_junit:" + str(e) tc_stderr = err_mgs print err_mgs tc_class = target_name + '.' + test_suite_name if result_text == 'SKIPPED': # Skipped test cases do not have logs and we do not want to put # whole log inside JUNIT for skipped test case tc_stderr = str() tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr) if result_text == 'FAIL': tc.add_failure_info(result_text) elif result_text == 'SKIPPED': tc.add_skipped_info(result_text) elif result_text != 'OK': tc.add_error_info(result_text) test_cases.append(tc) ts_name = target_name test_build_properties = test_suite_properties[target_name] if target_name in test_suite_properties else None ts = TestSuite(ts_name, test_cases, properties=test_build_properties) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def add_skipped_slow(self, behavior): super().add_skipped_slow(behavior) test_case = TestCase(self.get_behaviors_description(behavior)) test_case.add_skipped_info(message="Test ran too slowly and was skipped.") self.junit_cases.append(test_case)
def __init__ (self, cmd_list, nprocs=1, out=sys.stdout, log=None, verbosity=DEFAULT_VERBOSITY, output_junit_xml=False) : if (log is None) : log = null_out() self.out = multi_out() self.log = log self.out.register("stdout", out) self.out.register("log", log) self.verbosity = verbosity self.quiet = (verbosity == 0) self.results = [] # Filter cmd list for duplicates. self.cmd_list = [] for cmd in cmd_list : if (not cmd in self.cmd_list) : self.cmd_list.append(cmd) else : print >> self.out, "Test %s repeated, skipping"%cmd # Set number of processors. if (nprocs is Auto) : nprocs = cpu_count() nprocs = min(nprocs, len(self.cmd_list)) # Starting summary. if (self.verbosity > 0) : print >> self.out, "Running %d tests on %s processors:"%(len(self.cmd_list), nprocs) for cmd in self.cmd_list: print >> self.out, " %s"%cmd print >> self.out, "" t_start = time.time() if nprocs > 1: # Run the tests with multiprocessing pool. pool = Pool(processes=nprocs) for command in self.cmd_list: pool.apply_async( run_command, [command, verbosity, out], callback=self.save_result) try: pool.close() except KeyboardInterrupt: print >> self.out, "Caught KeyboardInterrupt, terminating" pool.terminate() finally: pool.join() else: # Run tests serially. for command in self.cmd_list: rc = run_command(command, verbosity=verbosity, out=out) self.save_result(rc) # Print ending summary. t_end = time.time() print >> self.out, "="*80 print >> self.out, "" print >> self.out, "Tests finished. Elapsed time: %.2fs" %(t_end-t_start) print >> self.out, "" test_cases = [] # Process results for errors and warnings. extra_stderr = len([result for result in self.results if result.stderr_lines]) longjobs = [result for result in self.results if result.wall_time > MAX_TIME] warnings = [result for result in self.results if self.check_alert(result) == 1] failures = [result for result in self.results if self.check_alert(result) == 2] self.finished = len(self.results) self.failure = len(failures) self.warning = len(warnings) # Output JUnit XML if output_junit_xml: from junit_xml import TestSuite, TestCase import re for result in self.results: test_name = reconstruct_test_name(result.command) output = '\n'.join(result.stdout_lines + result.stderr_lines) tc = TestCase(classname=test_name[0], name=test_name[1], elapsed_sec=result.wall_time, stdout='\n'.join(result.stdout_lines), stderr='\n'.join(result.stderr_lines)) if result.return_code == 0: # Identify skipped tests if re.search('skip', output, re.IGNORECASE): # find first line including word 'skip' and use it as message skipline = re.search('^((.*)skip(.*))$', output, re.IGNORECASE | re.MULTILINE).group(1) tc.add_skipped_info(skipline) else: # Test failed. Extract error message and stack trace if possible error_message = 'exit code %d' % result.return_code error_output = '\n'.join(result.stderr_lines) if len(result.stderr_lines): error_message = result.stderr_lines[-1] if len(result.stderr_lines) > 20: error_output = '\n'.join(result.stderr_lines[-20:]) tc.add_failure_info(message=error_message, output=error_output) test_cases.append(tc) ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases) with open('output.xml', 'wb') as f: print >> f, TestSuite.to_xml_string([ts], prettyprint=True) # Run time distribution. if (libtbx.env.has_module("scitbx")) : from scitbx.array_family import flex print >> self.out, "Distribution of test runtimes:" hist = flex.histogram(flex.double([result.wall_time for result in self.results]), n_slots=10) hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs") print >> self.out, "" # Long job warning. if longjobs: print >> self.out, "" print >> self.out, "Warning: the following jobs took at least %d seconds:"%MAX_TIME for result in sorted(longjobs, key=lambda result:result.wall_time): print >> self.out, " %s: %.1fs"%(result.command, result.wall_time) print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests." # Failures. if failures: print >> self.out, "" print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:" print >> self.out, "" for result in warnings: self.display_result(result, alert=1, out=self.out, log_return=self.out, log_stderr=self.out) for result in failures: self.display_result(result, alert=2, out=self.out, log_return=self.out, log_stderr=self.out) print >> self.out, "" print >> self.out, "Please verify these tests manually." print >> self.out, "" # Summary print >> self.out, "Summary:" print >> self.out, " Tests run :",self.finished print >> self.out, " Failures :",self.failure print >> self.out, " Warnings (possible failures) :",self.warning print >> self.out, " Stderr output (discouraged) :",extra_stderr if (self.finished != len(self.cmd_list)) : print >> self.out, "*" * 80 print >> self.out, " WARNING: NOT ALL TESTS FINISHED!" print >> self.out, "*" * 80
def run(self): # Clean up result file names results = [] for target in self.targets: results.append(target.replace('\\', '/')) # Dig through each result file, looking for details on pass/fail: for result_file in results: lines = list(map(lambda line: line.rstrip(), open(result_file, "r").read().split('\n'))) if len(lines) == 0: raise Exception("Empty test result file: %s" % result_file) # define an expression for your file reference entry_one = Combine( oneOf(list(alphas)) + ':/' + Word(alphanums + '_-./')) entry_two = Word(printables + ' ', excludeChars=':') entry = entry_one | entry_two delimiter = Literal(':').suppress() tc_result_line = Group(entry.setResultsName('tc_file_name') + delimiter + entry.setResultsName( 'tc_line_nr') + delimiter + entry.setResultsName('tc_name') + delimiter + entry.setResultsName( 'tc_status') + Optional( delimiter + entry.setResultsName('tc_msg'))).setResultsName("tc_line") eol = LineEnd().suppress() sol = LineStart().suppress() blank_line = sol + eol tc_summary_line = Group(Word(nums).setResultsName("num_of_tests") + "Tests" + Word(nums).setResultsName( "num_of_fail") + "Failures" + Word(nums).setResultsName("num_of_ignore") + "Ignored").setResultsName( "tc_summary") tc_end_line = Or(Literal("FAIL"), Literal('Ok')).setResultsName("tc_result") # run it and see... pp1 = tc_result_line | Optional(tc_summary_line | tc_end_line) pp1.ignore(blank_line | OneOrMore("-")) result = list() for l in lines: result.append((pp1.parseString(l)).asDict()) # delete empty results result = filter(None, result) tc_list = list() for r in result: if 'tc_line' in r: tmp_tc_line = r['tc_line'] # get only the file name which will be used as the classname file_name = tmp_tc_line['tc_file_name'].split('\\').pop().split('/').pop().rsplit('.', 1)[0] tmp_tc = TestCase(name=tmp_tc_line['tc_name'], classname=file_name) if 'tc_status' in tmp_tc_line: if str(tmp_tc_line['tc_status']) == 'IGNORE': if 'tc_msg' in tmp_tc_line: tmp_tc.add_skipped_info(message=tmp_tc_line['tc_msg'], output=r'[File]={0}, [Line]={1}'.format( tmp_tc_line['tc_file_name'], tmp_tc_line['tc_line_nr'])) else: tmp_tc.add_skipped_info(message=" ") elif str(tmp_tc_line['tc_status']) == 'FAIL': if 'tc_msg' in tmp_tc_line: tmp_tc.add_failure_info(message=tmp_tc_line['tc_msg'], output=r'[File]={0}, [Line]={1}'.format( tmp_tc_line['tc_file_name'], tmp_tc_line['tc_line_nr'])) else: tmp_tc.add_failure_info(message=" ") tc_list.append((str(result_file), tmp_tc)) for k, v in tc_list: try: self.test_suites[k].append(v) except KeyError: self.test_suites[k] = [v] ts = [] for suite_name in self.test_suites: ts.append(TestSuite(suite_name, self.test_suites[suite_name])) with open('result.xml', 'w') as f: TestSuite.to_file(f, ts, prettyprint='True', encoding='utf-8') return self.report