def main(): parser = argparse.ArgumentParser(description='dummy test') parser.add_argument('-classes', type=int, default=5, help='number of classes') parser.add_argument('-testcases', type=int, default=10, help='number of testcases') parser.add_argument('-pass_rate', type=int, default=75, help='pass rate') parser.add_argument('-error_rate', type=int, default=20, help='error rate') parser.add_argument('-failure_rate', type=int, default=10, help='failure rate') parser.add_argument('-skip_rate', type=int, default=10, help='skip rate') parser.add_argument('-outputfile', type=str, default='test_results.xml', help='output file') parser.add_argument('-print', action='store_true', help='print the test results') args = parser.parse_args() ts = TestSuite(name='my test suite', hostname=platform.node(), timestamp=datetime.now()) for i in range(args.classes): for j in range(args.testcases): tc = TestCase(classname=f"myclass{i}", name=f"mytest{j}", elapsed_sec=random.randint(100, 1000), stdout = "stdout output", stderr = "stderr output") if random.randint(0, 100) < args.pass_rate: if random.randint(0, 100) < args.error_rate: tc.add_error_info(message=f"error {i} {j}", output="error output message", error_type="ERR1") elif random.randint(0, 100) < args.failure_rate: tc.add_failure_info(message=f"failure {i} {j}", output="failure output message", failure_type="FAIL1") elif random.randint(0, 100) < args.skip_rate: tc.add_skipped_info(message=f"skipped {i} {j}", output="skipped output message") ts.test_cases.append(tc) # pretty printing is on by default but can be disabled using prettyprint=False if args.print: print(TestSuite.to_xml_string([ts])) with open(args.outputfile, 'w') as f: TestSuite.to_file(f, [ts], prettyprint=True)
def _collect_test_suite(self, scenario_result): if isinstance(scenario_result, GeneralError): test_case = TestCase("", "") test_case.add_error_info(scenario_result.message) test_suite = TestSuite("", "") test_suite.test_cases.append(test_case) return test_suite test_suite = TestSuite(scenario_result.name) for test_result in scenario_result.test_results: test_case = TestCase(test_result.name, test_result.name) for result in test_result.results: if isinstance(result, Failed): test_case.add_failure_info("ASSERTION {} failed".format(result.pretty_name), "EXPECTED {}\nGOT {}".format(result.expected, result.actual)) elif isinstance(result, (Error, ConnectionError)): test_case.add_error_info("ASSERTION {} failed".format(result.pretty_name), "ERROR {}".format(result.error)) elif isinstance(result, Passed): pass # TODO: What to do below? else: raise Exception("Unknown state") test_suite.test_cases.append(test_case) return test_suite
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] for platform in sorted(test_result_ext.keys()): # {platform : ['Platform', 'Result', 'Scope', 'Description']) test_cases = [] for tr_result in test_result_ext[platform]: result, name, scope, description = tr_result classname = 'test.ioper.%s.%s.%s' % (platform, name, scope) elapsed_sec = 0 _stdout = description _stderr = '' # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if result == 'FAIL': tc.add_failure_info(description, _stdout) elif result == 'ERROR': tc.add_error_info(description, _stdout) elif result == 'SKIP': tc.add_skipped_info(description, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def generate_junit_xml(inputfile): target = None suite = None infos = [] errors = [] testcases = [] for line in inputfile: tag = line[0:3] props = line[3:].split(':') if tag == "[!]": if len(props) == 2: if props[0].strip().lower() == "target": target = os.path.basename(props[1].strip()) elif props[0].strip().lower() == "group": suite = props[1].strip() else: infos.append(line) else: infos.append(line) if tag == "[x]": errors.append(line) if tag == "[+]": testcases.append(TestCase(name=props[0].strip(), classname=target, stdout=line)) if tag == "[-]": tc = TestCase(name=props[0].strip(), classname=target) tc.add_failure_info(message=props[1].strip(), output=line, failure_type="failed") testcases.append(tc) ts = TestSuite(name=suite, test_cases=testcases, stdout="\n".join(infos), stderr="\n".join(errors)) return TestSuite.to_xml_string([ts])
def test_init_illegal_unicode_char(): tc = Case("Failure-Message") tc.add_failure_info(u("failure message with illegal unicode char: [\x02]")) ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Failure-Message"}, failure_message=u("failure message with illegal unicode char: []"))
def write_junitxml(output_junitxml, results): """Write output file as JUnitXML format""" if not JUNIT_XML_FOUND: log = logging.getLogger(__name__ + ".write_junitxml") log.warning('junitxml output disabled: the `junit_xml` python module ' 'is missing.') return test_cases = [] duration_re = re.compile('([0-9]+):([0-9]+):([0-9]+).([0-9]+)') for vitem in results: if vitem.get('Validations'): parsed_duration = 0 test_duration = vitem.get('Duration', '') matched_duration = duration_re.match(test_duration) if matched_duration: parsed_duration = (int(matched_duration[1])*3600 + int(matched_duration[2])*60 + int(matched_duration[3]) + float('0.{}'.format(matched_duration[4]))) test_stdout = vitem.get('Status_by_Host', '') test_case = TestCase('validations', vitem['Validations'], parsed_duration, test_stdout) if vitem['Status'] == 'FAILED': test_case.add_failure_info('FAILED') test_cases.append(test_case) ts = TestSuite("Validations", test_cases) with open(output_junitxml, 'w') as output: output.write(to_xml_report_string([ts]))
def exporter_junit(test_result_ext, test_suite_properties=None): """! Export test results in JUnit XML compliant format @details This function will import junit_xml library to perform report conversion @return String containing Junit XML formatted test result output """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: test_cases = [] tests = sorted(test_result_ext[target].keys()) for test in tests: test_results = test_result_ext[target][test] classname = 'test.%s.%s' % (target, test) elapsed_sec = test_results['elapsed_time'] _stdout = test_results['single_test_output'] _stderr = '' # Test case tc = TestCase(test, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if test_results['single_test_result'] == 'FAIL': message = test_results['single_test_result'] tc.add_failure_info(message, _stdout) elif test_results['single_test_result'] != 'OK': message = test_results['single_test_result'] tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.%s" % target, test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def log_failed(test_name, actual, expected): if xml_out: tc = TestCase(test_name, executable, 0, actual, "") tc.add_failure_info(message="Test Failed\nExpected:\n%s\nActual:\n%s" % (expected, actual)) test_cases.append(tc) else: print "Test %s failed:\nexpected:\n%s\nactual:\n%s\n---------\n" % (test_name, expected, actual)
def run_test(instance, manifest, action): start = datetime.now() result = {} if 'launch' == action.name: pass else: instance.run_workflow(action.name, parameters=action.parameters) sleep(10) # Give some time to update status run_ok = instance.ready(timeout=120000) delta = duration(start) test = TestCase(action.name, manifest.name, delta, 'Workflow launched:' + manifest.source, '') if not run_ok: test.add_failure_info('Action ' + action.name + ' reached timeout before got "Running" state') test_cases.append(test) return False test_cases.append(test) log("Comparing results for action: %s" % action.name,'debug') for key in action.expected.keys(): try: result = compare(action.expected[key], instance.returnValues[key]) except KeyError: result['status'] = False result['details'] = "No key '%s' in returnValues" % key test = TestCase(action.name+'.'+key, manifest.name, 0, result['details'], '') if not result['status']: test.add_failure_info(key + ": " + result['details']) test_cases.append(test) return True
def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent) -> None: if isinstance(event, events.Initialized): self.start_time = event.start_time if isinstance(event, events.AfterExecution): test_case = TestCase( f"{event.result.method} {event.result.path}", elapsed_sec=event.elapsed_time, allow_multiple_subelements=True, ) if event.status == Status.failure: checks = deduplicate_failures(event.result.checks) for idx, check in enumerate(checks, 1): # `check.message` is always not empty for events with `failure` status test_case.add_failure_info( message=f"{idx}. {check.message}") if event.status == Status.error: test_case.add_error_info( message=event.result.errors[-1].exception, output=event.result.errors[-1].exception_with_traceback) self.test_cases.append(test_case) if isinstance(event, events.Finished): test_suites = [ TestSuite("schemathesis", test_cases=self.test_cases, hostname=platform.node()) ] to_xml_report_file(file_descriptor=self.file_handle, test_suites=test_suites, prettyprint=True)
def get_test_suites(self, use_bc_ids=False) -> List[TestSuite]: test_cases = defaultdict(list) test_suites = [] records = self.passed_checks + self.failed_checks + self.skipped_checks for record in records: check_name = f"{record.get_output_id(use_bc_ids)}/{record.check_name}" test_name = f"{self.check_type} {check_name} {record.resource}" test_case = TestCase(name=test_name, file=record.file_path, classname=record.check_class) if record.check_result["result"] == CheckResult.FAILED: if record.file_path and record.file_line_range: test_case.add_failure_info( f"Resource {record.resource} failed in check {check_name} - {record.file_path}:{record.file_line_range} - Guideline: {record.guideline}" ) else: test_case.add_failure_info( f"Resource {record.resource} failed in check {check_name}" ) if record.check_result["result"] == CheckResult.SKIPPED: test_case.add_skipped_info( f'Resource {record.resource} skipped in check {check_name} \n Suppress comment: {record.check_result["suppress_comment"]} - Guideline: {record.guideline}' ) test_cases[check_name].append(test_case) for key in test_cases.keys(): test_suites.append( TestSuite( name=key, test_cases=test_cases[key], package=test_cases[key][0].classname, )) return test_suites
def Reg_Create_Snow_order(self): test_case_result = TestCase(name="Reg_Create_Snow_order", classname=self.__class__.__name__) global serviceInstanceName provider = "snow" passed = False try: for i in range(1, 2): print "Placing order: " + str(i + 1) epoch_time = str(int(time.time())) serviceInstanceName = "testAPISnow" + epoch_time print "serviceInstanceName : " + str(serviceInstanceName) # To avoid 429 HTTP server error, we have to add a sleep between http requests time.sleep(4) print 111 orderNumber, passed = self.api_client.createOrder(orderURL, serviceInstanceName, provider, i + 1) print 2222 print orderNumber print 3333 print passed time.sleep(4) if passed: print "Approving order: " + str(orderNumber) print 444 passed = self.api_client.approveOrder(orderNumber) print 555 if not passed: responseBody = "Failure to create order" print "Softlayer order creation failed. Approve will be skipped" test_case_result.add_failure_info("Input " + str(i + 1) + " failed", responseBody) break except: print "An Error Occured" passed=False status['APITest'] = passed return passed, test_case_result
def run_tests(self): test_cases = [] for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info(message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output t_start = time.time() result = self.run_test(index) t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = test_output.getvalue() print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) test_output.close() test_cases += [test_case]; return TestSuite(self.name, test_cases)
def write_test_results(results, args): exit_code = ExitCodes.OK test_cases = [] for test_result in results["result"]: test_case = TestCase(test_result.name, elapsed_sec=test_result.elapsed_time, timestamp=test_result.timestamp) if test_result.name in args.ignore or test_result.state in [ TestStates.DISABLED, TestStates.UNCLEAR, TestStates.MANUAL, TestStates.NA, TestStates.OPTIONAL ]: test_case.add_skipped_info(test_result.detail) elif test_result.state in [TestStates.WARNING, TestStates.FAIL]: test_case.add_failure_info(test_result.detail, failure_type=str(test_result.state)) if test_result.state == TestStates.FAIL: exit_code = max(exit_code, ExitCodes.FAIL) elif test_result.state == TestStates.WARNING: exit_code = max(exit_code, ExitCodes.WARNING) elif test_result.state != TestStates.PASS: test_case.add_error_info(test_result.detail, error_type=str(test_result.state)) test_cases.append(test_case) ts = TestSuite(results["name"] + ": " + results["base_url"], test_cases) with open(args.output, "w") as f: TestSuite.to_file(f, [ts], prettyprint=False) print(" * Test results written to file: {}".format(args.output)) return exit_code
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] for platform in sorted(test_result_ext.keys()): # {platform : ['Platform', 'Result', 'Scope', 'Description']) test_cases = [] for tr_result in test_result_ext[platform]: result, name, scope, description = tr_result classname = "test.ioper.%s.%s.%s" % (platform, name, scope) elapsed_sec = 0 _stdout = description _stderr = "" # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if result == "FAIL": tc.add_failure_info(description, _stdout) elif result == "ERROR": tc.add_error_info(description, _stdout) elif result == "SKIP" or result == "NOT_SUPPORTED": tc.add_skipped_info(description, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def junit_report(results): test_cases = [] for result in results: if result["result"] != 'Failed' and result["data"] != 'Failed': tc = TestCase( name="browser: {}, version: {}".format(result["browser"], result["version"]), classname='TestCaseNum {}'.format(result["testCaseNum"]), elapsed_sec=round(result["elapsedTime"], 1), stdout="result: {}, data: {}".format(result["result"], result["data"])) else: tc = TestCase( name="browser: {}, version: {}".format(result["browser"], result["version"]), classname='TestCaseNum {}'.format(result["testCaseNum"]), elapsed_sec=round(result["elapsedTime"], 1), stdout="result: Failed, data: Failed") tc.add_failure_info("Failed") test_cases.append(tc) ts = TestSuite("autoupdate", test_cases) with open( os.path.abspath(os.path.dirname(__file__)) + '/junit_output.xml', 'w') as f: TestSuite.to_file(f, [ts])
def generate_junit_report_from_cfn_nag(report): total_failures = 0 """Generate Test Case from cfn_nag report""" test_cases = [] for file_findings in report: for violation in file_findings["file_results"]['violations']: total_failures += 1 for i, resource_id in enumerate(violation['logical_resource_ids']): test_case = TestCase("%s - %s" % (violation['id'], violation['message']), classname=resource_id) test_case.add_failure_info( output="%s#L%s" % (file_findings['filename'], violation['line_numbers'][i])) test_cases.append(test_case) test_suite = TestSuite("cfn-nag test suite", test_cases) if total_failures > 0: f = open("CFN_NAG_FAILURE", "a") f.close() return TestSuite.to_xml_string([test_suite], prettyprint=False)
def write_testcase(self, build, machine, test, path, passed, start_ts, end_ts): delta = end_ts - start_ts tc = { 'name': test.name, 'class': machine.getName(), 'time_elapsed': delta.total_seconds(), 'stdout': '\n'.join(self._harness.process_output(test, path)), 'stderr': "", 'passed': passed } if have_junit_xml: ju_tc = TestCase( tc['name'], tc['class'], tc['time_elapsed'], tc['stdout'], ) if not passed: errors = self._harness.extract_errors(test, path) errorstr = 'Failed' if errors is not None and len(errors) > 0: errorstr += ': ' + ''.join( [unicode(l, errors='replace') for l in errors]) ju_tc.add_failure_info(message=errorstr) return ju_tc else: return tc
def simple_report(self): """empty test report""" test_case_1 = TestCase("testcase1", elapsed_sec=1.5) test_case_2 = TestCase("testcase2", elapsed_sec=0.5) test_case_2.add_skipped_info("was skipped") test_case_3 = TestCase("testcase3", elapsed_sec=1.0) test_case_3.add_failure_info("failed") test_case_4 = TestCase("testcase4", elapsed_sec=0.25) test_case_4.add_error_info("errored") test_case_5 = TestCase("testcase5", elapsed_sec=0.1) test_cases = [ test_case_1, test_case_2, test_case_3, test_case_4, test_case_5 ] test_suites = [ TestSuite('testsuite1', test_cases, timestamp=datetime.datetime.utcnow()) ] return TestReport(NAME, {"module": test_suites}, BUILD_NUMBER, True)
def test_init_failure_output(self): tc = TestCase('Failure-Output') tc.add_failure_info(output="I failed!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Output'}, failure_output="I failed!")
def test_init_failure(self): tc = TestCase('Failure-Message-and-Output') tc.add_failure_info("failure message", "I failed!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Message-and-Output'}, failure_message="failure message", failure_output="I failed!")
def test_multiple_failures(self): """Tests multiple failures in one test case""" tc = TestCase('Multiple failures', allow_multiple_subelements=True) tc.add_failure_info("First failure", "First failure message") (_, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Multiple failures'}, failures=[{ "message": "First failure", "output": "First failure message", "type": "failure" }]) tc.add_failure_info("Second failure", "Second failure message") (_, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Multiple failures'}, failures=[{ "message": "First failure", "output": "First failure message", "type": "failure" }, { "message": "Second failure", "output": "Second failure message", "type": "failure" }])
def write_xml_file(self): test_cases = [] if os.path.isfile(self.output): logging.warn("File exists,deleting...") os.remove(self.output) with open(self.output, 'a') as f: for _, elements in self.log.items(): for j in elements.viewitems(): if j[0] == 'date' or j[0] == 'profile' or j[0] == 'score': # we really don't care pass else: try: test_case = TestCase(j[0], j[1]['descr'], '', '', '') if j[1]['status'] == 'Fail': test_case.add_failure_info(j[1]['output']) else: test_case = TestCase(j[0], '', '', '', '') test_cases.append(test_case) except KeyError: # the world's smallest violin playin' for KeyError pass ts = [TestSuite("Docker Security Benchmarks", test_cases)] TestSuite.to_file(f, ts)
def file_junit_report(rules, report): """ Output file Junit xml report :param rules: set of rules to verify :param report: report generated by drheader :return: None """ test_cases = [] for header in rules: tc = [] for item in report: if item.get('rule') == header: violation = item.copy() violation.pop('rule') message = violation.pop('message') tc = TestCase(name=header + ' :: ' + message) tc.add_failure_info(message, violation) test_cases.append(tc) if not tc: tc = TestCase(name=header) test_cases.append(tc) os.makedirs('reports', exist_ok=True) with open('reports/junit.xml', 'w') as f: TestSuite.to_file(f, [TestSuite(name='DrHeader', test_cases=test_cases)], prettyprint=False) f.close()
def generateJUnitReport(self, lstRunResult, runResultDir): #create junit xml report file use junit-xml 1.4 pip install junit-xml resultFileName = runResultDir + os.path.sep + 'RunResult.xml' previousCaseModuleName = '' rowIndex = 0 lstTestSuites = [] testSuite = [] for runResult in lstRunResult: #runResult (sheetName, moduleName, testCaseID, runResult, timeElapsedSec, failureMessage) #test testCaseName = runResult[2] className = runResult[1] + '.' + runResult[2] timeElapsedSec = runResult[4] failureMessage = runResult[5] testCase = TestCase(testCaseName, className, timeElapsedSec) testCase.add_failure_info(None, failureMessage) currTestCaseModuleName = runResult[1] if not currTestCaseModuleName == previousCaseModuleName: testSuite = TestSuite(currTestCaseModuleName) lstTestSuites.append(testSuite) testSuite.test_cases.append(testCase) #print TestSuite.to_xml_string(lstTestSuites) #Write the xml content to result file with open(runResultDir + os.path.sep + 'Result.xml', 'w') as f: TestSuite.to_file(f, lstTestSuites)
def _gen_cases(n_passes, n_fails, n_skips, n_errors): result = [] for i in range(n_passes): case = TestCase(name='TestPassed%s' % i, classname='generated.xml.test.case.passes', elapsed_sec=rand_duration()) result.append(case) for i in range(n_skips): case = TestCase(name='TestSkipped%s' % i, classname='generated.xml.test.case.skips', elapsed_sec=rand_duration()) case.add_skipped_info(message=rand_string('skipped!')) result.append(case) for i in range(n_fails): case = TestCase(name='TestFailed%s' % i, classname='generated.xml.test.case.fails', elapsed_sec=rand_duration()) case.add_failure_info(message=rand_string('failure!')) result.append(case) for i in range(n_errors): case = TestCase(name='TestErrored%s' % i, classname='generated.xml.test.case.errors', elapsed_sec=rand_duration()) case.add_error_info(message=rand_string('error!')) result.append(case) return result
def get_test_suites(self): test_cases = {} test_suites = [] records = self.passed_checks + self.failed_checks + self.skipped_checks for record in records: check_name = record.check_name if check_name not in test_cases: test_cases[check_name] = [] test_name = "{} {} {}".format(self.check_type, check_name, record.resource) test_case = TestCase(name=test_name, file=record.file_path, classname=record.check_class) if record.check_result['result'] == CheckResult.FAILED: test_case.add_failure_info( "Resource \"{}\" failed in check \"{}\"".format( record.resource, check_name)) if record.check_result['result'] == CheckResult.SKIPPED: test_case.add_skipped_info( "Resource \"{}\" skipped in check \"{}\"\n Suppress comment: {}" .format(record.resource, check_name, record.check_result['suppress_comment'])) test_cases[check_name].append(test_case) for key in test_cases.keys(): test_suites.append( TestSuite(name=key, test_cases=test_cases[key], package=test_cases[key][0].classname)) return test_suites
def test_init_failure_output(): tc = Case("Failure-Output") tc.add_failure_info(output="I failed!") #pylint: disable=unused-variable ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Failure-Output"}, failure_output="I failed!")
def test_multiple_failures(): """Tests multiple failures in one test case""" tc = Case("Multiple failures", allow_multiple_subelements=True) tc.add_failure_info("First failure", "First failure message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple failures"}, failures=[{ "message": "First failure", "output": "First failure message", "type": "failure" }], ) tc.add_failure_info("Second failure", "Second failure message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple failures"}, failures=[ { "message": "First failure", "output": "First failure message", "type": "failure" }, { "message": "Second failure", "output": "Second failure message", "type": "failure" }, ], )
def create_junit_test_file(tests_file: str): tests = [] for file in tests_file.split("\n"): file_log = "test_log_" + "_".join(file.split("_")[2:]) test = file_log_parser(file_log) with open(file) as file_buffer: # Use file to refer to the file object config = file_buffer.read() current_section = {} lines = config.split() date = lines[1] is_relaunch = lines[2] debug = lines[3] for _, test_name, status, _, _, duration in zip( *[iter(lines[6:])] * 6): if duration == "false": continue else: duration = duration[:-1] if test_name in test: tc = TestCase(name=test_name, elapsed_sec=int(duration), status=status.split("+")[0], stdout=test[test_name]) if status != "Passed++++": tc.add_failure_info(output=test[test_name]) else: tc = TestCase(name=test_name, elapsed_sec=int(duration), status=status.split("+")[0]) tests.append(tc) return tests
def _write_junit_report(site, config, output_file, total_time): pages = site.pages test_cases = [] for results, resource in pages.items(): origins = [source.origin.geturl() for source in resource.sources] if resource.status == 200: test_case = TestCase(name=resource.url_split.geturl(), classname=results.hostname, elapsed_sec=resource.response_time, stdout=resource.status, status="passed") else: stderr_message = "Link found on:\n{}".format("\n".join(origins)) test_case = TestCase(name=resource.url_split.geturl(), classname=results.hostname, elapsed_sec=resource.response_time, stderr=stderr_message, status="failed") if resource.exception: message = str(resource.exception) else: message = "Expected 200 OK but got {}".format(resource.status) test_case.add_failure_info(message=message, failure_type="UnexpectedStatusCode") test_cases.append(test_case) test_suite = TestSuite("pylinkvalidator test suite", test_cases) output_file.write(TestSuite.to_xml_string([test_suite])) print_summary(site, config, total_time)
def print_matches(self, matches, rules=None, filenames=None): """Output all the matches""" if not rules: return None test_cases = [] for rule in rules.all_rules: if not rules.is_rule_enabled(rule): if not rule.id: continue test_case = TestCase( name='{0} {1}'.format(rule.id, rule.shortdesc)) if rule.experimental: test_case.add_skipped_info( message='Experimental rule - not enabled') else: test_case.add_skipped_info(message='Ignored rule') test_cases.append(test_case) else: test_case = TestCase(name='{0} {1}'.format( rule.id, rule.shortdesc), allow_multiple_subelements=True, url=rule.source_url) for match in matches: if match.rule.id == rule.id: test_case.add_failure_info( message=self._failure_format(match), failure_type=match.message) test_cases.append(test_case) test_suite = TestSuite('CloudFormation Lint', test_cases) return to_xml_report_string([test_suite], prettyprint=True)
def pytest_runtest_makereport(item, call): # print('runtest makerepot') outcome = yield rep = outcome.get_result() testScript = rep.fspath scriptName = os.path.basename(testScript) testname = "" try: testname = rep.item['name'] except: testname = rep.nodeid if rep.when == 'call': scenario = rep.scenario steps = scenario["steps"] durInSeconds = rep.duration # for step in steps: # print("********" + step["name"] + ": " + str(step["failed"])) stepsSummary = generateTestStepsSummary(steps) # print("********name " + testname + " : " + rep.outcome ) # logging.getLogger().info('new Test CAse ' + testname) testCase = TestCase(testname, '', durInSeconds, stepsSummary) # logging.getLogger().info('Testname is ' + testname) if rep.failed: testCase.add_failure_info(rep.longreprtext) testCaseDictList[item.fspath.strpath].append(testCase)
def generate_junit_report(test_name, total_thresholds, report_name): test_cases = [] file_name = f"junit_report_{report_name}.xml" logger.info(f"Generate report {file_name}") for item in total_thresholds["details"]: message = item['message'] test_case = TestCase( item['name'], classname=f"{item['scope']}", status="PASSED", stdout= f"{item['scope']} {item['name'].lower()} {item['aggregation']} {item['actual']} " f"{item['rule']} {item['expected']}") if message: test_case.status = "FAILED" test_case.add_failure_info(message) test_cases.append(test_case) ts = TestSuite(test_name, test_cases) os.makedirs(f"{REPORT_PATH}/junit", exist_ok=True) with open(f"{REPORT_PATH}/junit/{file_name}", 'w') as f: TestSuite.to_file(f, [ts], prettyprint=True) return file_name
def test_init_failure_message(self): tc = TestCase('Failure-Message') tc.add_failure_info("failure message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Message'}, failure_message="failure message")
def run_test_configs(self, project_name): mstdout = [''] test_cases = TestCase('run_test_configs', '', '', mstdout, '') mproj = self.get_project_by_name(project_name) configs = self.get_all_configs(mproj['id']) for config in configs: inconfig = config logging.info("-----------------------------") logging.info(inconfig) logging.info(inconfig['name']) mstdout.insert(-1, str(inconfig['name'])) logging.info(inconfig['id']) mstdout.insert(-1, str(inconfig['id'])) for item in inconfig['configs']: mitems = self.toutf8(item) logging.info(".." + mitems['id']) mstdout.insert(-1, mitems['id']) logging.info(".." + mitems['name']) mstdout.insert(-1, mitems['name']) frdmk64f_id = self.get_config_item_id_by_name( mproj['id'], inconfig['name'], "frdm_k64f") frdmk64_name = self.get_config_item_name_by_id( mproj['id'], inconfig['id'], frdmk64f_id) logging.info("frdmk64f id is %s" % (frdmk64f_id)) mstdout.insert(-1, "frdmk64f id is %s" % (frdmk64f_id)) if frdmk64f_id == None or frdmk64_name == None: test_cases.add_failure_info( 'get_config_item_id_by_name failure') return test_cases
def print_result_cache_junitxml(dict_synonyms, suspicious_policy, untested_policy): test_cases = [] l = list(select(x for x in Mutant)) for filename, mutants in groupby(l, key=lambda x: x.line.sourcefile.filename): for mutant in mutants: tc = TestCase("Mutant #{}".format(mutant.id), file=filename, line=mutant.line.line_number, stdout=mutant.line.line) if mutant.status == BAD_SURVIVED: tc.add_failure_info(message=mutant.status, output=get_unified_diff( mutant.id, dict_synonyms)) if mutant.status == BAD_TIMEOUT: tc.add_error_info(message=mutant.status, error_type="timeout", output=get_unified_diff( mutant.id, dict_synonyms)) if mutant.status == OK_SUSPICIOUS: if suspicious_policy != 'ignore': func = getattr(tc, 'add_{}_info'.format(suspicious_policy)) func(message=mutant.status, output=get_unified_diff(mutant.id, dict_synonyms)) if mutant.status == UNTESTED: if untested_policy != 'ignore': func = getattr(tc, 'add_{}_info'.format(untested_policy)) func(message=mutant.status, output=get_unified_diff(mutant.id, dict_synonyms)) test_cases.append(tc) ts = TestSuite("mutmut", test_cases) print(TestSuite.to_xml_string([ts]))
def test(self): def Trim(string): if len(string) > 4096: return string[:4096] + " ..." else: return string varsdict = {} self.log("Assigning variables:") for var in self.variables: tmpdict = {} try: var.run(tmpdict) except: self.log("failure.") self.pass_status.append('F') return self.pass_status varsdict[var.name] = tmpdict[var.name] self.log("Assigning %s = %s" % (str(var.name), Trim(str(varsdict[var.name])))) if len(self.pass_tests) != 0: self.log("Running failure tests: ") for test in self.pass_tests: self.log("Running %s:" % test.name) status = test.run(varsdict) tc=TestCase(test.name, '%s.%s'%(self.length, self.filename[:-4])) if status == True: self.log("success.") self.pass_status.append('P') elif status == False: self.log("failure.") self.pass_status.append('F') tc.add_failure_info( "Failure" ) else: self.log("failure (info == %s)." % status) self.pass_status.append('F') tc.add_failure_info( "Failure", status ) self.xml_reports.append(tc) if len(self.warn_tests) != 0: self.log("Running warning tests: ") for test in self.warn_tests: self.log("Running %s:" % test.name) status = test.run(varsdict) if status == True: self.log("success.") self.warn_status.append('P') elif status == False: self.log("warning.") self.warn_status.append('W') else: self.log("warning (info == %s)." % status) self.warn_status.append('W') self.log(''.join(self.pass_status + self.warn_status)) return self.pass_status + self.warn_status
def test_init_illegal_unicode_char(self): tc = TestCase('Failure-Message') tc.add_failure_info( u("failure message with illegal unicode char: [\x02]")) (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Message'}, failure_message=u( "failure message with illegal unicode char: []"))
def execute(self, log, keyvals, testDef): testDef.logger.verbose_print("JunitXML Reporter") # pickup the options cmds = {} testDef.parseOptions(log, self.options, keyvals, cmds) if cmds['filename'] is not None: self.fh = open(cmds['filename'] if os.path.isabs(cmds['filename']) \ else os.path.join(cmds['scratch'],cmds['filename']), 'w') if testDef.options['description'] is not None: print(testDef.options['description'], file=self.fh) print(file=self.fh) # Use the Junit classname field to store the list of inifiles try: classname = testDef.log['inifiles'] except KeyError: classname = None # get the entire log of results fullLog = testDef.logger.getLog(None) testCases = [] # TODO: ain't nobody got time for that. 8-). time = 0 for lg in fullLog: if 'stdout' in lg and lg['stdout'] is not None: stdout = "\n".join(lg['stdout']) else: stdout = None if 'stderr' in lg and lg['stderr'] is not None: stderr = "\n".join(lg['stderr']) else: stderr = None if 'time' in lg and lg['time'] is not None: time = lg['time'] else: time = 0 tc = TestCase(lg['section'], classname, time, stdout, stderr) try: if 0 != lg['status']: # Find sections prefixed with 'TestRun' if re.match("TestRun", lg['section']): tc.add_failure_info("Test reported failure") else: tc.add_error_info("Test error") except KeyError: sys.exit(lg['section'] + " is missing status!") testCases.append(tc) # TODO: Pull in the resource manager jobid. jobid = "job1" ts = TestSuite(jobid, testCases) print(TestSuite.to_xml_string([ts]), file=self.fh) if cmds['filename'] is not None: self.fh.close() log['status'] = 0 return
def generate_generic_test_case(name, classname, time, message, result): default_pass_message = 'Test passed, check build log for additional details' default_skip_message = 'Test skipped, check build log for additional details' tc = TestCase(name, classname, time, default_pass_message if result == 'pass' else '') if result == 'fail': tc.add_failure_info(message=message) if result == 'skip': tc.add_skipped_info(message=default_skip_message) return tc
def take_action(self, args): test_cases = [] if args.playbook is not None: playbooks = args.playbook results = (models.TaskResult().query .join(models.Task) .filter(models.TaskResult.task_id == models.Task.id) .filter(models.Task.playbook_id.in_(playbooks))) else: results = models.TaskResult().query.all() for result in results: task_name = result.task.name if not task_name: task_name = result.task.action additional_results = { 'host': result.host.name, 'playbook_path': result.task.playbook.path } result_str = jsonutils.dumps(additional_results) test_path = \ u'{playbook_file}.{play_name}'.format( playbook_file=os.path.basename(result.task.playbook.path), play_name=result.task.play.name) test_case = TestCase( name=task_name, classname=test_path, elapsed_sec=result.duration.seconds, stdout=result_str) if result.status == 'skipped': test_case.add_skipped_info(message=result.result) elif ((result.status in ('failed', 'unreachable') and result.ignore_errors is False and 'EXPECTED FAILURE' not in task_name and 'TOGGLE RESULT' not in task_name) or (result.status == 'ok' and 'TOGGLE RESULT' in task_name)): test_case.add_failure_info(message=result.result) test_cases.append(test_case) test_suite = TestSuite('Ansible Tasks', test_cases) # TODO: junit_xml doesn't order the TestCase parameters. # This makes it so the order of the parameters for the same exact # TestCase is not guaranteed to be the same and thus results in a # different stdout (or file). This is easily reproducible on Py3. xml_string = six.text_type(test_suite.to_xml_string([test_suite])) if args.output_file == '-': if six.PY2: sys.stdout.write(encodeutils.safe_encode(xml_string)) else: sys.stdout.buffer.write(encodeutils.safe_encode(xml_string)) else: with open(args.output_file, 'wb') as f: f.write(encodeutils.safe_encode(xml_string))
def test_init_failure_type(self): tc = TestCase('Failure-Type') tc.add_failure_info(failure_type='com.example.Error') (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Type'}) tc.add_failure_info("failure message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Type'}, failure_message="failure message", failure_type='com.example.Error')
def run_tests(self): test_cases = [] should_update_env = True for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info(message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output print('running test "{}"'.format(name)) if should_update_env: res = self.update_env(self.env_vars) if res != BSTestRunner.SUCCESS: print('failed to set environment variables') break; should_update_env = False if name in self.mocks: debug_print('setting up mocks') self.mocks[name]['request_env'] = self.request_env self.mocks[name]['setup']() extra_env = mock_decorators.get_all_envs(name) if extra_env is not None: self.update_env(extra_env) t_start = time.time() result = self.run_test(index) if name in self.mocks: debug_print('tearing down mocks') try: self.mocks[name]['teardown']() except AssertionError: debug_print('teardown assert failure') result = BSTestRunner.FAIL t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = test_output.getvalue() print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) should_update_env = True test_output.close() test_cases += [test_case]; return TestSuite(self.name, test_cases)
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: toolchains = sorted(test_result_ext[target].keys()) for toolchain in toolchains: test_cases = [] tests = sorted(test_result_ext[target][toolchain].keys()) for test in tests: test_results = test_result_ext[target][toolchain][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result["description"] classname = "%s.%s.%s.%s" % (self.package, target, toolchain, test_result["id"]) elapsed_sec = test_result["elapsed_time"] _stdout = test_result["output"] if "target_name_unique" in test_result: _stderr = test_result["target_name_unique"] else: _stderr = test_result["target_name"] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info message = test_result["result"] if test_result["result"] == "FAIL": tc.add_failure_info(message, _stdout) elif test_result["result"] == "SKIP" or test_result["result"] == "NOT_SUPPORTED": tc.add_skipped_info(message, _stdout) elif test_result["result"] != "OK": tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite( "test.suite.%s.%s" % (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain], ) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def process(config): for mnf in config.keys(): log("Processing manifest: %s" % mnf, 'info') if config[mnf].has_key('submodule'): sub = config[mnf].pop('submodule') sub_manifest = guess_manifest(str(mnf), sub['source']) sub_app = qubell.organization.application(name=sub['name']) sub_app.upload(sub_manifest) if config[mnf].has_key('source'): manifest = guess_manifest(str(mnf), config[mnf].pop('source')) else: error("Missing manifest source directive") if not 'launch' == config[mnf].keys()[0]: error('Launch action missing or not first') if config[mnf]['launch'].has_key('parameters'): parameters = config[mnf]['launch']['parameters'] else: parameters = {} if config[mnf]['launch'].has_key('settings'): settings = config[mnf]['launch']['settings'] else: settings = {} if qubell.appid: app = qubell.organization.application(id=qubell.appid, manifest=manifest) elif qubell.appname: app = qubell.organization.application(name=qubell.appname, manifest=manifest) else: app = qubell.organization.application(name=manifest.name, manifest=manifest) assert app.upload(manifest) instance = qubell.organization.create_instance(application=app, parameters=parameters, **settings) assert instance for action in config[mnf].keys(): if config[mnf][action].has_key('parameters'): parameters = config[mnf][action]['parameters'] else: parameters = {} if not run_test(instance, manifest, Action(action, parameters, config[mnf][action]['expected'])): test = TestCase(action, manifest.name, 0, 'Manifest: ' + manifest.source, 'failed to start') test.add_failure_info(str(instance)) test_cases.append(test) return False
def _generate_junit_xml(self, config_name): testcases = [] tests=self.data_source.get_test_results(config_name) for test in tests: test_time = 0 if test.func_end_time != None and test.func_start_time != None: test_time = test.func_end_time - test.func_start_time tc = TestCase(test.name,config_name,test_time, test.description, test.message) if 'failed' in test.result: tc.add_failure_info(test.result) elif 'skipped' in test.result: tc.add_skipped_info(test.result) testcases.append(tc) testsuite = TestSuite(config_name+'_'+self.name, testcases) return testsuite
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: toolchains = sorted(test_result_ext[target].keys()) for toolchain in toolchains: test_cases = [] tests = sorted(test_result_ext[target][toolchain].keys()) for test in tests: test_results = test_result_ext[target][toolchain][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result['description'] classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id']) elapsed_sec = test_result['elapsed_time'] _stdout = test_result['output'] if 'target_name_unique' in test_result: _stderr = test_result['target_name_unique'] else: _stderr = test_result['target_name'] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info message = test_result['result'] if test_result['result'] == 'FAIL': tc.add_failure_info(message, _stdout) elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED': tc.add_skipped_info(message, _stdout) elif test_result['result'] != 'OK': tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def _output_normal(self, test_result): # Need refactor if test_result == {}: print '[what?!] there are not any test result, what is the test case id?' else: print xml_test_suites = [] summary_dict = self._get_summary_dict(test_result) self.report_create_time = str(time.strftime('%Y%m%d_%H%M%S', time.localtime())) for case_classify in test_result.keys(): xml_test_cases = [] if 'result' in test_result[case_classify].keys(): # Generate HTML report self._generate_html_file( case_classify, test_result[case_classify]['result'], test_result[case_classify]['summary']) # Save the result into the CSV self._output_result_to_csv(test_result) # Show in Console print '{0} {1} {2}'.format('='*16, case_classify, '='*16) test_case_result = test_result[case_classify]['result'] for case_id in test_case_result.keys(): print '[{0}][{1}] {2}, {3}, {4}'.format(case_classify, case_id, test_case_result[case_id][0], test_case_result[case_id][1], str(test_case_result[case_id][2])) # Produce xml file test_case = TestCase(case_id, case_classify, int(test_case_result[case_id][2])) if test_case_result[case_id][0] == 'Fail' or test_case_result[case_id][0] == 'Error': try: test_case.add_failure_info('msg' + test_case_result[case_id][1]) except: test_case.add_failure_info('msg' + str(test_case_result[case_id])) xml_test_cases.append(test_case) xml_test_suites.append(TestSuite(case_classify, xml_test_cases)) with open(os.path.join(self.latest_reports_dir, case_classify + '.xml'), 'w') as f: TestSuite.to_file(f, xml_test_suites, prettyprint=True) self._generate_summary_html_file(summary_dict) print '{0} {1} {2}'.format('='*16, 'Summary', '='*16) pprint.pprint(summary_dict)
def run_tests(TestClass): setup_logging() # Verify that we have needed input assert base_url, 'Please provide a value for base_url in runner.py' assert chromedriver_location, 'Please provide a value for chromedriver_location in runner.py' assert credentials['email'], 'Please provide credentials in runner.py' assert credentials['password'], 'Please provide credentials in runner.py' #WebDriver setup driver = webdriver.Chrome(chromedriver_location) # Init the provided class myTestClass = TestClass(driver, base_url, credentials) logging.info("Running: %s", myTestClass.__class__.__name__) # Run all the methods containing 'test' in name test_cases = [] for object_name in dir(myTestClass): if 'test' in object_name: test_method = getattr(myTestClass, object_name) if not hasattr(test_method, '__call__'): break # If the object is not callable this_test_case = TestCase(object_name, myTestClass.__class__.__name__) logging.info('#######################################################') logging.info("Running %s", object_name) try: # Run the actual method test_method() except Exception as e: logging.info('TEST FAILED') logging.exception(e) this_test_case.add_failure_info(output=traceback.format_exc()) else: logging.info("TEST PASSED") test_cases.append(this_test_case) logging.info('#######################################################') # Write the results to junit xml write_xml(test_cases, myTestClass.__class__.__name__)
def _build_test_case(self, task_data, host_data): """ build a TestCase from the given TaskData and HostData """ name = '[%s] %s: %s' % (host_data.name, task_data.play, task_data.name) duration = host_data.finish - task_data.start if self._task_class == 'true': junit_classname = re.sub('\.yml:[0-9]+$', '', task_data.path) else: junit_classname = task_data.path if host_data.status == 'included': return TestCase(name, junit_classname, duration, host_data.result) res = host_data.result._result rc = res.get('rc', 0) dump = self._dump_results(res, indent=0) dump = self._cleanse_string(dump) if host_data.status == 'ok': return TestCase(name, junit_classname, duration, dump) test_case = TestCase(name, junit_classname, duration) if host_data.status == 'failed': if 'exception' in res: message = res['exception'].strip().split('\n')[-1] output = res['exception'] test_case.add_error_info(message, output) elif 'msg' in res: message = res['msg'] test_case.add_failure_info(message, dump) else: test_case.add_failure_info('rc=%s' % rc, dump) elif host_data.status == 'skipped': if 'skip_reason' in res: message = res['skip_reason'] else: message = 'skipped' test_case.add_skipped_info(message) return test_case
def xmlwrite(target, everyline): xmltargetpath = os.getcwd() + os.sep + target + '_testresults.xml' xmltargetfile = open(xmltargetpath, 'w+') test_cases = [] for line in everyline.split('\n'): words = line.split(' ') if line.find('success') != -1: time_taken = get_time(line.split('-')[-1][7:]) name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip() print name test_cases.append(TestCase(name, target, time_taken, None)) elif line.find('failed') != -1: time_taken = get_time(line.split('-')[-1][7:]) name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip() message = ("-".join(line.split(' - ')[2:-1])).strip() tc = TestCase(name, target, time_taken, None) tc.add_failure_info(None, message) test_cases.append(tc) ts = TestSuite("testing this suite", test_cases) xmltargetfile.write(TestSuite.to_xml_string([ts])) xmltargetfile.close()
def report(self, report_dir=None): test_cases = [] if report_dir is None: report_dir = self.config.report_dir for report_file in os.listdir(report_dir): if report_file.endswith(".pkl"): f = open(os.path.join(report_dir, report_file), "r") result_dict = cPickle.load(f) f.close() tests = result_dict.keys() tests.sort() for test in tests: in_entry = result_dict[test] report_entry = TestCase(test, in_entry["CLASS_NAME"], in_entry["TIME"]) if in_entry["RESULT"] == "FAIL": report_entry.add_failure_info(in_entry["MESSAGE"], in_entry["TRACE"]) elif in_entry["RESULT"] == "ERROR": report_entry.add_error_info(in_entry["MESSAGE"], in_entry["TRACE"]) test_cases.append(report_entry) ts = TestSuite("my test suite", test_cases) f_xml = open(os.path.join(report_dir, "results.xml"), "w") f_xml.write(TestSuite.to_xml_string([ts])) f_xml.close()
def generate_junit_xml(file_name='junit.xml'): results = monitor_runner.get_latest_status() print(results) test_suites = [] # test_cases = [TestCase('Test1', 'some.class.name', 123.345, 'I am stdout!', 'I am stderr!')] # ts = TestSuite("my test suite", test_cases) for testsuite_name in results: test_cases = [] for test_case_name in results[testsuite_name]: try: name = results[testsuite_name][test_case_name]['name'] except: name = '.' success = results[testsuite_name][test_case_name]['success'] try: elapsed_sec = results[testsuite_name][test_case_name]['response_time'].total_seconds() except: elapsed_sec = -1 tc = TestCase( name=name, classname='{}.{}'.format(testsuite_name, test_case_name), elapsed_sec=elapsed_sec, stdout='{}'.format(success), ) if success is False: tc.add_failure_info('Failed') test_cases.append(tc) ts = TestSuite(testsuite_name, test_cases) test_suites.append(ts) pass with open(file_name, "w", encoding='utf-8-sig') as f: TestSuite.to_file(f, test_suites, prettyprint=True)
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] toolchains = sorted(test_result_ext.keys()) for toolchain in toolchains: targets = sorted(test_result_ext[toolchain].keys()) for target in targets: test_cases = [] tests = sorted(test_result_ext[toolchain][target].keys()) for test in tests: test_results = test_result_ext[toolchain][target][test] test_ids = sorted(test_results.keys()) for test_no in test_ids: test_result = test_results[test_no] name = test_result['test_description'] classname = 'test.%s.%s.%s'% (target, toolchain, test_result['test_id']) elapsed_sec = test_result['elapsed_time'] _stdout = test_result['single_test_output'] _stderr = '' # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if test_result['single_test_result'] == 'FAIL': message = test_result['single_test_result'] tc.add_failure_info(message, _stdout) elif test_result['single_test_result'] != 'OK': message = test_result['single_test_result'] tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def junit_test_case_from_compmake(db, job_id): from junit_xml import TestCase cache = get_job_cache(job_id, db=db) if cache.state == Cache.DONE: # and cache.done_iterations > 1: # elapsed_sec = cache.walltime_used elapsed_sec = cache.cputime_used else: elapsed_sec = None check_isinstance(cache.captured_stderr, (type(None), six.text_type)) check_isinstance(cache.captured_stdout, (type(None), six.text_type)) check_isinstance(cache.exception, (type(None), six.text_type)) stderr = remove_escapes(cache.captured_stderr) stdout = remove_escapes(cache.captured_stdout) tc = TestCase(name=job_id, classname=None, elapsed_sec=elapsed_sec, stdout=stdout, stderr=stderr) if cache.state == Cache.FAILED: message = cache.exception output = cache.exception + "\n" + cache.backtrace tc.add_failure_info(message, output) return tc
def write_xml_file(self): test_cases = [] if os.path.isfile(self.output): logging.warn("File exists,deleting...") os.remove(self.output) with open(self.output, "a") as f: for _, elements in self.log.items(): for j in elements.viewitems(): if j[0] == "date" or j[0] == "profile" or j[0] == "score": # we really don't care pass else: try: test_case = TestCase(j[0], j[1]["descr"], "", "", "") if j[1]["status"] == "Fail": test_case.add_failure_info(j[1]["output"]) else: test_case = TestCase(j[0], "", "", "", "") test_cases.append(test_case) except KeyError: # the world's smallest violin playin' for KeyError pass ts = [TestSuite("Docker Security Benchmarks", test_cases)] TestSuite.to_file(f, ts)