def print_matches(self, matches, rules=None, filenames=None): """Output all the matches""" if not rules: return None test_cases = [] for rule in rules.all_rules: if not rules.is_rule_enabled(rule): if not rule.id: continue test_case = TestCase( name='{0} {1}'.format(rule.id, rule.shortdesc)) if rule.experimental: test_case.add_skipped_info( message='Experimental rule - not enabled') else: test_case.add_skipped_info(message='Ignored rule') test_cases.append(test_case) else: test_case = TestCase(name='{0} {1}'.format( rule.id, rule.shortdesc), allow_multiple_subelements=True, url=rule.source_url) for match in matches: if match.rule.id == rule.id: test_case.add_failure_info( message=self._failure_format(match), failure_type=match.message) test_cases.append(test_case) test_suite = TestSuite('CloudFormation Lint', test_cases) return to_xml_report_string([test_suite], prettyprint=True)
def test_init_error(self): tc = TestCase('Error-Message-and-Output') tc.add_error_info("error message", "I errored!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Error-Message-and-Output'}, error_message="error message", error_output="I errored!")
def test_attribute_disable(): tc = Case("Disabled-Test") tc.is_enabled = False tss = [Suite("suite1", [tc])] suites = serialize_and_read(tss) assert suites[0][0].attributes["disabled"].value == "1"
def test_init_failure_message(self): tc = TestCase('Failure-Message') tc.add_failure_info("failure message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Message'}, failure_message="failure message")
def test_init_unicode(): tc = Case( name=decode("Test äöü", "utf-8"), classname=decode("some.class.name.äöü", "utf-8"), elapsed_sec=123.345, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), ) tc.add_skipped_info(message=decode("Skipped äöü", "utf-8"), output=decode("I skippäd!", "utf-8")) tc.add_error_info(message=decode("Skipped error äöü", "utf-8"), output=decode("I skippäd with an error!", "utf-8")) ts, tcs = serialize_and_read(Suite("Test Unicode", [tc]))[0] verify_test_case( tcs[0], { "name": decode("Test äöü", "utf-8"), "classname": decode("some.class.name.äöü", "utf-8"), "time": ("%f" % 123.345), }, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), skipped_message=decode("Skipped äöü", "utf-8"), skipped_output=decode("I skippäd!", "utf-8"), error_message=decode("Skipped error äöü", "utf-8"), error_output=decode("I skippäd with an error!", "utf-8"), )
def run_test_configs(self, project_name): mstdout = [''] test_cases = TestCase('run_test_configs', '', '', mstdout, '') mproj = self.get_project_by_name(project_name) configs = self.get_all_configs(mproj['id']) for config in configs: inconfig = config logging.info("-----------------------------") logging.info(inconfig) logging.info(inconfig['name']) mstdout.insert(-1, str(inconfig['name'])) logging.info(inconfig['id']) mstdout.insert(-1, str(inconfig['id'])) for item in inconfig['configs']: mitems = self.toutf8(item) logging.info(".." + mitems['id']) mstdout.insert(-1, mitems['id']) logging.info(".." + mitems['name']) mstdout.insert(-1, mitems['name']) frdmk64f_id = self.get_config_item_id_by_name( mproj['id'], inconfig['name'], "frdm_k64f") frdmk64_name = self.get_config_item_name_by_id( mproj['id'], inconfig['id'], frdmk64f_id) logging.info("frdmk64f id is %s" % (frdmk64f_id)) mstdout.insert(-1, "frdmk64f id is %s" % (frdmk64f_id)) if frdmk64f_id == None or frdmk64_name == None: test_cases.add_failure_info( 'get_config_item_id_by_name failure') return test_cases
def test_init_skipped(): tc = Case("Skipped-Message-and-Output") tc.add_skipped_info("skipped message", "I skipped!") ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Message-and-Output"}, skipped_message="skipped message", skipped_output="I skipped!")
def generate_junit_report(test_name, total_thresholds, report_name): test_cases = [] file_name = f"junit_report_{report_name}.xml" logger.info(f"Generate report {file_name}") for item in total_thresholds["details"]: message = item['message'] test_case = TestCase( item['name'], classname=f"{item['scope']}", status="PASSED", stdout= f"{item['scope']} {item['name'].lower()} {item['aggregation']} {item['actual']} " f"{item['rule']} {item['expected']}") if message: test_case.status = "FAILED" test_case.add_failure_info(message) test_cases.append(test_case) ts = TestSuite(test_name, test_cases) os.makedirs(f"{REPORT_PATH}/junit", exist_ok=True) with open(f"{REPORT_PATH}/junit/{file_name}", 'w') as f: TestSuite.to_file(f, [ts], prettyprint=True) return file_name
def test_init_error_message(self): tc = TestCase('Error-Message') tc.add_error_info("error message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Error-Message'}, error_message="error message")
def test_attribute_disable(self): tc = TestCase('Disabled-Test') tc.is_enabled = False tss = [TestSuite('suite1', [tc])] suites = serialize_and_read(tss) self.assertEqual('1', suites[0][0].attributes['disabled'].value)
def test_init_error_output(self): tc = TestCase('Error-Output') tc.add_error_info(output="I errored!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Error-Output'}, error_output="I errored!")
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] for platform in sorted(test_result_ext.keys()): # {platform : ['Platform', 'Result', 'Scope', 'Description']) test_cases = [] for tr_result in test_result_ext[platform]: result, name, scope, description = tr_result classname = "test.ioper.%s.%s.%s" % (platform, name, scope) elapsed_sec = 0 _stdout = description _stderr = "" # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if result == "FAIL": tc.add_failure_info(description, _stdout) elif result == "ERROR": tc.add_error_info(description, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def Reg_Create_Snow_order(self): test_case_result = TestCase(name="Reg_Create_Snow_order", classname=self.__class__.__name__) global serviceInstanceName provider = "snow" passed = False try: for i in range(1, 2): print "Placing order: " + str(i + 1) epoch_time = str(int(time.time())) serviceInstanceName = "testAPISnow" + epoch_time print "serviceInstanceName : " + str(serviceInstanceName) # To avoid 429 HTTP server error, we have to add a sleep between http requests time.sleep(4) print 111 orderNumber, passed = self.api_client.createOrder(orderURL, serviceInstanceName, provider, i + 1) print 2222 print orderNumber print 3333 print passed time.sleep(4) if passed: print "Approving order: " + str(orderNumber) print 444 passed = self.api_client.approveOrder(orderNumber) print 555 if not passed: responseBody = "Failure to create order" print "Softlayer order creation failed. Approve will be skipped" test_case_result.add_failure_info("Input " + str(i + 1) + " failed", responseBody) break except: print "An Error Occured" passed=False status['APITest'] = passed return passed, test_case_result
def log_failed(test_name, actual, expected): if xml_out: tc = TestCase(test_name, executable, 0, actual, "") tc.add_failure_info(message="Test Failed\nExpected:\n%s\nActual:\n%s" % (expected, actual)) test_cases.append(tc) else: print "Test %s failed:\nexpected:\n%s\nactual:\n%s\n---------\n" % (test_name, expected, actual)
def main(): parser = argparse.ArgumentParser() parser.add_argument('swift', help='path to swift executable') parser.add_argument('output', help='where to write xUnit output') args = parser.parse_args() test_cases = [ benchmark( TestCase('debug build'), [args.swift, 'build', '--product', 'TensorFlow'] ), benchmark( TestCase('release build'), [args.swift, 'build', '-c', 'release', '--product', 'TensorFlow'] ), # The point of "release build -Onone" is to compile TensorFlow in # "-whole-module-optimization" mode without "-O". benchmark( TestCase('release build -Onone'), [args.swift, 'build', '-c', 'release', '--product', 'TensorFlow', '-Xswiftc', '-Onone'] ), ] test_suite = TestSuite('swift-apis compile time', test_cases) with open(args.output, 'w') as f: TestSuite.to_file(f, [test_suite])
def log_errord(test_name, stderr, stdout): if xml_out: tc = TestCase(test_name, executable, 0, stdout, stderr) tc.add_error_info(message="Test Error\n", output=stdout) test_cases.append(tc) else: print "Test %s failed:\nstderr:\n %s\nstdout:\n%s\n----------\n" % (test_name, stderr, stdout)
def write_junitxml(output_junitxml, results): """Write output file as JUnitXML format""" if not JUNIT_XML_FOUND: log = logging.getLogger(__name__ + ".write_junitxml") log.warning('junitxml output disabled: the `junit_xml` python module ' 'is missing.') return test_cases = [] duration_re = re.compile('([0-9]+):([0-9]+):([0-9]+).([0-9]+)') for vitem in results: if vitem.get('Validations'): parsed_duration = 0 test_duration = vitem.get('Duration', '') matched_duration = duration_re.match(test_duration) if matched_duration: parsed_duration = (int(matched_duration[1])*3600 + int(matched_duration[2])*60 + int(matched_duration[3]) + float('0.{}'.format(matched_duration[4]))) test_stdout = vitem.get('Status_by_Host', '') test_case = TestCase('validations', vitem['Validations'], parsed_duration, test_stdout) if vitem['Status'] == 'FAILED': test_case.add_failure_info('FAILED') test_cases.append(test_case) ts = TestSuite("Validations", test_cases) with open(output_junitxml, 'w') as output: output.write(to_xml_report_string([ts]))
def write_testcase(self, build, machine, test, path, passed, start_ts, end_ts): delta = end_ts - start_ts tc = { 'name': test.name, 'class': machine.getName(), 'time_elapsed': delta.total_seconds(), 'stdout': '\n'.join(self._harness.process_output(test, path)), 'stderr': "", 'passed': passed } if have_junit_xml: ju_tc = TestCase( tc['name'], tc['class'], tc['time_elapsed'], tc['stdout'], ) if not passed: errors = self._harness.extract_errors(test, path) errorstr = 'Failed' if errors is not None and len(errors) > 0: errorstr += ': ' + ''.join( [unicode(l, errors='replace') for l in errors]) ju_tc.add_failure_info(message=errorstr) return ju_tc else: return tc
def test_multiple_failures(): """Tests multiple failures in one test case""" tc = Case("Multiple failures", allow_multiple_subelements=True) tc.add_failure_info("First failure", "First failure message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple failures"}, failures=[{ "message": "First failure", "output": "First failure message", "type": "failure" }], ) tc.add_failure_info("Second failure", "Second failure message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple failures"}, failures=[ { "message": "First failure", "output": "First failure message", "type": "failure" }, { "message": "Second failure", "output": "Second failure message", "type": "failure" }, ], )
def generateJUnitReport(self, lstRunResult, runResultDir): #create junit xml report file use junit-xml 1.4 pip install junit-xml resultFileName = runResultDir + os.path.sep + 'RunResult.xml' previousCaseModuleName = '' rowIndex = 0 lstTestSuites = [] testSuite = [] for runResult in lstRunResult: #runResult (sheetName, moduleName, testCaseID, runResult, timeElapsedSec, failureMessage) #test testCaseName = runResult[2] className = runResult[1] + '.' + runResult[2] timeElapsedSec = runResult[4] failureMessage = runResult[5] testCase = TestCase(testCaseName, className, timeElapsedSec) testCase.add_failure_info(None, failureMessage) currTestCaseModuleName = runResult[1] if not currTestCaseModuleName == previousCaseModuleName: testSuite = TestSuite(currTestCaseModuleName) lstTestSuites.append(testSuite) testSuite.test_cases.append(testCase) #print TestSuite.to_xml_string(lstTestSuites) #Write the xml content to result file with open(runResultDir + os.path.sep + 'Result.xml', 'w') as f: TestSuite.to_file(f, lstTestSuites)
def process_report(requests, thresholds): functional_test_cases, threshold_test_cases = [], [] test_suites = [] for req in requests: if requests[req]['KO'] != 0: functional_test_cases.append(TestCase(name=requests[req]['request_name'], stdout="PASSED: {}. FAILED: {}".format(str(requests[req]['OK']), str(requests[req]['KO'])), stderr="FAILED: {}".format(str(requests[req]['KO'])))) functional_test_cases[-1].add_failure_info("Request failed {} times".format(str(requests[req]['KO']))) else: functional_test_cases.append( TestCase(name=requests[req]['request_name'], stdout="PASSED: {}".format(str(requests[req]['OK'])), stderr="FAILED: {}".format(str(requests[req]['KO'])))) test_suites.append(TestSuite("Functional errors ", functional_test_cases)) for th in thresholds: threshold_test_cases.append(TestCase(name="Threshold for {}, target - {}".format(th['scope'], th['target']), stdout="Value: {} {}. Threshold value: {} {}".format(str(th['value']), th['metric'], str(th['threshold']), th['metric']))) if th['status'] == 'FAILED': threshold_test_cases[-1].add_failure_info("{} for {} exceeded threshold of {} {}. Test result - {} {}" .format(th['target'], th['scope'], str(th['threshold']), th['metric'], str(th['value']), th['metric'])) test_suites.append(TestSuite("Thresholds ", threshold_test_cases)) with open("/tmp/reports/jmeter.xml", 'w') as f: TestSuite.to_file(f, test_suites, prettyprint=True)
def exporter_junit(test_result_ext, test_suite_properties=None): """! Export test results in JUnit XML compliant format @details This function will import junit_xml library to perform report conversion @return String containing Junit XML formatted test result output """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] targets = sorted(test_result_ext.keys()) for target in targets: test_cases = [] tests = sorted(test_result_ext[target].keys()) for test in tests: test_results = test_result_ext[target][test] classname = 'test.%s.%s' % (target, test) elapsed_sec = test_results['elapsed_time'] _stdout = test_results['single_test_output'] _stderr = '' # Test case tc = TestCase(test, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if test_results['single_test_result'] == 'FAIL': message = test_results['single_test_result'] tc.add_failure_info(message, _stdout) elif test_results['single_test_result'] != 'OK': message = test_results['single_test_result'] tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite("test.suite.%s" % target, test_cases) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
class JunitReporter: REPORTS_DIRECTORY = "reports" def __init__(self): self._dummy_test_case = None def activate(self): subscribe_event_handlers(self) def on_suite_erred(self, suite_name, exception=None, **kwargs): self._dummy_test_case = TestCase(name=suite_name, status="error") if exception: self._dummy_test_case.add_error_info( message=exception_str(exception), output=format_exception(exception)) def on_suite_results_compiled(self, suite_results, **kwargs): suite_name = suite_results.suite_name or "NamelessSuite" test_cases = convert_tests(suite_results.tests) if self._dummy_test_case: test_cases.append(self._dummy_test_case) suite = dependency(TestSuite)(name=infer_package_name() + suite_name, timestamp=current_time().isoformat(), test_cases=test_cases) xml_report = ElementTree.tostring( suite.build_xml_doc(), encoding="utf-8").decode(encoding="utf-8") EventBroker.publish( event=TestEvent.report_created, suite=suite, cases=test_cases, report_filename=suite_name + ".xml", report_content=xml_report, )
def test_init_failure(self): tc = TestCase('Failure-Message-and-Output') tc.add_failure_info("failure message", "I failed!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Message-and-Output'}, failure_message="failure message", failure_output="I failed!")
def _gen_cases(n_passes, n_fails, n_skips, n_errors): result = [] for i in range(n_passes): case = TestCase(name='TestPassed%s' % i, classname='generated.xml.test.case.passes', elapsed_sec=rand_duration()) result.append(case) for i in range(n_skips): case = TestCase(name='TestSkipped%s' % i, classname='generated.xml.test.case.skips', elapsed_sec=rand_duration()) case.add_skipped_info(message=rand_string('skipped!')) result.append(case) for i in range(n_fails): case = TestCase(name='TestFailed%s' % i, classname='generated.xml.test.case.fails', elapsed_sec=rand_duration()) case.add_failure_info(message=rand_string('failure!')) result.append(case) for i in range(n_errors): case = TestCase(name='TestErrored%s' % i, classname='generated.xml.test.case.errors', elapsed_sec=rand_duration()) case.add_error_info(message=rand_string('error!')) result.append(case) return result
def get_log_entry_case(cls, entry: LogEntry, fail_cases: Dict[str, List[TestCase]], suite_name: str, failure_message: str) -> List[TestCase]: fail_case: List[TestCase] = list() if cls._is_duplicate_entry(entry, failure_message, fail_cases): return [] test_case = TestCase(name=entry.func, classname=suite_name, category=suite_name, timestamp=entry.time) test_case.failures.append( CaseFailure(message=failure_message, output=failure_message, type=entry.level)) fail_case.append(test_case) if entry.level != "fatal": # Add test case with the same name so it will be marked in PROW as flaky flaky_test_case = TestCase(name=entry.func, classname=suite_name, category=suite_name) fail_case.append(flaky_test_case) return fail_case
def generate_junit_xml(inputfile): target = None suite = None infos = [] errors = [] testcases = [] for line in inputfile: tag = line[0:3] props = line[3:].split(':') if tag == "[!]": if len(props) == 2: if props[0].strip().lower() == "target": target = os.path.basename(props[1].strip()) elif props[0].strip().lower() == "group": suite = props[1].strip() else: infos.append(line) else: infos.append(line) if tag == "[x]": errors.append(line) if tag == "[+]": testcases.append(TestCase(name=props[0].strip(), classname=target, stdout=line)) if tag == "[-]": tc = TestCase(name=props[0].strip(), classname=target) tc.add_failure_info(message=props[1].strip(), output=line, failure_type="failed") testcases.append(tc) ts = TestSuite(name=suite, test_cases=testcases, stdout="\n".join(infos), stderr="\n".join(errors)) return TestSuite.to_xml_string([ts])
def test_init_utf8(): tc = Case( name="Test äöü", classname="some.class.name.äöü", elapsed_sec=123.345, stdout="I am stdöüt!", stderr="I am stdärr!", ) tc.add_skipped_info(message="Skipped äöü", output="I skippäd!") tc.add_error_info(message="Skipped error äöü", output="I skippäd with an error!") test_suite = Suite("Test UTF-8", [tc]) #pylint: disable=unused-variable ts, tcs = serialize_and_read(test_suite, encoding="utf-8")[0] verify_test_case( tcs[0], { "name": decode("Test äöü", "utf-8"), "classname": decode("some.class.name.äöü", "utf-8"), "time": ("%f" % 123.345), }, stdout=decode("I am stdöüt!", "utf-8"), stderr=decode("I am stdärr!", "utf-8"), skipped_message=decode("Skipped äöü", "utf-8"), skipped_output=decode("I skippäd!", "utf-8"), error_message=decode("Skipped error äöü", "utf-8"), error_output=decode("I skippäd with an error!", "utf-8"), )
def pytest_runtest_makereport(item, call): # print('runtest makerepot') outcome = yield rep = outcome.get_result() testScript = rep.fspath scriptName = os.path.basename(testScript) testname = "" try: testname = rep.item['name'] except: testname = rep.nodeid if rep.when == 'call': scenario = rep.scenario steps = scenario["steps"] durInSeconds = rep.duration # for step in steps: # print("********" + step["name"] + ": " + str(step["failed"])) stepsSummary = generateTestStepsSummary(steps) # print("********name " + testname + " : " + rep.outcome ) # logging.getLogger().info('new Test CAse ' + testname) testCase = TestCase(testname, '', durInSeconds, stepsSummary) # logging.getLogger().info('Testname is ' + testname) if rep.failed: testCase.add_failure_info(rep.longreprtext) testCaseDictList[item.fspath.strpath].append(testCase)
def test_init_skipped(self): tc = TestCase('Skipped-Message-and-Output') tc.add_skipped_info("skipped message", "I skipped!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Skipped-Message-and-Output'}, skipped_message="skipped message", skipped_output="I skipped!")
def test_init_skipped_output(self): tc = TestCase('Skipped-Output') tc.add_skipped_info(output="I skipped!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Skipped-Output'}, skipped_output="I skipped!")
def test_init_illegal_unicode_char(): tc = Case("Failure-Message") tc.add_failure_info(u("failure message with illegal unicode char: [\x02]")) ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Failure-Message"}, failure_message=u("failure message with illegal unicode char: []"))
def test_init_skipped_message(self): tc = TestCase('Skipped-Message') tc.add_skipped_info("skipped message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Skipped-Message'}, skipped_message="skipped message")
def test_multiple_errors(): """Tests multiple errors in one test case""" tc = Case("Multiple error", allow_multiple_subelements=True) tc.add_error_info("First error", "First error message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple error"}, errors=[{ "message": "First error", "output": "First error message", "type": "error" }], ) tc.add_error_info("Second error", "Second error message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple error"}, errors=[ { "message": "First error", "output": "First error message", "type": "error" }, { "message": "Second error", "output": "Second error message", "type": "error" }, ], )
def _collect_test_suite(self, scenario_result): if isinstance(scenario_result, GeneralError): test_case = TestCase("", "") test_case.add_error_info(scenario_result.message) test_suite = TestSuite("", "") test_suite.test_cases.append(test_case) return test_suite test_suite = TestSuite(scenario_result.name) for test_result in scenario_result.test_results: test_case = TestCase(test_result.name, test_result.name) for result in test_result.results: if isinstance(result, Failed): test_case.add_failure_info("ASSERTION {} failed".format(result.pretty_name), "EXPECTED {}\nGOT {}".format(result.expected, result.actual)) elif isinstance(result, (Error, ConnectionError)): test_case.add_error_info("ASSERTION {} failed".format(result.pretty_name), "ERROR {}".format(result.error)) elif isinstance(result, Passed): pass # TODO: What to do below? else: raise Exception("Unknown state") test_suite.test_cases.append(test_case) return test_suite
def test_multiple_skipped(): """Tests multiple skipped messages in one test case""" tc = Case("Multiple skipped", allow_multiple_subelements=True) tc.add_skipped_info("First skipped", "First skipped message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Multiple skipped"}, skipped=[{ "message": "First skipped", "output": "First skipped message" }]) tc.add_skipped_info("Second skipped", "Second skipped message") (_, tcs) = serialize_and_read(Suite("test", [tc]))[0] verify_test_case( tcs[0], {"name": "Multiple skipped"}, skipped=[ { "message": "First skipped", "output": "First skipped message" }, { "message": "Second skipped", "output": "Second skipped message" }, ], )
def test_init_failure_output(self): tc = TestCase('Failure-Output') tc.add_failure_info(output="I failed!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Output'}, failure_output="I failed!")
def run_everything_else(xml = False): mega_suite = [] tests = [ run_test_arakoon_changes, run_tests_cli, run_test_big_object ] for x in tests: r = x () mega_suite.append(r) if is_true(xml): from junit_xml import TestSuite, TestCase test_cases = [] for (suite, results) in mega_suite: for (name,result, delta) in results: test_case = TestCase(name, suite, elapsed_sec = delta) if not result: test_case.add_error_info(message = "failed") test_cases.append(test_case) ts = [TestSuite("run_everything_else", test_cases)] with open('./testresults.xml', 'w') as f: TestSuite.to_file(f,ts) else: print mega_suite
def generate_junit_report_from_cfn_nag(report): total_failures = 0 """Generate Test Case from cfn_nag report""" test_cases = [] for file_findings in report: for violation in file_findings["file_results"]['violations']: total_failures += 1 for i, resource_id in enumerate(violation['logical_resource_ids']): test_case = TestCase("%s - %s" % (violation['id'], violation['message']), classname=resource_id) test_case.add_failure_info( output="%s#L%s" % (file_findings['filename'], violation['line_numbers'][i])) test_cases.append(test_case) test_suite = TestSuite("cfn-nag test suite", test_cases) if total_failures > 0: f = open("CFN_NAG_FAILURE", "a") f.close() return TestSuite.to_xml_string([test_suite], prettyprint=False)
def test_init_skipped_output(): tc = Case("Skipped-Output") tc.add_skipped_info(output="I skipped!") #pylint: disable=unused-variable ts, tcs = serialize_and_read(Suite("test", [tc]))[0] verify_test_case(tcs[0], {"name": "Skipped-Output"}, skipped_output="I skipped!")
def test(self): def Trim(string): if len(string) > 4096: return string[:4096] + " ..." else: return string varsdict = {} self.log("Assigning variables:") for var in self.variables: tmpdict = {} try: var.run(tmpdict) except: self.log("failure.") self.pass_status.append('F') return self.pass_status varsdict[var.name] = tmpdict[var.name] self.log("Assigning %s = %s" % (str(var.name), Trim(str(varsdict[var.name])))) if len(self.pass_tests) != 0: self.log("Running failure tests: ") for test in self.pass_tests: self.log("Running %s:" % test.name) status = test.run(varsdict) tc=TestCase(test.name, '%s.%s'%(self.length, self.filename[:-4])) if status == True: self.log("success.") self.pass_status.append('P') elif status == False: self.log("failure.") self.pass_status.append('F') tc.add_failure_info( "Failure" ) else: self.log("failure (info == %s)." % status) self.pass_status.append('F') tc.add_failure_info( "Failure", status ) self.xml_reports.append(tc) if len(self.warn_tests) != 0: self.log("Running warning tests: ") for test in self.warn_tests: self.log("Running %s:" % test.name) status = test.run(varsdict) if status == True: self.log("success.") self.warn_status.append('P') elif status == False: self.log("warning.") self.warn_status.append('W') else: self.log("warning (info == %s)." % status) self.warn_status.append('W') self.log(''.join(self.pass_status + self.warn_status)) return self.pass_status + self.warn_status
def test_init_illegal_unicode_char(self): tc = TestCase('Failure-Message') tc.add_failure_info( u("failure message with illegal unicode char: [\x02]")) (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Message'}, failure_message=u( "failure message with illegal unicode char: []"))
def execute(self, log, keyvals, testDef): testDef.logger.verbose_print("JunitXML Reporter") # pickup the options cmds = {} testDef.parseOptions(log, self.options, keyvals, cmds) if cmds['filename'] is not None: self.fh = open(cmds['filename'] if os.path.isabs(cmds['filename']) \ else os.path.join(cmds['scratch'],cmds['filename']), 'w') if testDef.options['description'] is not None: print(testDef.options['description'], file=self.fh) print(file=self.fh) # Use the Junit classname field to store the list of inifiles try: classname = testDef.log['inifiles'] except KeyError: classname = None # get the entire log of results fullLog = testDef.logger.getLog(None) testCases = [] # TODO: ain't nobody got time for that. 8-). time = 0 for lg in fullLog: if 'stdout' in lg and lg['stdout'] is not None: stdout = "\n".join(lg['stdout']) else: stdout = None if 'stderr' in lg and lg['stderr'] is not None: stderr = "\n".join(lg['stderr']) else: stderr = None if 'time' in lg and lg['time'] is not None: time = lg['time'] else: time = 0 tc = TestCase(lg['section'], classname, time, stdout, stderr) try: if 0 != lg['status']: # Find sections prefixed with 'TestRun' if re.match("TestRun", lg['section']): tc.add_failure_info("Test reported failure") else: tc.add_error_info("Test error") except KeyError: sys.exit(lg['section'] + " is missing status!") testCases.append(tc) # TODO: Pull in the resource manager jobid. jobid = "job1" ts = TestSuite(jobid, testCases) print(TestSuite.to_xml_string([ts]), file=self.fh) if cmds['filename'] is not None: self.fh.close() log['status'] = 0 return
def test_init_skipped_err_output(self): tc = TestCase('Skipped-Output') tc.add_skipped_info(output="I skipped!") tc.add_error_info(output="I skipped with an error!") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Skipped-Output'}, skipped_output="I skipped!", error_output="I skipped with an error!")
def simple_report(self): """empty test report""" test_case_1 = TestCase("testcase1", elapsed_sec=1.5) test_case_2 = TestCase("testcase2", elapsed_sec=0.5) test_case_2.add_skipped_info("was skipped") test_case_3 = TestCase("testcase3", elapsed_sec=1.0) test_case_3.add_failure_info("failed") test_case_4 = TestCase("testcase4", elapsed_sec=0.25) test_case_4.add_error_info("errored") test_case_5 = TestCase("testcase5", elapsed_sec=0.1) test_cases = [ test_case_1, test_case_2, test_case_3, test_case_4, test_case_5 ] test_suites = [ TestSuite('testsuite1', test_cases, timestamp=datetime.datetime.utcnow()) ] return TestReport(NAME, {"module": test_suites}, BUILD_NUMBER, True)
def generate_generic_test_case(name, classname, time, message, result): default_pass_message = 'Test passed, check build log for additional details' default_skip_message = 'Test skipped, check build log for additional details' tc = TestCase(name, classname, time, default_pass_message if result == 'pass' else '') if result == 'fail': tc.add_failure_info(message=message) if result == 'skip': tc.add_skipped_info(message=default_skip_message) return tc
def take_action(self, args): test_cases = [] if args.playbook is not None: playbooks = args.playbook results = (models.TaskResult().query .join(models.Task) .filter(models.TaskResult.task_id == models.Task.id) .filter(models.Task.playbook_id.in_(playbooks))) else: results = models.TaskResult().query.all() for result in results: task_name = result.task.name if not task_name: task_name = result.task.action additional_results = { 'host': result.host.name, 'playbook_path': result.task.playbook.path } result_str = jsonutils.dumps(additional_results) test_path = \ u'{playbook_file}.{play_name}'.format( playbook_file=os.path.basename(result.task.playbook.path), play_name=result.task.play.name) test_case = TestCase( name=task_name, classname=test_path, elapsed_sec=result.duration.seconds, stdout=result_str) if result.status == 'skipped': test_case.add_skipped_info(message=result.result) elif ((result.status in ('failed', 'unreachable') and result.ignore_errors is False and 'EXPECTED FAILURE' not in task_name and 'TOGGLE RESULT' not in task_name) or (result.status == 'ok' and 'TOGGLE RESULT' in task_name)): test_case.add_failure_info(message=result.result) test_cases.append(test_case) test_suite = TestSuite('Ansible Tasks', test_cases) # TODO: junit_xml doesn't order the TestCase parameters. # This makes it so the order of the parameters for the same exact # TestCase is not guaranteed to be the same and thus results in a # different stdout (or file). This is easily reproducible on Py3. xml_string = six.text_type(test_suite.to_xml_string([test_suite])) if args.output_file == '-': if six.PY2: sys.stdout.write(encodeutils.safe_encode(xml_string)) else: sys.stdout.buffer.write(encodeutils.safe_encode(xml_string)) else: with open(args.output_file, 'wb') as f: f.write(encodeutils.safe_encode(xml_string))
def test_init_failure_type(self): tc = TestCase('Failure-Type') tc.add_failure_info(failure_type='com.example.Error') (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Failure-Type'}) tc.add_failure_info("failure message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Failure-Type'}, failure_message="failure message", failure_type='com.example.Error')
def test_init_error_type(self): tc = TestCase('Error-Type') tc.add_error_info(error_type='com.example.Error') (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case(self, tcs[0], {'name': 'Error-Type'}) tc.add_error_info("error message") (ts, tcs) = serialize_and_read(TestSuite('test', [tc]))[0] verify_test_case( self, tcs[0], {'name': 'Error-Type'}, error_message="error message", error_type='com.example.Error')
def test_init_utf8(self): tc = TestCase('Test äöü', 'some.class.name.äöü', 123.345, 'I am stdöüt!', 'I am stdärr!') tc.add_skipped_info(message='Skipped äöü', output="I skippäd!") tc.add_error_info(message='Skipped error äöü', output="I skippäd with an error!") test_suite = TestSuite('Test UTF-8', [tc]) (ts, tcs) = serialize_and_read(test_suite, encoding='utf-8')[0] verify_test_case(self, tcs[0], {'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345)}, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def process(config): for mnf in config.keys(): log("Processing manifest: %s" % mnf, 'info') if config[mnf].has_key('submodule'): sub = config[mnf].pop('submodule') sub_manifest = guess_manifest(str(mnf), sub['source']) sub_app = qubell.organization.application(name=sub['name']) sub_app.upload(sub_manifest) if config[mnf].has_key('source'): manifest = guess_manifest(str(mnf), config[mnf].pop('source')) else: error("Missing manifest source directive") if not 'launch' == config[mnf].keys()[0]: error('Launch action missing or not first') if config[mnf]['launch'].has_key('parameters'): parameters = config[mnf]['launch']['parameters'] else: parameters = {} if config[mnf]['launch'].has_key('settings'): settings = config[mnf]['launch']['settings'] else: settings = {} if qubell.appid: app = qubell.organization.application(id=qubell.appid, manifest=manifest) elif qubell.appname: app = qubell.organization.application(name=qubell.appname, manifest=manifest) else: app = qubell.organization.application(name=manifest.name, manifest=manifest) assert app.upload(manifest) instance = qubell.organization.create_instance(application=app, parameters=parameters, **settings) assert instance for action in config[mnf].keys(): if config[mnf][action].has_key('parameters'): parameters = config[mnf][action]['parameters'] else: parameters = {} if not run_test(instance, manifest, Action(action, parameters, config[mnf][action]['expected'])): test = TestCase(action, manifest.name, 0, 'Manifest: ' + manifest.source, 'failed to start') test.add_failure_info(str(instance)) test_cases.append(test) return False
def _generate_junit_xml(self, config_name): testcases = [] tests=self.data_source.get_test_results(config_name) for test in tests: test_time = 0 if test.func_end_time != None and test.func_start_time != None: test_time = test.func_end_time - test.func_start_time tc = TestCase(test.name,config_name,test_time, test.description, test.message) if 'failed' in test.result: tc.add_failure_info(test.result) elif 'skipped' in test.result: tc.add_skipped_info(test.result) testcases.append(tc) testsuite = TestSuite(config_name+'_'+self.name, testcases) return testsuite
def _output_normal(self, test_result): # Need refactor if test_result == {}: print '[what?!] there are not any test result, what is the test case id?' else: print xml_test_suites = [] summary_dict = self._get_summary_dict(test_result) self.report_create_time = str(time.strftime('%Y%m%d_%H%M%S', time.localtime())) for case_classify in test_result.keys(): xml_test_cases = [] if 'result' in test_result[case_classify].keys(): # Generate HTML report self._generate_html_file( case_classify, test_result[case_classify]['result'], test_result[case_classify]['summary']) # Save the result into the CSV self._output_result_to_csv(test_result) # Show in Console print '{0} {1} {2}'.format('='*16, case_classify, '='*16) test_case_result = test_result[case_classify]['result'] for case_id in test_case_result.keys(): print '[{0}][{1}] {2}, {3}, {4}'.format(case_classify, case_id, test_case_result[case_id][0], test_case_result[case_id][1], str(test_case_result[case_id][2])) # Produce xml file test_case = TestCase(case_id, case_classify, int(test_case_result[case_id][2])) if test_case_result[case_id][0] == 'Fail' or test_case_result[case_id][0] == 'Error': try: test_case.add_failure_info('msg' + test_case_result[case_id][1]) except: test_case.add_failure_info('msg' + str(test_case_result[case_id])) xml_test_cases.append(test_case) xml_test_suites.append(TestSuite(case_classify, xml_test_cases)) with open(os.path.join(self.latest_reports_dir, case_classify + '.xml'), 'w') as f: TestSuite.to_file(f, xml_test_suites, prettyprint=True) self._generate_summary_html_file(summary_dict) print '{0} {1} {2}'.format('='*16, 'Summary', '='*16) pprint.pprint(summary_dict)
def run_tests(self): test_cases = [] for test in self.tests: desc = test['desc'] name = test['name'] index = test['id'] test_case = TestCase(name, self.name) if '[.]' in desc: print('skipping test "{}"'.format(name)) test_case.add_skipped_info(message="Skipped test marked with [.]") else: test_output = StringIO() self.sp.logfile = test_output t_start = time.time() result = self.run_test(index) t_stop = time.time() self.sp.logfile = None test_case.elapsed_sec = t_stop - t_start debug_print('test output was:') debug_print(test_output.getvalue()) if result == BSTestRunner.SUCCESS: test_case.stdout = test_output.getvalue() print('test "{}" passed'.format(name)) else: print('test "{}" failed'.format(name)) test_case.add_failure_info('Test failed', output=test_output.getvalue()) test_output.close() test_cases += [test_case]; return TestSuite(self.name, test_cases)
def test_init_unicode(self): tc = TestCase(decode('Test äöü', 'utf-8'), decode('some.class.name.äöü', 'utf-8'), 123.345, decode('I am stdöüt!', 'utf-8'), decode('I am stdärr!', 'utf-8')) tc.add_skipped_info(message=decode('Skipped äöü', 'utf-8'), output=decode('I skippäd!', 'utf-8')) tc.add_error_info(message=decode('Skipped error äöü', 'utf-8'), output=decode('I skippäd with an error!', 'utf-8')) (ts, tcs) = serialize_and_read(TestSuite('Test Unicode', [tc]))[0] verify_test_case(self, tcs[0], {'name': decode('Test äöü', 'utf-8'), 'classname': decode('some.class.name.äöü', 'utf-8'), 'time': ("%f" % 123.345)}, stdout=decode('I am stdöüt!', 'utf-8'), stderr=decode('I am stdärr!', 'utf-8'), skipped_message=decode('Skipped äöü', 'utf-8'), skipped_output=decode('I skippäd!', 'utf-8'), error_message=decode('Skipped error äöü', 'utf-8'), error_output=decode('I skippäd with an error!', 'utf-8'))
def run_tests(TestClass): setup_logging() # Verify that we have needed input assert base_url, 'Please provide a value for base_url in runner.py' assert chromedriver_location, 'Please provide a value for chromedriver_location in runner.py' assert credentials['email'], 'Please provide credentials in runner.py' assert credentials['password'], 'Please provide credentials in runner.py' #WebDriver setup driver = webdriver.Chrome(chromedriver_location) # Init the provided class myTestClass = TestClass(driver, base_url, credentials) logging.info("Running: %s", myTestClass.__class__.__name__) # Run all the methods containing 'test' in name test_cases = [] for object_name in dir(myTestClass): if 'test' in object_name: test_method = getattr(myTestClass, object_name) if not hasattr(test_method, '__call__'): break # If the object is not callable this_test_case = TestCase(object_name, myTestClass.__class__.__name__) logging.info('#######################################################') logging.info("Running %s", object_name) try: # Run the actual method test_method() except Exception as e: logging.info('TEST FAILED') logging.exception(e) this_test_case.add_failure_info(output=traceback.format_exc()) else: logging.info("TEST PASSED") test_cases.append(this_test_case) logging.info('#######################################################') # Write the results to junit xml write_xml(test_cases, myTestClass.__class__.__name__)
def exporter_junit(self, test_result_ext, test_suite_properties=None): """ Export test results in JUnit XML compliant format """ from junit_xml import TestSuite, TestCase test_suites = [] test_cases = [] toolchains = sorted(test_result_ext.keys()) for toolchain in toolchains: targets = sorted(test_result_ext[toolchain].keys()) for target in targets: test_cases = [] tests = sorted(test_result_ext[toolchain][target].keys()) for test in tests: test_results = test_result_ext[toolchain][target][test] for test_res in test_results: test_ids = sorted(test_res.keys()) for test_no in test_ids: test_result = test_res[test_no] name = test_result["test_description"] classname = "test.%s.%s.%s" % (target, toolchain, test_result["test_id"]) elapsed_sec = test_result["elapsed_time"] _stdout = test_result["single_test_output"] _stderr = test_result["target_name_unique"] # Test case tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) # Test case extra failure / error info if test_result["single_test_result"] == "FAIL": message = test_result["single_test_result"] tc.add_failure_info(message, _stdout) elif test_result["single_test_result"] != "OK": message = test_result["single_test_result"] tc.add_error_info(message, _stdout) test_cases.append(tc) ts = TestSuite( "test.suite.%s.%s" % (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain], ) test_suites.append(ts) return TestSuite.to_xml_string(test_suites)
def xmlwrite(target, everyline): xmltargetpath = os.getcwd() + os.sep + target + '_testresults.xml' xmltargetfile = open(xmltargetpath, 'w+') test_cases = [] for line in everyline.split('\n'): words = line.split(' ') if line.find('success') != -1: time_taken = get_time(line.split('-')[-1][7:]) name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip() print name test_cases.append(TestCase(name, target, time_taken, None)) elif line.find('failed') != -1: time_taken = get_time(line.split('-')[-1][7:]) name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip() message = ("-".join(line.split(' - ')[2:-1])).strip() tc = TestCase(name, target, time_taken, None) tc.add_failure_info(None, message) test_cases.append(tc) ts = TestSuite("testing this suite", test_cases) xmltargetfile.write(TestSuite.to_xml_string([ts])) xmltargetfile.close()