def test_console_pass_with_retries(self): saved_stdout = sys.stdout results = ResultList() res1 = Result() res1.set_verdict("fail", 1001, 1) res1.retries_left = 1 res2 = Result() res2.set_verdict("pass", 0, 1) res2.retries_left = 0 results.append(res1) results.append(res2) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 17) self.assertRegexpMatches(lines[3], "Yes", lines[4]) self.assertRegexpMatches(lines[4], "No", lines[5]) self.assertRegexpMatches(lines[9], "Final Verdict.*PASS", lines[9]) self.assertRegexpMatches(lines[10], "count.*2", lines[10]) self.assertRegexpMatches(lines[11], r"passrate.*50.00 \%", lines[11]) self.assertRegexpMatches(lines[12], r"passrate excluding retries.*100.00 \%", lines[12]) finally: sys.stdout = saved_stdout
def test_console_multiple_results_with_retries(self): saved_stdout = sys.stdout results = ResultList() res1 = Result() res1.set_verdict("fail", 1001, 1) res1.retries_left = 1 res2 = Result() res2.set_verdict("pass", 0, 1) res2.retries_left = 0 results.append(res1) results.append(res2) results.append(Result()) results.append(Result()) results.append(Result()) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 21) self.assertRegexpMatches(lines[3], "Yes", lines[4]) self.assertRegexpMatches(lines[4], "No", lines[5]) self.assertRegexpMatches(lines[12], "Final Verdict.*INCONCLUSIVE", lines[12]) self.assertRegexpMatches(lines[13], "count.*5", lines[13]) self.assertRegexpMatches(lines[14], r"passrate.*50.00 \%", lines[14]) finally: sys.stdout = saved_stdout
def test_clean_fails(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertTrue(resultlist.clean_fails()) dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res2.retries_left = 1 res3 = Result() res3.set_verdict("fail", 4, 5) res3.retries_left = 0 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertTrue(resultlist.clean_fails()) dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res2.retries_left = 1 res3 = Result() res3.set_verdict("pass", 4, 5) res3.retries_left = 0 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertFalse(resultlist.clean_fails())
def test_retries_count(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertEqual(resultlist.retry_count(), 1) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertEqual(resultlist.retry_count(), 1)
def test_get_summary(self): expected = { "count": 3, "pass": 1, "fail": 1, "skip": 0, "inconclusive": 1, "retries": 1, "duration": 10 } dictionary = {"retcode": 0} res = Result(kwargs=dictionary) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertDictEqual(resultlist.get_summary(), expected)
def test_junit_hides(self): str_should_be1 = '<testsuite failures="0" tests="3" errors="0" skipped="0">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="120"></testcase>\n\ <testcase classname="test-case-A6" name="unknown" time="2"></testcase>\n\ </testsuite>' results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) failres = Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "unknown", "duration": 120 }) failres.retries_left = 1 results.append(failres) results.append( Result({ "testcase": "test-case-A4", "verdict": "PASS", "duration": 120 })) incres = Result({ "testcase": "test-case-A6", "verdict": "INCONCLUSIVE", "reason": "unknown", "duration": 2 }) incres.retries_left = 1 results.append(incres) results.append( Result({ "testcase": "test-case-A6", "verdict": "PASS", "duration": 2 })) junit = ReportJunit(results) str_report = junit.to_string() report_xml = ET.fromstring(str_report) shouldbe_xml = ET.fromstring(str_should_be1) self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib) self.assertEqual(len(report_xml.findall("testcase")), len(shouldbe_xml.findall("testcase"))) str_should_be2 = '<testsuite failures="0" tests="3" errors="1" skipped="0">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="12"></testcase>\n\ <testcase classname="test-case-A6" name="unknown" time="2">\n\ <error message="unknown"></error>\n\ </testcase>\n\ </testsuite>' results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) failres = Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "unknown", "duration": 120 }) failres.retries_left = 1 results.append(failres) results.append( Result({ "testcase": "test-case-A4", "verdict": "PASS", "duration": 12 })) results.append( Result({ "testcase": "test-case-A6", "verdict": "INCONCLUSIVE", "reason": "unknown", "duration": 2 })) junit = ReportJunit(results) str_report = junit.to_string() report_xml = ET.fromstring(str_report) shouldbe_xml = ET.fromstring(str_should_be2) self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib) self.assertEqual(len(report_xml.findall("testcase")), len(shouldbe_xml.findall("testcase"))) errors = [] for elem in report_xml.findall("testcase"): if elem.find("error") is not None: errors.append(elem) self.assertEqual(len(errors), 1)