def test_console_pass_with_retries(self):
saved_stdout = sys.stdout
results = ResultList()
res1 = Result()
res1.set_verdict("fail", 1001, 1)
res1.retries_left = 1
res2 = Result()
res2.set_verdict("pass", 0, 1)
res2.retries_left = 0
results.append(res1)
results.append(res2)
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 17)
self.assertRegexpMatches(lines[3], "Yes", lines[4])
self.assertRegexpMatches(lines[4], "No", lines[5])
self.assertRegexpMatches(lines[9], "Final Verdict.*PASS", lines[9])
self.assertRegexpMatches(lines[10], "count.*2", lines[10])
self.assertRegexpMatches(lines[11], r"passrate.*50.00 \%",
lines[11])
self.assertRegexpMatches(lines[12],
r"passrate excluding retries.*100.00 \%",
lines[12])
finally:
sys.stdout = saved_stdout
def test_console_multiple_results_with_retries(self):
saved_stdout = sys.stdout
results = ResultList()
res1 = Result()
res1.set_verdict("fail", 1001, 1)
res1.retries_left = 1
res2 = Result()
res2.set_verdict("pass", 0, 1)
res2.retries_left = 0
results.append(res1)
results.append(res2)
results.append(Result())
results.append(Result())
results.append(Result())
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 21)
self.assertRegexpMatches(lines[3], "Yes", lines[4])
self.assertRegexpMatches(lines[4], "No", lines[5])
self.assertRegexpMatches(lines[12], "Final Verdict.*INCONCLUSIVE",
lines[12])
self.assertRegexpMatches(lines[13], "count.*5", lines[13])
self.assertRegexpMatches(lines[14], r"passrate.*50.00 \%",
lines[14])
finally:
sys.stdout = saved_stdout
def test_clean_fails(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertTrue(resultlist.clean_fails())
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res2.retries_left = 1
res3 = Result()
res3.set_verdict("fail", 4, 5)
res3.retries_left = 0
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertTrue(resultlist.clean_fails())
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res2.retries_left = 1
res3 = Result()
res3.set_verdict("pass", 4, 5)
res3.retries_left = 0
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertFalse(resultlist.clean_fails())
def test_retries_count(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertEqual(resultlist.retry_count(), 1)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertEqual(resultlist.retry_count(), 1)
def test_get_summary(self):
expected = {
"count": 3,
"pass": 1,
"fail": 1,
"skip": 0,
"inconclusive": 1,
"retries": 1,
"duration": 10
}
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertDictEqual(resultlist.get_summary(), expected)
def test_junit_hides(self):
str_should_be1 = '<testsuite failures="0" tests="3" errors="0" skipped="0">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="120"></testcase>\n\
<testcase classname="test-case-A6" name="unknown" time="2"></testcase>\n\
</testsuite>'
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
failres = Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "unknown",
"duration": 120
})
failres.retries_left = 1
results.append(failres)
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "PASS",
"duration": 120
}))
incres = Result({
"testcase": "test-case-A6",
"verdict": "INCONCLUSIVE",
"reason": "unknown",
"duration": 2
})
incres.retries_left = 1
results.append(incres)
results.append(
Result({
"testcase": "test-case-A6",
"verdict": "PASS",
"duration": 2
}))
junit = ReportJunit(results)
str_report = junit.to_string()
report_xml = ET.fromstring(str_report)
shouldbe_xml = ET.fromstring(str_should_be1)
self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib)
self.assertEqual(len(report_xml.findall("testcase")),
len(shouldbe_xml.findall("testcase")))
str_should_be2 = '<testsuite failures="0" tests="3" errors="1" skipped="0">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="12"></testcase>\n\
<testcase classname="test-case-A6" name="unknown" time="2">\n\
<error message="unknown"></error>\n\
</testcase>\n\
</testsuite>'
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
failres = Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "unknown",
"duration": 120
})
failres.retries_left = 1
results.append(failres)
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "PASS",
"duration": 12
}))
results.append(
Result({
"testcase": "test-case-A6",
"verdict": "INCONCLUSIVE",
"reason": "unknown",
"duration": 2
}))
junit = ReportJunit(results)
str_report = junit.to_string()
report_xml = ET.fromstring(str_report)
shouldbe_xml = ET.fromstring(str_should_be2)
self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib)
self.assertEqual(len(report_xml.findall("testcase")),
len(shouldbe_xml.findall("testcase")))
errors = []
for elem in report_xml.findall("testcase"):
if elem.find("error") is not None:
errors.append(elem)
self.assertEqual(len(errors), 1)
