def test_haslogs(self):
result = Result()
result.logpath = os.path.join(os.path.dirname(__file__),
"test_logpath")
files = result.has_logs()
self.assertTrue(files)
self.assertEqual(len(files), 2)
result = Result()
result.logpath = None
files = result.has_logs()
self.assertListEqual(files, [])
def test_get_verdict(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
reslist = ResultList()
reslist.append(res)
self.assertEquals(reslist.get_verdict(), "pass")
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=10)
reslist.append(res2)
self.assertEquals(reslist.get_verdict(), "fail")
res3 = Result()
res3.set_verdict("inconclusive", 4, 1)
reslist = ResultList()
reslist.append(res3)
self.assertEquals(reslist.get_verdict(), "inconclusive")
reslist.append(res2)
self.assertEquals(reslist.get_verdict(), "fail")
def test_reportconsole_multiple_results(self): # pylint: disable=invalid-name
saved_stdout = sys.stdout
results = ResultList()
results.append(Result())
results.append(Result())
results.append(Result())
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 17)
self.assertRegexpMatches(lines[10], r"Final Verdict.*INCONCLUSIVE",
lines[10])
self.assertRegexpMatches(lines[11], r"count.*3", lines[11])
self.assertRegexpMatches(lines[12], r"passrate.*0.00 \%",
lines[12])
finally:
sys.stdout = saved_stdout
def test_init(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
self.assertEqual(res.get_verdict(), "pass")
dictionary = {"retcode": 1}
res = Result(kwargs=dictionary)
self.assertEqual(res.get_verdict(), "fail")
def test_get_summary(self):
expected = {
"count": 3,
"pass": 1,
"fail": 1,
"skip": 0,
"inconclusive": 1,
"retries": 1,
"duration": 10
}
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertDictEqual(resultlist.get_summary(), expected)
def test_reportconsole_skip(self):
saved_stdout = sys.stdout
results = ResultList()
res = Result()
res.skip_reason = "Skip_reason"
res.set_verdict("skip", -1, -1)
results.append(res)
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 15)
self.assertRegexpMatches(lines[3], r"skip.*Skip_reason")
self.assertRegexpMatches(lines[8], r"Final Verdict.*PASS",
lines[8])
self.assertRegexpMatches(lines[9], r"count.*1", lines[9])
self.assertRegexpMatches(lines[11], r"passrate.*0.00 \%",
lines[10])
self.assertRegexpMatches(lines[12], r"skip.*1", lines[10])
finally:
sys.stdout = saved_stdout
def test_reportconsole_decodefail(self):
saved_stdout = sys.stdout
failing_message = "\x00\x00\x00\x00\x00\x00\x01\xc8"
results = ResultList()
res = Result()
res.set_verdict("fail", 1001, 0)
res.fail_reason = failing_message
results.append(res)
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 15)
self.assertRegexpMatches(lines[8], r"Final Verdict.*FAIL",
lines[8])
self.assertRegexpMatches(lines[9], r"count.*1", lines[9])
self.assertRegexpMatches(lines[10], r"passrate.*0.00 \%",
lines[10])
finally:
sys.stdout = saved_stdout
def fail_testcase(self, error):
"""
Workaround that overwrites icetea's result value in order to
store the test binary name in the results. Without this workaround
the testcase name would be "pal_test_runner" which is not descriptive.
:param error: Python error that failed the test case
"""
# Overwrite test case result
self._results = ResultList()
self._results.append(
Result({
"testcase": (self.config["requirements"]
["duts"]["*"]["application"]["bin"]),
"reason": repr(error),
"verdict": "FAIL"
})
)
def test_reportconsole_one_results(self):
saved_stdout = sys.stdout
results = ResultList()
results.append(Result())
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 15)
self.assertRegexpMatches(lines[8], r"Final Verdict.*INCONCLUSIVE",
lines[8])
self.assertRegexpMatches(lines[9], r"count.*1", lines[9])
self.assertRegexpMatches(lines[10], r"passrate.*0.00 \%",
lines[10])
finally:
sys.stdout = saved_stdout
def parse_unity_result(self, dut):
"""
Parses given DUTs stdout for Unity test results.
:param dut: icetea DUT to inspect the results from
"""
re_unity_test = re.compile(
r".*\<\*\*\*UnityTest\*\*\*\>TEST\((?P<suite>.*?), (?P<test>.*?)\)\<\/\*\*\*UnityTest\*\*\*\>") # noqa: E501 # pylint: disable=line-too-long
re_unity_result = re.compile(
r".*\<\*\*\*UnityResult\*\*\*\>(?P<result>.*?)\<\/\*\*\*UnityResult\*\*\*\>") # noqa: E501 # pylint: disable=line-too-long
test_output = ""
test_active = False
test_results = ResultList()
for line in dut.traces:
line = line.strip()
# Activate test output logging
match = re_unity_test.match(line)
if match:
test_active = True
test_output = ""
unity_name = match.group("test")
unity_suite = match.group("suite")
self.logger.info("parsing %s.%s", unity_suite, unity_name)
# Log test output
if test_active:
test_output += line + "\n"
# Check if test is over
match = re_unity_result.match(line)
if match and test_active:
unity_result = match.group("result")
# Create icetea Result()
test_result = Result(
{
"testcase": unity_suite + "." + unity_name,
"stdout": test_output,
"reason": line if unity_result == "FAIL" else ""
})
test_result.build_result_metadata({"toolchain": unity_name})
# Would need to do runtime analysis to get duration
test_result.set_verdict(unity_result, 0, 0.0)
test_results.append(test_result)
test_active = False
self.logger.info("finished %s", unity_name)
return test_results
def test_result_metainfo_generation(self):
pass_result = Result()
pass_result.set_verdict('pass', 0, 10)
dinfo = DutInformation("Test_platform", "123456", "1")
dinfo.build = Build(ref="test_file", type="file")
pass_result.add_dutinformation(dinfo)
self.args_tc.branch = "test_branch"
self.args_tc.commitId = "123456"
self.args_tc.gitUrl = "url"
self.args_tc.buildUrl = "url2"
self.args_tc.campaign = "campaign"
self.args_tc.jobId = "test_job"
self.args_tc.toolchain = "toolchain"
self.args_tc.buildDate = "today"
pass_result.build_result_metadata(args=self.args_tc)
self.assertEqual(pass_result.build_branch, "test_branch")
self.assertEqual(pass_result.buildcommit, "123456")
self.assertEqual(pass_result.build_git_url, "url")
self.assertEqual(pass_result.build_url, "url2")
self.assertEqual(pass_result.campaign, "campaign")
self.assertEqual(pass_result.job_id, "test_job")
self.assertEqual(pass_result.toolchain, "toolchain")
self.assertEqual(pass_result.build_date, "today")
def test_junit_default(self):
str_should_be = '<testsuite failures="0" tests="1" errors="0" skipped="0">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
</testsuite>'
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
junit = ReportJunit(results)
str_report = junit.to_string()
xml_shouldbe = ET.fromstring(str_should_be)
report_xml = ET.fromstring(str_report)
self.assertDictEqual(xml_shouldbe.attrib, report_xml.attrib)
self.assertDictEqual(
xml_shouldbe.find("testcase").attrib,
report_xml.find("testcase").attrib)
self.assertEqual(len(xml_shouldbe.findall("testcase")),
len(report_xml.findall("testcase")))
def get_result(self):
return Result()
def test_run(self, mock_tclist): # pylint: disable=too-many-statements
testsuite = TestSuite(args=self.args_tc)
cont1 = mock.MagicMock()
pass_result = Result()
pass_result.set_verdict('pass', 0, 10)
fail_result = Result()
fail_result.set_verdict('fail', 1000, 10)
skipped_result = Result()
skipped_result.set_verdict('skip', 0, 1)
resultlist = ResultList()
resultlist.append(pass_result)
resultlist.save = mock.MagicMock()
testsuite._default_configs["retryCount"] = 1
cont1.run.side_effect = [pass_result,
fail_result,
skipped_result,
KeyboardInterrupt,
fail_result,
pass_result]
cont_reslist = mock.MagicMock()
cont_reslist.run = mock.MagicMock()
cont_reslist.run.return_value = resultlist
# Passing result
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results.get(0).get_verdict(), "pass")
self.assertTrue(self.args_tc.forceflash) # pylint: disable=no-member
self.assertEquals(testsuite._results.save.call_count, 1)
# ResultList as result
testsuite._testcases = []
testsuite._testcases.append(cont_reslist)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results.get(0).get_verdict(), "pass")
# Failing result, no retry
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results.get(0).get_verdict(), "fail")
# skipped result
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results.get(0).get_verdict(), "skip")
# Interrupt
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._testcases.append(cont2)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 0)
cont2.run.assert_not_called()
# Failing result, retried
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite._default_configs["retryReason"] = "includeFailures"
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "fail")
self.assertEqual(testsuite._results.get(1).get_verdict(), "pass")
# TC not exist result, retried
testsuite._testcases = []
contx = mock.MagicMock()
inconc_res = Result()
inconc_res.set_verdict("inconclusive", 1015, 0)
contx.run = mock.MagicMock(return_value=inconc_res)
testsuite._testcases.append(contx)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite._default_configs["retryReason"] = "includeFailures"
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results.get(0).get_verdict(), "inconclusive")
# Failing result, retried, from a result list.
testsuite._testcases = []
fail_result_2 = Result()
fail_result_2.set_verdict('fail', 1000, 10)
reslist = ResultList()
reslist.append(fail_result_2)
cont_retry = mock.MagicMock()
cont_retry.run.side_effect = [reslist, resultlist]
testsuite._testcases.append(cont_retry)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite._default_configs["retryReason"] = "includeFailures"
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "fail")
self.assertEqual(testsuite._results.get(1).get_verdict(), "pass")
self.assertEqual(testsuite._results.get(0).retries_left, 1)
self.assertEqual(testsuite._results.get(1).retries_left, 0)
# Inconclusive result, retried, from a result list.
testsuite._testcases = []
fail_result_3 = Result()
fail_result_3.set_verdict('inconclusive', 1000, 10)
reslist = ResultList()
reslist.append(fail_result_3)
cont_retry = mock.MagicMock()
cont_retry.run.side_effect = [reslist, resultlist]
testsuite._testcases.append(cont_retry)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
testsuite._default_configs["retryReason"] = "includeFailures"
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "inconclusive")
self.assertEqual(testsuite._results.get(1).get_verdict(), "pass")
self.assertEqual(testsuite._results.get(0).retries_left, 1)
self.assertEqual(testsuite._results.get(1).retries_left, 0)
self.args_tc.repeat = 2
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
cont1.run.side_effect = [pass_result, pass_result, pass_result, pass_result]
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "pass")
self.assertFalse(self.args_tc.forceflash) # pylint: disable=no-member
# Failing result, stop_on_failure
self.args_tc.stop_on_failure = True
self.args_tc.repeat = 1
testsuite._default_configs["retryCount"] = 0
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
cont1.run.side_effect = [pass_result]
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
cont2.run.side_effect = [fail_result]
cont3 = mock.MagicMock()
cont3.run = mock.MagicMock()
cont3.run.side_effect = [pass_result]
testsuite._testcases.append(cont2)
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "pass")
self.assertEqual(testsuite._results.get(1).get_verdict(), "fail")
self.assertEquals(testsuite._results.save.call_count, 2)
# Skipped result, stop_on_failure
self.args_tc.stop_on_failure = True
self.args_tc.repeat = 0
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = ResultList()
testsuite._results.save = mock.MagicMock()
cont1.run.side_effect = [skipped_result]
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
cont2.run.side_effect = [pass_result]
testsuite._testcases.append(cont2)
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results.get(0).get_verdict(), "skip")
self.assertEqual(testsuite._results.get(1).get_verdict(), "pass")
self.assertEquals(testsuite._results.save.call_count, 2)
def test_retries_count(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertEqual(resultlist.retry_count(), 1)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertEqual(resultlist.retry_count(), 1)
def test_junit_hides(self):
str_should_be1 = '<testsuite failures="0" tests="3" errors="0" skipped="0">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="120"></testcase>\n\
<testcase classname="test-case-A6" name="unknown" time="2"></testcase>\n\
</testsuite>'
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
failres = Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "unknown",
"duration": 120
})
failres.retries_left = 1
results.append(failres)
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "PASS",
"duration": 120
}))
incres = Result({
"testcase": "test-case-A6",
"verdict": "INCONCLUSIVE",
"reason": "unknown",
"duration": 2
})
incres.retries_left = 1
results.append(incres)
results.append(
Result({
"testcase": "test-case-A6",
"verdict": "PASS",
"duration": 2
}))
junit = ReportJunit(results)
str_report = junit.to_string()
report_xml = ET.fromstring(str_report)
shouldbe_xml = ET.fromstring(str_should_be1)
self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib)
self.assertEqual(len(report_xml.findall("testcase")),
len(shouldbe_xml.findall("testcase")))
str_should_be2 = '<testsuite failures="0" tests="3" errors="1" skipped="0">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="12"></testcase>\n\
<testcase classname="test-case-A6" name="unknown" time="2">\n\
<error message="unknown"></error>\n\
</testcase>\n\
</testsuite>'
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
failres = Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "unknown",
"duration": 120
})
failres.retries_left = 1
results.append(failres)
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "PASS",
"duration": 12
}))
results.append(
Result({
"testcase": "test-case-A6",
"verdict": "INCONCLUSIVE",
"reason": "unknown",
"duration": 2
}))
junit = ReportJunit(results)
str_report = junit.to_string()
report_xml = ET.fromstring(str_report)
shouldbe_xml = ET.fromstring(str_should_be2)
self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib)
self.assertEqual(len(report_xml.findall("testcase")),
len(shouldbe_xml.findall("testcase")))
errors = []
for elem in report_xml.findall("testcase"):
if elem.find("error") is not None:
errors.append(elem)
self.assertEqual(len(errors), 1)
def test_junit_multiple(self):
str_should_be = '<testsuite failures="2" tests="7" errors="1" skipped="1">\n\
<testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\
<testcase classname="test-case-A2" name="unknown" time="50"></testcase>\n\
<testcase classname="test-case-A3" name="unknown" time="120"></testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="120">\n\
<failure message="unknown"></failure>\n\
</testcase>\n\
<testcase classname="test-case-A5" name="unknown" time="1">\n\
<skipped></skipped>\n\
</testcase>\n\
<testcase classname="test-case-A6" name="unknown" time="2">\n\
<error message="unknown"></error>\n\
</testcase>\n\
<testcase classname="test-case-A4" name="unknown" time="1220">\n\
<failure message="WIN blue screen"></failure>\n\
</testcase>\n\
</testsuite>'
shouldbe_xml = ET.fromstring(str_should_be)
results = ResultList()
results.append(
Result({
"testcase": "test-case-A1",
"verdict": "PASS",
"duration": 20
}))
results.append(
Result({
"testcase": "test-case-A2",
"verdict": "PASS",
"duration": 50
}))
results.append(
Result({
"testcase": "test-case-A3",
"verdict": "PASS",
"duration": 120
}))
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "unknown",
"duration": 120
}))
results.append(
Result({
"testcase": "test-case-A5",
"verdict": "SKIP",
"reason": "unknown",
"duration": 1
}))
results.append(
Result({
"testcase": "test-case-A6",
"verdict": "INCONCLUSIVE",
"reason": "unknown",
"duration": 2
}))
results.append(
Result({
"testcase": "test-case-A4",
"verdict": "FAIL",
"reason": "WIN blue screen",
"duration": 1220
}))
junit = ReportJunit(results)
str_report = junit.to_string()
is_xml = ET.fromstring(str_report)
self.assertDictEqual(is_xml.attrib, {
"failures": "2",
"tests": "7",
"errors": "1",
"skipped": "1"
})
self.assertEqual(len(is_xml.findall("testcase")),
len(shouldbe_xml.findall("testcase")))
self.assertEqual(len(is_xml.findall("failure")),
len(shouldbe_xml.findall("failure")))
self.assertEqual(len(is_xml.findall("skipped")),
len(shouldbe_xml.findall("skipped")))
self.assertEqual(len(is_xml.findall("error")),
len(shouldbe_xml.findall("error")))
def test_clean_inconcs(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertTrue(resultlist.clean_inconcs())
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res2.retries_left = 1
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res3.retries_left = 1
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
res4 = Result()
res4.set_verdict("inconclusive", 4, 5)
resultlist.append(res4)
self.assertTrue(resultlist.clean_inconcs())
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="inconclusive", retcode=1, duration=5)
res2.retries_left = 1
res3 = Result()
res3.set_verdict("pass", 4, 5)
res3.retries_left = 0
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertFalse(resultlist.clean_inconcs())
class DummyContainer(TestcaseContainer):
"""
Class DummyContainer
subclasses TestcaseContainer, acts as a dummy object for listing test cases
that were not found when importing test cases.
"""
def __init__(self, logger=None):
"""
Just initialize the super class.
:param logger: logger to use.
"""
super(DummyContainer, self).__init__(logger)
@staticmethod
def find_testcases(modulename,
moduleroot,
tc_meta_schema,
path=None,
suiteconfig=None,
logger=None):
"""
Static method find_testcases. Returns a DummyContainer with attributes collected from
function params.
"""
dc = DummyContainer(logger)
dc.tcname = modulename
dc._modulename = modulename
dc.status = TestStatus.PENDING
dc._instance = None
dc._final_configuration = {}
dc._moduleroot = moduleroot
dc._meta_schema = tc_meta_schema
dc._result = None
dc._filepath = path
dc._suiteconfig = suiteconfig if suiteconfig else {}
return dc
def run(self, forceflash=False):
"""
Just returns the Result object for this Dummy.
"""
return self._result
def get(self, field):
return None
def set_final_config(self):
pass
def set_result_verdict(self, reason):
"""
Sets the inconclusive verdict for this DummyContainer with reason reason.
:param reason: String reason for why this dummy exists.
:return: Nothing
"""
if not self._result:
self._result = Result()
self._result.set_verdict(verdict="inconclusive",
retcode=ReturnCodes.RETCODE_FAIL_INCONCLUSIVE,
duration=0)
self._result.tc_metadata["name"] = self.tcname
self._result.fail_reason = reason
def validate_tc_instance(self):
return True
def test_console_multiple_results_with_retries(self):
saved_stdout = sys.stdout
results = ResultList()
res1 = Result()
res1.set_verdict("fail", 1001, 1)
res1.retries_left = 1
res2 = Result()
res2.set_verdict("pass", 0, 1)
res2.retries_left = 0
results.append(res1)
results.append(res2)
results.append(Result())
results.append(Result())
results.append(Result())
try:
out = StringIO()
sys.stdout = out
report = ReportConsole(results)
report.generate()
output = out.getvalue().strip()
lines = output.split("\n")
self.assertEqual(len(lines), 21)
self.assertRegexpMatches(lines[3], "Yes", lines[4])
self.assertRegexpMatches(lines[4], "No", lines[5])
self.assertRegexpMatches(lines[12], "Final Verdict.*INCONCLUSIVE",
lines[12])
self.assertRegexpMatches(lines[13], "count.*5", lines[13])
self.assertRegexpMatches(lines[14], r"passrate.*50.00 \%",
lines[14])
finally:
sys.stdout = saved_stdout
def test_run(self, mock_tclist):
testsuite = TestSuite(args=self.args_tc)
cont1 = mock.MagicMock()
pass_result = Result()
pass_result.set_verdict('pass', 0, 10)
fail_result = Result()
fail_result.set_verdict('fail', 1000, 10)
skipped_result = Result()
skipped_result.set_verdict('skip', 0, 1)
resultlist = ResultList()
resultlist.append(pass_result)
testsuite._default_configs["retryCount"] = 1
cont1.run.side_effect = [
pass_result, fail_result, skipped_result, KeyboardInterrupt,
fail_result, pass_result
]
cont_reslist = mock.MagicMock()
cont_reslist.run = mock.MagicMock()
cont_reslist.run.return_value = resultlist
# Passing result
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results[0].get_verdict(), "pass")
self.assertTrue(self.args_tc.forceflash)
# ResultList as result
testsuite._testcases = []
testsuite._testcases.append(cont_reslist)
testsuite._results = []
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results[0].get_verdict(), "pass")
# Failing result, no retry
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results[0].get_verdict(), "fail")
# skipped result
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 1)
self.assertEqual(testsuite._results[0].get_verdict(), "skip")
# Interrupt
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._testcases.append(cont2)
testsuite._results = []
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 0)
cont2.run.assert_not_called()
# Failing result, retried
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
testsuite._default_configs["retryReason"] = "includeFailures"
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results[0].get_verdict(), "fail")
self.assertEqual(testsuite._results[1].get_verdict(), "pass")
self.args_tc.repeat = 2
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
cont1.run.side_effect = [
pass_result, pass_result, pass_result, pass_result
]
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results[0].get_verdict(), "pass")
self.assertFalse(self.args_tc.forceflash)
# Failing result, stop_on_failure
self.args_tc.stop_on_failure = True
self.args_tc.repeat = 1
testsuite._default_configs["retryCount"] = 0
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
cont1.run.side_effect = [pass_result]
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
cont2.run.side_effect = [fail_result]
cont3 = mock.MagicMock()
cont3.run = mock.MagicMock()
cont3.run.side_effect = [pass_result]
testsuite._testcases.append(cont2)
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results[0].get_verdict(), "pass")
self.assertEqual(testsuite._results[1].get_verdict(), "fail")
# Skipped result, stop_on_failure
self.args_tc.stop_on_failure = True
self.args_tc.repeat = 0
testsuite._testcases = []
testsuite._testcases.append(cont1)
testsuite._results = []
cont1.run.side_effect = [skipped_result]
cont2 = mock.MagicMock()
cont2.run = mock.MagicMock()
cont2.run.side_effect = [pass_result]
testsuite._testcases.append(cont2)
testsuite.run()
self.assertEqual(testsuite.status, TestStatus.FINISHED)
self.assertEqual(len(testsuite._results), 2)
self.assertEqual(testsuite._results[0].get_verdict(), "skip")
self.assertEqual(testsuite._results[1].get_verdict(), "pass")
def test_pass_rate(self):
dictionary = {"retcode": 0}
res = Result(kwargs=dictionary)
res.set_verdict(verdict="pass", retcode=0, duration=0)
dictionary = {"retcode": 1}
res2 = Result(kwargs=dictionary)
res2.set_verdict(verdict="fail", retcode=1, duration=5)
res3 = Result()
res3.set_verdict("inconclusive", 4, 5)
res4 = Result(kwargs=dictionary)
res4.set_verdict(verdict="skip", retcode=1, duration=5)
resultlist = ResultList()
resultlist.append(res)
resultlist.append(res2)
resultlist.append(res3)
self.assertEquals(resultlist.pass_rate(), "50.00 %")
self.assertEquals(resultlist.pass_rate(include_inconclusive=True),
"33.33 %")
self.assertEquals(resultlist.pass_rate(include_skips=True), "50.00 %")
resultlist.append(res4)
self.assertEquals(
resultlist.pass_rate(include_skips=True,
include_inconclusive=True), "25.00 %")
def test_cloudresult_full(self):
test_res = {
"testcase": "tc_name",
"verdict": "fail",
"reason": "ohnou",
"retcode": 0,
"duration": 1,
"fw_name": "Icetea",
"fw_version": "0.10.2"
}
result = Result(test_res)
class DummyBuild(object):
@property
def branch(self):
return 'master'
@property
def commit_id(self):
return '1234'
@property
def build_url(self):
return 'url'
@property
def giturl(self):
return 'url'
@property
def date(self):
return "22.22.2222"
@property
def sha1(self):
return "asdv"
class DummyDut(object):
@property
def resource_id(self):
return '123'
@property
def platform(self):
return 'K64F'
@property
def build(self):
return DummyBuild()
result.set_dutinformation([DummyDut()])
res = create_result_object(result)
compare = {
'exec': {
'verdict': 'pass',
'note': 'ohnou',
'duration': 1,
'env': {
'framework': {
'ver': '0.10.2',
'name': 'Icetea'
}
},
'sut': {
'commitId': '1234',
'buildUrl': 'url',
'gitUrl': 'url',
'branch': 'master',
"buildDate": "22.22.2222",
"buildSha1": "asdv"
},
'dut': {
'sn': '123',
'model': 'K64F'
}
},
'tcid': 'tc_name'
}
self.assertDictContainsSubset(compare, res)
def append_result(self, tc_file=None):
"""
Append a new fully constructed Result to the internal ResultList.
:param tc_file: Test case file path
:return: Nothing
"""
result = Result()
result.set_tc_metadata(self._configuration.config)
tc_rev = get_git_info(self._configuration.get_tc_abspath(tc_file),
verbose=self._args.verbose)
if self._logger:
self._logger.debug(tc_rev)
result.set_tc_git_info(tc_rev)
result.component = self._configuration.get_test_component()
result.feature = self._configuration.get_features_under_test()
result.skip_reason = self._configuration.skip_reason() if self._configuration.skip() else ''
result.fail_reason = self._failreason
result.logpath = os.path.abspath(LogManager.get_base_dir())
result.logfiles = LogManager.get_logfiles()
result.retcode = self.retcode
result.set_dutinformation(self._resources.dutinformations)
# pylint: disable=unused-variable
for platform, serialnumber in zip(self._resources.get_platforms(),
self._resources.get_serialnumbers()):
# Zipping done to keep platforms and serial numbers aligned in case some sn:s are
# missing
result.dut_vendor.append('')
result.dut_resource_id.append(serialnumber)
result.dut_count = self._resources.get_dut_count()
result.duts = self._resources.resource_configuration.get_dut_configuration()
if self._resources.resource_configuration.count_hardware() > 0:
result.dut_type = 'hw'
elif self._resources.resource_configuration.count_process() > 0:
result.dut_type = 'process'
else:
result.dut_type = None
self._result_list.append(result)
