def test_haslogs(self): result = Result() result.logpath = os.path.join(os.path.dirname(__file__), "test_logpath") files = result.has_logs() self.assertTrue(files) self.assertEqual(len(files), 2) result = Result() result.logpath = None files = result.has_logs() self.assertListEqual(files, [])
def test_get_verdict(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) reslist = ResultList() reslist.append(res) self.assertEquals(reslist.get_verdict(), "pass") dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=10) reslist.append(res2) self.assertEquals(reslist.get_verdict(), "fail") res3 = Result() res3.set_verdict("inconclusive", 4, 1) reslist = ResultList() reslist.append(res3) self.assertEquals(reslist.get_verdict(), "inconclusive") reslist.append(res2) self.assertEquals(reslist.get_verdict(), "fail")
def test_reportconsole_multiple_results(self): # pylint: disable=invalid-name saved_stdout = sys.stdout results = ResultList() results.append(Result()) results.append(Result()) results.append(Result()) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 17) self.assertRegexpMatches(lines[10], r"Final Verdict.*INCONCLUSIVE", lines[10]) self.assertRegexpMatches(lines[11], r"count.*3", lines[11]) self.assertRegexpMatches(lines[12], r"passrate.*0.00 \%", lines[12]) finally: sys.stdout = saved_stdout
def test_init(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) self.assertEqual(res.get_verdict(), "pass") dictionary = {"retcode": 1} res = Result(kwargs=dictionary) self.assertEqual(res.get_verdict(), "fail")
def test_get_summary(self): expected = { "count": 3, "pass": 1, "fail": 1, "skip": 0, "inconclusive": 1, "retries": 1, "duration": 10 } dictionary = {"retcode": 0} res = Result(kwargs=dictionary) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertDictEqual(resultlist.get_summary(), expected)
def test_reportconsole_skip(self): saved_stdout = sys.stdout results = ResultList() res = Result() res.skip_reason = "Skip_reason" res.set_verdict("skip", -1, -1) results.append(res) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 15) self.assertRegexpMatches(lines[3], r"skip.*Skip_reason") self.assertRegexpMatches(lines[8], r"Final Verdict.*PASS", lines[8]) self.assertRegexpMatches(lines[9], r"count.*1", lines[9]) self.assertRegexpMatches(lines[11], r"passrate.*0.00 \%", lines[10]) self.assertRegexpMatches(lines[12], r"skip.*1", lines[10]) finally: sys.stdout = saved_stdout
def test_reportconsole_decodefail(self): saved_stdout = sys.stdout failing_message = "\x00\x00\x00\x00\x00\x00\x01\xc8" results = ResultList() res = Result() res.set_verdict("fail", 1001, 0) res.fail_reason = failing_message results.append(res) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 15) self.assertRegexpMatches(lines[8], r"Final Verdict.*FAIL", lines[8]) self.assertRegexpMatches(lines[9], r"count.*1", lines[9]) self.assertRegexpMatches(lines[10], r"passrate.*0.00 \%", lines[10]) finally: sys.stdout = saved_stdout
def fail_testcase(self, error): """ Workaround that overwrites icetea's result value in order to store the test binary name in the results. Without this workaround the testcase name would be "pal_test_runner" which is not descriptive. :param error: Python error that failed the test case """ # Overwrite test case result self._results = ResultList() self._results.append( Result({ "testcase": (self.config["requirements"] ["duts"]["*"]["application"]["bin"]), "reason": repr(error), "verdict": "FAIL" }) )
def test_reportconsole_one_results(self): saved_stdout = sys.stdout results = ResultList() results.append(Result()) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 15) self.assertRegexpMatches(lines[8], r"Final Verdict.*INCONCLUSIVE", lines[8]) self.assertRegexpMatches(lines[9], r"count.*1", lines[9]) self.assertRegexpMatches(lines[10], r"passrate.*0.00 \%", lines[10]) finally: sys.stdout = saved_stdout
def parse_unity_result(self, dut): """ Parses given DUTs stdout for Unity test results. :param dut: icetea DUT to inspect the results from """ re_unity_test = re.compile( r".*\<\*\*\*UnityTest\*\*\*\>TEST\((?P<suite>.*?), (?P<test>.*?)\)\<\/\*\*\*UnityTest\*\*\*\>") # noqa: E501 # pylint: disable=line-too-long re_unity_result = re.compile( r".*\<\*\*\*UnityResult\*\*\*\>(?P<result>.*?)\<\/\*\*\*UnityResult\*\*\*\>") # noqa: E501 # pylint: disable=line-too-long test_output = "" test_active = False test_results = ResultList() for line in dut.traces: line = line.strip() # Activate test output logging match = re_unity_test.match(line) if match: test_active = True test_output = "" unity_name = match.group("test") unity_suite = match.group("suite") self.logger.info("parsing %s.%s", unity_suite, unity_name) # Log test output if test_active: test_output += line + "\n" # Check if test is over match = re_unity_result.match(line) if match and test_active: unity_result = match.group("result") # Create icetea Result() test_result = Result( { "testcase": unity_suite + "." + unity_name, "stdout": test_output, "reason": line if unity_result == "FAIL" else "" }) test_result.build_result_metadata({"toolchain": unity_name}) # Would need to do runtime analysis to get duration test_result.set_verdict(unity_result, 0, 0.0) test_results.append(test_result) test_active = False self.logger.info("finished %s", unity_name) return test_results
def test_result_metainfo_generation(self): pass_result = Result() pass_result.set_verdict('pass', 0, 10) dinfo = DutInformation("Test_platform", "123456", "1") dinfo.build = Build(ref="test_file", type="file") pass_result.add_dutinformation(dinfo) self.args_tc.branch = "test_branch" self.args_tc.commitId = "123456" self.args_tc.gitUrl = "url" self.args_tc.buildUrl = "url2" self.args_tc.campaign = "campaign" self.args_tc.jobId = "test_job" self.args_tc.toolchain = "toolchain" self.args_tc.buildDate = "today" pass_result.build_result_metadata(args=self.args_tc) self.assertEqual(pass_result.build_branch, "test_branch") self.assertEqual(pass_result.buildcommit, "123456") self.assertEqual(pass_result.build_git_url, "url") self.assertEqual(pass_result.build_url, "url2") self.assertEqual(pass_result.campaign, "campaign") self.assertEqual(pass_result.job_id, "test_job") self.assertEqual(pass_result.toolchain, "toolchain") self.assertEqual(pass_result.build_date, "today")
def test_junit_default(self): str_should_be = '<testsuite failures="0" tests="1" errors="0" skipped="0">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ </testsuite>' results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) junit = ReportJunit(results) str_report = junit.to_string() xml_shouldbe = ET.fromstring(str_should_be) report_xml = ET.fromstring(str_report) self.assertDictEqual(xml_shouldbe.attrib, report_xml.attrib) self.assertDictEqual( xml_shouldbe.find("testcase").attrib, report_xml.find("testcase").attrib) self.assertEqual(len(xml_shouldbe.findall("testcase")), len(report_xml.findall("testcase")))
def get_result(self): return Result()
def test_run(self, mock_tclist): # pylint: disable=too-many-statements testsuite = TestSuite(args=self.args_tc) cont1 = mock.MagicMock() pass_result = Result() pass_result.set_verdict('pass', 0, 10) fail_result = Result() fail_result.set_verdict('fail', 1000, 10) skipped_result = Result() skipped_result.set_verdict('skip', 0, 1) resultlist = ResultList() resultlist.append(pass_result) resultlist.save = mock.MagicMock() testsuite._default_configs["retryCount"] = 1 cont1.run.side_effect = [pass_result, fail_result, skipped_result, KeyboardInterrupt, fail_result, pass_result] cont_reslist = mock.MagicMock() cont_reslist.run = mock.MagicMock() cont_reslist.run.return_value = resultlist # Passing result testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results.get(0).get_verdict(), "pass") self.assertTrue(self.args_tc.forceflash) # pylint: disable=no-member self.assertEquals(testsuite._results.save.call_count, 1) # ResultList as result testsuite._testcases = [] testsuite._testcases.append(cont_reslist) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results.get(0).get_verdict(), "pass") # Failing result, no retry testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results.get(0).get_verdict(), "fail") # skipped result testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results.get(0).get_verdict(), "skip") # Interrupt cont2 = mock.MagicMock() cont2.run = mock.MagicMock() testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._testcases.append(cont2) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 0) cont2.run.assert_not_called() # Failing result, retried testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite._default_configs["retryReason"] = "includeFailures" testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "fail") self.assertEqual(testsuite._results.get(1).get_verdict(), "pass") # TC not exist result, retried testsuite._testcases = [] contx = mock.MagicMock() inconc_res = Result() inconc_res.set_verdict("inconclusive", 1015, 0) contx.run = mock.MagicMock(return_value=inconc_res) testsuite._testcases.append(contx) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite._default_configs["retryReason"] = "includeFailures" testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results.get(0).get_verdict(), "inconclusive") # Failing result, retried, from a result list. testsuite._testcases = [] fail_result_2 = Result() fail_result_2.set_verdict('fail', 1000, 10) reslist = ResultList() reslist.append(fail_result_2) cont_retry = mock.MagicMock() cont_retry.run.side_effect = [reslist, resultlist] testsuite._testcases.append(cont_retry) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite._default_configs["retryReason"] = "includeFailures" testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "fail") self.assertEqual(testsuite._results.get(1).get_verdict(), "pass") self.assertEqual(testsuite._results.get(0).retries_left, 1) self.assertEqual(testsuite._results.get(1).retries_left, 0) # Inconclusive result, retried, from a result list. testsuite._testcases = [] fail_result_3 = Result() fail_result_3.set_verdict('inconclusive', 1000, 10) reslist = ResultList() reslist.append(fail_result_3) cont_retry = mock.MagicMock() cont_retry.run.side_effect = [reslist, resultlist] testsuite._testcases.append(cont_retry) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() testsuite._default_configs["retryReason"] = "includeFailures" testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "inconclusive") self.assertEqual(testsuite._results.get(1).get_verdict(), "pass") self.assertEqual(testsuite._results.get(0).retries_left, 1) self.assertEqual(testsuite._results.get(1).retries_left, 0) self.args_tc.repeat = 2 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() cont1.run.side_effect = [pass_result, pass_result, pass_result, pass_result] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "pass") self.assertFalse(self.args_tc.forceflash) # pylint: disable=no-member # Failing result, stop_on_failure self.args_tc.stop_on_failure = True self.args_tc.repeat = 1 testsuite._default_configs["retryCount"] = 0 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() cont1.run.side_effect = [pass_result] cont2 = mock.MagicMock() cont2.run = mock.MagicMock() cont2.run.side_effect = [fail_result] cont3 = mock.MagicMock() cont3.run = mock.MagicMock() cont3.run.side_effect = [pass_result] testsuite._testcases.append(cont2) testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "pass") self.assertEqual(testsuite._results.get(1).get_verdict(), "fail") self.assertEquals(testsuite._results.save.call_count, 2) # Skipped result, stop_on_failure self.args_tc.stop_on_failure = True self.args_tc.repeat = 0 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = ResultList() testsuite._results.save = mock.MagicMock() cont1.run.side_effect = [skipped_result] cont2 = mock.MagicMock() cont2.run = mock.MagicMock() cont2.run.side_effect = [pass_result] testsuite._testcases.append(cont2) testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results.get(0).get_verdict(), "skip") self.assertEqual(testsuite._results.get(1).get_verdict(), "pass") self.assertEquals(testsuite._results.save.call_count, 2)
def test_retries_count(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertEqual(resultlist.retry_count(), 1) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertEqual(resultlist.retry_count(), 1)
def test_junit_hides(self): str_should_be1 = '<testsuite failures="0" tests="3" errors="0" skipped="0">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="120"></testcase>\n\ <testcase classname="test-case-A6" name="unknown" time="2"></testcase>\n\ </testsuite>' results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) failres = Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "unknown", "duration": 120 }) failres.retries_left = 1 results.append(failres) results.append( Result({ "testcase": "test-case-A4", "verdict": "PASS", "duration": 120 })) incres = Result({ "testcase": "test-case-A6", "verdict": "INCONCLUSIVE", "reason": "unknown", "duration": 2 }) incres.retries_left = 1 results.append(incres) results.append( Result({ "testcase": "test-case-A6", "verdict": "PASS", "duration": 2 })) junit = ReportJunit(results) str_report = junit.to_string() report_xml = ET.fromstring(str_report) shouldbe_xml = ET.fromstring(str_should_be1) self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib) self.assertEqual(len(report_xml.findall("testcase")), len(shouldbe_xml.findall("testcase"))) str_should_be2 = '<testsuite failures="0" tests="3" errors="1" skipped="0">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="12"></testcase>\n\ <testcase classname="test-case-A6" name="unknown" time="2">\n\ <error message="unknown"></error>\n\ </testcase>\n\ </testsuite>' results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) failres = Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "unknown", "duration": 120 }) failres.retries_left = 1 results.append(failres) results.append( Result({ "testcase": "test-case-A4", "verdict": "PASS", "duration": 12 })) results.append( Result({ "testcase": "test-case-A6", "verdict": "INCONCLUSIVE", "reason": "unknown", "duration": 2 })) junit = ReportJunit(results) str_report = junit.to_string() report_xml = ET.fromstring(str_report) shouldbe_xml = ET.fromstring(str_should_be2) self.assertDictEqual(report_xml.attrib, shouldbe_xml.attrib) self.assertEqual(len(report_xml.findall("testcase")), len(shouldbe_xml.findall("testcase"))) errors = [] for elem in report_xml.findall("testcase"): if elem.find("error") is not None: errors.append(elem) self.assertEqual(len(errors), 1)
def test_junit_multiple(self): str_should_be = '<testsuite failures="2" tests="7" errors="1" skipped="1">\n\ <testcase classname="test-case-A1" name="unknown" time="20"></testcase>\n\ <testcase classname="test-case-A2" name="unknown" time="50"></testcase>\n\ <testcase classname="test-case-A3" name="unknown" time="120"></testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="120">\n\ <failure message="unknown"></failure>\n\ </testcase>\n\ <testcase classname="test-case-A5" name="unknown" time="1">\n\ <skipped></skipped>\n\ </testcase>\n\ <testcase classname="test-case-A6" name="unknown" time="2">\n\ <error message="unknown"></error>\n\ </testcase>\n\ <testcase classname="test-case-A4" name="unknown" time="1220">\n\ <failure message="WIN blue screen"></failure>\n\ </testcase>\n\ </testsuite>' shouldbe_xml = ET.fromstring(str_should_be) results = ResultList() results.append( Result({ "testcase": "test-case-A1", "verdict": "PASS", "duration": 20 })) results.append( Result({ "testcase": "test-case-A2", "verdict": "PASS", "duration": 50 })) results.append( Result({ "testcase": "test-case-A3", "verdict": "PASS", "duration": 120 })) results.append( Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "unknown", "duration": 120 })) results.append( Result({ "testcase": "test-case-A5", "verdict": "SKIP", "reason": "unknown", "duration": 1 })) results.append( Result({ "testcase": "test-case-A6", "verdict": "INCONCLUSIVE", "reason": "unknown", "duration": 2 })) results.append( Result({ "testcase": "test-case-A4", "verdict": "FAIL", "reason": "WIN blue screen", "duration": 1220 })) junit = ReportJunit(results) str_report = junit.to_string() is_xml = ET.fromstring(str_report) self.assertDictEqual(is_xml.attrib, { "failures": "2", "tests": "7", "errors": "1", "skipped": "1" }) self.assertEqual(len(is_xml.findall("testcase")), len(shouldbe_xml.findall("testcase"))) self.assertEqual(len(is_xml.findall("failure")), len(shouldbe_xml.findall("failure"))) self.assertEqual(len(is_xml.findall("skipped")), len(shouldbe_xml.findall("skipped"))) self.assertEqual(len(is_xml.findall("error")), len(shouldbe_xml.findall("error")))
def test_clean_inconcs(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertTrue(resultlist.clean_inconcs()) dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res2.retries_left = 1 res3 = Result() res3.set_verdict("inconclusive", 4, 5) res3.retries_left = 1 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) res4 = Result() res4.set_verdict("inconclusive", 4, 5) resultlist.append(res4) self.assertTrue(resultlist.clean_inconcs()) dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="inconclusive", retcode=1, duration=5) res2.retries_left = 1 res3 = Result() res3.set_verdict("pass", 4, 5) res3.retries_left = 0 resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertFalse(resultlist.clean_inconcs())
class DummyContainer(TestcaseContainer): """ Class DummyContainer subclasses TestcaseContainer, acts as a dummy object for listing test cases that were not found when importing test cases. """ def __init__(self, logger=None): """ Just initialize the super class. :param logger: logger to use. """ super(DummyContainer, self).__init__(logger) @staticmethod def find_testcases(modulename, moduleroot, tc_meta_schema, path=None, suiteconfig=None, logger=None): """ Static method find_testcases. Returns a DummyContainer with attributes collected from function params. """ dc = DummyContainer(logger) dc.tcname = modulename dc._modulename = modulename dc.status = TestStatus.PENDING dc._instance = None dc._final_configuration = {} dc._moduleroot = moduleroot dc._meta_schema = tc_meta_schema dc._result = None dc._filepath = path dc._suiteconfig = suiteconfig if suiteconfig else {} return dc def run(self, forceflash=False): """ Just returns the Result object for this Dummy. """ return self._result def get(self, field): return None def set_final_config(self): pass def set_result_verdict(self, reason): """ Sets the inconclusive verdict for this DummyContainer with reason reason. :param reason: String reason for why this dummy exists. :return: Nothing """ if not self._result: self._result = Result() self._result.set_verdict(verdict="inconclusive", retcode=ReturnCodes.RETCODE_FAIL_INCONCLUSIVE, duration=0) self._result.tc_metadata["name"] = self.tcname self._result.fail_reason = reason def validate_tc_instance(self): return True
def test_console_multiple_results_with_retries(self): saved_stdout = sys.stdout results = ResultList() res1 = Result() res1.set_verdict("fail", 1001, 1) res1.retries_left = 1 res2 = Result() res2.set_verdict("pass", 0, 1) res2.retries_left = 0 results.append(res1) results.append(res2) results.append(Result()) results.append(Result()) results.append(Result()) try: out = StringIO() sys.stdout = out report = ReportConsole(results) report.generate() output = out.getvalue().strip() lines = output.split("\n") self.assertEqual(len(lines), 21) self.assertRegexpMatches(lines[3], "Yes", lines[4]) self.assertRegexpMatches(lines[4], "No", lines[5]) self.assertRegexpMatches(lines[12], "Final Verdict.*INCONCLUSIVE", lines[12]) self.assertRegexpMatches(lines[13], "count.*5", lines[13]) self.assertRegexpMatches(lines[14], r"passrate.*50.00 \%", lines[14]) finally: sys.stdout = saved_stdout
def test_run(self, mock_tclist): testsuite = TestSuite(args=self.args_tc) cont1 = mock.MagicMock() pass_result = Result() pass_result.set_verdict('pass', 0, 10) fail_result = Result() fail_result.set_verdict('fail', 1000, 10) skipped_result = Result() skipped_result.set_verdict('skip', 0, 1) resultlist = ResultList() resultlist.append(pass_result) testsuite._default_configs["retryCount"] = 1 cont1.run.side_effect = [ pass_result, fail_result, skipped_result, KeyboardInterrupt, fail_result, pass_result ] cont_reslist = mock.MagicMock() cont_reslist.run = mock.MagicMock() cont_reslist.run.return_value = resultlist # Passing result testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results[0].get_verdict(), "pass") self.assertTrue(self.args_tc.forceflash) # ResultList as result testsuite._testcases = [] testsuite._testcases.append(cont_reslist) testsuite._results = [] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results[0].get_verdict(), "pass") # Failing result, no retry testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results[0].get_verdict(), "fail") # skipped result testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 1) self.assertEqual(testsuite._results[0].get_verdict(), "skip") # Interrupt cont2 = mock.MagicMock() cont2.run = mock.MagicMock() testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._testcases.append(cont2) testsuite._results = [] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 0) cont2.run.assert_not_called() # Failing result, retried testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] testsuite._default_configs["retryReason"] = "includeFailures" testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results[0].get_verdict(), "fail") self.assertEqual(testsuite._results[1].get_verdict(), "pass") self.args_tc.repeat = 2 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] cont1.run.side_effect = [ pass_result, pass_result, pass_result, pass_result ] testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results[0].get_verdict(), "pass") self.assertFalse(self.args_tc.forceflash) # Failing result, stop_on_failure self.args_tc.stop_on_failure = True self.args_tc.repeat = 1 testsuite._default_configs["retryCount"] = 0 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] cont1.run.side_effect = [pass_result] cont2 = mock.MagicMock() cont2.run = mock.MagicMock() cont2.run.side_effect = [fail_result] cont3 = mock.MagicMock() cont3.run = mock.MagicMock() cont3.run.side_effect = [pass_result] testsuite._testcases.append(cont2) testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results[0].get_verdict(), "pass") self.assertEqual(testsuite._results[1].get_verdict(), "fail") # Skipped result, stop_on_failure self.args_tc.stop_on_failure = True self.args_tc.repeat = 0 testsuite._testcases = [] testsuite._testcases.append(cont1) testsuite._results = [] cont1.run.side_effect = [skipped_result] cont2 = mock.MagicMock() cont2.run = mock.MagicMock() cont2.run.side_effect = [pass_result] testsuite._testcases.append(cont2) testsuite.run() self.assertEqual(testsuite.status, TestStatus.FINISHED) self.assertEqual(len(testsuite._results), 2) self.assertEqual(testsuite._results[0].get_verdict(), "skip") self.assertEqual(testsuite._results[1].get_verdict(), "pass")
def test_pass_rate(self): dictionary = {"retcode": 0} res = Result(kwargs=dictionary) res.set_verdict(verdict="pass", retcode=0, duration=0) dictionary = {"retcode": 1} res2 = Result(kwargs=dictionary) res2.set_verdict(verdict="fail", retcode=1, duration=5) res3 = Result() res3.set_verdict("inconclusive", 4, 5) res4 = Result(kwargs=dictionary) res4.set_verdict(verdict="skip", retcode=1, duration=5) resultlist = ResultList() resultlist.append(res) resultlist.append(res2) resultlist.append(res3) self.assertEquals(resultlist.pass_rate(), "50.00 %") self.assertEquals(resultlist.pass_rate(include_inconclusive=True), "33.33 %") self.assertEquals(resultlist.pass_rate(include_skips=True), "50.00 %") resultlist.append(res4) self.assertEquals( resultlist.pass_rate(include_skips=True, include_inconclusive=True), "25.00 %")
def test_cloudresult_full(self): test_res = { "testcase": "tc_name", "verdict": "fail", "reason": "ohnou", "retcode": 0, "duration": 1, "fw_name": "Icetea", "fw_version": "0.10.2" } result = Result(test_res) class DummyBuild(object): @property def branch(self): return 'master' @property def commit_id(self): return '1234' @property def build_url(self): return 'url' @property def giturl(self): return 'url' @property def date(self): return "22.22.2222" @property def sha1(self): return "asdv" class DummyDut(object): @property def resource_id(self): return '123' @property def platform(self): return 'K64F' @property def build(self): return DummyBuild() result.set_dutinformation([DummyDut()]) res = create_result_object(result) compare = { 'exec': { 'verdict': 'pass', 'note': 'ohnou', 'duration': 1, 'env': { 'framework': { 'ver': '0.10.2', 'name': 'Icetea' } }, 'sut': { 'commitId': '1234', 'buildUrl': 'url', 'gitUrl': 'url', 'branch': 'master', "buildDate": "22.22.2222", "buildSha1": "asdv" }, 'dut': { 'sn': '123', 'model': 'K64F' } }, 'tcid': 'tc_name' } self.assertDictContainsSubset(compare, res)
def append_result(self, tc_file=None): """ Append a new fully constructed Result to the internal ResultList. :param tc_file: Test case file path :return: Nothing """ result = Result() result.set_tc_metadata(self._configuration.config) tc_rev = get_git_info(self._configuration.get_tc_abspath(tc_file), verbose=self._args.verbose) if self._logger: self._logger.debug(tc_rev) result.set_tc_git_info(tc_rev) result.component = self._configuration.get_test_component() result.feature = self._configuration.get_features_under_test() result.skip_reason = self._configuration.skip_reason() if self._configuration.skip() else '' result.fail_reason = self._failreason result.logpath = os.path.abspath(LogManager.get_base_dir()) result.logfiles = LogManager.get_logfiles() result.retcode = self.retcode result.set_dutinformation(self._resources.dutinformations) # pylint: disable=unused-variable for platform, serialnumber in zip(self._resources.get_platforms(), self._resources.get_serialnumbers()): # Zipping done to keep platforms and serial numbers aligned in case some sn:s are # missing result.dut_vendor.append('') result.dut_resource_id.append(serialnumber) result.dut_count = self._resources.get_dut_count() result.duts = self._resources.resource_configuration.get_dut_configuration() if self._resources.resource_configuration.count_hardware() > 0: result.dut_type = 'hw' elif self._resources.resource_configuration.count_process() > 0: result.dut_type = 'process' else: result.dut_type = None self._result_list.append(result)