def test_printErrorsStdout(self): """ printErrors() prints out the captured stdout. """ self.args.verbose = 1 self.args.termcolor = False gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test spit out to stdout' gtr.recordStdout(pt, output) gtr.addSuccess(pt) gtr.printErrors() self.assertIn(output, self.stream.getvalue())
def test_printErrorsStdout(self): """ printErrors() prints out the captured stdout. """ self.args.verbose = 1 self.args.termcolor = False gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test spit out to stdout' gtr.recordStdout(pt, output) gtr.addSuccess(pt) gtr.printErrors() self.assertIn(output, self.stream.getvalue())
def test_printErrorsStdoutQuietStdoutOnSuccess(self): """ printErrors() prints out the captured stdout except when quiet_stdout is set to True for successful tests. """ self.args.quiet_stdout = True gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test should not spit out to stdout' gtr.recordStdout(pt, output) gtr.addSuccess(pt) gtr.printErrors() self.assertNotIn(output, self.stream.getvalue())
def test_printErrorsStdoutQuietStdoutOnSuccess(self): """ printErrors() prints out the captured stdout except when quiet_stdout is set to True for successful tests. """ self.args.quiet_stdout = True gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test should not spit out to stdout' gtr.recordStdout(pt, output) gtr.addSuccess(pt) gtr.printErrors() self.assertNotIn(output, self.stream.getvalue())
def test_tryRecordingStdoutStderr(self): """ Recording stdout and stderr works correctly. """ gtr = GreenTestResult(self.args, GreenStream(self.stream)) gtr.recordStdout = MagicMock() gtr.recordStderr = MagicMock() output = 'apple' test1 = MagicMock() ptr1 = MagicMock() ptr1.stdout_output = {test1:output} ptr1.stderr_errput = {} errput = 'banana' test2 = MagicMock() ptr2 = MagicMock() ptr2.stdout_output = {} ptr2.stderr_errput = {test2:errput} gtr.tryRecordingStdoutStderr(test1, ptr1) gtr.recordStdout.assert_called_with(test1, output) gtr.tryRecordingStdoutStderr(test2, ptr2) gtr.recordStderr.assert_called_with(test2, errput)
def test_tryRecordingStdoutStderr_SubTest(self): """ Recording stdout and stderr works correctly for failed/errored SubTests. """ gtr = GreenTestResult(self.args, GreenStream(self.stream)) gtr.recordStdout = MagicMock() gtr.recordStderr = MagicMock() output = "apple" test1 = MagicMock() test1.dotted_name = "test 1" subtest1 = MagicMock() subtest1.dotted_name = "test 1: the subtest" subtest1.class_name = "SubTest" ptr1 = MagicMock() ptr1.stdout_output = {test1: output} ptr1.stderr_errput = {} errput = "banana" test2 = MagicMock() test2.dotted_name = "test 2" subtest2 = MagicMock() subtest2.dotted_name = "test 2: subtests are annoying" subtest2.class_name = "SubTest" ptr2 = MagicMock() ptr2.stdout_output = {} ptr2.stderr_errput = {test2: errput} gtr.tryRecordingStdoutStderr(subtest1, ptr1, err=True) gtr.recordStdout.assert_called_with(subtest1, output) gtr.tryRecordingStdoutStderr(subtest2, ptr2, err=True) gtr.recordStderr.assert_called_with(subtest2, errput)
def test_printErrorsStdoutQuietStdoutOnError(self): """ printErrors() prints out the captured stdout except when quiet_stdout is set to True for successful tests, but here we are on a failing test. """ self.args.quiet_stdout = True try: raise Exception except: err = sys.exc_info() gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test should spit out to stdout' gtr.recordStdout(pt, output) gtr.addError(pt, proto_error(err)) gtr.printErrors() self.assertIn(output, self.stream.getvalue())
def test_printErrorsStdoutQuietStdoutOnError(self): """ printErrors() prints out the captured stdout except when quiet_stdout is set to True for successful tests, but here we are on a failling test. """ self.args.quiet_stdout = True try: raise Exception except: err = sys.exc_info() gtr = GreenTestResult(self.args, GreenStream(self.stream)) pt = MyProtoTest() output = 'this is what the test should spit out to stdout' gtr.recordStdout(pt, output) gtr.addError(pt, proto_error(err)) gtr.printErrors() self.assertIn(output, self.stream.getvalue())
class JUnitXMLReportIsGenerated(TestCase): def setUp(self): self._destination = StringIO() self._test_results = GreenTestResult(default_args, GreenStream(StringIO())) self._adapter = JUnitXML() self._test = ProtoTest() self._test.module = "my_module" self._test.class_name = "MyClass" self._test.method_name = "my_method" def test_when_the_results_contain_only_one_successful_test(self): self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.PASSED } } } }) def test_when_the_results_contain_tests_with_various_verdict(self): self._test_results.addSuccess( test("my.module", "MyClass", "test_method1")) self._test_results.addSuccess( test("my.module", "MyClass", "test_method2")) self._record_failure(test("my.module", "MyClass", "test_method3")) self._record_failure(test("my.module", "MyClass", "test_method4")) self._record_error(test("my.module", "MyClass", "test_method5")) self._test_results.addSkip( test("my.module", "MyClass", "test_method6"), "Take too long") self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my.module.MyClass": { "#tests": "6", "#failures": "2", "#errors": "1", "#skipped": "1", "tests": { "test_method1": { "verdict": Verdict.PASSED }, "test_method2": { "verdict": Verdict.PASSED }, "test_method3": { "verdict": Verdict.FAILED }, "test_method4": { "verdict": Verdict.FAILED }, "test_method5": { "verdict": Verdict.ERROR }, "test_method6": { "verdict": Verdict.SKIPPED } } }, }) def _record_failure(self, test): try: raise ValueError("Wrong value") except: error = proto_error(exc_info()) self._test_results.addFailure(test, error) def _record_error(self, test): try: raise ValueError("Wrong value") except: error = proto_error(exc_info()) self._test_results.addError(test, error) def test_when_the_results_contain_only_one_test_with_output(self): output = "This is the output of the test" self._test_results.recordStdout(self._test, output) self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.PASSED, "stdout": output } } } }) def test_when_the_results_contain_only_one_test_with_errput(self): errput = "This is the errput of the test" self._test_results.recordStderr(self._test, errput) self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.PASSED, "stderr": errput } } } }) def test_when_the_results_contain_only_one_failed_test(self): self._record_failure(test("my_module", "MyClass", "my_method")) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.FAILED } } } }) def test_when_the_results_contain_only_one_erroneous_test(self): self._record_error(test("my_module", "MyClass", "my_method")) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.ERROR } } } }) def test_when_the_results_contain_only_one_skipped_test(self): self._test_results.addSkip(self._test, "reason for skipping") self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.SKIPPED } } } }) def _assert_report_is(self, report): """ Verify the structure of the generated XML text against the given 'report' structure. """ root = from_xml(self._destination.getvalue()) test_suites = root.findall(JUnitDialect.TEST_SUITE) self.assertEqual(len(report), len(test_suites)) for each_suite in test_suites: self._assert_suite(report, each_suite) def _assert_suite(self, expected_report, suite): """ Verify that the given 'suite' matches one in the expected test report. """ name = suite.get(JUnitDialect.NAME) self.assertIsNotNone(name) self.assertIn(name, expected_report) expected_suite = expected_report[name] # Check the count of tests if "#tests" in expected_suite: self.assertEqual(expected_suite["#tests"], suite.get(JUnitDialect.TEST_COUNT)) # Check the count of failures if "#failures" in expected_suite: self.assertEqual(expected_suite["#failures"], suite.get(JUnitDialect.FAILURE_COUNT)) # Check the count of errors if "#errors" in expected_suite: self.assertEqual(expected_suite["#errors"], suite.get(JUnitDialect.ERROR_COUNT)) # Check the count of skipped tests if "#skipped" in expected_suite: self.assertEqual(expected_suite["#skipped"], suite.get(JUnitDialect.SKIPPED_COUNT)) # Check individual test reports self.assertEqual(len(expected_suite["tests"]), len(suite)) for each_test in suite: self._assert_test(expected_suite["tests"], each_test) def _assert_test(self, expected_suite, test): """ Verify that the given 'test' matches one in the expected test suite. """ name = test.get(JUnitDialect.NAME) self.assertIsNotNone(test) self.assertIn(name, expected_suite) expected_test = expected_suite[name] test_passed = True for key, expected in expected_test.items(): if key == "verdict": self._assert_verdict(expected, test) elif key == "stdout": system_out = test.find(JUnitDialect.SYSTEM_OUT) self.assertIsNotNone(system_out) self.assertEqual(expected, system_out.text) elif key == "stderr": system_err = test.find(JUnitDialect.SYSTEM_ERR) self.assertIsNotNone(system_err) self.assertEqual(expected, system_err.text) def _assert_verdict(self, expected_verdict, test): failure = test.find(JUnitDialect.FAILURE) error = test.find(JUnitDialect.ERROR) skipped = test.find(JUnitDialect.SKIPPED) if expected_verdict == Verdict.FAILED: self.assertIsNotNone(failure) self.assertIsNone(error) self.assertIsNone(skipped) elif expected_verdict == Verdict.ERROR: self.assertIsNone(failure) self.assertIsNotNone(error) self.assertIsNone(skipped) elif expected_verdict == Verdict.SKIPPED: self.assertIsNone(failure) self.assertIsNone(error) self.assertIsNotNone(skipped) else: # Verdict == PASSED self.assertIsNone(failure) self.assertIsNone(error) self.assertIsNone(skipped)
class JUnitXMLReportIsGenerated(TestCase): def setUp(self): self._destination = StringIO() self._test_results = GreenTestResult(default_args, GreenStream(StringIO())) self._adapter = JUnitXML() self._test = ProtoTest() self._test.module = "my_module" self._test.class_name = "MyClass" self._test.method_name = "my_method" def test_when_the_results_contain_only_one_successful_test(self): self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": {"verdict": Verdict.PASSED} } } }) def test_when_the_results_contain_tests_with_various_verdict(self): self._test_results.addSuccess( test("my.module", "MyClass", "test_method1")) self._test_results.addSuccess( test("my.module", "MyClass", "test_method2")) self._record_failure( test("my.module", "MyClass", "test_method3")) self._record_failure( test("my.module", "MyClass", "test_method4")) self._record_error( test("my.module", "MyClass", "test_method5")) self._test_results.addSkip( test("my.module", "MyClass", "test_method6"), "Take too long") self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my.module.MyClass": { "#tests": "6", "#failures": "2", "#errors": "1", "#skipped": "1", "tests": { "test_method1": { "verdict": Verdict.PASSED }, "test_method2": { "verdict": Verdict.PASSED }, "test_method3": { "verdict": Verdict.FAILED }, "test_method4": { "verdict": Verdict.FAILED }, "test_method5": { "verdict": Verdict.ERROR }, "test_method6": { "verdict": Verdict.SKIPPED } } }, }) def _record_failure(self, test): try: raise ValueError("Wrong value") except: error = proto_error(exc_info()) self._test_results.addFailure(test, error) def _record_error(self, test): try: raise ValueError("Wrong value") except: error = proto_error(exc_info()) self._test_results.addError(test, error) def test_when_the_results_contain_only_one_test_with_output(self): output = "This is the output of the test" self._test_results.recordStdout(self._test, output) self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.PASSED, "stdout": output } } } }) def test_when_the_results_contain_only_one_test_with_errput(self): errput = "This is the errput of the test" self._test_results.recordStderr(self._test, errput) self._test_results.addSuccess(self._test) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": { "verdict": Verdict.PASSED, "stderr": errput } } } }) def test_when_the_results_contain_only_one_failed_test(self): self._record_failure( test("my_module", "MyClass", "my_method")) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests" : { "my_method": {"verdict": Verdict.FAILED} } } }) def test_when_the_results_contain_only_one_erroneous_test(self): self._record_error( test("my_module", "MyClass", "my_method")) self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": {"verdict": Verdict.ERROR} } } }) def test_when_the_results_contain_only_one_skipped_test(self): self._test_results.addSkip(self._test, "reason for skipping") self._adapter.save_as(self._test_results, self._destination) self._assert_report_is({ "my_module.MyClass": { "tests": { "my_method": {"verdict": Verdict.SKIPPED} } } }) def _assert_report_is(self, report): """ Verify the structure of the generated XML text against the given 'report' structure. """ root = from_xml(self._destination.getvalue()) test_suites = root.findall(JUnitDialect.TEST_SUITE) self.assertEqual(len(report), len(test_suites)) for each_suite in test_suites: self._assert_suite(report, each_suite) def _assert_suite(self, expected_report, suite): """ Verify that the given 'suite' matches one in the expected test report. """ name = suite.get(JUnitDialect.NAME) self.assertIsNotNone(name) self.assertIn(name, expected_report) expected_suite = expected_report[name] # Check the count of tests if "#tests" in expected_suite: self.assertEqual(expected_suite["#tests"], suite.get(JUnitDialect.TEST_COUNT)) # Check the count of failures if "#failures" in expected_suite: self.assertEqual(expected_suite["#failures"], suite.get(JUnitDialect.FAILURE_COUNT)) # Check the count of errors if "#errors" in expected_suite: self.assertEqual(expected_suite["#errors"], suite.get(JUnitDialect.ERROR_COUNT)) # Check the count of skipped tests if "#skipped" in expected_suite: self.assertEqual(expected_suite["#skipped"], suite.get(JUnitDialect.SKIPPED_COUNT)) # Check individual test reports self.assertEqual(len(expected_suite["tests"]), len(suite)) for each_test in suite: self._assert_test(expected_suite["tests"], each_test) def _assert_test(self, expected_suite, test): """ Verify that the given 'test' matches one in the expected test suite. """ name = test.get(JUnitDialect.NAME) self.assertIsNotNone(test) self.assertIn(name, expected_suite) expected_test = expected_suite[name] test_passed = True for key, expected in expected_test.items(): if key == "verdict": self._assert_verdict(expected, test) elif key == "stdout": system_out = test.find(JUnitDialect.SYSTEM_OUT) self.assertIsNotNone(system_out) self.assertEqual(expected, system_out.text) elif key == "stderr": system_err = test.find(JUnitDialect.SYSTEM_ERR) self.assertIsNotNone(system_err) self.assertEqual(expected, system_err.text) def _assert_verdict(self, expected_verdict, test): failure = test.find(JUnitDialect.FAILURE) error = test.find(JUnitDialect.ERROR) skipped = test.find(JUnitDialect.SKIPPED) if expected_verdict == Verdict.FAILED: self.assertIsNotNone(failure) self.assertIsNone(error) self.assertIsNone(skipped) elif expected_verdict == Verdict.ERROR: self.assertIsNone(failure) self.assertIsNotNone(error) self.assertIsNone(skipped) elif expected_verdict == Verdict.SKIPPED: self.assertIsNone(failure) self.assertIsNone(error) self.assertIsNotNone(skipped) else: # Verdict == PASSED self.assertIsNone(failure) self.assertIsNone(error) self.assertIsNone(skipped)