def test_report_progress_and_results(self, tc_prog, tc_state, tc_results, fm_results): fm = Formatter() r0 = make_runner(RunCommand, name='test0', command=('sleep', '0.2')) tc0 = TestCase(runner=r0) tc_prog.reset_mock() _report_progress_and_results(tc0, fm, None, None, None) tc_prog.assert_not_called() tc_state.assert_not_called() tc_results.assert_not_called() fm_results.assert_not_called() tc_prog.reset_mock() _report_progress_and_results(tc0, fm, TestCase.Progress.RUNNING, None, None) tc_prog.assert_called_once_with(TestCase.Progress.RUNNING) tc_state.assert_not_called() tc_results.assert_not_called() fm_results.assert_not_called() tc_prog.reset_mock() _report_progress_and_results(tc0, fm, TestCase.Progress.FINISHED, None, None) tc_prog.assert_called_once_with(TestCase.Progress.FINISHED) tc_state.assert_called_once_with(None) tc_results.assert_called_once_with(None) fm_results.assert_called_once_with(tc0)
def testReportProgress(self, pstate): # Runner fm = Formatter(progress_interval=0) ct = TestController() rr = make_runner(TestRunner, [ct], name='r') tc = TestCase(runner=rr, controllers=(ct, )) fm.reportProgress(tc) pstate.assert_not_called() tc.setProgress(TestCase.Progress.RUNNING) fm.reportProgress(tc) pstate.assert_called_with(tc, rr, TestCase.Progress.RUNNING, None) pstate.reset_mock() tc.setProgress(TestCase.Progress.FINISHED) fm.reportProgress(tc) pstate.assert_not_called() # Error tc._TestCase__progress = None fm.reportProgress(tc) pstate.assert_not_called() self.assertEqual(tc.progress, TestCase.Progress.FINISHED) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The progress has not been set via the `setProgress` method.", r['r'].stderr) self.assertEqual(r['r'].reasons, None)
def test_execute_testcase(self): r = make_runner(RunCommand, name='test', command=('sleep', '0')) tc = TestCase(runner=r) # No error conn = PipeProxy() _execute_testcase(tc, conn) print(conn.result['test'].stderr) self.assertEqual(conn.state, TestCase.Result.PASS) self.assertIn('test', conn.result) data = conn.result['test'] self.assertEqual(data.state, TestCase.Result.PASS) self.assertEqual(data.returncode, 0) self.assertEqual(data.stderr, "") self.assertIn("sleep 0", data.stdout) self.assertEqual(data.reasons, []) # Exception with mock.patch('moosetools.moosetest.base.TestCase.execute', side_effect=Exception("wrong")): _execute_testcase(tc, conn) self.assertEqual(conn.state, TestCase.Result.FATAL) self.assertIn('test', conn.result) data = conn.result['test'] self.assertEqual(data.state, TestCase.Result.FATAL) self.assertEqual(data.returncode, None) self.assertEqual(data.stdout, "") self.assertIn("wrong", data.stderr) self.assertEqual(data.reasons, None)
def testMinFailState(self): # SKIP as failure c = TestController(skip=True) r = make_runner(TestRunner, (c, ), name='Andrew') fm = Formatter() rcode = run([[r]], (c, ), fm, min_fail_state=TestCase.Result.SKIP) self.assertEqual(rcode, 1) # this is what is being tested self.assertEqual(self._r_state.call_count, 1) self.assertEqual(self._r_results.call_count, 1) self.assertCall(self._r_state.call_args_list[0], name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall(self._r_results.call_args_list[0], name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=TestRun.ANY, duration=TestRun.ANY, returncode=None, stdout='', stderr='')
def testProgressTime(self, pstate): rr = make_runner(TestRunner, name='r') tc = TestCase(runner=rr) tc.setProgress(TestCase.Progress.RUNNING) fm = Formatter(progress_interval=1) fm.reportProgress(tc) pstate.assert_not_called() time.sleep(1.1) fm.reportProgress(tc) pstate.assert_called()
def testSetResults(self): ct = TestController() dr = make_differ(TestDiffer, [ct], name='d') rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r') tc = TestCase(runner=rr, controllers=(ct, )) # Wrong type tc.setResults('wrong') self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The supplied result must be of type `dict`.", r['r'].stderr) self.assertEqual(r['r'].reasons, None) tc.setResults({'r': 'wrong'}) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The supplied result values must be of type `TestCase.Data`.", r['r'].stderr) self.assertEqual(r['r'].reasons, None) tc.setResults({'wrong': TestCase.Data()}) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn( "The supplied result keys must be the names of the `Runner` or `Differ` object(s).", r['r'].stderr) self.assertEqual(r['r'].reasons, None) tc.setResults({'r': TestCase.Data(TestCase.Result.PASS, None, 'out', 'err', None)}) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, 'out') self.assertEqual(r['r'].stderr, 'err') self.assertEqual(r['r'].reasons, None)
def testMaxFail(self): r0 = make_runner(TestRunner, name='Just Andrew', error=True) r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5) r2 = make_runner(TestRunner, name='Best Andrew') fm = Formatter() # This test helped me catch a logic bug. This is a single group, so only a single worker # is created. Thus, even if max fail is hit the worker should finish, thus nothing reported # as max failed. At one point both the max fail and dependency message were being dumped # with this test, but it should only be the dependency. rcode = run([[r0, r1, r2]], tuple(), fm, max_fails=1) self.assertEqual(rcode, 1) self.assertEqual(self._r_state.call_count, 3) self.assertEqual(self._r_results.call_count, 3) self.assertCall(self._r_state.call_args_list[0], name='Just Andrew', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall(self._r_results.call_args_list[0], name='Just Andrew', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY, returncode=2011, stdout='', stderr=TestRun.IN('runner error')) self.assertCall(self._r_state.call_args_list[1], name='Other Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall( self._r_results.call_args_list[1], name='Other Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=TestRun.ANY, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN("A previous test case (Just Andrew)")) self.assertCall(self._r_state.call_args_list[2], name='Best Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=100, duration=TestRun.ANY) self.assertCall( self._r_results.call_args_list[2], name='Best Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN("A previous test case (Just Andrew)")) # Similar to above, but with individual groups groups = list() for i in range(5): r = make_runner(TestRunner, name=str(i), sleep=1) groups.append([r]) groups[0][0].setValue('error', True) groups[0][0].setValue('sleep', 0) self.resetMockObjects() rcode = run(groups, tuple(), fm, n_threads=1, max_fails=1) self.assertEqual(rcode, 1) self.assertEqual(self._r_state.call_count, 5) self.assertEqual(self._r_results.call_count, 5) # Only look at first and last, the middle can change depending how fast works fire up self.assertCall(self._r_state.call_args_list[0], name='0', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall(self._r_results.call_args_list[0], name='0', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY, returncode=2011, stdout='', stderr=TestRun.IN('runner error')) self.assertCall(self._r_state.call_args_list[-1], name='4', state=TestCase.Result.SKIP, reasons=['max failures reached'], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results.call_args_list[-1], name='4', state=TestCase.Result.SKIP, reasons=['max failures reached'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN("Max failures of 1 exceeded."))
def testRunnerWithDiffersWithControllers(self): c = TestController() d0 = make_differ(TestDiffer, (c, ), name='a') d1 = make_differ(TestDiffer, (c, ), name='b') r = make_runner(TestRunner, (c, ), name='Andrew', differs=(d0, d1)) fm = Formatter() # SKIP, RUNNER c.setValue('skip', True) rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 0) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr='') # SKIP, DIFFER self.resetMockObjects() c.setValue('object_name', d0.name()) rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 0) self._r_state.assert_called_once() self._r_results.assert_called_once() self.assertEqual(self._d_state.call_count, 2) self.assertEqual(self._d_results.call_count, 2) self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, returncode=2011, stdout='', stderr='') self.assertCall(self._d_state.call_args_list[0], name='a', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY) self.assertCall(self._d_state.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_results.call_args_list[0], name='a', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY, stdout='', stderr='') self.assertCall(self._d_results.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='')
def testRunnerWithDiffers(self): d0 = make_differ(TestDiffer, name='a', stderr=True) d1 = make_differ(TestDiffer, name='b', stdout=True) r = make_runner(TestRunner, name='Andrew', differs=(d0, d1)) fm = Formatter() # PASS rcode = run([[r]], tuple(), fm) self.assertEqual(rcode, 0) self._r_state.assert_called_once() self._r_results.assert_called_once() self.assertEqual(self._d_state.call_count, 2) self.assertEqual(self._d_results.call_count, 2) self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='') self.assertCall(self._d_state.call_args_list[0], name='a', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_state.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_results.call_args_list[0], name='a', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr=TestRun.IN('differ stderr')) self.assertCall(self._d_results.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout=TestRun.IN('differ stdout'), stderr='') # ERROR, DIFFER 0 self.resetMockObjects() d0.setValue('stderr', False) d1.setValue('stdout', False) d0.setValue('error', True) rcode = run([[r]], tuple(), fm) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self.assertEqual(self._d_state.call_count, 2) self.assertEqual(self._d_results.call_count, 2) self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.DIFF, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='') self.assertCall(self._d_state.call_args_list[0], name='a', state=TestCase.Result.DIFF, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_state.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_results.call_args_list[0], name='a', state=TestCase.Result.DIFF, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr=TestRun.IN('differ error')) self.assertCall(self._d_results.call_args_list[1], name='b', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='') # EXCEPTION, DIFFER 1 self.resetMockObjects() d0.setValue('error', False) d1.setValue('raise', True) rcode = run([[r]], tuple(), fm) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self.assertEqual(self._d_state.call_count, 2) self.assertEqual(self._d_results.call_count, 2) self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.EXCEPTION, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='') self.assertCall(self._d_state.call_args_list[0], name='a', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._d_state.call_args_list[1], name='b', state=TestCase.Result.EXCEPTION, reasons=None, percent=100, duration=TestRun.ANY) self.assertCall(self._d_results.call_args_list[0], name='a', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout='', stderr='') self.assertCall(self._d_results.call_args_list[1], name='b', state=TestCase.Result.EXCEPTION, reasons=None, percent=100, duration=TestRun.ANY, stdout='', stderr=TestRun.IN('differ raise')) # TIMEOUT, DIFFER 1 self.resetMockObjects() d1.setValue('sleep', 1) d1.setValue('raise', False) rcode = run([[r]], tuple(), fm, None, 0.5) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self.assertEqual(self._d_state.call_count, 0) self.assertEqual(self._d_results.call_count, 0) self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.TIMEOUT, reasons=['max time (0.5) exceeded'], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.TIMEOUT, reasons=['max time (0.5) exceeded'], percent=100, duration=TestRun.ANY, stdout='', stderr='')
def testExecuteObject_Controller(self): ctrl = TestController() obj = make_runner(TestRunner, [ ctrl, ], name='a') tc = TestCase(runner=obj, controllers=(ctrl, )) # No error, no output out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.PASS) self.assertEqual(out.returncode, 2011) self.assertEqual(out.stdout, '') self.assertEqual(out.stderr, '') self.assertEqual(out.reasons, []) with mock.patch("moosetools.moosetest.base.Controller.reset") as func: func.side_effect = Exception("raise") out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn('raise', out.stderr) self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'a' object.", out.stderr) self.assertEqual(out.reasons, None) ctrl.setValue('raise', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn('raise', out.stderr) self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'a' object.", out.stderr) self.assertEqual(out.reasons, None) ctrl.setValue('raise', False) with mock.patch("moosetools.moosetest.base.Controller.status") as func: func.return_value = 1 out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn( "An error occurred, on the controller, within the `execute` method of the TestController controller with 'a' object.", out.stderr) self.assertEqual(out.reasons, []) with mock.patch("moosetools.moosetest.base.Runner.status") as func: func.return_value = 1 out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn( "An error occurred, on the object, within the `execute` method of the TestController controller with 'a' object.", out.stderr) self.assertEqual(out.reasons, []) ctrl.setValue('skip', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.SKIP) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertEqual(out.stderr, '') self.assertEqual(out.reasons, ['a reason'])
def testRunnerWithController(self): c = TestController(stdout=True, stderr=True) r = make_runner(TestRunner, (c, ), name='Andrew') fm = Formatter() # PASS rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 0) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.PASS, reasons=[], percent=100, duration=TestRun.ANY, stdout=TestRun.IN('controller stdout'), stderr=TestRun.IN('controller stderr')) # ERROR, CONTROLLER self.resetMockObjects() c.setValue('error', True) c.setValue('stderr', False) c.setValue('stdout', False) rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.FATAL, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall( self._r_results, name='Andrew', state=TestCase.Result.FATAL, reasons=[], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN('An error occurred, on the controller')) # ERROR, RUNNER (during execution of Controller) self.resetMockObjects() c.setValue('error', False) with mock.patch('moosetools.moosetest.base.Runner.status', return_value=1): rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.FATAL, reasons=[], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.FATAL, reasons=[], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN('An error occurred, on the object')) # EXCEPTION self.resetMockObjects() c.setValue('error', False) c.setValue('raise', True) rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.FATAL, reasons=None, percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.FATAL, reasons=None, percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN('An exception occurred')) # TIMEOUT (because of Controller) self.resetMockObjects() c.setValue('raise', False) c.setValue('sleep', 1) rcode = run([[r]], (c, ), fm, None, 0.5) self.assertEqual(rcode, 1) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.TIMEOUT, reasons=['max time (0.5) exceeded'], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.TIMEOUT, reasons=['max time (0.5) exceeded'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr='') # SKIP self.resetMockObjects() c.setValue('skip', True) c.setValue('sleep', 0) rcode = run([[r]], (c, ), fm) self.assertEqual(rcode, 0) self._r_state.assert_called_once() self._r_results.assert_called_once() self._d_state.assert_not_called() self._d_results.assert_not_called() self._complete.assert_called_once() self.assertCall(self._r_state, name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY) self.assertCall(self._r_results, name='Andrew', state=TestCase.Result.SKIP, reasons=['a reason'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN(''))
def testExecute(self): ct = TestController() dr = make_differ(TestDiffer, [ct], name='d') rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r') tc = TestCase(runner=rr, controllers=(ct, )) # No error, no output s, r = tc.execute() self.assertEqual(s, TestCase.Result.PASS) self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'], TestCase.Data(TestCase.Result.PASS, 2011, '', '', [])) self.assertEqual(r['d'], TestCase.Data(TestCase.Result.PASS, 2013, '', '', [])) ## RUNNER ################################ # Reset Exception, Runner with mock.patch("moosetools.moosetest.base.Runner.reset") as func: func.side_effect = Exception("runner reset raise") s, r = tc.execute() self.assertEqual(s, TestCase.Result.FATAL) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("An exception occurred within the `reset` method of the 'r' object.", r['r'].stderr) self.assertIn("runner reset raise", r['r'].stderr) self.assertEqual(r['r'].reasons, None) # Reset Exception, Controller with Runner with mock.patch("moosetools.moosetest.base.Controller.reset") as func: func.side_effect = Exception("controller reset raise") s, r = tc.execute() self.assertEqual(s, TestCase.Result.FATAL) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'r' object.", r['r'].stderr) self.assertIn("controller reset raise", r['r'].stderr) self.assertEqual(r['r'].reasons, None) # Execute Exception, Controller with Runner ct.setValue('raise', True) s, r = tc.execute() self.assertEqual(s, TestCase.Result.FATAL) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'r' object.", r['r'].stderr) self.assertIn("controller raise", r['r'].stderr) self.assertEqual(r['r'].reasons, None) # Error Object, Controller with Runner ct.setValue('raise', False) with mock.patch("moosetools.moosetest.base.Runner.status") as func: func.return_value = 1 s, r = tc.execute() self.assertEqual(s, TestCase.Result.FATAL) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn( "An error occurred, on the object, within the `execute` method of the TestController controller with 'r' object.", r['r'].stderr) self.assertEqual(r['r'].reasons, []) # Skip, Controller with Runner ct.setValue('skip', True) s, r = tc.execute() self.assertEqual(s, TestCase.Result.SKIP) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.SKIP) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stderr, '') self.assertEqual(r['r'].reasons, ['a reason']) ct.setValue('skip', False) # Error on Runner rr.setValue('error', True) s, r = tc.execute() self.assertEqual(s, TestCase.Result.ERROR) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.ERROR) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertIn("runner error", r['r'].stderr) self.assertIn("An error occurred within the `execute` method of the 'r' object", r['r'].stderr) self.assertEqual(r['r'].reasons, []) rr.setValue('error', False) # Exception on Runner rr.setValue('raise', True) s, r = tc.execute() self.assertEqual(s, TestCase.Result.EXCEPTION) self.assertEqual(list(r.keys()), ['r']) self.assertEqual(r['r'].state, TestCase.Result.EXCEPTION) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("runner raise", r['r'].stderr) self.assertIn("An exception occurred within the `execute` method of the 'r' object", r['r'].stderr) self.assertEqual(r['r'].reasons, None) rr.setValue('raise', False) ## DIFFER ################################ # Reset Exception, Differ with mock.patch("moosetools.moosetest.base.Differ.reset") as func: func.side_effect = Exception("differ reset raise") s, r = tc.execute() self.assertEqual(s, TestCase.Result.FATAL) self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.FATAL) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertIn("An exception occurred within the `reset` method of the 'd' object.", r['d'].stderr) self.assertIn("differ reset raise", r['d'].stderr) self.assertEqual(r['d'].reasons, None) # Reset Exception, Controller with Differ with mock.patch("moosetools.moosetest.base.Controller.reset") as func: func.side_effect = [None, Exception("controller reset raise")] s, r = tc.execute() self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.FATAL) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'd' object.", r['d'].stderr) self.assertIn("controller reset raise", r['d'].stderr) self.assertEqual(r['d'].reasons, None) # Execute Exception, Controller with Differ ct.setValue('raise', True) ct.setValue('object_name', 'd') s, r = tc.execute() self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.FATAL) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertIn( "An exception occurred within the `execute` method of the TestController controller with 'd' object.", r['d'].stderr) self.assertIn("controller raise", r['d'].stderr) self.assertEqual(r['d'].reasons, None) ct.setValue('raise', False) ct.setValue('object_name', None) # Error Object, Controller with Differ with mock.patch("moosetools.moosetest.base.Differ.status") as func: func.return_value = 1 s, r = tc.execute() self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.FATAL) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertIn( "An error occurred, on the object, within the `execute` method of the TestController controller with 'd' object.", r['d'].stderr) self.assertEqual(r['d'].reasons, []) # Skip, Controller with Differ ct.setValue('skip', True) ct.setValue('object_name', 'd') s, r = tc.execute() self.assertEqual(s, TestCase.Result.PASS) self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.SKIP) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertEqual(r['d'].stderr, '') self.assertEqual(r['d'].reasons, ['a reason']) ct.setValue('skip', False) ct.setValue('object_name', None) # Error on Differ dr.setValue('error', True) s, r = tc.execute() print(s, r) self.assertEqual(s, TestCase.Result.DIFF) self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.DIFF) self.assertEqual(r['d'].returncode, 2013) self.assertEqual(r['d'].stdout, '') self.assertIn("differ error", r['d'].stderr) self.assertIn("An error occurred within the `execute` method of the 'd' object", r['d'].stderr) self.assertEqual(r['d'].reasons, []) dr.setValue('error', False) # Exception on Differ dr.setValue('raise', True) s, r = tc.execute() self.assertEqual(s, TestCase.Result.EXCEPTION) self.assertEqual(list(r.keys()), ['r', 'd']) self.assertEqual(r['r'].state, TestCase.Result.PASS) self.assertEqual(r['r'].returncode, 2011) self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].stdout, '') self.assertEqual(r['r'].reasons, []) self.assertEqual(r['d'].state, TestCase.Result.EXCEPTION) self.assertEqual(r['d'].returncode, None) self.assertEqual(r['d'].stdout, '') self.assertIn("differ raise", r['d'].stderr) self.assertIn("An exception occurred within the `execute` method of the 'd' object", r['d'].stderr) self.assertEqual(r['d'].reasons, None)
def testGroupSkip(self): # Same as first test in testMaxFails, but without the max fails r0 = make_runner(TestRunner, name='Just Andrew', error=True) r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5) r2 = make_runner(TestRunner, name='Best Andrew') fm = Formatter() rcode = run([[r0, r1, r2]], tuple(), fm) self.assertEqual(rcode, 1) self.assertEqual(self._r_state.call_count, 3) self.assertEqual(self._r_results.call_count, 3) self.assertCall(self._r_state.call_args_list[0], name='Just Andrew', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall(self._r_results.call_args_list[0], name='Just Andrew', state=TestCase.Result.ERROR, reasons=[], percent=TestRun.ANY, duration=TestRun.ANY, returncode=2011, stdout='', stderr=TestRun.IN('runner error')) self.assertCall(self._r_state.call_args_list[1], name='Other Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=TestRun.ANY, duration=TestRun.ANY) self.assertCall( self._r_results.call_args_list[1], name='Other Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=TestRun.ANY, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN("A previous test case (Just Andrew)")) self.assertCall(self._r_state.call_args_list[2], name='Best Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=100, duration=TestRun.ANY) self.assertCall( self._r_results.call_args_list[2], name='Best Andrew', state=TestCase.Result.SKIP, reasons=['dependency'], percent=100, duration=TestRun.ANY, returncode=None, stdout='', stderr=TestRun.IN("A previous test case (Just Andrew)"))
def testExecuteObject_Differ(self): obj = make_differ(TestDiffer, name='a') r = make_runner(TestRunner, name='a', differs=(obj, )) tc = TestCase(runner=r) # No error, no output out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.PASS) self.assertEqual(out.returncode, 2013) self.assertEqual(out.stdout, '') self.assertEqual(out.stderr, '') self.assertEqual(out.reasons, []) # No error, with stdout and stderr obj.setValue('stdout', True) obj.setValue('stderr', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.PASS) self.assertEqual(out.returncode, 2013) self.assertIn('differ stdout\n', out.stdout) self.assertIn('differ stderr\n', out.stderr) self.assertEqual(out.reasons, []) # Error obj.setValue('error', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.DIFF) self.assertEqual(out.returncode, 2013) self.assertEqual(out.stdout, 'differ stdout\n') self.assertIn('differ stderr\n', out.stderr) self.assertIn('differ error', out.stderr) self.assertIn("An error occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, []) # Exception obj.setValue('raise', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.EXCEPTION) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'differ stdout\n') self.assertIn('differ stderr\n', out.stderr) self.assertIn('differ error', out.stderr) self.assertIn('differ raise', out.stderr) self.assertIn("An exception occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None) # Exception during reset def side_effect(): print('print text') raise Execption("reset failed") with mock.patch("moosetools.moosetest.base.Differ.reset") as reset: reset.side_effect = side_effect out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'print text\n') self.assertIn('reset failed', out.stderr) self.assertIn("An exception occurred within the `reset` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None)
def test_execute_testcases(self): r0 = make_runner(RunCommand, name='test0', command=('sleep', '0.2')) r1 = make_runner(RunCommand, name='test1', command=('sleep', '0.3')) tc0 = TestCase(runner=r0) tc1 = TestCase(runner=r1) # No error q = queue.Queue() _execute_testcases([tc0, tc1], q, 2) u, p, s, r = q.get() self.assertEqual(u, tc0.unique_id) self.assertEqual(p, TestCase.Progress.RUNNING) self.assertEqual(s, None) self.assertEqual(r, None) u, p, s, r = q.get() self.assertEqual(u, tc0.unique_id) self.assertEqual(p, TestCase.Progress.FINISHED) self.assertEqual(s, TestCase.Result.PASS) self.assertIn('test0', r) data = r['test0'] self.assertEqual(data.state, TestCase.Result.PASS) self.assertEqual(data.returncode, 0) self.assertEqual(data.stderr, "") self.assertIn("sleep 0.2", data.stdout) self.assertEqual(data.reasons, []) u, p, s, r = q.get() self.assertEqual(u, tc1.unique_id) self.assertEqual(p, TestCase.Progress.RUNNING) self.assertEqual(s, None) self.assertEqual(r, None) u, p, s, r = q.get() self.assertEqual(u, tc1.unique_id) self.assertEqual(p, TestCase.Progress.FINISHED) self.assertEqual(s, TestCase.Result.PASS) self.assertIn('test1', r) data = r['test1'] self.assertEqual(data.state, TestCase.Result.PASS) self.assertEqual(data.returncode, 0) self.assertEqual(data.stderr, "") self.assertIn("sleep 0.3", data.stdout) self.assertEqual(data.reasons, []) # Exception and skip with mock.patch('moosetools.moosetest.base.TestCase.execute', side_effect=[Exception("wrong"), None]): _execute_testcases([tc0, tc1], q, 2) u, p, s, r = q.get() self.assertEqual(u, tc0.unique_id) self.assertEqual(p, TestCase.Progress.RUNNING) self.assertEqual(s, None) self.assertEqual(r, None) u, p, s, r = q.get() self.assertEqual(u, tc0.unique_id) self.assertEqual(p, TestCase.Progress.FINISHED) self.assertEqual(s, TestCase.Result.FATAL) self.assertIn('test0', r) data = r['test0'] self.assertEqual(data.state, TestCase.Result.FATAL) self.assertEqual(data.returncode, None) self.assertEqual(data.stdout, "") self.assertIn("wrong", data.stderr) self.assertEqual(data.reasons, None) u, p, s, r = q.get() self.assertEqual(u, tc1.unique_id) self.assertEqual(p, TestCase.Progress.FINISHED) self.assertEqual(s, TestCase.Result.SKIP) self.assertIn('test1', r) data = r['test1'] self.assertEqual(data.state, TestCase.Result.SKIP) self.assertEqual(data.returncode, None) self.assertEqual(data.stdout, "") self.assertIn("A previous", data.stderr) self.assertEqual(data.reasons, ['dependency']) # Timeout r2 = make_runner(RunCommand, name='test2', command=('sleep', '2')) tc2 = TestCase(runner=r2) _execute_testcases([tc2], q, 1) u, p, s, r = q.get() self.assertEqual(u, tc2.unique_id) self.assertEqual(p, TestCase.Progress.RUNNING) self.assertEqual(s, None) self.assertEqual(r, None) u, p, s, r = q.get() self.assertEqual(u, tc2.unique_id) self.assertEqual(p, TestCase.Progress.FINISHED) self.assertEqual(s, TestCase.Result.TIMEOUT) self.assertIn('test2', r) data = r['test2'] self.assertEqual(data.state, TestCase.Result.TIMEOUT) self.assertEqual(data.returncode, None) self.assertEqual(data.stdout, "") self.assertEqual(data.stderr, "") self.assertEqual(data.reasons, ['max time (1) exceeded'])
def testPrintState(self, r_state, d_state, r_result, d_result): # Runner ct = TestController() fm = Formatter() dr = make_differ(TestDiffer, [ct], name='d') rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r') tc = TestCase(runner=rr, controllers=(ct, )) tc.setProgress(TestCase.Progress.RUNNING) # Runner, progress fm._printState(tc, rr, TestCase.Progress.RUNNING, ["all the reasons"]) kwargs = r_state.call_args[1] # .call_args.kwargs (python > 3.7) self.assertEqual(kwargs['name'], 'r') self.assertEqual(kwargs['state'], TestCase.Progress.RUNNING) self.assertEqual(kwargs['reasons'], ["all the reasons"]) self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested self.assertIsInstance(kwargs['percent'], float) # Differ, progress tc.setProgress(TestCase.Progress.FINISHED) # call this to use execute time fm._printState(tc, dr, TestCase.Progress.FINISHED, ["all the reasons"]) kwargs = d_state.call_args[1] # .call_args.kwargs (python > 3.7) self.assertEqual(kwargs['name'], 'd') self.assertEqual(kwargs['state'], TestCase.Progress.FINISHED) self.assertEqual(kwargs['reasons'], ["all the reasons"]) self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested self.assertIsInstance(kwargs['percent'], float) # Runner, results tc.setProgress(TestCase.Progress.FINISHED) tc.setState(TestCase.Result.PASS) tc.setResults({ 'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None), 'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None) }) fm._printResult(tc, rr, tc.results['r']) kwargs = r_result.call_args[1] # .call_args.kwargs (python > 3.7) self.assertEqual(kwargs['name'], 'r') self.assertEqual(kwargs['state'], TestCase.Result.PASS) self.assertEqual(kwargs['reasons'], None) self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested self.assertIsInstance(kwargs['percent'], float) self.assertEqual(kwargs['stdout'], 'r_out') self.assertEqual(kwargs['stderr'], 'r_err') # Differ, results tc.setState(TestCase.Result.PASS) tc.setResults({ 'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None), 'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None) }) fm._printResult(tc, dr, tc.results['d']) kwargs = d_result.call_args[1] # .call_args.kwargs (python > 3.7) self.assertEqual(kwargs['name'], 'd') self.assertEqual(kwargs['state'], TestCase.Result.PASS) self.assertEqual(kwargs['reasons'], None) self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested self.assertIsInstance(kwargs['percent'], float) self.assertEqual(kwargs['stdout'], 'd_out') self.assertEqual(kwargs['stderr'], 'd_err')
def fuzzer(seed=1980, timeout=(3, 10), max_fails=(15, 100), progress_interval=(3, 15), group_num=(15, 50), group_name_len=(6, 25), controller_num=(1, 6), controller_skip=0.05, controller_raise=0.05, controller_error=0.1, differ_num=(0, 3), differ_raise=0.01, differ_error=0.1, differ_fatal=0.1, differ_platform=0.1, differ_name_len=(6, 15), runner_num=(1, 3), runner_raise=0.01, runner_error=0.1, runner_fatal=0.05, runner_sleep=(0.5, 10), runner_platform=0.1, runner_name_len=(4, 29)): """ A tool for calling `run` function with randomized test cases. """ # This is more of a test object, so I wanted to keep the testing related import out of the # main functions for the run command. import random import string from moosetools.moosetest.formatters import BasicFormatter from moosetools.moosetest.base import make_runner, make_differ sys.path.append(os.path.join(os.path.dirname(__file__), 'tests')) from _helpers import TestController, TestRunner, TestDiffer def gen_name(rng): return ''.join(random.sample(string.ascii_letters, random.randint(*rng))) def gen_platform(ctrls, prob, kwargs): if random.uniform(0, 1) < prob: prefix = "{}_platform".format(random.choice(ctrls).getParam('prefix')) value = tuple( set(random.choices(['Darwin', 'Linux', 'Windows'], k=random.randint(1, 3)))) kwargs[prefix] = value def gen_bool_with_odds(prob): return random.uniform(0, 1) < prob # Controller objects controllers = list() for i, n_controllers in enumerate(range(random.randint(*controller_num))): name_start = random.choice(string.ascii_letters) kwargs = dict() kwargs['stdout'] = True kwargs['stderr'] = True kwargs['prefix'] = "ctrl{:0.0f}".format(i) kwargs['skip'] = gen_bool_with_odds(controller_skip) kwargs['error'] = gen_bool_with_odds(controller_error) kwargs['raise'] = gen_bool_with_odds(controller_raise) controllers.append(TestController(object_name=name_start, **kwargs)) controllers = tuple(controllers) # Runners/Differs groups = list() for n_groups in range(random.randint(*group_num)): runners = list() group_name = gen_name(group_name_len) for n_runners in range(random.randint(*runner_num)): differs = list() for n_differs in range(random.randint(*differ_num)): kwargs = dict() kwargs['name'] = gen_name(differ_name_len) kwargs['stdout'] = True kwargs['stderr'] = True kwargs['error'] = gen_bool_with_odds(differ_error) kwargs['raise'] = gen_bool_with_odds(differ_raise) kwargs['fatal'] = gen_bool_with_odds(differ_fatal) gen_platform(controllers, differ_platform, kwargs) differs.append(make_differ(TestDiffer, controllers, **kwargs)) kwargs = dict() kwargs['name'] = f"{group_name}/{gen_name(runner_name_len)}" kwargs['differs'] = tuple(differs) kwargs['stdout'] = True kwargs['stderr'] = True kwargs['error'] = gen_bool_with_odds(runner_error) kwargs['raise'] = gen_bool_with_odds(runner_raise) kwargs['fatal'] = gen_bool_with_odds(runner_fatal) kwargs['sleep'] = random.uniform(*runner_sleep) gen_platform(controllers, runner_platform, kwargs) runners.append(make_runner(TestRunner, controllers, **kwargs)) groups.append(runners) # Formatter kwargs = dict() kwargs['progress_interval'] = random.randint(*progress_interval) formatter = BasicFormatter(**kwargs) # Run kwargs = dict() kwargs['timeout'] = random.randint(*timeout) kwargs['max_fails'] = random.randint(*max_fails) kwargs['min_fail_state'] = random.choice([r for r in TestCase.Result]) return run(groups, controllers, formatter, **kwargs)
def testReportResults(self, pstate, presult): # Runner fm = moosetest.base.Formatter(progress_interval=0) ct = TestController() rr = make_runner(TestRunner, [ct], name='r') tc = TestCase(runner=rr, controllers=(ct, )) tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'out', 'err', None)}) tc.setProgress(TestCase.Progress.FINISHED) tc.setState(TestCase.Result.PASS) fm.reportResults(tc) pstate.assert_called_with(tc, rr, tc.state, None) presult.assert_called_with(tc, rr, tc.results['r']) # Differ dr = make_differ(TestDiffer, [ct], name='d') rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r') tc = TestCase(runner=rr, controllers=(ct, )) tc.setProgress(TestCase.Progress.FINISHED) tc.setState(TestCase.Result.PASS) tc.setResults({ 'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None), 'd': TestCase.Data(TestCase.Result.TIMEOUT, None, 'd_out', 'd_err', None) }) fm.reportResults(tc) pstate.assert_called_with(tc, dr, tc.results['d'].state, None) presult.assert_called_with(tc, dr, tc.results['d']) # Errors ct = TestController() rr = make_runner(TestRunner, [ct], name='r') tc = TestCase(runner=rr, controllers=(ct, )) fm.reportResults(tc) self.assertEqual(tc.progress, TestCase.Progress.FINISHED) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The state has not been set via the `setState` method.", r['r'].stderr) self.assertEqual(r['r'].reasons, None) ct = TestController() rr = make_runner(TestRunner, [ct], name='r') tc = TestCase(runner=rr, controllers=(ct, )) tc.setState(TestCase.Result.PASS) fm.reportResults(tc) self.assertEqual(tc.progress, TestCase.Progress.FINISHED) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The results have not been set via the `setResults` method.", r['r'].stderr) self.assertEqual(r['r'].reasons, None) ct = TestController() rr = make_runner(TestRunner, [ct], name='r') tc = TestCase(runner=rr, controllers=(ct, )) tc.setState(TestCase.Result.PASS) tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None)}) fm.reportResults(tc) self.assertEqual(tc.progress, TestCase.Progress.FINISHED) self.assertEqual(tc.state, TestCase.Result.FATAL) r = tc.results self.assertEqual(r['r'].state, TestCase.Result.FATAL) self.assertEqual(r['r'].returncode, None) self.assertEqual(r['r'].stdout, '') self.assertIn("The execution has not finished, so results cannot be reported.", r['r'].stderr) self.assertEqual(r['r'].reasons, None)
def testExecuteObject_Runner(self): obj = make_runner(TestRunner, name='a') tc = TestCase(runner=obj) # No error, no output out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.PASS) self.assertEqual(out.returncode, 2011) self.assertEqual(out.stdout, '') self.assertEqual(out.stderr, '') self.assertEqual(out.reasons, []) # No error, with stdout and stderr obj.setValue('stdout', True) obj.setValue('stderr', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.PASS) self.assertEqual(out.returncode, 2011) self.assertIn('runner stdout\n', out.stdout) self.assertIn('runner stderr\n', out.stderr) self.assertEqual(out.reasons, []) # Error obj.setValue('error', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.ERROR) self.assertEqual(out.returncode, 2011) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn('runner stderr\n', out.stderr) self.assertIn('runner error', out.stderr) self.assertIn("An error occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, []) # Exception obj.setValue('raise', True) out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.EXCEPTION) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn('runner stderr\n', out.stderr) self.assertIn('runner error', out.stderr) self.assertIn('runner raise', out.stderr) self.assertIn("An exception occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None) # Exception during reset def side_effect(): print('print text') raise Execption("reset failed") with mock.patch("moosetools.moosetest.base.Runner.reset") as reset: reset.side_effect = side_effect out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'print text\n') self.assertIn('reset failed', out.stderr) self.assertIn("An exception occurred within the `reset` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None) # Error on preExecute with mock.patch('moosetools.moosetest.base.Runner.status', return_value=1): out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn("An error occurred within the `preExecute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, []) # Exception on preExecute with mock.patch('moosetools.moosetest.base.Runner.preExecute', side_effect=Exception()): out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, '') self.assertIn("An exception occurred within the `preExecute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None) # Error on postExecute obj.setValue('raise', False) with mock.patch('moosetools.moosetest.base.Runner.status', side_effect=[0, 0, 1]): out = tc._executeObject(obj) print(out.stdout) print(out.stderr) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn("An error occurred within the `postExecute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, []) # Exception on postExecute with mock.patch('moosetools.moosetest.base.Runner.postExecute', side_effect=Exception()): out = tc._executeObject(obj) self.assertEqual(out.state, TestCase.Result.FATAL) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn("An exception occurred within the `postExecute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None) # Error on execute, postExecute still runs obj.setValue('error', True) with mock.patch('moosetools.moosetest.base.Runner.postExecute') as mock_postExecute: out = tc._executeObject(obj) mock_postExecute.assert_called_once() self.assertEqual(out.state, TestCase.Result.ERROR) self.assertEqual(out.returncode, 2011) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn('runner stderr\n', out.stderr) self.assertIn('runner error', out.stderr) self.assertIn("An error occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, []) # Error on execute, postExecute still runs obj.setValue('error', False) obj.setValue('raise', True) with mock.patch('moosetools.moosetest.base.Runner.postExecute') as mock_postExecute: out = tc._executeObject(obj) mock_postExecute.assert_called_once() self.assertEqual(out.state, TestCase.Result.EXCEPTION) self.assertEqual(out.returncode, None) self.assertEqual(out.stdout, 'runner stdout\n') self.assertIn('runner stderr\n', out.stderr) self.assertIn('runner raise', out.stderr) self.assertIn("An exception occurred within the `execute` method of the 'a' object.", out.stderr) self.assertEqual(out.reasons, None)