def test_report_progress_and_results(self, tc_prog, tc_state, tc_results,
fm_results):
fm = Formatter()
r0 = make_runner(RunCommand, name='test0', command=('sleep', '0.2'))
tc0 = TestCase(runner=r0)
tc_prog.reset_mock()
_report_progress_and_results(tc0, fm, None, None, None)
tc_prog.assert_not_called()
tc_state.assert_not_called()
tc_results.assert_not_called()
fm_results.assert_not_called()
tc_prog.reset_mock()
_report_progress_and_results(tc0, fm, TestCase.Progress.RUNNING, None,
None)
tc_prog.assert_called_once_with(TestCase.Progress.RUNNING)
tc_state.assert_not_called()
tc_results.assert_not_called()
fm_results.assert_not_called()
tc_prog.reset_mock()
_report_progress_and_results(tc0, fm, TestCase.Progress.FINISHED, None,
None)
tc_prog.assert_called_once_with(TestCase.Progress.FINISHED)
tc_state.assert_called_once_with(None)
tc_results.assert_called_once_with(None)
fm_results.assert_called_once_with(tc0)
def testReportProgress(self, pstate):
# Runner
fm = Formatter(progress_interval=0)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
fm.reportProgress(tc)
pstate.assert_not_called()
tc.setProgress(TestCase.Progress.RUNNING)
fm.reportProgress(tc)
pstate.assert_called_with(tc, rr, TestCase.Progress.RUNNING, None)
pstate.reset_mock()
tc.setProgress(TestCase.Progress.FINISHED)
fm.reportProgress(tc)
pstate.assert_not_called()
# Error
tc._TestCase__progress = None
fm.reportProgress(tc)
pstate.assert_not_called()
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The progress has not been set via the `setProgress` method.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
def test_execute_testcase(self):
r = make_runner(RunCommand, name='test', command=('sleep', '0'))
tc = TestCase(runner=r)
# No error
conn = PipeProxy()
_execute_testcase(tc, conn)
print(conn.result['test'].stderr)
self.assertEqual(conn.state, TestCase.Result.PASS)
self.assertIn('test', conn.result)
data = conn.result['test']
self.assertEqual(data.state, TestCase.Result.PASS)
self.assertEqual(data.returncode, 0)
self.assertEqual(data.stderr, "")
self.assertIn("sleep 0", data.stdout)
self.assertEqual(data.reasons, [])
# Exception
with mock.patch('moosetools.moosetest.base.TestCase.execute',
side_effect=Exception("wrong")):
_execute_testcase(tc, conn)
self.assertEqual(conn.state, TestCase.Result.FATAL)
self.assertIn('test', conn.result)
data = conn.result['test']
self.assertEqual(data.state, TestCase.Result.FATAL)
self.assertEqual(data.returncode, None)
self.assertEqual(data.stdout, "")
self.assertIn("wrong", data.stderr)
self.assertEqual(data.reasons, None)
def testMinFailState(self):
# SKIP as failure
c = TestController(skip=True)
r = make_runner(TestRunner, (c, ), name='Andrew')
fm = Formatter()
rcode = run([[r]], (c, ), fm, min_fail_state=TestCase.Result.SKIP)
self.assertEqual(rcode, 1) # this is what is being tested
self.assertEqual(self._r_state.call_count, 1)
self.assertEqual(self._r_results.call_count, 1)
self.assertCall(self._r_state.call_args_list[0],
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
def testProgressTime(self, pstate):
rr = make_runner(TestRunner, name='r')
tc = TestCase(runner=rr)
tc.setProgress(TestCase.Progress.RUNNING)
fm = Formatter(progress_interval=1)
fm.reportProgress(tc)
pstate.assert_not_called()
time.sleep(1.1)
fm.reportProgress(tc)
pstate.assert_called()
def testSetResults(self):
ct = TestController()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
# Wrong type
tc.setResults('wrong')
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The supplied result must be of type `dict`.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'r': 'wrong'})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The supplied result values must be of type `TestCase.Data`.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'wrong': TestCase.Data()})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"The supplied result keys must be the names of the `Runner` or `Differ` object(s).",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'r': TestCase.Data(TestCase.Result.PASS, None, 'out', 'err', None)})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, 'out')
self.assertEqual(r['r'].stderr, 'err')
self.assertEqual(r['r'].reasons, None)
def testMaxFail(self):
r0 = make_runner(TestRunner, name='Just Andrew', error=True)
r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5)
r2 = make_runner(TestRunner, name='Best Andrew')
fm = Formatter()
# This test helped me catch a logic bug. This is a single group, so only a single worker
# is created. Thus, even if max fail is hit the worker should finish, thus nothing reported
# as max failed. At one point both the max fail and dependency message were being dumped
# with this test, but it should only be the dependency.
rcode = run([[r0, r1, r2]], tuple(), fm, max_fails=1)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 3)
self.assertEqual(self._r_results.call_count, 3)
self.assertCall(self._r_state.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
self.assertCall(self._r_state.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
# Similar to above, but with individual groups
groups = list()
for i in range(5):
r = make_runner(TestRunner, name=str(i), sleep=1)
groups.append([r])
groups[0][0].setValue('error', True)
groups[0][0].setValue('sleep', 0)
self.resetMockObjects()
rcode = run(groups, tuple(), fm, n_threads=1, max_fails=1)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 5)
self.assertEqual(self._r_results.call_count, 5)
# Only look at first and last, the middle can change depending how fast works fire up
self.assertCall(self._r_state.call_args_list[0],
name='0',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='0',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[-1],
name='4',
state=TestCase.Result.SKIP,
reasons=['max failures reached'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[-1],
name='4',
state=TestCase.Result.SKIP,
reasons=['max failures reached'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("Max failures of 1 exceeded."))
def testRunnerWithDiffersWithControllers(self):
c = TestController()
d0 = make_differ(TestDiffer, (c, ), name='a')
d1 = make_differ(TestDiffer, (c, ), name='b')
r = make_runner(TestRunner, (c, ), name='Andrew', differs=(d0, d1))
fm = Formatter()
# SKIP, RUNNER
c.setValue('skip', True)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
# SKIP, DIFFER
self.resetMockObjects()
c.setValue('object_name', d0.name())
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testRunnerWithDiffers(self):
d0 = make_differ(TestDiffer, name='a', stderr=True)
d1 = make_differ(TestDiffer, name='b', stdout=True)
r = make_runner(TestRunner, name='Andrew', differs=(d0, d1))
fm = Formatter()
# PASS
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ stderr'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout=TestRun.IN('differ stdout'),
stderr='')
# ERROR, DIFFER 0
self.resetMockObjects()
d0.setValue('stderr', False)
d1.setValue('stdout', False)
d0.setValue('error', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ error'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
# EXCEPTION, DIFFER 1
self.resetMockObjects()
d0.setValue('error', False)
d1.setValue('raise', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.EXCEPTION,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ raise'))
# TIMEOUT, DIFFER 1
self.resetMockObjects()
d1.setValue('sleep', 1)
d1.setValue('raise', False)
rcode = run([[r]], tuple(), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 0)
self.assertEqual(self._d_results.call_count, 0)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testExecuteObject_Controller(self):
ctrl = TestController()
obj = make_runner(TestRunner, [
ctrl,
], name='a')
tc = TestCase(runner=obj, controllers=(ctrl, ))
# No error, no output
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2011)
self.assertEqual(out.stdout, '')
self.assertEqual(out.stderr, '')
self.assertEqual(out.reasons, [])
with mock.patch("moosetools.moosetest.base.Controller.reset") as func:
func.side_effect = Exception("raise")
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn('raise', out.stderr)
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
ctrl.setValue('raise', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn('raise', out.stderr)
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
ctrl.setValue('raise', False)
with mock.patch("moosetools.moosetest.base.Controller.status") as func:
func.return_value = 1
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn(
"An error occurred, on the controller, within the `execute` method of the TestController controller with 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
with mock.patch("moosetools.moosetest.base.Runner.status") as func:
func.return_value = 1
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn(
"An error occurred, on the object, within the `execute` method of the TestController controller with 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
ctrl.setValue('skip', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.SKIP)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertEqual(out.stderr, '')
self.assertEqual(out.reasons, ['a reason'])
def testRunnerWithController(self):
c = TestController(stdout=True, stderr=True)
r = make_runner(TestRunner, (c, ), name='Andrew')
fm = Formatter()
# PASS
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout=TestRun.IN('controller stdout'),
stderr=TestRun.IN('controller stderr'))
# ERROR, CONTROLLER
self.resetMockObjects()
c.setValue('error', True)
c.setValue('stderr', False)
c.setValue('stdout', False)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An error occurred, on the controller'))
# ERROR, RUNNER (during execution of Controller)
self.resetMockObjects()
c.setValue('error', False)
with mock.patch('moosetools.moosetest.base.Runner.status',
return_value=1):
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An error occurred, on the object'))
# EXCEPTION
self.resetMockObjects()
c.setValue('error', False)
c.setValue('raise', True)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=None,
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An exception occurred'))
# TIMEOUT (because of Controller)
self.resetMockObjects()
c.setValue('raise', False)
c.setValue('sleep', 1)
rcode = run([[r]], (c, ), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
# SKIP
self.resetMockObjects()
c.setValue('skip', True)
c.setValue('sleep', 0)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN(''))
def testExecute(self):
ct = TestController()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
# No error, no output
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.PASS)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'], TestCase.Data(TestCase.Result.PASS, 2011, '', '', []))
self.assertEqual(r['d'], TestCase.Data(TestCase.Result.PASS, 2013, '', '', []))
## RUNNER ################################
# Reset Exception, Runner
with mock.patch("moosetools.moosetest.base.Runner.reset") as func:
func.side_effect = Exception("runner reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("An exception occurred within the `reset` method of the 'r' object.",
r['r'].stderr)
self.assertIn("runner reset raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Reset Exception, Controller with Runner
with mock.patch("moosetools.moosetest.base.Controller.reset") as func:
func.side_effect = Exception("controller reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertIn("controller reset raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Execute Exception, Controller with Runner
ct.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertIn("controller raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Error Object, Controller with Runner
ct.setValue('raise', False)
with mock.patch("moosetools.moosetest.base.Runner.status") as func:
func.return_value = 1
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An error occurred, on the object, within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertEqual(r['r'].reasons, [])
# Skip, Controller with Runner
ct.setValue('skip', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.SKIP)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.SKIP)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stderr, '')
self.assertEqual(r['r'].reasons, ['a reason'])
ct.setValue('skip', False)
# Error on Runner
rr.setValue('error', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.ERROR)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.ERROR)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertIn("runner error", r['r'].stderr)
self.assertIn("An error occurred within the `execute` method of the 'r' object",
r['r'].stderr)
self.assertEqual(r['r'].reasons, [])
rr.setValue('error', False)
# Exception on Runner
rr.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.EXCEPTION)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.EXCEPTION)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("runner raise", r['r'].stderr)
self.assertIn("An exception occurred within the `execute` method of the 'r' object",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
rr.setValue('raise', False)
## DIFFER ################################
# Reset Exception, Differ
with mock.patch("moosetools.moosetest.base.Differ.reset") as func:
func.side_effect = Exception("differ reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn("An exception occurred within the `reset` method of the 'd' object.",
r['d'].stderr)
self.assertIn("differ reset raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
# Reset Exception, Controller with Differ
with mock.patch("moosetools.moosetest.base.Controller.reset") as func:
func.side_effect = [None, Exception("controller reset raise")]
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertIn("controller reset raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
# Execute Exception, Controller with Differ
ct.setValue('raise', True)
ct.setValue('object_name', 'd')
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertIn("controller raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
ct.setValue('raise', False)
ct.setValue('object_name', None)
# Error Object, Controller with Differ
with mock.patch("moosetools.moosetest.base.Differ.status") as func:
func.return_value = 1
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An error occurred, on the object, within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertEqual(r['d'].reasons, [])
# Skip, Controller with Differ
ct.setValue('skip', True)
ct.setValue('object_name', 'd')
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.PASS)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.SKIP)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertEqual(r['d'].stderr, '')
self.assertEqual(r['d'].reasons, ['a reason'])
ct.setValue('skip', False)
ct.setValue('object_name', None)
# Error on Differ
dr.setValue('error', True)
s, r = tc.execute()
print(s, r)
self.assertEqual(s, TestCase.Result.DIFF)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.DIFF)
self.assertEqual(r['d'].returncode, 2013)
self.assertEqual(r['d'].stdout, '')
self.assertIn("differ error", r['d'].stderr)
self.assertIn("An error occurred within the `execute` method of the 'd' object",
r['d'].stderr)
self.assertEqual(r['d'].reasons, [])
dr.setValue('error', False)
# Exception on Differ
dr.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.EXCEPTION)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.EXCEPTION)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn("differ raise", r['d'].stderr)
self.assertIn("An exception occurred within the `execute` method of the 'd' object",
r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
def testGroupSkip(self):
# Same as first test in testMaxFails, but without the max fails
r0 = make_runner(TestRunner, name='Just Andrew', error=True)
r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5)
r2 = make_runner(TestRunner, name='Best Andrew')
fm = Formatter()
rcode = run([[r0, r1, r2]], tuple(), fm)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 3)
self.assertEqual(self._r_results.call_count, 3)
self.assertCall(self._r_state.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
self.assertCall(self._r_state.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
def testExecuteObject_Differ(self):
obj = make_differ(TestDiffer, name='a')
r = make_runner(TestRunner, name='a', differs=(obj, ))
tc = TestCase(runner=r)
# No error, no output
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2013)
self.assertEqual(out.stdout, '')
self.assertEqual(out.stderr, '')
self.assertEqual(out.reasons, [])
# No error, with stdout and stderr
obj.setValue('stdout', True)
obj.setValue('stderr', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2013)
self.assertIn('differ stdout\n', out.stdout)
self.assertIn('differ stderr\n', out.stderr)
self.assertEqual(out.reasons, [])
# Error
obj.setValue('error', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.DIFF)
self.assertEqual(out.returncode, 2013)
self.assertEqual(out.stdout, 'differ stdout\n')
self.assertIn('differ stderr\n', out.stderr)
self.assertIn('differ error', out.stderr)
self.assertIn("An error occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Exception
obj.setValue('raise', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.EXCEPTION)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'differ stdout\n')
self.assertIn('differ stderr\n', out.stderr)
self.assertIn('differ error', out.stderr)
self.assertIn('differ raise', out.stderr)
self.assertIn("An exception occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Exception during reset
def side_effect():
print('print text')
raise Execption("reset failed")
with mock.patch("moosetools.moosetest.base.Differ.reset") as reset:
reset.side_effect = side_effect
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'print text\n')
self.assertIn('reset failed', out.stderr)
self.assertIn("An exception occurred within the `reset` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
def test_execute_testcases(self):
r0 = make_runner(RunCommand, name='test0', command=('sleep', '0.2'))
r1 = make_runner(RunCommand, name='test1', command=('sleep', '0.3'))
tc0 = TestCase(runner=r0)
tc1 = TestCase(runner=r1)
# No error
q = queue.Queue()
_execute_testcases([tc0, tc1], q, 2)
u, p, s, r = q.get()
self.assertEqual(u, tc0.unique_id)
self.assertEqual(p, TestCase.Progress.RUNNING)
self.assertEqual(s, None)
self.assertEqual(r, None)
u, p, s, r = q.get()
self.assertEqual(u, tc0.unique_id)
self.assertEqual(p, TestCase.Progress.FINISHED)
self.assertEqual(s, TestCase.Result.PASS)
self.assertIn('test0', r)
data = r['test0']
self.assertEqual(data.state, TestCase.Result.PASS)
self.assertEqual(data.returncode, 0)
self.assertEqual(data.stderr, "")
self.assertIn("sleep 0.2", data.stdout)
self.assertEqual(data.reasons, [])
u, p, s, r = q.get()
self.assertEqual(u, tc1.unique_id)
self.assertEqual(p, TestCase.Progress.RUNNING)
self.assertEqual(s, None)
self.assertEqual(r, None)
u, p, s, r = q.get()
self.assertEqual(u, tc1.unique_id)
self.assertEqual(p, TestCase.Progress.FINISHED)
self.assertEqual(s, TestCase.Result.PASS)
self.assertIn('test1', r)
data = r['test1']
self.assertEqual(data.state, TestCase.Result.PASS)
self.assertEqual(data.returncode, 0)
self.assertEqual(data.stderr, "")
self.assertIn("sleep 0.3", data.stdout)
self.assertEqual(data.reasons, [])
# Exception and skip
with mock.patch('moosetools.moosetest.base.TestCase.execute',
side_effect=[Exception("wrong"), None]):
_execute_testcases([tc0, tc1], q, 2)
u, p, s, r = q.get()
self.assertEqual(u, tc0.unique_id)
self.assertEqual(p, TestCase.Progress.RUNNING)
self.assertEqual(s, None)
self.assertEqual(r, None)
u, p, s, r = q.get()
self.assertEqual(u, tc0.unique_id)
self.assertEqual(p, TestCase.Progress.FINISHED)
self.assertEqual(s, TestCase.Result.FATAL)
self.assertIn('test0', r)
data = r['test0']
self.assertEqual(data.state, TestCase.Result.FATAL)
self.assertEqual(data.returncode, None)
self.assertEqual(data.stdout, "")
self.assertIn("wrong", data.stderr)
self.assertEqual(data.reasons, None)
u, p, s, r = q.get()
self.assertEqual(u, tc1.unique_id)
self.assertEqual(p, TestCase.Progress.FINISHED)
self.assertEqual(s, TestCase.Result.SKIP)
self.assertIn('test1', r)
data = r['test1']
self.assertEqual(data.state, TestCase.Result.SKIP)
self.assertEqual(data.returncode, None)
self.assertEqual(data.stdout, "")
self.assertIn("A previous", data.stderr)
self.assertEqual(data.reasons, ['dependency'])
# Timeout
r2 = make_runner(RunCommand, name='test2', command=('sleep', '2'))
tc2 = TestCase(runner=r2)
_execute_testcases([tc2], q, 1)
u, p, s, r = q.get()
self.assertEqual(u, tc2.unique_id)
self.assertEqual(p, TestCase.Progress.RUNNING)
self.assertEqual(s, None)
self.assertEqual(r, None)
u, p, s, r = q.get()
self.assertEqual(u, tc2.unique_id)
self.assertEqual(p, TestCase.Progress.FINISHED)
self.assertEqual(s, TestCase.Result.TIMEOUT)
self.assertIn('test2', r)
data = r['test2']
self.assertEqual(data.state, TestCase.Result.TIMEOUT)
self.assertEqual(data.returncode, None)
self.assertEqual(data.stdout, "")
self.assertEqual(data.stderr, "")
self.assertEqual(data.reasons, ['max time (1) exceeded'])
def testPrintState(self, r_state, d_state, r_result, d_result):
# Runner
ct = TestController()
fm = Formatter()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setProgress(TestCase.Progress.RUNNING)
# Runner, progress
fm._printState(tc, rr, TestCase.Progress.RUNNING, ["all the reasons"])
kwargs = r_state.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'r')
self.assertEqual(kwargs['state'], TestCase.Progress.RUNNING)
self.assertEqual(kwargs['reasons'], ["all the reasons"])
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
# Differ, progress
tc.setProgress(TestCase.Progress.FINISHED) # call this to use execute time
fm._printState(tc, dr, TestCase.Progress.FINISHED, ["all the reasons"])
kwargs = d_state.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'd')
self.assertEqual(kwargs['state'], TestCase.Progress.FINISHED)
self.assertEqual(kwargs['reasons'], ["all the reasons"])
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
# Runner, results
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None)
})
fm._printResult(tc, rr, tc.results['r'])
kwargs = r_result.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'r')
self.assertEqual(kwargs['state'], TestCase.Result.PASS)
self.assertEqual(kwargs['reasons'], None)
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
self.assertEqual(kwargs['stdout'], 'r_out')
self.assertEqual(kwargs['stderr'], 'r_err')
# Differ, results
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None)
})
fm._printResult(tc, dr, tc.results['d'])
kwargs = d_result.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'd')
self.assertEqual(kwargs['state'], TestCase.Result.PASS)
self.assertEqual(kwargs['reasons'], None)
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
self.assertEqual(kwargs['stdout'], 'd_out')
self.assertEqual(kwargs['stderr'], 'd_err')
def fuzzer(seed=1980,
timeout=(3, 10),
max_fails=(15, 100),
progress_interval=(3, 15),
group_num=(15, 50),
group_name_len=(6, 25),
controller_num=(1, 6),
controller_skip=0.05,
controller_raise=0.05,
controller_error=0.1,
differ_num=(0, 3),
differ_raise=0.01,
differ_error=0.1,
differ_fatal=0.1,
differ_platform=0.1,
differ_name_len=(6, 15),
runner_num=(1, 3),
runner_raise=0.01,
runner_error=0.1,
runner_fatal=0.05,
runner_sleep=(0.5, 10),
runner_platform=0.1,
runner_name_len=(4, 29)):
"""
A tool for calling `run` function with randomized test cases.
"""
# This is more of a test object, so I wanted to keep the testing related import out of the
# main functions for the run command.
import random
import string
from moosetools.moosetest.formatters import BasicFormatter
from moosetools.moosetest.base import make_runner, make_differ
sys.path.append(os.path.join(os.path.dirname(__file__), 'tests'))
from _helpers import TestController, TestRunner, TestDiffer
def gen_name(rng):
return ''.join(random.sample(string.ascii_letters, random.randint(*rng)))
def gen_platform(ctrls, prob, kwargs):
if random.uniform(0, 1) < prob:
prefix = "{}_platform".format(random.choice(ctrls).getParam('prefix'))
value = tuple(
set(random.choices(['Darwin', 'Linux', 'Windows'], k=random.randint(1, 3))))
kwargs[prefix] = value
def gen_bool_with_odds(prob):
return random.uniform(0, 1) < prob
# Controller objects
controllers = list()
for i, n_controllers in enumerate(range(random.randint(*controller_num))):
name_start = random.choice(string.ascii_letters)
kwargs = dict()
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['prefix'] = "ctrl{:0.0f}".format(i)
kwargs['skip'] = gen_bool_with_odds(controller_skip)
kwargs['error'] = gen_bool_with_odds(controller_error)
kwargs['raise'] = gen_bool_with_odds(controller_raise)
controllers.append(TestController(object_name=name_start, **kwargs))
controllers = tuple(controllers)
# Runners/Differs
groups = list()
for n_groups in range(random.randint(*group_num)):
runners = list()
group_name = gen_name(group_name_len)
for n_runners in range(random.randint(*runner_num)):
differs = list()
for n_differs in range(random.randint(*differ_num)):
kwargs = dict()
kwargs['name'] = gen_name(differ_name_len)
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['error'] = gen_bool_with_odds(differ_error)
kwargs['raise'] = gen_bool_with_odds(differ_raise)
kwargs['fatal'] = gen_bool_with_odds(differ_fatal)
gen_platform(controllers, differ_platform, kwargs)
differs.append(make_differ(TestDiffer, controllers, **kwargs))
kwargs = dict()
kwargs['name'] = f"{group_name}/{gen_name(runner_name_len)}"
kwargs['differs'] = tuple(differs)
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['error'] = gen_bool_with_odds(runner_error)
kwargs['raise'] = gen_bool_with_odds(runner_raise)
kwargs['fatal'] = gen_bool_with_odds(runner_fatal)
kwargs['sleep'] = random.uniform(*runner_sleep)
gen_platform(controllers, runner_platform, kwargs)
runners.append(make_runner(TestRunner, controllers, **kwargs))
groups.append(runners)
# Formatter
kwargs = dict()
kwargs['progress_interval'] = random.randint(*progress_interval)
formatter = BasicFormatter(**kwargs)
# Run
kwargs = dict()
kwargs['timeout'] = random.randint(*timeout)
kwargs['max_fails'] = random.randint(*max_fails)
kwargs['min_fail_state'] = random.choice([r for r in TestCase.Result])
return run(groups, controllers, formatter, **kwargs)
def testReportResults(self, pstate, presult):
# Runner
fm = moosetest.base.Formatter(progress_interval=0)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'out', 'err', None)})
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
fm.reportResults(tc)
pstate.assert_called_with(tc, rr, tc.state, None)
presult.assert_called_with(tc, rr, tc.results['r'])
# Differ
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.TIMEOUT, None, 'd_out', 'd_err', None)
})
fm.reportResults(tc)
pstate.assert_called_with(tc, dr, tc.results['d'].state, None)
presult.assert_called_with(tc, dr, tc.results['d'])
# Errors
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The state has not been set via the `setState` method.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setState(TestCase.Result.PASS)
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The results have not been set via the `setResults` method.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setState(TestCase.Result.PASS)
tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None)})
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The execution has not finished, so results cannot be reported.",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
def testExecuteObject_Runner(self):
obj = make_runner(TestRunner, name='a')
tc = TestCase(runner=obj)
# No error, no output
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2011)
self.assertEqual(out.stdout, '')
self.assertEqual(out.stderr, '')
self.assertEqual(out.reasons, [])
# No error, with stdout and stderr
obj.setValue('stdout', True)
obj.setValue('stderr', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2011)
self.assertIn('runner stdout\n', out.stdout)
self.assertIn('runner stderr\n', out.stderr)
self.assertEqual(out.reasons, [])
# Error
obj.setValue('error', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.ERROR)
self.assertEqual(out.returncode, 2011)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn('runner stderr\n', out.stderr)
self.assertIn('runner error', out.stderr)
self.assertIn("An error occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Exception
obj.setValue('raise', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.EXCEPTION)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn('runner stderr\n', out.stderr)
self.assertIn('runner error', out.stderr)
self.assertIn('runner raise', out.stderr)
self.assertIn("An exception occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Exception during reset
def side_effect():
print('print text')
raise Execption("reset failed")
with mock.patch("moosetools.moosetest.base.Runner.reset") as reset:
reset.side_effect = side_effect
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'print text\n')
self.assertIn('reset failed', out.stderr)
self.assertIn("An exception occurred within the `reset` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Error on preExecute
with mock.patch('moosetools.moosetest.base.Runner.status', return_value=1):
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn("An error occurred within the `preExecute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Exception on preExecute
with mock.patch('moosetools.moosetest.base.Runner.preExecute', side_effect=Exception()):
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, '')
self.assertIn("An exception occurred within the `preExecute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Error on postExecute
obj.setValue('raise', False)
with mock.patch('moosetools.moosetest.base.Runner.status', side_effect=[0, 0, 1]):
out = tc._executeObject(obj)
print(out.stdout)
print(out.stderr)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn("An error occurred within the `postExecute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Exception on postExecute
with mock.patch('moosetools.moosetest.base.Runner.postExecute', side_effect=Exception()):
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn("An exception occurred within the `postExecute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Error on execute, postExecute still runs
obj.setValue('error', True)
with mock.patch('moosetools.moosetest.base.Runner.postExecute') as mock_postExecute:
out = tc._executeObject(obj)
mock_postExecute.assert_called_once()
self.assertEqual(out.state, TestCase.Result.ERROR)
self.assertEqual(out.returncode, 2011)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn('runner stderr\n', out.stderr)
self.assertIn('runner error', out.stderr)
self.assertIn("An error occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Error on execute, postExecute still runs
obj.setValue('error', False)
obj.setValue('raise', True)
with mock.patch('moosetools.moosetest.base.Runner.postExecute') as mock_postExecute:
out = tc._executeObject(obj)
mock_postExecute.assert_called_once()
self.assertEqual(out.state, TestCase.Result.EXCEPTION)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'runner stdout\n')
self.assertIn('runner stderr\n', out.stderr)
self.assertIn('runner raise', out.stderr)
self.assertIn("An exception occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
