def testMinFailState(self):
# SKIP as failure
c = TestController(skip=True)
r = make_runner(TestRunner, (c, ), name='Andrew')
fm = Formatter()
rcode = run([[r]], (c, ), fm, min_fail_state=TestCase.Result.SKIP)
self.assertEqual(rcode, 1) # this is what is being tested
self.assertEqual(self._r_state.call_count, 1)
self.assertEqual(self._r_results.call_count, 1)
self.assertCall(self._r_state.call_args_list[0],
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
def testFutureException(self):
r = TestRunner(name='Andrew', stderr=True, stdout=True)
fm = Formatter()
with mock.patch('concurrent.futures.Future.exception', return_value=Exception('future exception')), \
self.assertRaises(Exception) as cm:
rcode = run([[r]], tuple(), fm)
self.assertIn('future exception', str(cm.exception))
def main():
"""
Complete function for automatically detecting and performing tests based on test specifications.
This function exists for the use by the `moosetest` executable in the bin directory of the
moosetools repository.
"""
# Extract command-line arguments
args = cli_args()
if args.demo:
return fuzzer()
# Locate the config
filename = _locate_config(args.config)
# Load the config (pyhit.Node)
root = _load_config(filename)
# Create the TestHarness object from the configuration, after this point the cli_args should
# no longer be used. They are applied to the TestHarness object in this function by calling
# the TestHarness.applyCommandLineArguments method.
harness = make_harness(filename, root, args)
del args # just to avoid accidental use in the future
# Create the Controller objects and Formatter
controllers = make_controllers(filename, root,
harness.getParam('plugin_dirs'))
formatter = make_formatter(filename, root, harness.getParam('plugin_dirs'))
# Locate the tests to execute
groups = discover(os.getcwd(), harness.getParam('spec_file_names'),
harness.getParam('spec_file_blocks'),
harness.getParam('plugin_dirs'), controllers,
harness.getParam('n_threads'))
# Execute the tests
rcode = run(groups, controllers, formatter, harness.getParam('n_threads'),
harness.getParam('timeout'), harness.getParam('max_failures'))
return rcode
def testMaxFail(self):
r0 = make_runner(TestRunner, name='Just Andrew', error=True)
r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5)
r2 = make_runner(TestRunner, name='Best Andrew')
fm = Formatter()
# This test helped me catch a logic bug. This is a single group, so only a single worker
# is created. Thus, even if max fail is hit the worker should finish, thus nothing reported
# as max failed. At one point both the max fail and dependency message were being dumped
# with this test, but it should only be the dependency.
rcode = run([[r0, r1, r2]], tuple(), fm, max_fails=1)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 3)
self.assertEqual(self._r_results.call_count, 3)
self.assertCall(self._r_state.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
self.assertCall(self._r_state.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
# Similar to above, but with individual groups
groups = list()
for i in range(5):
r = make_runner(TestRunner, name=str(i), sleep=1)
groups.append([r])
groups[0][0].setValue('error', True)
groups[0][0].setValue('sleep', 0)
self.resetMockObjects()
rcode = run(groups, tuple(), fm, n_threads=1, max_fails=1)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 5)
self.assertEqual(self._r_results.call_count, 5)
# Only look at first and last, the middle can change depending how fast works fire up
self.assertCall(self._r_state.call_args_list[0],
name='0',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='0',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[-1],
name='4',
state=TestCase.Result.SKIP,
reasons=['max failures reached'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[-1],
name='4',
state=TestCase.Result.SKIP,
reasons=['max failures reached'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("Max failures of 1 exceeded."))
def testRunnerWithDiffersWithControllers(self):
c = TestController()
d0 = make_differ(TestDiffer, (c, ), name='a')
d1 = make_differ(TestDiffer, (c, ), name='b')
r = make_runner(TestRunner, (c, ), name='Andrew', differs=(d0, d1))
fm = Formatter()
# SKIP, RUNNER
c.setValue('skip', True)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
# SKIP, DIFFER
self.resetMockObjects()
c.setValue('object_name', d0.name())
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testRunnerWithDiffers(self):
d0 = make_differ(TestDiffer, name='a', stderr=True)
d1 = make_differ(TestDiffer, name='b', stdout=True)
r = make_runner(TestRunner, name='Andrew', differs=(d0, d1))
fm = Formatter()
# PASS
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ stderr'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout=TestRun.IN('differ stdout'),
stderr='')
# ERROR, DIFFER 0
self.resetMockObjects()
d0.setValue('stderr', False)
d1.setValue('stdout', False)
d0.setValue('error', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ error'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
# EXCEPTION, DIFFER 1
self.resetMockObjects()
d0.setValue('error', False)
d1.setValue('raise', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.EXCEPTION,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ raise'))
# TIMEOUT, DIFFER 1
self.resetMockObjects()
d1.setValue('sleep', 1)
d1.setValue('raise', False)
rcode = run([[r]], tuple(), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 0)
self.assertEqual(self._d_results.call_count, 0)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testRunnerWithController(self):
c = TestController(stdout=True, stderr=True)
r = make_runner(TestRunner, (c, ), name='Andrew')
fm = Formatter()
# PASS
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout=TestRun.IN('controller stdout'),
stderr=TestRun.IN('controller stderr'))
# ERROR, CONTROLLER
self.resetMockObjects()
c.setValue('error', True)
c.setValue('stderr', False)
c.setValue('stdout', False)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An error occurred, on the controller'))
# ERROR, RUNNER (during execution of Controller)
self.resetMockObjects()
c.setValue('error', False)
with mock.patch('moosetools.moosetest.base.Runner.status',
return_value=1):
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An error occurred, on the object'))
# EXCEPTION
self.resetMockObjects()
c.setValue('error', False)
c.setValue('raise', True)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.FATAL,
reasons=None,
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('An exception occurred'))
# TIMEOUT (because of Controller)
self.resetMockObjects()
c.setValue('raise', False)
c.setValue('sleep', 1)
rcode = run([[r]], (c, ), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
# SKIP
self.resetMockObjects()
c.setValue('skip', True)
c.setValue('sleep', 0)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN(''))
def testRunnerOnly(self):
r = TestRunner(name='Andrew', stderr=True, stdout=True)
fm = Formatter()
# PASS
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=2011,
stdout=TestRun.IN('runner stdout'),
stderr=TestRun.IN('runner stderr'))
# ERROR
self.resetMockObjects()
r.setValue('error', True)
r.setValue('stderr', False)
r.setValue('stdout', False)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error\n'))
# EXCEPTION
self.resetMockObjects()
r.setValue('error', False)
r.setValue('raise', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN('runner raise\n'))
# TIMEOUT
self.resetMockObjects()
r.setValue('raise', False)
r.setValue('sleep', 1)
rcode = run([[r]], tuple(), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN(''))
def testGroupSkip(self):
# Same as first test in testMaxFails, but without the max fails
r0 = make_runner(TestRunner, name='Just Andrew', error=True)
r1 = make_runner(TestRunner, name='Other Andrew', sleep=0.5)
r2 = make_runner(TestRunner, name='Best Andrew')
fm = Formatter()
rcode = run([[r0, r1, r2]], tuple(), fm)
self.assertEqual(rcode, 1)
self.assertEqual(self._r_state.call_count, 3)
self.assertEqual(self._r_results.call_count, 3)
self.assertCall(self._r_state.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(self._r_results.call_args_list[0],
name='Just Andrew',
state=TestCase.Result.ERROR,
reasons=[],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr=TestRun.IN('runner error'))
self.assertCall(self._r_state.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[1],
name='Other Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=TestRun.ANY,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
self.assertCall(self._r_state.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY)
self.assertCall(
self._r_results.call_args_list[2],
name='Best Andrew',
state=TestCase.Result.SKIP,
reasons=['dependency'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr=TestRun.IN("A previous test case (Just Andrew)"))
