def test_getConfigItem(self):
config_file = os.path.join(os.path.dirname(__file__), 'TestConfig.h')
ctrl = TestConfig(config_files=(config_file, ), log_level='DEBUG')
obj = make_differ(TestDiffer, (ctrl, ))
m_value, r_value, r_name = ctrl.getConfigItem(obj.getParam('moose'), 'ad_mode')
self.assertEqual(m_value, 'SPARSE')
self.assertEqual(r_value, '1')
self.assertEqual(r_name, 'MOOSE_SPARSE_AD')
m_value, r_value, r_name = ctrl.getConfigItem(obj.getParam('moose'), 'not_in_config')
self.assertEqual(m_value, 'YES')
self.assertEqual(r_value, '1')
self.assertEqual(r_name, 'MOOSE_NOT_CONFIGURED')
with self.assertRaises(RuntimeError) as e:
ctrl.getConfigItem(obj.getParam('moose'), 'no_user_data')
self.assertEqual(
"The parameter 'no_user_data' does not contain a `AutotoolsConfigItem` object within the parameter 'user_data'.",
str(e.exception))
ctrl._AutotoolsConfigController__config_items['MOOSE_NO_MAP'] = '42'
with self.assertRaises(RuntimeError) as e:
ctrl.getConfigItem(obj.getParam('moose'), 'no_map')
self.assertEqual(
"The value of 'no_map' in the loaded file does not have a registered value in the mapping for '42'. The available mapping values are: 0, 1",
str(e.exception))
m_value, r_value, r_name = ctrl.getConfigItem(obj.getParam('moose'), 'value_from_func')
self.assertEqual(m_value, 1980)
self.assertEqual(r_value, '1980')
self.assertEqual(r_name, 'MOOSE_VALUE')
def testDefault(self):
ctrl = EnvironmentController()
obj = make_differ(TestObject, (ctrl, ))
self.assertEqual(ctrl.name(), 'EnvironmentController')
self.assertEqual(ctrl.getParam('prefix'), 'env')
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), True)
self.assertEqual(ctrl.getReasons(), [])
def test_checkConfig(self):
config_file = os.path.join(os.path.dirname(__file__), 'TestConfig.h')
ctrl = TestConfig(config_files=(config_file, ), log_level='DEBUG')
obj = make_differ(TestDiffer, (ctrl, ))
ctrl.checkConfig(obj.getParam('moose'), 'ad_mode')
obj.parameters().setValue('moose', 'ad_mode', 'NONSPARSE')
with self.assertLogs(level='DEBUG') as log:
ctrl.execute(obj, obj.getParam('moose'))
self.assertEqual(len(log.output), 1)
self.assertIn(
"The application is configured with 'MOOSE_SPARSE_AD' equal to '1', which maps to a value of 'SPARSE'. However, the associated 'ad_mode' parameter for this test requires 'NONSPARSE'.",
log.output[0])
def testMaxVersion(self):
ctrl = EnvironmentController()
obj = make_differ(TestObject, (ctrl, ),
env_python_maximum_version='1980.6.24')
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), True)
self.assertEqual(ctrl.getReasons(), [])
with mock.patch('platform.python_version', return_value='2013.5.15'):
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), False)
self.assertEqual(ctrl.getReasons(), ['Python 1980.6.24 < 2013.5.15'])
def testPlatform(self):
ctrl = EnvironmentController()
obj = make_differ(TestObject, (ctrl, ), env_platform=('Darwin', ))
with mock.patch('platform.system', return_value='Darwin'):
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), True)
self.assertEqual(ctrl.getReasons(), [])
with mock.patch('platform.system', return_value='Linux'):
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), False)
self.assertEqual(ctrl.getReasons(), ["'Linux' not in ('Darwin',)"])
def testRequired(self):
ctrl = EnvironmentController()
obj = make_differ(TestObject, (ctrl, ),
env_python_required_packages=('sys', 'io',
'unittest'))
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), True)
self.assertEqual(ctrl.getReasons(), [])
with mock.patch('moosetools.mooseutils.check_configuration',
return_value=['sys', 'io']):
ctrl.execute(obj, obj.getParam('env'))
self.assertEqual(ctrl.isRunnable(), False)
self.assertEqual(ctrl.getReasons(), ['missing python package(s)'])
def testSetResults(self):
ct = TestController()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
# Wrong type
tc.setResults('wrong')
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The supplied result must be of type `dict`.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'r': 'wrong'})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The supplied result values must be of type `TestCase.Data`.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'wrong': TestCase.Data()})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"The supplied result keys must be the names of the `Runner` or `Differ` object(s).",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
tc.setResults({'r': TestCase.Data(TestCase.Result.PASS, None, 'out', 'err', None)})
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, 'out')
self.assertEqual(r['r'].stderr, 'err')
self.assertEqual(r['r'].reasons, None)
def testRunnerWithDiffersWithControllers(self):
c = TestController()
d0 = make_differ(TestDiffer, (c, ), name='a')
d1 = make_differ(TestDiffer, (c, ), name='b')
r = make_runner(TestRunner, (c, ), name='Andrew', differs=(d0, d1))
fm = Formatter()
# SKIP, RUNNER
c.setValue('skip', True)
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self._d_state.assert_not_called()
self._d_results.assert_not_called()
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
returncode=None,
stdout='',
stderr='')
# SKIP, DIFFER
self.resetMockObjects()
c.setValue('object_name', d0.name())
rcode = run([[r]], (c, ), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
returncode=2011,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.SKIP,
reasons=['a reason'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testRunnerWithDiffers(self):
d0 = make_differ(TestDiffer, name='a', stderr=True)
d1 = make_differ(TestDiffer, name='b', stdout=True)
r = make_runner(TestRunner, name='Andrew', differs=(d0, d1))
fm = Formatter()
# PASS
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 0)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ stderr'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout=TestRun.IN('differ stdout'),
stderr='')
# ERROR, DIFFER 0
self.resetMockObjects()
d0.setValue('stderr', False)
d1.setValue('stdout', False)
d0.setValue('error', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.DIFF,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ error'))
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
# EXCEPTION, DIFFER 1
self.resetMockObjects()
d0.setValue('error', False)
d1.setValue('raise', True)
rcode = run([[r]], tuple(), fm)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 2)
self.assertEqual(self._d_results.call_count, 2)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.EXCEPTION,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_state.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_state.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY)
self.assertCall(self._d_results.call_args_list[0],
name='a',
state=TestCase.Result.PASS,
reasons=[],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
self.assertCall(self._d_results.call_args_list[1],
name='b',
state=TestCase.Result.EXCEPTION,
reasons=None,
percent=100,
duration=TestRun.ANY,
stdout='',
stderr=TestRun.IN('differ raise'))
# TIMEOUT, DIFFER 1
self.resetMockObjects()
d1.setValue('sleep', 1)
d1.setValue('raise', False)
rcode = run([[r]], tuple(), fm, None, 0.5)
self.assertEqual(rcode, 1)
self._r_state.assert_called_once()
self._r_results.assert_called_once()
self.assertEqual(self._d_state.call_count, 0)
self.assertEqual(self._d_results.call_count, 0)
self._complete.assert_called_once()
self.assertCall(self._r_state,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY)
self.assertCall(self._r_results,
name='Andrew',
state=TestCase.Result.TIMEOUT,
reasons=['max time (0.5) exceeded'],
percent=100,
duration=TestRun.ANY,
stdout='',
stderr='')
def testReportResults(self, pstate, presult):
# Runner
fm = moosetest.base.Formatter(progress_interval=0)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'out', 'err', None)})
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
fm.reportResults(tc)
pstate.assert_called_with(tc, rr, tc.state, None)
presult.assert_called_with(tc, rr, tc.results['r'])
# Differ
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.TIMEOUT, None, 'd_out', 'd_err', None)
})
fm.reportResults(tc)
pstate.assert_called_with(tc, dr, tc.results['d'].state, None)
presult.assert_called_with(tc, dr, tc.results['d'])
# Errors
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The state has not been set via the `setState` method.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setState(TestCase.Result.PASS)
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The results have not been set via the `setResults` method.", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
ct = TestController()
rr = make_runner(TestRunner, [ct], name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setState(TestCase.Result.PASS)
tc.setResults({'r': TestCase.Data(TestCase.Result.ERROR, None, 'r_out', 'r_err', None)})
fm.reportResults(tc)
self.assertEqual(tc.progress, TestCase.Progress.FINISHED)
self.assertEqual(tc.state, TestCase.Result.FATAL)
r = tc.results
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("The execution has not finished, so results cannot be reported.",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
def testPrintState(self, r_state, d_state, r_result, d_result):
# Runner
ct = TestController()
fm = Formatter()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
tc.setProgress(TestCase.Progress.RUNNING)
# Runner, progress
fm._printState(tc, rr, TestCase.Progress.RUNNING, ["all the reasons"])
kwargs = r_state.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'r')
self.assertEqual(kwargs['state'], TestCase.Progress.RUNNING)
self.assertEqual(kwargs['reasons'], ["all the reasons"])
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
# Differ, progress
tc.setProgress(TestCase.Progress.FINISHED) # call this to use execute time
fm._printState(tc, dr, TestCase.Progress.FINISHED, ["all the reasons"])
kwargs = d_state.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'd')
self.assertEqual(kwargs['state'], TestCase.Progress.FINISHED)
self.assertEqual(kwargs['reasons'], ["all the reasons"])
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
# Runner, results
tc.setProgress(TestCase.Progress.FINISHED)
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None)
})
fm._printResult(tc, rr, tc.results['r'])
kwargs = r_result.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'r')
self.assertEqual(kwargs['state'], TestCase.Result.PASS)
self.assertEqual(kwargs['reasons'], None)
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
self.assertEqual(kwargs['stdout'], 'r_out')
self.assertEqual(kwargs['stderr'], 'r_err')
# Differ, results
tc.setState(TestCase.Result.PASS)
tc.setResults({
'r': TestCase.Data(TestCase.Result.PASS, None, 'r_out', 'r_err', None),
'd': TestCase.Data(TestCase.Result.PASS, None, 'd_out', 'd_err', None)
})
fm._printResult(tc, dr, tc.results['d'])
kwargs = d_result.call_args[1] # .call_args.kwargs (python > 3.7)
self.assertEqual(kwargs['name'], 'd')
self.assertEqual(kwargs['state'], TestCase.Result.PASS)
self.assertEqual(kwargs['reasons'], None)
self.assertIsInstance(kwargs['duration'], float) # exact number can't be tested
self.assertIsInstance(kwargs['percent'], float)
self.assertEqual(kwargs['stdout'], 'd_out')
self.assertEqual(kwargs['stderr'], 'd_err')
def fuzzer(seed=1980,
timeout=(3, 10),
max_fails=(15, 100),
progress_interval=(3, 15),
group_num=(15, 50),
group_name_len=(6, 25),
controller_num=(1, 6),
controller_skip=0.05,
controller_raise=0.05,
controller_error=0.1,
differ_num=(0, 3),
differ_raise=0.01,
differ_error=0.1,
differ_fatal=0.1,
differ_platform=0.1,
differ_name_len=(6, 15),
runner_num=(1, 3),
runner_raise=0.01,
runner_error=0.1,
runner_fatal=0.05,
runner_sleep=(0.5, 10),
runner_platform=0.1,
runner_name_len=(4, 29)):
"""
A tool for calling `run` function with randomized test cases.
"""
# This is more of a test object, so I wanted to keep the testing related import out of the
# main functions for the run command.
import random
import string
from moosetools.moosetest.formatters import BasicFormatter
from moosetools.moosetest.base import make_runner, make_differ
sys.path.append(os.path.join(os.path.dirname(__file__), 'tests'))
from _helpers import TestController, TestRunner, TestDiffer
def gen_name(rng):
return ''.join(random.sample(string.ascii_letters, random.randint(*rng)))
def gen_platform(ctrls, prob, kwargs):
if random.uniform(0, 1) < prob:
prefix = "{}_platform".format(random.choice(ctrls).getParam('prefix'))
value = tuple(
set(random.choices(['Darwin', 'Linux', 'Windows'], k=random.randint(1, 3))))
kwargs[prefix] = value
def gen_bool_with_odds(prob):
return random.uniform(0, 1) < prob
# Controller objects
controllers = list()
for i, n_controllers in enumerate(range(random.randint(*controller_num))):
name_start = random.choice(string.ascii_letters)
kwargs = dict()
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['prefix'] = "ctrl{:0.0f}".format(i)
kwargs['skip'] = gen_bool_with_odds(controller_skip)
kwargs['error'] = gen_bool_with_odds(controller_error)
kwargs['raise'] = gen_bool_with_odds(controller_raise)
controllers.append(TestController(object_name=name_start, **kwargs))
controllers = tuple(controllers)
# Runners/Differs
groups = list()
for n_groups in range(random.randint(*group_num)):
runners = list()
group_name = gen_name(group_name_len)
for n_runners in range(random.randint(*runner_num)):
differs = list()
for n_differs in range(random.randint(*differ_num)):
kwargs = dict()
kwargs['name'] = gen_name(differ_name_len)
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['error'] = gen_bool_with_odds(differ_error)
kwargs['raise'] = gen_bool_with_odds(differ_raise)
kwargs['fatal'] = gen_bool_with_odds(differ_fatal)
gen_platform(controllers, differ_platform, kwargs)
differs.append(make_differ(TestDiffer, controllers, **kwargs))
kwargs = dict()
kwargs['name'] = f"{group_name}/{gen_name(runner_name_len)}"
kwargs['differs'] = tuple(differs)
kwargs['stdout'] = True
kwargs['stderr'] = True
kwargs['error'] = gen_bool_with_odds(runner_error)
kwargs['raise'] = gen_bool_with_odds(runner_raise)
kwargs['fatal'] = gen_bool_with_odds(runner_fatal)
kwargs['sleep'] = random.uniform(*runner_sleep)
gen_platform(controllers, runner_platform, kwargs)
runners.append(make_runner(TestRunner, controllers, **kwargs))
groups.append(runners)
# Formatter
kwargs = dict()
kwargs['progress_interval'] = random.randint(*progress_interval)
formatter = BasicFormatter(**kwargs)
# Run
kwargs = dict()
kwargs['timeout'] = random.randint(*timeout)
kwargs['max_fails'] = random.randint(*max_fails)
kwargs['min_fail_state'] = random.choice([r for r in TestCase.Result])
return run(groups, controllers, formatter, **kwargs)
def testExecute(self):
ct = TestController()
dr = make_differ(TestDiffer, [ct], name='d')
rr = make_runner(TestRunner, [ct], differs=(dr, ), name='r')
tc = TestCase(runner=rr, controllers=(ct, ))
# No error, no output
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.PASS)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'], TestCase.Data(TestCase.Result.PASS, 2011, '', '', []))
self.assertEqual(r['d'], TestCase.Data(TestCase.Result.PASS, 2013, '', '', []))
## RUNNER ################################
# Reset Exception, Runner
with mock.patch("moosetools.moosetest.base.Runner.reset") as func:
func.side_effect = Exception("runner reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("An exception occurred within the `reset` method of the 'r' object.",
r['r'].stderr)
self.assertIn("runner reset raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Reset Exception, Controller with Runner
with mock.patch("moosetools.moosetest.base.Controller.reset") as func:
func.side_effect = Exception("controller reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertIn("controller reset raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Execute Exception, Controller with Runner
ct.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertIn("controller raise", r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
# Error Object, Controller with Runner
ct.setValue('raise', False)
with mock.patch("moosetools.moosetest.base.Runner.status") as func:
func.return_value = 1
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.FATAL)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn(
"An error occurred, on the object, within the `execute` method of the TestController controller with 'r' object.",
r['r'].stderr)
self.assertEqual(r['r'].reasons, [])
# Skip, Controller with Runner
ct.setValue('skip', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.SKIP)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.SKIP)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stderr, '')
self.assertEqual(r['r'].reasons, ['a reason'])
ct.setValue('skip', False)
# Error on Runner
rr.setValue('error', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.ERROR)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.ERROR)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertIn("runner error", r['r'].stderr)
self.assertIn("An error occurred within the `execute` method of the 'r' object",
r['r'].stderr)
self.assertEqual(r['r'].reasons, [])
rr.setValue('error', False)
# Exception on Runner
rr.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.EXCEPTION)
self.assertEqual(list(r.keys()), ['r'])
self.assertEqual(r['r'].state, TestCase.Result.EXCEPTION)
self.assertEqual(r['r'].returncode, None)
self.assertEqual(r['r'].stdout, '')
self.assertIn("runner raise", r['r'].stderr)
self.assertIn("An exception occurred within the `execute` method of the 'r' object",
r['r'].stderr)
self.assertEqual(r['r'].reasons, None)
rr.setValue('raise', False)
## DIFFER ################################
# Reset Exception, Differ
with mock.patch("moosetools.moosetest.base.Differ.reset") as func:
func.side_effect = Exception("differ reset raise")
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.FATAL)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn("An exception occurred within the `reset` method of the 'd' object.",
r['d'].stderr)
self.assertIn("differ reset raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
# Reset Exception, Controller with Differ
with mock.patch("moosetools.moosetest.base.Controller.reset") as func:
func.side_effect = [None, Exception("controller reset raise")]
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertIn("controller reset raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
# Execute Exception, Controller with Differ
ct.setValue('raise', True)
ct.setValue('object_name', 'd')
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An exception occurred within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertIn("controller raise", r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
ct.setValue('raise', False)
ct.setValue('object_name', None)
# Error Object, Controller with Differ
with mock.patch("moosetools.moosetest.base.Differ.status") as func:
func.return_value = 1
s, r = tc.execute()
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.FATAL)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn(
"An error occurred, on the object, within the `execute` method of the TestController controller with 'd' object.",
r['d'].stderr)
self.assertEqual(r['d'].reasons, [])
# Skip, Controller with Differ
ct.setValue('skip', True)
ct.setValue('object_name', 'd')
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.PASS)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.SKIP)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertEqual(r['d'].stderr, '')
self.assertEqual(r['d'].reasons, ['a reason'])
ct.setValue('skip', False)
ct.setValue('object_name', None)
# Error on Differ
dr.setValue('error', True)
s, r = tc.execute()
print(s, r)
self.assertEqual(s, TestCase.Result.DIFF)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.DIFF)
self.assertEqual(r['d'].returncode, 2013)
self.assertEqual(r['d'].stdout, '')
self.assertIn("differ error", r['d'].stderr)
self.assertIn("An error occurred within the `execute` method of the 'd' object",
r['d'].stderr)
self.assertEqual(r['d'].reasons, [])
dr.setValue('error', False)
# Exception on Differ
dr.setValue('raise', True)
s, r = tc.execute()
self.assertEqual(s, TestCase.Result.EXCEPTION)
self.assertEqual(list(r.keys()), ['r', 'd'])
self.assertEqual(r['r'].state, TestCase.Result.PASS)
self.assertEqual(r['r'].returncode, 2011)
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].stdout, '')
self.assertEqual(r['r'].reasons, [])
self.assertEqual(r['d'].state, TestCase.Result.EXCEPTION)
self.assertEqual(r['d'].returncode, None)
self.assertEqual(r['d'].stdout, '')
self.assertIn("differ raise", r['d'].stderr)
self.assertIn("An exception occurred within the `execute` method of the 'd' object",
r['d'].stderr)
self.assertEqual(r['d'].reasons, None)
def testExecuteObject_Differ(self):
obj = make_differ(TestDiffer, name='a')
r = make_runner(TestRunner, name='a', differs=(obj, ))
tc = TestCase(runner=r)
# No error, no output
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2013)
self.assertEqual(out.stdout, '')
self.assertEqual(out.stderr, '')
self.assertEqual(out.reasons, [])
# No error, with stdout and stderr
obj.setValue('stdout', True)
obj.setValue('stderr', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.PASS)
self.assertEqual(out.returncode, 2013)
self.assertIn('differ stdout\n', out.stdout)
self.assertIn('differ stderr\n', out.stderr)
self.assertEqual(out.reasons, [])
# Error
obj.setValue('error', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.DIFF)
self.assertEqual(out.returncode, 2013)
self.assertEqual(out.stdout, 'differ stdout\n')
self.assertIn('differ stderr\n', out.stderr)
self.assertIn('differ error', out.stderr)
self.assertIn("An error occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, [])
# Exception
obj.setValue('raise', True)
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.EXCEPTION)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'differ stdout\n')
self.assertIn('differ stderr\n', out.stderr)
self.assertIn('differ error', out.stderr)
self.assertIn('differ raise', out.stderr)
self.assertIn("An exception occurred within the `execute` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)
# Exception during reset
def side_effect():
print('print text')
raise Execption("reset failed")
with mock.patch("moosetools.moosetest.base.Differ.reset") as reset:
reset.side_effect = side_effect
out = tc._executeObject(obj)
self.assertEqual(out.state, TestCase.Result.FATAL)
self.assertEqual(out.returncode, None)
self.assertEqual(out.stdout, 'print text\n')
self.assertIn('reset failed', out.stderr)
self.assertIn("An exception occurred within the `reset` method of the 'a' object.",
out.stderr)
self.assertEqual(out.reasons, None)