def test_evaluate_errors(self):
error_confs = (
{
'val_a': 'undefined_a'
}, # No such variable
{
'val_b': 'parse_error ++'
}, # Parse error
{
'val_c': '"hello" + 3'
}, # Value Error
{
'val_d': 'val_e + 3',
'val_e': 'val_d + 1'
}, # Reference loop.
{
'val_f': 'really.complicated.*.variable.error'
})
for error_conf in error_confs:
cfg = self._quick_test_cfg()
cfg['result_evaluate'] = error_conf
test = self._quick_test(cfg)
test.run()
with self.assertRaises(pavilion.result.common.ResultError):
result.evaluate_results({}, error_conf, utils.IndentedLog())
def gather_results(self, run_result, regather=False, log_file=None):
"""Process and log the results of the test, including the default set
of result keys.
:param int run_result: The return code of the test run.
:param bool regather: Gather results without performing any changes to the
test itself.
:param IO[str] log_file: The file to save result logs to.
"""
import pprint
if self.finished is None:
raise RuntimeError(
"test.gather_results can't be run unless the test was run"
"(or an attempt was made to run it. "
"This occurred for test {s.name}, #{s.id}".format(s=self))
parser_configs = self.config['result_parse']
result_log = utils.IndentedLog(log_file)
result_log("Gathering base results.")
results = result.base_results(self)
results['return_value'] = run_result
result_log("Base results:")
result_log.indent = 1
result_log(pprint.pformat(results))
if not regather:
self.status.set(
STATES.RESULTS,
"Parsing {} result types.".format(len(parser_configs)))
try:
result.parse_results(self, results, log=result_log)
except pavilion.result.common.ResultError as err:
results['result'] = self.ERROR
results['pav_result_errors'].append(
"Error parsing results: {}".format(err.args[0]))
if not regather:
self.status.set(STATES.RESULTS_ERROR,
results['pav_result_errors'][-1])
if not regather:
self.status.set(
STATES.RESULTS, "Performing {} result evaluations.".format(
len(self.config['result_evaluate'])))
try:
result.evaluate_results(results, self.config['result_evaluate'],
result_log)
except pavilion.result.common.ResultError as err:
results['result'] = self.ERROR
results['pav_result_errors'].append(err.args[0])
if not regather:
self.status.set(STATES.RESULTS_ERROR,
results['pav_result_errors'][-1])
if results['result'] is True:
results['result'] = self.PASS
elif results['result'] is False:
results['result'] = self.FAIL
else:
results['pav_result_errors'].append(
"The value for the 'result' key in the results must be a "
"boolean. Got '{}' instead".format(results['result']))
results['result'] = self.ERROR
result_log("Set final result key to: '{}'".format(results['result']))
result_log("See results.json for the final result json.")
result_log("Removing temporary values.")
result_log.indent = 1
result.remove_temp_results(results, result_log)
self._results = results
return results
def gather_results(self, run_result, regather=False):
"""Process and log the results of the test, including the default set
of result keys.
:param int run_result: The return code of the test run.
:param bool regather: Gather results without performing any changes to the
test itself.
"""
if self.finished is None:
raise RuntimeError(
"test.gather_results can't be run unless the test was run"
"(or an attempt was made to run it. "
"This occurred for test {s.name}, #{s.id}"
.format(s=self)
)
parser_configs = self.config['result_parse']
results = result.base_results(self)
results['return_value'] = run_result
if not regather:
self.status.set(STATES.RESULTS,
"Parsing {} result types."
.format(len(parser_configs)))
try:
result.parse_results(self, results)
except result.ResultError as err:
results['result'] = self.ERROR
results['pav_result_errors'].append(
"Error parsing results: {}".format(err.args[0]))
if not regather:
self.status.set(STATES.RESULTS_ERROR,
results['pav_result_errors'][-1])
return results
if not regather:
self.status.set(STATES.RESULTS,
"Performing {} result evaluations."
.format(len(self.config['result_evaluate'])))
try:
result.evaluate_results(
results,
self.config['result_evaluate'])
except result.ResultError as err:
results['result'] = self.ERROR
results['pav_result_errors'].append(err.args[0])
results['result'] = self.ERROR
if not regather:
self.status.set(STATES.RESULTS_ERROR,
results['pav_result_errors'][-1])
return results
if results['result'] is True:
results['result'] = self.PASS
elif results['result'] is False:
results['result'] = self.FAIL
else:
results['result'] = self.ERROR
results['pav_result_errors'].append(
"The value for the 'result' key in the results must be a "
"boolean. Got '{}' instead".format(results['result']))
return results
self._results = results
return results