def _load_test(self,
name: str,
host: str = 'this',
modes: List[str] = None,
build=True,
finalize=True) -> List[TestRun]:
"""Load the named test config from file. Returns a list of the
resulting configs."""
if modes is None:
modes = []
res = resolver.TestConfigResolver(self.pav_cfg)
test_cfgs = res.load([name], host, modes)
tests = []
for ptest in test_cfgs:
test = TestRun(self.pav_cfg, ptest.config, var_man=ptest.var_man)
if build:
test.build()
if finalize:
fin_sys = system_variables.SysVarDict(unique=True)
fin_var_man = VariableSetManager()
fin_var_man.add_var_set('sys', fin_sys)
res.finalize(test, fin_var_man)
tests.append(test)
return tests
def _quick_test(self,
cfg=None,
name="quick_test",
build=True,
finalize=True,
sched_vars=None):
"""Create a test run object to work with.
The default is a simple hello world test with the raw scheduler.
:param dict cfg: An optional config dict to create the test from.
:param str name: The name of the test.
:param bool build: Build this test, while we're at it.
:param bool finalize: Finalize this test.
:param dict sched_vars: Add these scheduler variables to our var set.
:rtype: TestRun
"""
if cfg is None:
cfg = self._quick_test_cfg()
cfg = copy.deepcopy(cfg)
loader = TestConfigLoader()
cfg = loader.validate(loader.normalize(cfg))
cfg['name'] = name
var_man = VariableSetManager()
var_man.add_var_set('var', cfg['variables'])
var_man.add_var_set('sys', system_variables.get_vars(defer=True))
var_man.add_var_set('pav', self.pav_cfg.pav_vars)
if sched_vars is not None:
var_man.add_var_set('sched', sched_vars)
var_man.resolve_references()
cfg = resolver.TestConfigResolver.resolve_test_vars(cfg, var_man)
test = TestRun(
pav_cfg=self.pav_cfg,
config=cfg,
var_man=var_man,
)
if build:
test.build()
if finalize:
fin_sys = system_variables.SysVarDict(unique=True)
fin_var_man = VariableSetManager()
fin_var_man.add_var_set('sys', fin_sys)
resolver.TestConfigResolver.finalize(test, fin_var_man)
return test
def _get_var_man(test, sched):
"""Get the variable manager for the given test.
:param TestRun test: The test run object
:param sched: The scheduler for this test.
:rtype VariableSetManager
"""
# Re-add var sets that may have had deferred variables.
try:
var_man = VariableSetManager()
var_man.add_var_set('sys', system_variables.get_vars(defer=False))
sched_config = test.config[test.scheduler]
var_man.add_var_set('sched', sched.get_vars(sched_config))
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error getting pavilion variables at "
"run time.")
raise
return var_man
def test_deferred(self):
# The following tests make sure deferred variables are
# interpreted correctly by the conditional checks.
test_list = []
base_cfg = self._quick_test_cfg()
# Test 1:
# Not_if with deferred variable that resolves to skip.
test_cfg = base_cfg.copy()
test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']}
test_list.append(test_cfg)
# Test 2:
# Only_if with deferred variable that resolves to skip.
test_cfg = base_cfg.copy()
test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']}
test_list.append(test_cfg)
# Test 3:
# Not_if that fails to skip with deferred only_if that skips.
test_cfg = base_cfg.copy()
test_cfg['not_if'] = {
'{{dumb_user}}': ['nivlac', 'notcalvin'],
'{{dumb_os}}': ['blurg']
}
test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']}
test_list.append(test_cfg)
# Test 4:
# Only_if that fails to skip with deferred not_if that skips.
test_cfg = base_cfg.copy()
test_cfg['only_if'] = {
'{{dumb_user}}': ['nivlac', 'calvin'],
'{{dumb_os}}': ['bieber']
}
test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']}
test_list.append(test_cfg)
# Run through scenario of deferred(no-skip) into skip.
for test_cfg in test_list:
test = self._quick_test(cfg=test_cfg, finalize=False)
self.assertFalse(test.skipped,
msg="dumb_sys_var should be deferred"
" with skip not assigned to"
" the test")
fin_sys = system_variables.SysVarDict(defer=False, unique=True)
fin_var_man = VariableSetManager()
fin_var_man.add_var_set('sys', fin_sys)
test.finalize(fin_var_man)
self.assertTrue(test.skipped, msg="Now it should skip")
test_list = []
# Test 5:
# Not_if with deferred variable that resolves to no skip.
test_cfg = base_cfg.copy()
test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']}
test_list.append(test_cfg)
# Test 6:
# Only_if with deferred variable that resolves to no skip.
test_cfg = base_cfg.copy()
test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']}
test_list.append(test_cfg)
# Test 7:
# Not_if and only_if-deferred that fails to skip.
test_cfg = base_cfg.copy()
test_cfg['not_if'] = {
'{{dumb_user}}': ['nivlac', 'notcalvin'],
'{{dumb_os}}': ['blurg']
}
test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']}
test_list.append(test_cfg)
# Test 8:
# Only_if and not_if-deferred that fails to skip.
test_cfg = base_cfg.copy()
test_cfg['only_if'] = {
'{{dumb_user}}': ['nivlac', 'calvin'],
'{{dumb_os}}': ['bieber']
}
test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']}
test_list.append(test_cfg)
# Run through scenario of deferred(no-skip) into no skip.
for test_cfg in test_list:
test = self._quick_test(cfg=test_cfg, finalize=False)
self.assertFalse(test.skipped,
msg="dumb_sys_var should be deferred"
" with skip not assigned to"
" the test.")
fin_sys = system_variables.SysVarDict(defer=False, unique=True)
fin_var_man = VariableSetManager()
fin_var_man.add_var_set('sys', fin_sys)
test.finalize(fin_var_man)
self.assertFalse(test.skipped, msg="Test Should NOT skip.")
def _run(self, pav_cfg, test):
"""Run an already prepped test in the current environment.
"""
try:
sched = schedulers.get_plugin(test.scheduler)
except Exception:
test.status.set(STATES.BUILD_ERROR,
"Unknown error getting the scheduler. Refer to "
"the kickoff log.")
raise
# Re-add var sets that may have had deferred variables.
try:
var_man = VariableSetManager()
var_man.add_var_set('sys', system_variables.get_vars(defer=False))
sched_config = test.config[test.scheduler]
var_man.add_var_set('sched', sched.get_vars(sched_config))
except Exception:
test.status.set(STATES.RUN_ERROR,
"Unknown error getting pavilion variables at "
"run time.")
raise
try:
test.finalize(var_man)
except Exception:
test.status.set(STATES.RUN_ERROR,
"Unknown error finalizing test.")
raise
try:
if test.config['build']['on_nodes'] in ['true', 'True']:
if not test.build():
self.logger.warning(
"Test {t.id} failed to build:"
)
except Exception:
test.status.set(STATES.BUILD_ERROR,
"Unknown build error. Refer to the kickoff log.")
raise
# Optionally wait on other tests running under the same scheduler.
# This depends on the scheduler and the test configuration.
lock = sched.lock_concurrency(pav_cfg, test)
try:
run_result = test.run()
except TestRunError as err:
test.status.set(STATES.RUN_ERROR, err)
return 1
except TimeoutError:
return 1
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error while running test. Refer to the kickoff log.")
raise
finally:
sched.unlock_concurrency(lock)
try:
rp_errors = []
# Make sure the result parsers have reasonable arguments.
# We check here because the parser code itself will likely assume
# the args are valid form _check_args, but those might not be
# checkable before kickoff due to deferred variables.
try:
result_parsers.check_args(test.config['results'])
except TestRunError as err:
rp_errors.append(str(err))
if rp_errors:
for msg in rp_errors:
test.status.set(STATES.RESULTS_ERROR, msg)
test.set_run_complete()
return 1
results = test.gather_results(run_result)
except result_parsers.ResultParserError as err:
self.logger.error("Unexpected error gathering results: %s", err)
test.status.set(STATES.RESULTS_ERROR,
"Error parsing results: {}".format(err))
return 1
try:
test.save_results(results)
result_logger = logging.getLogger('results')
result_logger.info(output.json_dumps(results))
except Exception:
test.status.set(
STATES.RESULTS_ERROR,
"Unknown error while saving results. Refer to the kickoff log.")
raise
try:
test.status.set(STATES.COMPLETE,
"The test completed with result: {}"
.format(results.get('result', '<unknown>')))
except Exception:
test.status.set(
STATES.UNKNOWN,
"Unknown error while setting test completion. Refer to the "
"kickoff log.")
raise