def test_loading_tests(self):
"""Make sure get_tests can find tests and resolve inheritance."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world'])
self.assertEqual(sorted(['narf', 'hello', 'world']),
sorted([test['name'] for test in tests]))
tests = load_test_configs(self.pav_cfg, 'this', [],
['hello_world.hello'])
hello = tests.pop()
# There should have only been 1
self.assertFalse(tests)
# Check some basic test attributes.
self.assertEqual(hello['scheduler'], 'raw')
self.assertEqual(hello['suite'], 'hello_world')
# Make sure the variable from the host config got propagated.
self.assertIn('hosty', hello['variables'])
tests = load_test_configs(self.pav_cfg, 'this', [],
['hello_world.narf'])
narf = tests.pop()
# Make sure this got overridden from 'world'
self.assertEqual(narf['scheduler'], 'dummy')
# Make sure this didn't get lost.
self.assertEqual(narf['run']['cmds'], ['echo "Running World"'])
plugins._reset_plugins()
def test_loading_hidden(self):
"""Make sure we only get hidden tests when specifically requested."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden'])
names = sorted([t['name'] for t in tests])
self.assertEqual(names, ['hello', 'narf'])
tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden._hidden'])
names = sorted([t['name'] for t in tests])
self.assertEqual(names, ['_hidden'])
plugins._reset_plugins()
def test_env_order(self):
"""Make sure environment variables keep their order from the test
config to the final run scripts."""
test_conf = load_test_configs(self.pav_cfg, 'this', [], ['order'])[0]
test = self._quick_test(test_conf, "order")
# Each exported variable in this config has a value that denotes its
# expected place in the order. The variable names are random letter
# sequences in a random order; hash order should be decidedly different
# than the listed order.
exports = []
with (test.path / 'build.sh').open() as build_script:
for line in build_script:
if line.startswith('export') and 'TEST_ID' not in line:
_, var_val = line.split()
var, val = line.split('=')
val = int(val)
exports.append((var, val))
# The values should be the numbers for 0..99, in that order.
self.assertEqual(list(range(len(exports))),
[val for var, val in exports],
msg="Environment variable order not preserved. \n"
"Got the following instead: \n{}".format(''.join(
["{}: {}\n".format(*v) for v in exports])))
def test_defaulted_variables(self):
"""Make sure default variables work as expected."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg,
host='defaulted',
modes=['defaulted'],
tests=['defaulted.test'])
test = tests[0]
self.assertEqual(test['variables']['host_def'], ['host'])
self.assertEqual(test['variables']['mode_def'], ['mode'])
self.assertEqual(test['variables']['test_def'], ['test'])
self.assertNotIn('no_val', test['variables'])
with self.assertRaises(TestConfigError):
tests = load_test_configs(self.pav_cfg,
host='defaulted',
modes=['defaulted'],
tests=['defaulted_error.error'])
plugins._reset_plugins()
def test_apply_overrides(self):
"""Make sure overrides get applied to test configs correctly."""
tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world'])
overrides = {'run': {'env': {'foo': 'bar'}}, 'scheduler': 'fuzzy'}
for test in tests:
otest = test.copy()
apply_overrides(test, overrides)
# Make sure the overrides were applied
self.assertEqual(test['scheduler'], 'fuzzy')
self.assertEqual(test['run']['env']['foo'], 'bar')
# Make sure other stuff wasn't changed.
self.assertEqual(test['build'], otest['build'])
self.assertEqual(test['run']['cmds'], otest['run']['cmds'])
def test_extended_variables(self):
"""Make sure default variables work as expected."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg,
host='extended',
modes=['extended'],
tests=['extended.test'])
test = tests[0]
self.assertEqual(test['variables']['long_base'],
['what', 'are', 'you', 'up', 'to', 'punk?'])
self.assertEqual(test['variables']['single_base'],
['host', 'mode', 'test'])
self.assertEqual(test['variables']['no_base_mode'], ['mode'])
self.assertEqual(test['variables']['no_base'], ['test'])
self.assertEqual(test['variables']['null_base_mode'], ['mode'])
self.assertEqual(test['variables']['null_base'], ['test'])
plugins._reset_plugins()
def test_layering(self):
"""Make sure test config layering works as expected."""
plugins.initialize_plugins(self.pav_cfg)
for host in ('this', 'layer_host'):
for modes in ([], ['layer_mode']):
for test in ('layer_tests.layer_test',
'layer_tests.layer_test_part'):
answer = 'standard'
if host == 'layer_host':
answer = 'host'
if modes:
answer = 'mode'
if test.endswith('part'):
answer = 'test'
tests = load_test_configs(pav_cfg=self.pav_cfg,
host=host,
modes=modes,
tests=[test])
self.assertEqual(tests[0]['slurm']['partition'], answer)
plugins._reset_plugins()
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides,
sys_vars):
"""Translate a general set of pavilion test configs into the final,
resolved configurations. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:param pav_cfg: The pavilion config
:param str host: The host config to target these tests with
:param list(str) modes: The mode configs to use.
:param list(Path) test_files: Files containing a newline separated
list of tests.
:param list(str) tests: The tests to run.
:param list(str) overrides: Overrides to apply to the configurations.
:param system_variables.SysVarDict sys_vars: The system variables dict.
:returns: A dictionary (by scheduler type name) of lists of test
configs.
"""
self.logger.debug("Finding Configs")
# Use the sys_host if a host isn't specified.
if host is None:
host = sys_vars.get('sys_name')
tests = list(tests)
for file in test_files:
try:
with file.open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
try:
raw_tests = test_config.load_test_configs(pav_cfg, host, modes,
tests)
except test_config.TestConfigError as err:
self.logger.error(str(err))
raise commands.CommandError(str(err))
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
test_config.apply_overrides(test_cfg, overrides)
except test_config.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
p_cfg, permutes = test_config.resolve_permutations(
test_cfg, pav_cfg.pav_vars, sys_vars)
for p_var_man in permutes:
sched = p_cfg['scheduler'].resolve(p_var_man)
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except test_config.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_scheduler_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the scheduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
test_var_man.add_var_set('sched', sched.get_vars(test_cfg))
# Resolve all variables for the test.
try:
resolved_config = test_config.resolve_all_vars(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = "Error resolving variables in config at '{}': {}"\
.format(test_cfg['suite_path'].resolve(test_var_man),
err)
self.logger.error(msg)
raise commands.CommandError(msg)
tests_by_scheduler[sched.name].append(resolved_config)
return tests_by_scheduler
