def resolve_test_vars(cls, config, var_man):
"""Recursively resolve the variables in the value strings in the given
configuration.
Deferred Variable Handling
When a config value references a deferred variable, it is left
unresolved and prepended with the DEFERRED_PREFIX. To complete
these, use resolve_deferred().
:param dict config: The config dict to resolve recursively.
:param variables.VariableSetManager var_man: A variable manager. (
Presumably a permutation of the base var_man)
:return: The resolved config,
"""
no_deferred_allowed = schedulers.list_plugins()
# This can eventually be allowed if the build is non-local.
no_deferred_allowed.append('build')
no_deferred_allowed.append('scheduler')
# This can be allowed, eventually.
no_deferred_allowed.append('only_if')
no_deferred_allowed.append('not_if')
resolved_dict = {}
for section in config:
resolved_dict[section] = cls.resolve_section_values(
component=config[section],
var_man=var_man,
allow_deferred=section not in no_deferred_allowed,
key_parts=(section, ),
)
for section in ('only_if', 'not_if'):
if section in config:
resolved_dict[section] = cls.resolve_keys(
base_dict=resolved_dict.get(section, {}),
var_man=var_man,
section_name=section)
return resolved_dict
def _scheduler_cmd(self, _, args):
"""
:param argparse.Namespace args:
"""
sched = None # type : schedulers.SchedulerPlugin
sched_name = None
if args.vars is not None or args.config is not None:
sched_name = args.vars if args.vars is not None else args.config
try:
sched = schedulers.get_plugin(sched_name)
except schedulers.SchedulerPluginError:
output.fprint(
"Invalid scheduler plugin '{}'.".format(sched_name),
color=output.RED,
)
return errno.EINVAL
if args.vars is not None:
sched_vars = []
empty_config = file_format.TestConfigLoader().load_empty()
svars = sched.get_vars(empty_config[sched_name])
for key in sorted(list(svars.keys())):
sched_vars.append(svars.info(key))
output.draw_table(
self.outfile,
fields=['name', 'deferred', 'example', 'help'],
rows=sched_vars,
title="Variables for the {} scheduler plugin.".format(
args.vars))
elif args.config is not None:
sched_config = sched.get_conf()
class Loader(yaml_config.YamlConfigLoader):
"""Loader for just a scheduler's config."""
ELEMENTS = [sched_config]
defaults = Loader().load_empty()
Loader().dump(self.outfile, values=defaults)
else:
# Assuming --list was given
scheds = []
for sched_name in schedulers.list_plugins():
sched = schedulers.get_plugin(sched_name)
scheds.append({
'name': sched_name,
'description': sched.description,
'path': sched.path
})
fields = ['name', 'description']
if args.verbose:
fields.append('path')
output.draw_table(self.outfile,
fields=fields,
rows=scheds,
title="Available Scheduler Plugins")
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides,
sys_vars):
"""Translate a general set of pavilion test configs into the final,
resolved configurations. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:param pav_cfg: The pavilion config
:param str host: The host config to target these tests with
:param list(str) modes: The mode configs to use.
:param list(Path) test_files: Files containing a newline separated
list of tests.
:param list(str) tests: The tests to run.
:param list(str) overrides: Overrides to apply to the configurations.
:param system_variables.SysVarDict sys_vars: The system variables dict.
:returns: A dictionary (by scheduler type name) of lists of tuples
test configs and their variable managers.
"""
self.logger.debug("Finding Configs")
# Use the sys_host if a host isn't specified.
if host is None:
host = sys_vars.get('sys_name')
tests = list(tests)
for file in test_files:
try:
with pathlib.PosixPath(file).open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
try:
raw_tests = test_config.load_test_configs(pav_cfg, host, modes,
tests)
except test_config.TestConfigError as err:
self.logger.error(str(err))
raise commands.CommandError(str(err))
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
test_config.apply_overrides(test_cfg, overrides)
except test_config.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}' \
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
p_cfg, permutes = test_config.resolve_permutations(
test_cfg, pav_cfg.pav_vars, sys_vars)
for p_var_man in permutes:
# Get the scheduler from the config.
sched = p_cfg['scheduler']
sched = test_config.resolve_section_vars(
component=sched,
var_man=p_var_man,
allow_deferred=False,
deferred_only=False,
)
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except test_config.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}' \
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the scheduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
sched_config = test_config.resolve_section_vars(
component=test_cfg[sched_name],
var_man=test_var_man,
allow_deferred=False,
deferred_only=False,
)
test_var_man.add_var_set('sched', sched.get_vars(sched_config))
# Resolve all variables for the test (that aren't deferred).
try:
resolved_config = test_config.resolve_config(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = "Error resolving variables in config at '{}': {}" \
.format(test_cfg['suite_path'].resolve(test_var_man),
err)
self.logger.error(msg)
raise commands.CommandError(msg)
tests_by_scheduler[sched.name].append(
(resolved_config, test_var_man))
return tests_by_scheduler