def resolve_test_vars(cls, config, var_man): """Recursively resolve the variables in the value strings in the given configuration. Deferred Variable Handling When a config value references a deferred variable, it is left unresolved and prepended with the DEFERRED_PREFIX. To complete these, use resolve_deferred(). :param dict config: The config dict to resolve recursively. :param variables.VariableSetManager var_man: A variable manager. ( Presumably a permutation of the base var_man) :return: The resolved config, """ no_deferred_allowed = schedulers.list_plugins() # This can eventually be allowed if the build is non-local. no_deferred_allowed.append('build') no_deferred_allowed.append('scheduler') # This can be allowed, eventually. no_deferred_allowed.append('only_if') no_deferred_allowed.append('not_if') resolved_dict = {} for section in config: resolved_dict[section] = cls.resolve_section_values( component=config[section], var_man=var_man, allow_deferred=section not in no_deferred_allowed, key_parts=(section, ), ) for section in ('only_if', 'not_if'): if section in config: resolved_dict[section] = cls.resolve_keys( base_dict=resolved_dict.get(section, {}), var_man=var_man, section_name=section) return resolved_dict
def _scheduler_cmd(self, _, args): """ :param argparse.Namespace args: """ sched = None # type : schedulers.SchedulerPlugin sched_name = None if args.vars is not None or args.config is not None: sched_name = args.vars if args.vars is not None else args.config try: sched = schedulers.get_plugin(sched_name) except schedulers.SchedulerPluginError: output.fprint( "Invalid scheduler plugin '{}'.".format(sched_name), color=output.RED, ) return errno.EINVAL if args.vars is not None: sched_vars = [] empty_config = file_format.TestConfigLoader().load_empty() svars = sched.get_vars(empty_config[sched_name]) for key in sorted(list(svars.keys())): sched_vars.append(svars.info(key)) output.draw_table( self.outfile, fields=['name', 'deferred', 'example', 'help'], rows=sched_vars, title="Variables for the {} scheduler plugin.".format( args.vars)) elif args.config is not None: sched_config = sched.get_conf() class Loader(yaml_config.YamlConfigLoader): """Loader for just a scheduler's config.""" ELEMENTS = [sched_config] defaults = Loader().load_empty() Loader().dump(self.outfile, values=defaults) else: # Assuming --list was given scheds = [] for sched_name in schedulers.list_plugins(): sched = schedulers.get_plugin(sched_name) scheds.append({ 'name': sched_name, 'description': sched.description, 'path': sched.path }) fields = ['name', 'description'] if args.verbose: fields.append('path') output.draw_table(self.outfile, fields=fields, rows=scheds, title="Available Scheduler Plugins")
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides, sys_vars): """Translate a general set of pavilion test configs into the final, resolved configurations. These objects will be organized in a dictionary by scheduler, and have a scheduler object instantiated and attached. :param pav_cfg: The pavilion config :param str host: The host config to target these tests with :param list(str) modes: The mode configs to use. :param list(Path) test_files: Files containing a newline separated list of tests. :param list(str) tests: The tests to run. :param list(str) overrides: Overrides to apply to the configurations. :param system_variables.SysVarDict sys_vars: The system variables dict. :returns: A dictionary (by scheduler type name) of lists of tuples test configs and their variable managers. """ self.logger.debug("Finding Configs") # Use the sys_host if a host isn't specified. if host is None: host = sys_vars.get('sys_name') tests = list(tests) for file in test_files: try: with pathlib.PosixPath(file).open() as test_file: for line in test_file.readlines(): line = line.strip() if line and not line.startswith('#'): tests.append(line) except (OSError, IOError) as err: msg = "Could not read test file {}: {}".format(file, err) self.logger.error(msg) raise commands.CommandError(msg) try: raw_tests = test_config.load_test_configs(pav_cfg, host, modes, tests) except test_config.TestConfigError as err: self.logger.error(str(err)) raise commands.CommandError(str(err)) raw_tests_by_sched = defaultdict(lambda: []) tests_by_scheduler = defaultdict(lambda: []) # Apply config overrides. for test_cfg in raw_tests: # Apply the overrides to each of the config values. try: test_config.apply_overrides(test_cfg, overrides) except test_config.TestConfigError as err: msg = 'Error applying overrides to test {} from {}: {}' \ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Resolve all configuration permutations. try: p_cfg, permutes = test_config.resolve_permutations( test_cfg, pav_cfg.pav_vars, sys_vars) for p_var_man in permutes: # Get the scheduler from the config. sched = p_cfg['scheduler'] sched = test_config.resolve_section_vars( component=sched, var_man=p_var_man, allow_deferred=False, deferred_only=False, ) raw_tests_by_sched[sched].append((p_cfg, p_var_man)) except test_config.TestConfigError as err: msg = 'Error resolving permutations for test {} from {}: {}' \ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Get the schedulers for the tests, and the scheduler variables. # The scheduler variables are based on all of the for sched_name in raw_tests_by_sched.keys(): try: sched = schedulers.get_plugin(sched_name) except KeyError: msg = "Could not find scheduler '{}'.".format(sched_name) self.logger.error(msg) raise commands.CommandError(msg) nondeferred_cfg_sctns = schedulers.list_plugins() # Builds must have the values of all their variables now. nondeferred_cfg_sctns.append('build') # Set the scheduler variables for each test. for test_cfg, test_var_man in raw_tests_by_sched[sched_name]: sched_config = test_config.resolve_section_vars( component=test_cfg[sched_name], var_man=test_var_man, allow_deferred=False, deferred_only=False, ) test_var_man.add_var_set('sched', sched.get_vars(sched_config)) # Resolve all variables for the test (that aren't deferred). try: resolved_config = test_config.resolve_config( test_cfg, test_var_man, no_deferred_allowed=nondeferred_cfg_sctns) except (ResolveError, KeyError) as err: msg = "Error resolving variables in config at '{}': {}" \ .format(test_cfg['suite_path'].resolve(test_var_man), err) self.logger.error(msg) raise commands.CommandError(msg) tests_by_scheduler[sched.name].append( (resolved_config, test_var_man)) return tests_by_scheduler