def test_loading_tests(self): """Make sure get_tests can find tests and resolve inheritance.""" plugins.initialize_plugins(self.pav_cfg) tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world']) self.assertEqual(sorted(['narf', 'hello', 'world']), sorted([test['name'] for test in tests])) tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world.hello']) hello = tests.pop() # There should have only been 1 self.assertFalse(tests) # Check some basic test attributes. self.assertEqual(hello['scheduler'], 'raw') self.assertEqual(hello['suite'], 'hello_world') # Make sure the variable from the host config got propagated. self.assertIn('hosty', hello['variables']) tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world.narf']) narf = tests.pop() # Make sure this got overridden from 'world' self.assertEqual(narf['scheduler'], 'dummy') # Make sure this didn't get lost. self.assertEqual(narf['run']['cmds'], ['echo "Running World"']) plugins._reset_plugins()
def test_loading_hidden(self): """Make sure we only get hidden tests when specifically requested.""" plugins.initialize_plugins(self.pav_cfg) tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden']) names = sorted([t['name'] for t in tests]) self.assertEqual(names, ['hello', 'narf']) tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden._hidden']) names = sorted([t['name'] for t in tests]) self.assertEqual(names, ['_hidden']) plugins._reset_plugins()
def test_env_order(self): """Make sure environment variables keep their order from the test config to the final run scripts.""" test_conf = load_test_configs(self.pav_cfg, 'this', [], ['order'])[0] test = self._quick_test(test_conf, "order") # Each exported variable in this config has a value that denotes its # expected place in the order. The variable names are random letter # sequences in a random order; hash order should be decidedly different # than the listed order. exports = [] with (test.path / 'build.sh').open() as build_script: for line in build_script: if line.startswith('export') and 'TEST_ID' not in line: _, var_val = line.split() var, val = line.split('=') val = int(val) exports.append((var, val)) # The values should be the numbers for 0..99, in that order. self.assertEqual(list(range(len(exports))), [val for var, val in exports], msg="Environment variable order not preserved. \n" "Got the following instead: \n{}".format(''.join( ["{}: {}\n".format(*v) for v in exports])))
def test_defaulted_variables(self): """Make sure default variables work as expected.""" plugins.initialize_plugins(self.pav_cfg) tests = load_test_configs(self.pav_cfg, host='defaulted', modes=['defaulted'], tests=['defaulted.test']) test = tests[0] self.assertEqual(test['variables']['host_def'], ['host']) self.assertEqual(test['variables']['mode_def'], ['mode']) self.assertEqual(test['variables']['test_def'], ['test']) self.assertNotIn('no_val', test['variables']) with self.assertRaises(TestConfigError): tests = load_test_configs(self.pav_cfg, host='defaulted', modes=['defaulted'], tests=['defaulted_error.error']) plugins._reset_plugins()
def test_apply_overrides(self): """Make sure overrides get applied to test configs correctly.""" tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world']) overrides = {'run': {'env': {'foo': 'bar'}}, 'scheduler': 'fuzzy'} for test in tests: otest = test.copy() apply_overrides(test, overrides) # Make sure the overrides were applied self.assertEqual(test['scheduler'], 'fuzzy') self.assertEqual(test['run']['env']['foo'], 'bar') # Make sure other stuff wasn't changed. self.assertEqual(test['build'], otest['build']) self.assertEqual(test['run']['cmds'], otest['run']['cmds'])
def test_extended_variables(self): """Make sure default variables work as expected.""" plugins.initialize_plugins(self.pav_cfg) tests = load_test_configs(self.pav_cfg, host='extended', modes=['extended'], tests=['extended.test']) test = tests[0] self.assertEqual(test['variables']['long_base'], ['what', 'are', 'you', 'up', 'to', 'punk?']) self.assertEqual(test['variables']['single_base'], ['host', 'mode', 'test']) self.assertEqual(test['variables']['no_base_mode'], ['mode']) self.assertEqual(test['variables']['no_base'], ['test']) self.assertEqual(test['variables']['null_base_mode'], ['mode']) self.assertEqual(test['variables']['null_base'], ['test']) plugins._reset_plugins()
def test_layering(self): """Make sure test config layering works as expected.""" plugins.initialize_plugins(self.pav_cfg) for host in ('this', 'layer_host'): for modes in ([], ['layer_mode']): for test in ('layer_tests.layer_test', 'layer_tests.layer_test_part'): answer = 'standard' if host == 'layer_host': answer = 'host' if modes: answer = 'mode' if test.endswith('part'): answer = 'test' tests = load_test_configs(pav_cfg=self.pav_cfg, host=host, modes=modes, tests=[test]) self.assertEqual(tests[0]['slurm']['partition'], answer) plugins._reset_plugins()
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides, sys_vars): """Translate a general set of pavilion test configs into the final, resolved configurations. These objects will be organized in a dictionary by scheduler, and have a scheduler object instantiated and attached. :param pav_cfg: The pavilion config :param str host: The host config to target these tests with :param list(str) modes: The mode configs to use. :param list(Path) test_files: Files containing a newline separated list of tests. :param list(str) tests: The tests to run. :param list(str) overrides: Overrides to apply to the configurations. :param system_variables.SysVarDict sys_vars: The system variables dict. :returns: A dictionary (by scheduler type name) of lists of test configs. """ self.logger.debug("Finding Configs") # Use the sys_host if a host isn't specified. if host is None: host = sys_vars.get('sys_name') tests = list(tests) for file in test_files: try: with file.open() as test_file: for line in test_file.readlines(): line = line.strip() if line and not line.startswith('#'): tests.append(line) except (OSError, IOError) as err: msg = "Could not read test file {}: {}".format(file, err) self.logger.error(msg) raise commands.CommandError(msg) try: raw_tests = test_config.load_test_configs(pav_cfg, host, modes, tests) except test_config.TestConfigError as err: self.logger.error(str(err)) raise commands.CommandError(str(err)) raw_tests_by_sched = defaultdict(lambda: []) tests_by_scheduler = defaultdict(lambda: []) # Apply config overrides. for test_cfg in raw_tests: # Apply the overrides to each of the config values. try: test_config.apply_overrides(test_cfg, overrides) except test_config.TestConfigError as err: msg = 'Error applying overrides to test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Resolve all configuration permutations. try: p_cfg, permutes = test_config.resolve_permutations( test_cfg, pav_cfg.pav_vars, sys_vars) for p_var_man in permutes: sched = p_cfg['scheduler'].resolve(p_var_man) raw_tests_by_sched[sched].append((p_cfg, p_var_man)) except test_config.TestConfigError as err: msg = 'Error resolving permutations for test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Get the schedulers for the tests, and the scheduler variables. # The scheduler variables are based on all of the for sched_name in raw_tests_by_sched.keys(): try: sched = schedulers.get_scheduler_plugin(sched_name) except KeyError: msg = "Could not find scheduler '{}'.".format(sched_name) self.logger.error(msg) raise commands.CommandError(msg) nondeferred_cfg_sctns = schedulers.list_scheduler_plugins() # Builds must have the values of all their variables now. nondeferred_cfg_sctns.append('build') # Set the scheduler variables for each test. for test_cfg, test_var_man in raw_tests_by_sched[sched_name]: test_var_man.add_var_set('sched', sched.get_vars(test_cfg)) # Resolve all variables for the test. try: resolved_config = test_config.resolve_all_vars( test_cfg, test_var_man, no_deferred_allowed=nondeferred_cfg_sctns) except (ResolveError, KeyError) as err: msg = "Error resolving variables in config at '{}': {}"\ .format(test_cfg['suite_path'].resolve(test_var_man), err) self.logger.error(msg) raise commands.CommandError(msg) tests_by_scheduler[sched.name].append(resolved_config) return tests_by_scheduler