def _load_test(self, name: str, host: str = 'this', modes: List[str] = None, build=True, finalize=True) -> List[TestRun]: """Load the named test config from file. Returns a list of the resulting configs.""" if modes is None: modes = [] res = resolver.TestConfigResolver(self.pav_cfg) test_cfgs = res.load([name], host, modes) tests = [] for ptest in test_cfgs: test = TestRun(self.pav_cfg, ptest.config, var_man=ptest.var_man) if build: test.build() if finalize: fin_sys = system_variables.SysVarDict(unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) res.finalize(test, fin_var_man) tests.append(test) return tests
def _quick_test(self, cfg=None, name="quick_test", build=True, finalize=True, sched_vars=None): """Create a test run object to work with. The default is a simple hello world test with the raw scheduler. :param dict cfg: An optional config dict to create the test from. :param str name: The name of the test. :param bool build: Build this test, while we're at it. :param bool finalize: Finalize this test. :param dict sched_vars: Add these scheduler variables to our var set. :rtype: TestRun """ if cfg is None: cfg = self._quick_test_cfg() cfg = copy.deepcopy(cfg) loader = TestConfigLoader() cfg = loader.validate(loader.normalize(cfg)) cfg['name'] = name var_man = VariableSetManager() var_man.add_var_set('var', cfg['variables']) var_man.add_var_set('sys', system_variables.get_vars(defer=True)) var_man.add_var_set('pav', self.pav_cfg.pav_vars) if sched_vars is not None: var_man.add_var_set('sched', sched_vars) var_man.resolve_references() cfg = resolver.TestConfigResolver.resolve_test_vars(cfg, var_man) test = TestRun( pav_cfg=self.pav_cfg, config=cfg, var_man=var_man, ) if build: test.build() if finalize: fin_sys = system_variables.SysVarDict(unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) resolver.TestConfigResolver.finalize(test, fin_var_man) return test
def _get_var_man(test, sched): """Get the variable manager for the given test. :param TestRun test: The test run object :param sched: The scheduler for this test. :rtype VariableSetManager """ # Re-add var sets that may have had deferred variables. try: var_man = VariableSetManager() var_man.add_var_set('sys', system_variables.get_vars(defer=False)) sched_config = test.config[test.scheduler] var_man.add_var_set('sched', sched.get_vars(sched_config)) except Exception: test.status.set( STATES.RUN_ERROR, "Unknown error getting pavilion variables at " "run time.") raise return var_man
def test_deferred(self): # The following tests make sure deferred variables are # interpreted correctly by the conditional checks. test_list = [] base_cfg = self._quick_test_cfg() # Test 1: # Not_if with deferred variable that resolves to skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 2: # Only_if with deferred variable that resolves to skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 3: # Not_if that fails to skip with deferred only_if that skips. test_cfg = base_cfg.copy() test_cfg['not_if'] = { '{{dumb_user}}': ['nivlac', 'notcalvin'], '{{dumb_os}}': ['blurg'] } test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 4: # Only_if that fails to skip with deferred not_if that skips. test_cfg = base_cfg.copy() test_cfg['only_if'] = { '{{dumb_user}}': ['nivlac', 'calvin'], '{{dumb_os}}': ['bieber'] } test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Run through scenario of deferred(no-skip) into skip. for test_cfg in test_list: test = self._quick_test(cfg=test_cfg, finalize=False) self.assertFalse(test.skipped, msg="dumb_sys_var should be deferred" " with skip not assigned to" " the test") fin_sys = system_variables.SysVarDict(defer=False, unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) test.finalize(fin_var_man) self.assertTrue(test.skipped, msg="Now it should skip") test_list = [] # Test 5: # Not_if with deferred variable that resolves to no skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 6: # Only_if with deferred variable that resolves to no skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 7: # Not_if and only_if-deferred that fails to skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = { '{{dumb_user}}': ['nivlac', 'notcalvin'], '{{dumb_os}}': ['blurg'] } test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 8: # Only_if and not_if-deferred that fails to skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = { '{{dumb_user}}': ['nivlac', 'calvin'], '{{dumb_os}}': ['bieber'] } test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Run through scenario of deferred(no-skip) into no skip. for test_cfg in test_list: test = self._quick_test(cfg=test_cfg, finalize=False) self.assertFalse(test.skipped, msg="dumb_sys_var should be deferred" " with skip not assigned to" " the test.") fin_sys = system_variables.SysVarDict(defer=False, unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) test.finalize(fin_var_man) self.assertFalse(test.skipped, msg="Test Should NOT skip.")
def _run(self, pav_cfg, test): """Run an already prepped test in the current environment. """ try: sched = schedulers.get_plugin(test.scheduler) except Exception: test.status.set(STATES.BUILD_ERROR, "Unknown error getting the scheduler. Refer to " "the kickoff log.") raise # Re-add var sets that may have had deferred variables. try: var_man = VariableSetManager() var_man.add_var_set('sys', system_variables.get_vars(defer=False)) sched_config = test.config[test.scheduler] var_man.add_var_set('sched', sched.get_vars(sched_config)) except Exception: test.status.set(STATES.RUN_ERROR, "Unknown error getting pavilion variables at " "run time.") raise try: test.finalize(var_man) except Exception: test.status.set(STATES.RUN_ERROR, "Unknown error finalizing test.") raise try: if test.config['build']['on_nodes'] in ['true', 'True']: if not test.build(): self.logger.warning( "Test {t.id} failed to build:" ) except Exception: test.status.set(STATES.BUILD_ERROR, "Unknown build error. Refer to the kickoff log.") raise # Optionally wait on other tests running under the same scheduler. # This depends on the scheduler and the test configuration. lock = sched.lock_concurrency(pav_cfg, test) try: run_result = test.run() except TestRunError as err: test.status.set(STATES.RUN_ERROR, err) return 1 except TimeoutError: return 1 except Exception: test.status.set( STATES.RUN_ERROR, "Unknown error while running test. Refer to the kickoff log.") raise finally: sched.unlock_concurrency(lock) try: rp_errors = [] # Make sure the result parsers have reasonable arguments. # We check here because the parser code itself will likely assume # the args are valid form _check_args, but those might not be # checkable before kickoff due to deferred variables. try: result_parsers.check_args(test.config['results']) except TestRunError as err: rp_errors.append(str(err)) if rp_errors: for msg in rp_errors: test.status.set(STATES.RESULTS_ERROR, msg) test.set_run_complete() return 1 results = test.gather_results(run_result) except result_parsers.ResultParserError as err: self.logger.error("Unexpected error gathering results: %s", err) test.status.set(STATES.RESULTS_ERROR, "Error parsing results: {}".format(err)) return 1 try: test.save_results(results) result_logger = logging.getLogger('results') result_logger.info(output.json_dumps(results)) except Exception: test.status.set( STATES.RESULTS_ERROR, "Unknown error while saving results. Refer to the kickoff log.") raise try: test.status.set(STATES.COMPLETE, "The test completed with result: {}" .format(results.get('result', '<unknown>'))) except Exception: test.status.set( STATES.UNKNOWN, "Unknown error while setting test completion. Refer to the " "kickoff log.") raise