def _load_test(self, name: str, host: str = 'this', modes: List[str] = None, build=True, finalize=True) -> List[TestRun]: """Load the named test config from file. Returns a list of the resulting configs.""" if modes is None: modes = [] res = resolver.TestConfigResolver(self.pav_cfg) test_cfgs = res.load([name], host, modes) tests = [] for ptest in test_cfgs: test = TestRun(self.pav_cfg, ptest.config, var_man=ptest.var_man) if build: test.build() if finalize: fin_sys = system_variables.SysVarDict(unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) res.finalize(test, fin_var_man) tests.append(test) return tests
def test_run(self): var_man = VariableSetManager() config1 = { 'name': 'run_test', 'scheduler': 'raw', 'build': { 'timeout': '30', }, 'run': { 'timeout': None, 'env': { 'foo': 'bar', }, # 'cmds': ['echo "I ran, punks"'], }, } test = TestRun(self.pav_cfg, config1, var_man) self.assert_(test.build()) test.finalize(VariableSetManager()) self.assertTrue(test.run(), msg="Test failed to run.") config2 = config1.copy() config2['run']['modules'] = ['asdlfkjae', 'adjwerloijeflkasd'] test = TestRun(self.pav_cfg, config2, var_man) self.assert_(test.build()) test.finalize(VariableSetManager()) self.assertEqual( test.run(), False, msg="Test should have failed because a module couldn't be " "loaded. {}".format(test.path)) # TODO: Make sure this is the exact reason for the failure # (doesn't work currently). # Make sure the test fails properly on a timeout. config3 = { 'name': 'sleep_test', 'scheduler': 'raw', 'run': { 'timeout': '1', 'cmds': ['sleep 10'] } } test = TestRun(self.pav_cfg, config3, VariableSetManager()) self.assert_(test.build()) test.finalize(VariableSetManager()) with self.assertRaises(TimeoutError, msg="Test should have failed due " "to timeout. {}"): test.run()
def test_obj(self): """Test pavtest object initialization.""" # Initializing with a mostly blank config config = { # The only required param. 'name': 'blank_test', 'scheduler': 'raw', } # Making sure this doesn't throw errors from missing params. TestRun(self.pav_cfg, config, VariableSetManager()) config = { 'subtest': 'st', 'name': 'test', 'scheduler': 'raw', 'build': { 'modules': ['gcc'], 'cmds': ['echo "Hello World"'], 'timeout': '30', }, 'run': { 'modules': ['gcc', 'openmpi'], 'cmds': ['echo "Running dis stuff"'], 'env': { 'BLARG': 'foo' }, 'timeout': '30', } } # Make sure we can create a test from a fairly populated config. t = TestRun(self.pav_cfg, config, VariableSetManager()) t.build() # Make sure we can recreate the object from id. t2 = TestRun.load(self.pav_cfg, t.id) # Make sure the objects are identical # This tests the following functions # - from_id # - save_config, load_config # - get_test_path # - write_tmpl for key in set(t.__dict__.keys()).union(t2.__dict__.keys()): if key != 'var_man': self.assertEqual(t.__dict__[key], t2.__dict__[key], msg="Mismatch for key {}".format(key))
def test_status_command_with_sched(self): """Test status command when test is 'SCHEDULED'.""" test = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['sleep 1'], }, }) test['name'] = 'testytest' test = TestRun(self.pav_cfg, test, VariableSetManager()) test.build() schedulers.get_plugin(test.scheduler) \ .schedule_test(self.pav_cfg, test) status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args([str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args(['-j', str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def test_src_urls(self): base_config = { 'name': 'test', 'scheduler': 'raw', 'build': { 'modules': ['gcc'], } } config = copy.deepcopy(base_config) config['build']['source_location'] = self.TEST_URL # remove existing downloads, and replace the directory. downloads_path = self.pav_cfg.working_dir / 'downloads' shutil.rmtree(str(downloads_path)) downloads_path.mkdir() test = TestRun(self.pav_cfg, config, VariableSetManager()) if test.build_origin.exists(): shutil.rmtree(str(test.build_origin)) test._setup_build_dir(test.build_origin) self.assertEqual(self.README_HASH, self.get_hash(test.build_origin / 'README.md'))
def test_sched_vars(self): """Make sure the scheduler variable class works as expected.""" class TestVars(schedulers.SchedulerVariables): @schedulers.var_method def hello(self): return 'hello' @schedulers.var_method def foo(self): return self.sched_data['foo'] @schedulers.dfr_var_method() def bar(self): return 'bar' def not_a_key(self): pass class DummySched(schedulers.SchedulerPlugin): VAR_CLASS = TestVars def __init__(self): super().__init__('dummy', 'more dumb') self.in_alloc_var = False def _get_data(self): return {'foo': 'baz'} def _in_alloc(self): return self.in_alloc_var test = TestRun( self.pav_cfg, { 'name': 'sched-vars', 'scheduler': 'dummy' }, VariableSetManager(), ) dummy_sched = DummySched() svars = dummy_sched.get_vars(test) # There should only be three keys. self.assertEqual(len(list(svars.keys())), 3) self.assertEqual(svars['hello'], 'hello') self.assertEqual(svars['foo'], 'baz') # Make sure we get a deferred variable when outside of an allocation self.assert_(isinstance(svars['bar'], variables.DeferredVariable)) # And the real thing inside dummy_sched.in_alloc_var = True del svars['bar'] self.assertEqual(svars['bar'], 'bar')
def test_suites(self): """Test suite creation and regeneration.""" config1 = { 'name': 'run_test', 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, # 'cmds': ['echo "I ran, punks"'], }, } tests = [] for i in range(3): tests.append(TestRun(self.pav_cfg, config1, VariableSetManager())) # Make sure this doesn't explode suite = TestSeries(self.pav_cfg, tests) # Make sure we got all the tests self.assertEqual(len(suite.tests), 3) test_paths = [Path(suite.path, p) for p in os.listdir(str(suite.path))] # And that the test paths are unique self.assertEqual(len(set(test_paths)), len([p.resolve() for p in test_paths])) self._is_softlink_dir(suite.path) suite2 = TestSeries.from_id(self.pav_cfg, suite._id) self.assertEqual(sorted(suite.tests.keys()), sorted(suite2.tests.keys())) self.assertEqual(sorted([t.id for t in suite.tests.values()]), sorted([t.id for t in suite2.tests.values()])) self.assertEqual(suite.path, suite2.path) self.assertEqual(suite.id, suite2.id)
def test_kickoff_env(self): pav_cfg = self.pav_cfg pav_cfg['env_setup'] = ['test1', 'test2', 'test3'] test = TestRun( self.pav_cfg, { 'name': 'sched-vars', 'scheduler': 'dummy' }, VariableSetManager(), ) dummy_sched = schedulers.get_plugin('dummy') path = dummy_sched._create_kickoff_script(pav_cfg, test) with path.open() as file: lines = file.readlines() for i in range(0, len(lines)): lines[i] = lines[i].strip() testlist = pav_cfg['env_setup'] self.assertTrue(set(testlist).issubset(lines)) self.assertTrue(re.match(r'pav _run.*', lines[-1]))
def _get_var_man(test, sched): """Get the variable manager for the given test. :param TestRun test: The test run object :param sched: The scheduler for this test. :rtype VariableSetManager """ # Re-add var sets that may have had deferred variables. try: var_man = VariableSetManager() var_man.add_var_set('sys', system_variables.get_vars(defer=False)) sched_config = test.config[test.scheduler] var_man.add_var_set('sched', sched.get_vars(sched_config)) except Exception: test.status.set( STATES.RUN_ERROR, "Unknown error getting pavilion variables at " "run time.") raise return var_man
def test_deferred(self): # The following tests make sure deferred variables are # interpreted correctly by the conditional checks. test_list = [] base_cfg = self._quick_test_cfg() # Test 1: # Not_if with deferred variable that resolves to skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 2: # Only_if with deferred variable that resolves to skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 3: # Not_if that fails to skip with deferred only_if that skips. test_cfg = base_cfg.copy() test_cfg['not_if'] = { '{{dumb_user}}': ['nivlac', 'notcalvin'], '{{dumb_os}}': ['blurg'] } test_cfg['only_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 4: # Only_if that fails to skip with deferred not_if that skips. test_cfg = base_cfg.copy() test_cfg['only_if'] = { '{{dumb_user}}': ['nivlac', 'calvin'], '{{dumb_os}}': ['bieber'] } test_cfg['not_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Run through scenario of deferred(no-skip) into skip. for test_cfg in test_list: test = self._quick_test(cfg=test_cfg, finalize=False) self.assertFalse(test.skipped, msg="dumb_sys_var should be deferred" " with skip not assigned to" " the test") fin_sys = system_variables.SysVarDict(defer=False, unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) test.finalize(fin_var_man) self.assertTrue(test.skipped, msg="Now it should skip") test_list = [] # Test 5: # Not_if with deferred variable that resolves to no skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Test 6: # Only_if with deferred variable that resolves to no skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 7: # Not_if and only_if-deferred that fails to skip. test_cfg = base_cfg.copy() test_cfg['not_if'] = { '{{dumb_user}}': ['nivlac', 'notcalvin'], '{{dumb_os}}': ['blurg'] } test_cfg['only_if'] = {'{{dumb_sys_var}}': ['stupid']} test_list.append(test_cfg) # Test 8: # Only_if and not_if-deferred that fails to skip. test_cfg = base_cfg.copy() test_cfg['only_if'] = { '{{dumb_user}}': ['nivlac', 'calvin'], '{{dumb_os}}': ['bieber'] } test_cfg['not_if'] = {'{{dumb_sys_var}}': ['notstupid']} test_list.append(test_cfg) # Run through scenario of deferred(no-skip) into no skip. for test_cfg in test_list: test = self._quick_test(cfg=test_cfg, finalize=False) self.assertFalse(test.skipped, msg="dumb_sys_var should be deferred" " with skip not assigned to" " the test.") fin_sys = system_variables.SysVarDict(defer=False, unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) test.finalize(fin_var_man) self.assertFalse(test.skipped, msg="Test Should NOT skip.")
def test_status_command(self): """Test status command by generating a suite of tests.""" config1 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['echo "I $foo, punks"'], }, }) config1['name'] = 'run_test0' config2 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['echo "I $too, punks"'], }, }) config2['name'] = 'run_test1' config3 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['sleep 10'], }, }) config3['name'] = 'run_test2' configs = [config1, config2, config3] var_man = VariableSetManager() tests = [self._quick_test(cfg) for cfg in configs] for test in tests: test.RUN_SILENT_TIMEOUT = 1 # Make sure this doesn't explode suite = TestSeries(self.pav_cfg, tests) test_str = " ".join([str(test) for test in suite.tests]) status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() # Testing for individual tests with json output for test in suite.tests: parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) arg_list = ['-j', str(test)] args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with json output parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) arg_list = ['-j'] + test_str.split() args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) # Testing for individual tests with tabular output for test in suite.tests: parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args([str(test)]) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with tabular output parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) arg_list = test_str.split() args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def _run(self, pav_cfg, test): """Run an already prepped test in the current environment. """ try: sched = schedulers.get_plugin(test.scheduler) except Exception: test.status.set(STATES.BUILD_ERROR, "Unknown error getting the scheduler. Refer to " "the kickoff log.") raise # Re-add var sets that may have had deferred variables. try: var_man = VariableSetManager() var_man.add_var_set('sys', system_variables.get_vars(defer=False)) sched_config = test.config[test.scheduler] var_man.add_var_set('sched', sched.get_vars(sched_config)) except Exception: test.status.set(STATES.RUN_ERROR, "Unknown error getting pavilion variables at " "run time.") raise try: test.finalize(var_man) except Exception: test.status.set(STATES.RUN_ERROR, "Unknown error finalizing test.") raise try: if test.config['build']['on_nodes'] in ['true', 'True']: if not test.build(): self.logger.warning( "Test {t.id} failed to build:" ) except Exception: test.status.set(STATES.BUILD_ERROR, "Unknown build error. Refer to the kickoff log.") raise # Optionally wait on other tests running under the same scheduler. # This depends on the scheduler and the test configuration. lock = sched.lock_concurrency(pav_cfg, test) try: run_result = test.run() except TestRunError as err: test.status.set(STATES.RUN_ERROR, err) return 1 except TimeoutError: return 1 except Exception: test.status.set( STATES.RUN_ERROR, "Unknown error while running test. Refer to the kickoff log.") raise finally: sched.unlock_concurrency(lock) try: rp_errors = [] # Make sure the result parsers have reasonable arguments. # We check here because the parser code itself will likely assume # the args are valid form _check_args, but those might not be # checkable before kickoff due to deferred variables. try: result_parsers.check_args(test.config['results']) except TestRunError as err: rp_errors.append(str(err)) if rp_errors: for msg in rp_errors: test.status.set(STATES.RESULTS_ERROR, msg) test.set_run_complete() return 1 results = test.gather_results(run_result) except result_parsers.ResultParserError as err: self.logger.error("Unexpected error gathering results: %s", err) test.status.set(STATES.RESULTS_ERROR, "Error parsing results: {}".format(err)) return 1 try: test.save_results(results) result_logger = logging.getLogger('results') result_logger.info(output.json_dumps(results)) except Exception: test.status.set( STATES.RESULTS_ERROR, "Unknown error while saving results. Refer to the kickoff log.") raise try: test.status.set(STATES.COMPLETE, "The test completed with result: {}" .format(results.get('result', '<unknown>'))) except Exception: test.status.set( STATES.UNKNOWN, "Unknown error while setting test completion. Refer to the " "kickoff log.") raise
def test_build(self): """Make sure building works.""" config1 = { 'name': 'build_test', 'scheduler': 'raw', 'build': { 'timeout': '12', 'cmds': ['echo "Hello World [\x1esched.num_nodes\x1e]"'], 'source_location': 'binfile.gz', }, } var_man = VariableSetManager() test = TestRun(self.pav_cfg, config1, var_man) # Test a basic build, with a gzip file and an actual build script. self.assertTrue(test.build(), msg="Build failed") # Make sure the build path and build origin contain softlinks to the # same files. self._cmp_tree(test.build_origin, test.build_path) self._is_softlink_dir(test.build_path) # We're going to time out this build on purpose, to test the code # that waits for builds to complete. config = { 'name': 'build_test', 'scheduler': 'raw', 'build': { 'timeout': '1', 'cmds': ['sleep 10'], 'source_location': 'binfile.gz', }, } test = TestRun(self.pav_cfg, config, var_man) # This build should fail. self.assertFalse(test.build(), "Build succeeded when it should have timed out.") current_note = test.status.current().note self.assertTrue(current_note.startswith("Build timed out")) # Test general build failure. config = { 'name': 'build_test', 'scheduler': 'raw', 'build': { 'timeout': '12', 'cmds': ['exit 0'], 'source_location': 'binfile.gz', }, } # Check that building, and then re-using, a build directory works. test = TestRun(self.pav_cfg, config, var_man) # Remove the build tree to ensure we do the build fresh. if test.build_origin.is_dir(): shutil.rmtree(str(test.build_origin)) self.assertTrue(test.build()) test2 = TestRun(self.pav_cfg, config, var_man) self.assertTrue(test2.build()) self.assertEqual(test.build_origin, test2.build_origin) config3 = copy.deepcopy(config) config3['build']['cmds'] = ['exit 1'] # This should fail because the build exits non-zero test3 = TestRun(self.pav_cfg, config3, var_man) self.assertFalse(test3.build(), "Build succeeded when it should have failed.") current_note = test3.status.current().note self.assertTrue( current_note.startswith("Build returned a non-zero result."))
def test_setup_build_dir(self): """Make sure we can correctly handle all of the various archive formats.""" base_config = { 'name': 'test', 'scheduler': 'raw', 'build': { 'modules': ['gcc'], } } # Check that decompression and setup works for all accepted types. archives = [ 'src.tar.gz', 'src.xz', # A bz2 archive 'src.extensions_dont_matter', 'src.zip', # These archives don't have a containing directory. 'no_encaps.tgz', 'no_encaps.zip', 'softlink.zip', ] test_archives = self.TEST_DATA_ROOT / 'pav_config_dir' / 'test_src' original_tree = test_archives / 'src' for archive in archives: config = copy.deepcopy(base_config) config['build']['source_location'] = archive test = TestRun(self.pav_cfg, config, VariableSetManager()) if test.build_origin.exists(): shutil.rmtree(str(test.build_origin)) test._setup_build_dir(test.build_origin) # Make sure the extracted archive is identical to the original # (Though the containing directory will have a different name) try: self._cmp_tree(test.build_origin, original_tree) except AssertionError as err: raise AssertionError("Error extracting {}".format(archive), *err.args) # Check directory copying config = copy.deepcopy(base_config) config['build']['source_location'] = 'src' test = TestRun(self.pav_cfg, config, VariableSetManager()) if test.build_origin.exists(): shutil.rmtree(str(test.build_origin)) test._setup_build_dir(test.build_origin) self._cmp_tree(test.build_origin, original_tree) # Test single compressed files. files = [ 'binfile.gz', 'binfile.bz2', 'binfile.xz', ] for file in files: config = copy.deepcopy(base_config) config['build']['source_location'] = file test = TestRun(self.pav_cfg, config, VariableSetManager()) if test.build_origin.exists(): shutil.rmtree(str(test.build_origin)) test._setup_build_dir(test.build_origin) self._cmp_files(test.build_origin / 'binfile', original_tree / 'binfile') # Make sure extra files are getting copied over. config = copy.deepcopy(base_config) config['build']['source_location'] = 'src.tar.gz' config['build']['extra_files'] = [ 'src.tar.gz', 'src.xz', ] test = TestRun(self.pav_cfg, config, VariableSetManager()) if test.build_origin.exists(): shutil.rmtree(str(test.build_origin)) test._setup_build_dir(test.build_origin) for file in config['build']['extra_files']: self._cmp_files(test_archives / file, test.build_origin / file)
def test_wait_command(self): """Test wait command.""" config1 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['echo 0'], }, }) config1['name'] = 'run_test0' config2 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['echo 1'], }, }) config2['name'] = 'run_test1' config3 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['sleep 1'], }, }) config3['name'] = 'run_test2' configs = [config1, config2, config3] tests = [TestRun(self.pav_cfg, test, VariableSetManager()) for test in configs] for test in tests: test.RUN_SILENT_TIMEOUT = 1 # Make sure this doesn't explode suite = TestSeries(self.pav_cfg, tests) test_str = " ".join([str(test) for test in suite.tests]) wait_cmd = commands.get_command('wait') wait_cmd.outfile = io.StringIO() # Testing for individual tests with json output for test in suite.tests: parser = argparse.ArgumentParser() wait_cmd._setup_arguments(parser) arg_list = ['-j', '-t', '1', str(test)] args = parser.parse_args(arg_list) self.assertEqual(wait_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with json output parser = argparse.ArgumentParser() wait_cmd._setup_arguments(parser) arg_list = ['-j', '-t', '1'] + test_str.split() args = parser.parse_args(arg_list) self.assertEqual(wait_cmd.run(self.pav_cfg, args), 0) # Testing for individual tests with tabular output for test in suite.tests: parser = argparse.ArgumentParser() wait_cmd._setup_arguments(parser) arg_list = ['-j', '-t', '1', str(test)] args = parser.parse_args(arg_list) self.assertEqual(wait_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with tabular output parser = argparse.ArgumentParser() wait_cmd._setup_arguments(parser) arg_list = ['-j', '-t', '1'] + test_str.split() args = parser.parse_args(arg_list) self.assertEqual(wait_cmd.run(self.pav_cfg, args), 0)
def _quick_test(self, cfg=None, name="quick_test", build=True, finalize=True, sched_vars=None): """Create a test run object to work with. The default is a simple hello world test with the raw scheduler. :param dict cfg: An optional config dict to create the test from. :param str name: The name of the test. :param bool build: Build this test, while we're at it. :param bool finalize: Finalize this test. :param dict sched_vars: Add these scheduler variables to our var set. :rtype: TestRun """ if cfg is None: cfg = self._quick_test_cfg() cfg = copy.deepcopy(cfg) loader = TestConfigLoader() cfg = loader.validate(loader.normalize(cfg)) cfg['name'] = name var_man = VariableSetManager() var_man.add_var_set('var', cfg['variables']) var_man.add_var_set('sys', system_variables.get_vars(defer=True)) var_man.add_var_set('pav', self.pav_cfg.pav_vars) if sched_vars is not None: var_man.add_var_set('sched', sched_vars) var_man.resolve_references() cfg = resolver.TestConfigResolver.resolve_test_vars(cfg, var_man) test = TestRun( pav_cfg=self.pav_cfg, config=cfg, var_man=var_man, ) if build: test.build() if finalize: fin_sys = system_variables.SysVarDict(unique=True) fin_var_man = VariableSetManager() fin_var_man.add_var_set('sys', fin_sys) resolver.TestConfigResolver.finalize(test, fin_var_man) return test
def test_set_status_command(self): """Test set status command by generating a suite of tests.""" config1 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['echo "I $foo, punks"'], }, }) config1['name'] = 'run_test0' config2 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['echo "I $too, punks"'], }, }) config2['name'] = 'run_test1' config3 = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'too': 'tar', }, 'cmds': ['sleep 10'], }, }) config3['name'] = 'run_test2' configs = [config1, config2, config3] var_man = VariableSetManager() tests = [TestRun(self.pav_cfg, test, var_man) for test in configs] for test in tests: test.RUN_SILENT_TIMEOUT = 1 set_status_cmd = commands.get_command('set_status') set_status_cmd.outfile = io.StringIO() # Testing for individual tests with json output for test in tests: start_status = test.status.current() parser = argparse.ArgumentParser() set_status_cmd._setup_arguments(parser) arg_list = ['-s', 'RUN_USER', '-n', 'tacos are delicious', str(test.id)] args = parser.parse_args(arg_list) self.assertEqual(set_status_cmd.run(self.pav_cfg, args), 0) end_status = test.status.current() self.assertNotEqual(end_status.state, start_status.state) self.assertNotEqual(end_status.note, start_status.note) self.assertEqual(end_status.state, 'RUN_USER') self.assertEqual(end_status.note, 'tacos are delicious')