def test_cancel_job(self): """Create a series of tests and kill them under different circumstances.""" # This test will just sleep for a bit. cfg = self._quick_test_cfg() cfg['run']['cmds'] = ['sleep 100'] test = self._quick_test(cfg=cfg) test.build() raw = schedulers.get_scheduler_plugin('raw') raw.schedule_test(self.pav_cfg, test) timeout = time.time() + 1 while (raw.job_status(self.pav_cfg, test).state == STATES.SCHEDULED and time.time() < timeout): time.sleep(.1) # The test should be running self.assertEqual(test.status.current().state, STATES.RUNNING) _, pid = test.job_id.split('_') self.assertEqual(raw.cancel_job(test).state, STATES.SCHED_CANCELLED)
def test_job_status(self): """Make sure all the slurm scheduler variable methods work when not on a node.""" slurm = schedulers.get_scheduler_plugin('slurm') cfg = TestConfigLoader().validate({ 'scheduler': 'slurm', 'run': { 'cmds': ['echo "Hello World."'] }, }) cfg['name'] = 'slurm_test' test = PavTest(self.pav_cfg, cfg, {}) test.status.set(STATES.SCHEDULED, "not really though.") # Grab a random jobid, and get the status of it. jobs = subprocess.check_output(['squeue', '-o', "%i %T"]) jobs = jobs.decode('utf-8') try: last_job = jobs.strip().split('\n')[-1] jobid, status = last_job.strip().split() except Exception: raise RuntimeError("No available test from which to borrow a" "job id.") test.job_id = jobid
def test_job_status(self): """Make sure we can get a slurm job status.""" cfg = self._quick_test_cfg() cfg['scheduler'] = 'slurm' test = self._quick_test(cfg, name='slurm_job_status', finalize=False) slurm = schedulers.get_scheduler_plugin('slurm') # Steal a running job's ID, and then check our status. test.status.set(STATES.SCHEDULED, "not really though.") test.job_id = self._get_job('JobState=RUNNING') status = slurm.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHEDULED) self.assertIn('RUNNING', status.note) # Steal a canceled jobs id test.status.set(STATES.SCHEDULED, "not really though.") test.job_id = self._get_job('JobState=CANCELLED') sched_status = slurm.job_status(self.pav_cfg, test) self.assertEqual(sched_status.state, STATES.SCHED_CANCELLED) status = test.status.current() self.assertEqual(status.state, STATES.SCHED_CANCELLED) # Check another random state. In this case, all pavilion will # just consider the test still scheduled. test.status.set(STATES.SCHEDULED, "not really though.") test.job_id = self._get_job('JobState=COMPLETED') sched_status = slurm.job_status(self.pav_cfg, test) self.assertEqual(sched_status.state, STATES.SCHEDULED) self.assertIn('COMPLETED', sched_status.note)
def status_from_test_obj(pav_cfg, test_obj): """Takes a test object or list of test objects and creates the dictionary expected by the print_status function. :param dict pav_cfg: Pavilion base configuration. :param pav_test.PavTest test_obj: Pavilion test object. :return list List of dictionary objects containing the test ID, name, state, time of state update, and note associated with that state. """ if not isinstance(test_obj, list): test_obj = [test_obj] test_statuses = [] for test in test_obj: status_f = test.status.current() if status_f.state == STATES.SCHEDULED: sched = schedulers.get_scheduler_plugin(test.scheduler) status_f = sched.job_status(pav_cfg, test) test_statuses.append({ 'test_id': test.id, 'name': test.name, 'state': status_f.state, 'time': status_f.when.strftime("%d %b %Y %H:%M:%S %Z"), 'note': status_f.note, }) test_statuses.sort(key=lambda x: x['test_id']) return test_statuses
def test_cancel_sched_check(self): """Cancel Test and make sure it is cancelled through scheduler.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this' 'hello_world2']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel']) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() test = [] series_id = series.TestSeries.load_user_series_id(self.pav_cfg) test.append(series_id) test_list = [] test_list.extend( series.TestSeries.from_id(self.pav_cfg, int(test[0][1:])).tests) for test_id in test_list: test = PavTest.load(self.pav_cfg, test_id) if test.status.current().state != STATES.COMPLETE: sched = schedulers.get_scheduler_plugin(test.scheduler) sched_status = sched.job_status(self.pav_cfg, test) self.assertIn("SCHED_CANCELLED", str(sched_status))
def _cancel_all(tests_by_sched): """Cancel each of the given tests using the appropriate scheduler.""" for sched_name, tests in tests_by_sched.items(): sched = schedulers.get_scheduler_plugin(sched_name) for test in tests: sched.cancel_job(test)
def test_sched_vars(self): """Make sure all the slurm scheduler variable methods work when not on a node.""" raw = schedulers.get_scheduler_plugin('raw') svars = raw.get_vars(self._quick_test()) for key, value in svars.items(): self.assertNotEqual(int(value), 0)
def test_log_arguments(self): log_cmd = commands.get_command('log') parser = argparse.ArgumentParser() log_cmd._setup_arguments(parser) # run a simple test test = self._quick_test() test.build() raw = schedulers.get_scheduler_plugin('raw') raw.schedule_test(self.pav_cfg, test) state = test.status.current().state end = time.time() + 1 while ('ERROR' not in state and 'FAIL' not in state and state != STATES.COMPLETE and time.time() < end): time.sleep(.1) # test `pav log run test` args = parser.parse_args(['run', str(test.id)]) self.assertEqual(args.test, test.id) out = io.StringIO() err = io.StringIO() result = log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err) err.seek(0) out.seek(0) self.assertEqual(err.read(), '') self.assertEqual(out.read(), 'Hello World.\n') self.assertEqual(result, 0) # test `pav log build test` # note: echo-ing hello world should not require anything to be built out.truncate(0) err.truncate(0) args = parser.parse_args(['build', str(test.id)]) log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err) out.seek(0) err.seek(0) self.assertEqual(out.read(), '') # test `pav log kickoff test` # note: in general, kickoff.log should be an empty file out.truncate(0) err.truncate(0) args = parser.parse_args(['kickoff', str(test.id)]) result = log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err) out.seek(0) err.seek(0) self.assertEqual(out.read(), '') self.assertEqual(err.read(), '') self.assertEqual(result, 0)
def test_status_command_with_sched(self): """Test status command when test is 'SCHEDULED'.""" test = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['sleep 1'], }, }) test['name'] = 'testytest' sys_vars = system_variables.get_vars(False) test = PavTest(self.pav_cfg, test, sys_vars) test.build() schedulers.get_scheduler_plugin(test.scheduler) \ .schedule_test(self.pav_cfg, test) status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args([str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args(['-j', str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def run(self, pav_config, args): """Resolve the test configurations into individual tests and assign to schedulers. Have those schedulers kick off jobs to run the individual tests themselves.""" # 1. Resolve the test configs # - Get sched vars from scheduler. # - Compile variables. # test_configs = self.get_tests(pav_config, args) for sched_name, tests in test_configs.items(): sched = schedulers.get_scheduler_plugin(sched_name) sched.run_tests(tests)
def test_slurm_vars(self): """Make sure all the slurm scheduler variable methods work when not on a node.""" slurm = schedulers.get_scheduler_plugin('slurm') from pavilion.utils import cprint cprint(slurm.get_data()['summary']) # Grab a random jobid, and get the status of it. jobs = subprocess.check_output(['squeue', '-o', "%i %T"]) jobs = jobs.decode('utf-8') last_job = jobs.strip().split('\n')[-1] jobid, status = last_job.strip().split() slurm.check_job(jobid)
def test_schedule_test(self): """Make sure the scheduler can run a test.""" raw = schedulers.get_scheduler_plugin('raw') test = self._quick_test() self.assertTrue(test.build()) raw.schedule_tests(self.pav_cfg, [test]) try: test.wait(2) except: self.fail() self.assertEqual(test.status.current().state, STATES.COMPLETE)
def test_sched_vars(self): """Make sure the scheduler vars are reasonable.""" slurm = schedulers.get_scheduler_plugin('slurm') cfg = TestConfigLoader().validate({ 'scheduler': 'slurm', 'run': { 'cmds': ['echo "Hello World."'] }, }) cfg['name'] = 'slurm_test' test = PavTest(self.pav_cfg, cfg, {}) for k, v in slurm.get_vars(test).items(): # Make sure everything has a value of some sort. self.assertNotIn(v, ['None', ''])
def test_kickoff_env(self): pav_cfg = self.pav_cfg pav_cfg['env_setup'] = ['test1', 'test2', 'test3'] test = PavTest(self.pav_cfg, { 'name': 'sched-vars', 'scheduler': 'dummy' }, {}) dummy_sched = schedulers.get_scheduler_plugin('dummy') path = dummy_sched._create_kickoff_script(pav_cfg, test) with path.open() as file: lines = file.readlines() for i in range(0, len(lines)): lines[i] = lines[i].strip() testlist = pav_cfg['env_setup'] self.assertTrue(set(testlist).issubset(lines)) self.assertTrue(re.match(r'pav _run.*', lines[-1]))
def test_check_job(self): """Make sure we can get the test's scheduler status.""" cfg = self._quick_test_cfg() cfg['run']['cmds'] = ['sleep 2'] test = self._quick_test(cfg=cfg) test.status.set('SCHEDULED', 'but not really') with Path('/proc/sys/kernel/pid_max').open() as pid_max_file: max_pid = int(pid_max_file.read()) hostname = socket.gethostname() raw = schedulers.get_scheduler_plugin('raw') # Make a test from another host. test.job_id = 'garbledhostnameasldfkjasd_{}'.format(os.getpid()) status = raw.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHEDULED) # Make a test with a non-existent pid. test.job_id = '{}_{}'.format(hostname, max_pid + 1) status = raw.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHED_ERROR) # Check the 'race condition' case of check_job test.status.set(STATES.COMPLETE, 'not really this either.') status = raw.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.COMPLETE) test.status.set(STATES.SCHEDULED, "reseting.") # Make a test with a re-used pid. test.job_id = '{}_{}'.format(hostname, os.getpid()) status = raw.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHED_ERROR) raw.schedule_test(self.pav_cfg, test) status = raw.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHEDULED)
def test_schedule_test(self): """Try to schedule a test. It doesn't have to run (but it can!) """ slurm = schedulers.get_scheduler_plugin('slurm') cfg = TestConfigLoader().validate({ 'scheduler': 'slurm', 'run': { 'cmds': ['echo "Hello World."'] }, }) cfg['name'] = 'slurm_test' test = PavTest(self.pav_cfg, cfg, {}) slurm.schedule_test(self.pav_cfg, test) status = slurm.job_status(self.pav_cfg, test) self.assertEqual(status.state, STATES.SCHEDULED) status = slurm.cancel_job(test) self.assertEqual(status.state, STATES.SCHED_CANCELLED)
def _scheduler_cmd(_, args, outfile=sys.stdout): """ :param argparse.Namespace args: """ sched = None # type : schedulers.SchedulerPlugin if args.vars is not None or args.config is not None: sched_name = args.vars if args.vars is not None else args.config try: sched = schedulers.get_scheduler_plugin(sched_name) except schedulers.SchedulerPluginError: utils.fprint( "Invalid scheduler plugin '{}'.".format(sched_name), color=utils.RED, ) return errno.EINVAL if args.vars is not None: sched_vars = [] svars = sched.get_vars(None) for key in sorted(list(svars.keys())): sched_vars.append(svars.info(key)) utils.draw_table( outfile, field_info={}, fields=['name', 'deferred', 'help'], rows=sched_vars, title="Variables for the {} scheduler plugin.".format(args.vars) ) elif args.config is not None: sched_config = sched.get_conf() class Loader(yaml_config.YamlConfigLoader): ELEMENTS = [sched_config] defaults = Loader().load_empty() Loader().dump(sys.stdout, values=defaults) else: # Assuming --list was given scheds = [] for sched_name in schedulers.list_scheduler_plugins(): sched = schedulers.get_scheduler_plugin(sched_name) scheds.append({ 'name': sched_name, 'description': sched.description, 'path': sched.path }) fields = ['name', 'description'] if args.verbose: fields.append('path') utils.draw_table( outfile, field_info={}, fields=fields, rows=scheds, title="Available Scheduler Plugins" )
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr): """Resolve the test configurations into individual tests and assign to schedulers. Have those schedulers kick off jobs to run the individual tests themselves. :param pav_cfg: The pavilion configuration. :param args: The parsed command line argument object. :param out_file: The file object to output to (stdout) :param err_file: The file object to output errors to (stderr) """ # 1. Resolve the test configs # - Get sched vars from scheduler. # - Compile variables. # overrides = {} for ovr in args.overrides: if '=' not in ovr: fprint( "Invalid override value. Must be in the form: " "<key>=<value>. Ex. -c run.modules=['gcc'] ", file=self.errfile) return errno.EINVAL key, value = ovr.split('=', 1) overrides[key] = value sys_vars = system_variables.get_vars(True) try: configs_by_sched = self._get_tests( pav_cfg=pav_cfg, host=args.host, test_files=args.files, tests=args.tests, modes=args.modes, overrides=overrides, sys_vars=sys_vars, ) tests_by_sched = self._configs_to_tests( pav_cfg=pav_cfg, sys_vars=sys_vars, configs_by_sched=configs_by_sched, ) except commands.CommandError as err: fprint(err, file=self.errfile) return errno.EINVAL all_tests = sum(tests_by_sched.values(), []) if not all_tests: fprint("You must specify at least one test.", file=self.errfile) return errno.EINVAL series = TestSeries(pav_cfg, all_tests) rp_errors = [] for test in all_tests: # Make sure the result parsers have reasonable arguments. try: result_parsers.check_args(test.config['results']) except PavTestError as err: rp_errors.append(str(err)) if rp_errors: fprint("Result Parser configurations had errors:", file=self.errfile, color=utils.RED) for msg in rp_errors: fprint(msg, bullet=' - ', file=self.errfile) return errno.EINVAL # Building any tests that specify that they should be built before for test in all_tests: if test.config['build']['on_nodes'] not in ['true', 'True']: if not test.build(): for oth_test in all_tests: if oth_test.build_hash != test.build_hash: oth_test.status.set( STATES.BUILD_ERROR, "Build cancelled because build {} failed.". format(test.id)) fprint("Error building test: ", file=self.errfile, color=utils.RED) fprint("status {status.state} - {status.note}".format( status=test.status.current()), file=self.errfile) fprint( "For more information, run 'pav log build {}'".format( test.id), file=self.errfile) return errno.EINVAL for sched_name, tests in tests_by_sched.items(): sched = schedulers.get_scheduler_plugin(sched_name) try: sched.schedule_tests(pav_cfg, tests) except schedulers.SchedulerPluginError as err: fprint('Error scheduling tests:', file=self.errfile, color=utils.RED) fprint(err, bullet=' ', file=self.errfile) fprint('Cancelling already kicked off tests.', file=self.errfile) self._cancel_all(tests_by_sched) # Tests should all be scheduled now, and have the SCHEDULED state # (at some point, at least). Wait until something isn't scheduled # anymore (either running or dead), or our timeout expires. wait_result = None if args.wait is not None: end_time = time.time() + args.wait while time.time() < end_time and wait_result is None: last_time = time.time() for sched_name, tests in tests_by_sched.items(): sched = schedulers.get_scheduler_plugin(sched_name) for test in tests: status = test.status.current() if status == STATES.SCHEDULED: status = sched.job_status(pav_cfg, test) if status != STATES.SCHEDULED: # The test has moved past the scheduled state. wait_result = None break break if wait_result is None: # Sleep at most SLEEP INTERVAL seconds, minus the time # we spent checking our jobs. time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time)) fprint("{} test{} started as test series {}.".format( len(all_tests), 's' if len(all_tests) > 1 else '', series.id), file=self.outfile, color=utils.GREEN) if args.status: tests = list(series.tests.keys()) tests, _ = test_obj_from_id(pav_cfg, tests) return print_from_test_obj(pav_cfg, tests, self.outfile, args.json) return 0
def get_tests(self, pav_config, args): """Translate a general set of pavilion test configs into the final, resolved configuration objects. These objects will be organized in a dictionary by scheduler, and have a scheduler object instantiated and attached. :returns: A dictionary (by scheduler type name) of lists of test objects """ self.logger.DEBUG("Finding Configs") # Use the sys_host if a host isn't specified. if args.host is None: host = pav_config.sys_vars.get('sys_host') else: host = args.host tests = args.tests for file in args.files: try: with open(file) as test_file: for line in test_file.readlines(): line = line.strip() if line and not line.startswith('#'): tests.append(line) except (OSError, IOError) as err: msg = "Could not read test file {}: {}".format(file, err) self.logger.error(msg) raise commands.CommandError(msg) raw_tests = config_utils.get_tests(pav_config, host, args.mode, tests) raw_tests_by_sched = defaultdict(lambda: []) tests_by_scheduler = defaultdict(lambda: []) # Apply config overrides. for test_cfg in raw_tests: # Apply the overrides to each of the config values. try: config_utils.apply_overrides(test_cfg, args.overrides) except config_utils.TestConfigError as err: msg = 'Error applying overrides to test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Resolve all configuration permutations. try: for p_cfg, p_var_man in config_utils.resolve_permutations( test_cfg, pav_config.pav_vars, pav_config.sys_vars): sched = p_cfg['scheduler'] raw_tests_by_sched[sched].append((p_cfg, p_var_man)) except config_utils.TestConfigError as err: msg = 'Error resolving permutations for test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Get the schedulers for the tests, and the scheduler variables. # The scheduler variables are based on all of the for sched_name in raw_tests_by_sched.keys(): try: sched = schedulers.get_scheduler_plugin(sched_name) except KeyError: msg = "Could not find scheduler '{}'.".format(sched_name) self.logger.error(msg) raise commands.CommandError(msg) nondeferred_cfg_sctns = schedulers.list_scheduler_plugins() # Builds must have the values of all their variables now. nondeferred_cfg_sctns.append('build') # Set the echeduler variables for each test. for test_cfg, test_var_man in raw_tests_by_sched[sched_name]: test_var_man.add_var_set('sched', sched) # Resolve all variables for the test. try: resolved_config = config_utils.resolve_all_vars( test_cfg, test_var_man, no_deferred_allowed=nondeferred_cfg_sctns) except (ResolveError, KeyError) as err: msg = 'Error resolving variables in config: {}'.format(err) self.logger.error(msg) raise commands.CommandError(msg) test = PavTest(pav_config, resolved_config) tests_by_scheduler[sched.name].append(test) return tests_by_scheduler
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides, sys_vars): """Translate a general set of pavilion test configs into the final, resolved configurations. These objects will be organized in a dictionary by scheduler, and have a scheduler object instantiated and attached. :param pav_cfg: The pavilion config :param str host: The host config to target these tests with :param list(str) modes: The mode configs to use. :param list(Path) test_files: Files containing a newline separated list of tests. :param list(str) tests: The tests to run. :param list(str) overrides: Overrides to apply to the configurations. :param system_variables.SysVarDict sys_vars: The system variables dict. :returns: A dictionary (by scheduler type name) of lists of test configs. """ self.logger.debug("Finding Configs") # Use the sys_host if a host isn't specified. if host is None: host = sys_vars.get('sys_name') tests = list(tests) for file in test_files: try: with file.open() as test_file: for line in test_file.readlines(): line = line.strip() if line and not line.startswith('#'): tests.append(line) except (OSError, IOError) as err: msg = "Could not read test file {}: {}".format(file, err) self.logger.error(msg) raise commands.CommandError(msg) try: raw_tests = test_config.load_test_configs(pav_cfg, host, modes, tests) except test_config.TestConfigError as err: self.logger.error(str(err)) raise commands.CommandError(str(err)) raw_tests_by_sched = defaultdict(lambda: []) tests_by_scheduler = defaultdict(lambda: []) # Apply config overrides. for test_cfg in raw_tests: # Apply the overrides to each of the config values. try: test_config.apply_overrides(test_cfg, overrides) except test_config.TestConfigError as err: msg = 'Error applying overrides to test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Resolve all configuration permutations. try: p_cfg, permutes = test_config.resolve_permutations( test_cfg, pav_cfg.pav_vars, sys_vars) for p_var_man in permutes: sched = p_cfg['scheduler'].resolve(p_var_man) raw_tests_by_sched[sched].append((p_cfg, p_var_man)) except test_config.TestConfigError as err: msg = 'Error resolving permutations for test {} from {}: {}'\ .format(test_cfg['name'], test_cfg['suite_path'], err) self.logger.error(msg) raise commands.CommandError(msg) # Get the schedulers for the tests, and the scheduler variables. # The scheduler variables are based on all of the for sched_name in raw_tests_by_sched.keys(): try: sched = schedulers.get_scheduler_plugin(sched_name) except KeyError: msg = "Could not find scheduler '{}'.".format(sched_name) self.logger.error(msg) raise commands.CommandError(msg) nondeferred_cfg_sctns = schedulers.list_scheduler_plugins() # Builds must have the values of all their variables now. nondeferred_cfg_sctns.append('build') # Set the scheduler variables for each test. for test_cfg, test_var_man in raw_tests_by_sched[sched_name]: test_var_man.add_var_set('sched', sched.get_vars(test_cfg)) # Resolve all variables for the test. try: resolved_config = test_config.resolve_all_vars( test_cfg, test_var_man, no_deferred_allowed=nondeferred_cfg_sctns) except (ResolveError, KeyError) as err: msg = "Error resolving variables in config at '{}': {}"\ .format(test_cfg['suite_path'].resolve(test_var_man), err) self.logger.error(msg) raise commands.CommandError(msg) tests_by_scheduler[sched.name].append(resolved_config) return tests_by_scheduler
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr): if not args.tests: # Get the last series ran by this user. series_id = series.TestSeries.load_user_series_id(pav_cfg) if series_id is not None: args.tests.append(series_id) test_list = [] for test_id in args.tests: if test_id.startswith('s'): try: test_list.extend(series.TestSeries.from_id(pav_cfg, int(test_id[1:])) .tests) except series.TestSeriesError as err: utils.fprint( "Series {} could not be found.\n{}".format(test_id[1:], err), file=self.errfile, color=utils.RED ) return errno.EINVAL except ValueError as err: utils.fprint( "Series {} is not a valid series.\n{}" .format(test_id[1:], err), file=self.errfile, color=utils.RED ) return errno.EINVAL else: try: test_list.append(int(test_id)) except ValueError as err: utils.fprint( "Test {} is not a valid test.\n{}".format(test_id, err), file=self.errfile, color=utils.RED ) return errno.EINVAL test_object_list = [] for test_id in test_list: try: test = PavTest.load(pav_cfg, test_id) sched = schedulers.get_scheduler_plugin(test.scheduler) test_object_list.append(test) status = test.status.current() # Won't try to cancel a completed job or a job that was # previously cancelled. if status.state != STATES.COMPLETE and \ status.state != STATES.SCHED_CANCELLED: # Sets status based on the result of sched.cancel_job. # Ran into trouble when 'cancelling' jobs that never # actually started, ie. build errors/created job states. test.status.set(sched.cancel_job(test).state, sched.cancel_job(test).note) utils.fprint("test {} cancelled." .format(test_id), file=self.outfile, color=utils.GREEN) else: utils.fprint("test {} could not be cancelled has state: {}." .format(test_id, status.state), file=self.outfile, color=utils.RED) except PavTestError as err: utils.fprint("Test {} could not be cancelled, cannot be" \ " found. \n{}".format(test_id, err), file=self.errfile, color=utils.RED) return errno.EINVAL # Only prints statuses of tests if option is selected # and test_list is not empty if args.status and test_object_list: return print_from_test_obj(pav_cfg, test_object_list, self.outfile, args.json) return 0
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr): """Load and run an already prepped test in the current environment. :param out_file: :param err_file: """ try: test = PavTest.load(pav_cfg, args.test_id) except PavTestError as err: self.logger.error("Error loading test '%s': %s", args.test_id, err) raise try: if test.config['build']['on_nodes'] in ['true', 'True']: if not test.build(): self.logger.warning("Test {t.id} failed to build:") except Exception: test.status.set(STATES.BUILD_ERROR, "Unknown build error. Refer to the kickoff log.") raise try: sched = schedulers.get_scheduler_plugin(test.scheduler) except Exception: test.status.set( STATES.BUILD_ERROR, "Unknown error getting the scheduler. Refer to " "the kickoff log.") raise # Optionally wait on other tests running under the same scheduler. # This depends on the scheduler and the test configuration. lock = sched.lock_concurrency(pav_cfg, test) try: run_result = test.run(sched.get_vars(test), system_variables.get_vars(defer=False)) except PavTestError as err: test.status.set(STATES.RUN_ERROR, err) test.set_run_complete() return 1 except Exception: test.status.set( STATES.RUN_ERROR, "Unknown error while running test. Refer to the kickoff log.") raise finally: sched.unlock_concurrency(lock) # The test.run() method should have already logged the error and # set an appropriate status. if run_result in (STATES.RUN_ERROR, STATES.RUN_TIMEOUT): return 1 try: rp_errors = [] # Make sure the result parsers have reasonable arguments. # We check here because the parser code itself will likely assume # the args are valid form _check_args, but those might not be # checkable before kickoff due to deferred variables. try: result_parsers.check_args(test.config['results']) except PavTestError as err: rp_errors.append(str(err)) if rp_errors: for msg in rp_errors: test.status.set(STATES.RESULTS_ERROR, msg) test.set_run_complete() return 1 results = test.gather_results(run_result) except result_parsers.ResultParserError as err: self.logger.error("Unexpected error gathering results: %s", err) test.status.set(STATES.RESULTS_ERROR, "Error parsing results: {}".format(err)) test.set_run_complete() return 1 try: test.save_results(results) result_logger = logging.getLogger('results') result_logger.info(utils.json_dumps(results)) except Exception: test.status.set( STATES.RESULTS_ERROR, "Unknown error while saving results. Refer to the kickoff log." ) raise try: test.status.set( STATES.COMPLETE, "The test completed with result: {}".format( results.get('result', '<unknown>'))) test.set_run_complete() except Exception: test.status.set( STATES.UNKNOWN, "Unknown error while setting test completion. Refer to the " "kickoff log.") raise