def test_cancel_job(self):
"""Create a series of tests and kill them under different
circumstances."""
# This test will just sleep for a bit.
cfg = self._quick_test_cfg()
cfg['run']['cmds'] = ['sleep 100']
test = self._quick_test(cfg=cfg)
test.build()
raw = schedulers.get_scheduler_plugin('raw')
raw.schedule_test(self.pav_cfg, test)
timeout = time.time() + 1
while (raw.job_status(self.pav_cfg, test).state == STATES.SCHEDULED
and time.time() < timeout):
time.sleep(.1)
# The test should be running
self.assertEqual(test.status.current().state,
STATES.RUNNING)
_, pid = test.job_id.split('_')
self.assertEqual(raw.cancel_job(test).state, STATES.SCHED_CANCELLED)
def test_job_status(self):
"""Make sure all the slurm scheduler variable methods work when
not on a node."""
slurm = schedulers.get_scheduler_plugin('slurm')
cfg = TestConfigLoader().validate({
'scheduler': 'slurm',
'run': {
'cmds': ['echo "Hello World."']
},
})
cfg['name'] = 'slurm_test'
test = PavTest(self.pav_cfg, cfg, {})
test.status.set(STATES.SCHEDULED, "not really though.")
# Grab a random jobid, and get the status of it.
jobs = subprocess.check_output(['squeue', '-o', "%i %T"])
jobs = jobs.decode('utf-8')
try:
last_job = jobs.strip().split('\n')[-1]
jobid, status = last_job.strip().split()
except Exception:
raise RuntimeError("No available test from which to borrow a"
"job id.")
test.job_id = jobid
def test_job_status(self):
"""Make sure we can get a slurm job status."""
cfg = self._quick_test_cfg()
cfg['scheduler'] = 'slurm'
test = self._quick_test(cfg, name='slurm_job_status', finalize=False)
slurm = schedulers.get_scheduler_plugin('slurm')
# Steal a running job's ID, and then check our status.
test.status.set(STATES.SCHEDULED, "not really though.")
test.job_id = self._get_job('JobState=RUNNING')
status = slurm.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHEDULED)
self.assertIn('RUNNING', status.note)
# Steal a canceled jobs id
test.status.set(STATES.SCHEDULED, "not really though.")
test.job_id = self._get_job('JobState=CANCELLED')
sched_status = slurm.job_status(self.pav_cfg, test)
self.assertEqual(sched_status.state, STATES.SCHED_CANCELLED)
status = test.status.current()
self.assertEqual(status.state, STATES.SCHED_CANCELLED)
# Check another random state. In this case, all pavilion will
# just consider the test still scheduled.
test.status.set(STATES.SCHEDULED, "not really though.")
test.job_id = self._get_job('JobState=COMPLETED')
sched_status = slurm.job_status(self.pav_cfg, test)
self.assertEqual(sched_status.state, STATES.SCHEDULED)
self.assertIn('COMPLETED', sched_status.note)
def status_from_test_obj(pav_cfg, test_obj):
"""Takes a test object or list of test objects and creates the dictionary
expected by the print_status function.
:param dict pav_cfg: Pavilion base configuration.
:param pav_test.PavTest test_obj: Pavilion test object.
:return list List of dictionary objects containing the test ID, name, state,
time of state update, and note associated with that state.
"""
if not isinstance(test_obj, list):
test_obj = [test_obj]
test_statuses = []
for test in test_obj:
status_f = test.status.current()
if status_f.state == STATES.SCHEDULED:
sched = schedulers.get_scheduler_plugin(test.scheduler)
status_f = sched.job_status(pav_cfg, test)
test_statuses.append({
'test_id':
test.id,
'name':
test.name,
'state':
status_f.state,
'time':
status_f.when.strftime("%d %b %Y %H:%M:%S %Z"),
'note':
status_f.note,
})
test_statuses.sort(key=lambda x: x['test_id'])
return test_statuses
def test_cancel_sched_check(self):
"""Cancel Test and make sure it is cancelled through scheduler."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this' 'hello_world2'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel'])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
test = []
series_id = series.TestSeries.load_user_series_id(self.pav_cfg)
test.append(series_id)
test_list = []
test_list.extend(
series.TestSeries.from_id(self.pav_cfg, int(test[0][1:])).tests)
for test_id in test_list:
test = PavTest.load(self.pav_cfg, test_id)
if test.status.current().state != STATES.COMPLETE:
sched = schedulers.get_scheduler_plugin(test.scheduler)
sched_status = sched.job_status(self.pav_cfg, test)
self.assertIn("SCHED_CANCELLED", str(sched_status))
def _cancel_all(tests_by_sched):
"""Cancel each of the given tests using the appropriate scheduler."""
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
for test in tests:
sched.cancel_job(test)
def test_sched_vars(self):
"""Make sure all the slurm scheduler variable methods work when
not on a node."""
raw = schedulers.get_scheduler_plugin('raw')
svars = raw.get_vars(self._quick_test())
for key, value in svars.items():
self.assertNotEqual(int(value), 0)
def test_log_arguments(self):
log_cmd = commands.get_command('log')
parser = argparse.ArgumentParser()
log_cmd._setup_arguments(parser)
# run a simple test
test = self._quick_test()
test.build()
raw = schedulers.get_scheduler_plugin('raw')
raw.schedule_test(self.pav_cfg, test)
state = test.status.current().state
end = time.time() + 1
while ('ERROR' not in state and 'FAIL' not in state
and state != STATES.COMPLETE and time.time() < end):
time.sleep(.1)
# test `pav log run test`
args = parser.parse_args(['run', str(test.id)])
self.assertEqual(args.test, test.id)
out = io.StringIO()
err = io.StringIO()
result = log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err)
err.seek(0)
out.seek(0)
self.assertEqual(err.read(), '')
self.assertEqual(out.read(), 'Hello World.\n')
self.assertEqual(result, 0)
# test `pav log build test`
# note: echo-ing hello world should not require anything to be built
out.truncate(0)
err.truncate(0)
args = parser.parse_args(['build', str(test.id)])
log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err)
out.seek(0)
err.seek(0)
self.assertEqual(out.read(), '')
# test `pav log kickoff test`
# note: in general, kickoff.log should be an empty file
out.truncate(0)
err.truncate(0)
args = parser.parse_args(['kickoff', str(test.id)])
result = log_cmd.run(self.pav_cfg, args, out_file=out, err_file=err)
out.seek(0)
err.seek(0)
self.assertEqual(out.read(), '')
self.assertEqual(err.read(), '')
self.assertEqual(result, 0)
def test_status_command_with_sched(self):
"""Test status command when test is 'SCHEDULED'."""
test = file_format.TestConfigLoader().validate({
'scheduler': 'raw',
'run': {
'env': {
'foo': 'bar',
},
'cmds': ['sleep 1'],
},
})
test['name'] = 'testytest'
sys_vars = system_variables.get_vars(False)
test = PavTest(self.pav_cfg, test, sys_vars)
test.build()
schedulers.get_scheduler_plugin(test.scheduler) \
.schedule_test(self.pav_cfg, test)
status_cmd = commands.get_command('status')
status_cmd.outfile = io.StringIO()
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
args = parser.parse_args([str(test.id)])
test.status.set(status_file.STATES.SCHEDULED, "faker")
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
args = parser.parse_args(['-j', str(test.id)])
test.status.set(status_file.STATES.SCHEDULED, "faker")
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def run(self, pav_config, args):
"""Resolve the test configurations into individual tests and assign to
schedulers. Have those schedulers kick off jobs to run the individual
tests themselves."""
# 1. Resolve the test configs
# - Get sched vars from scheduler.
# - Compile variables.
#
test_configs = self.get_tests(pav_config, args)
for sched_name, tests in test_configs.items():
sched = schedulers.get_scheduler_plugin(sched_name)
sched.run_tests(tests)
def test_slurm_vars(self):
"""Make sure all the slurm scheduler variable methods work when
not on a node."""
slurm = schedulers.get_scheduler_plugin('slurm')
from pavilion.utils import cprint
cprint(slurm.get_data()['summary'])
# Grab a random jobid, and get the status of it.
jobs = subprocess.check_output(['squeue', '-o', "%i %T"])
jobs = jobs.decode('utf-8')
last_job = jobs.strip().split('\n')[-1]
jobid, status = last_job.strip().split()
slurm.check_job(jobid)
def test_schedule_test(self):
"""Make sure the scheduler can run a test."""
raw = schedulers.get_scheduler_plugin('raw')
test = self._quick_test()
self.assertTrue(test.build())
raw.schedule_tests(self.pav_cfg, [test])
try:
test.wait(2)
except:
self.fail()
self.assertEqual(test.status.current().state, STATES.COMPLETE)
def test_sched_vars(self):
"""Make sure the scheduler vars are reasonable."""
slurm = schedulers.get_scheduler_plugin('slurm')
cfg = TestConfigLoader().validate({
'scheduler': 'slurm',
'run': {
'cmds': ['echo "Hello World."']
},
})
cfg['name'] = 'slurm_test'
test = PavTest(self.pav_cfg, cfg, {})
for k, v in slurm.get_vars(test).items():
# Make sure everything has a value of some sort.
self.assertNotIn(v, ['None', ''])
def test_kickoff_env(self):
pav_cfg = self.pav_cfg
pav_cfg['env_setup'] = ['test1', 'test2', 'test3']
test = PavTest(self.pav_cfg, {
'name': 'sched-vars',
'scheduler': 'dummy'
}, {})
dummy_sched = schedulers.get_scheduler_plugin('dummy')
path = dummy_sched._create_kickoff_script(pav_cfg, test)
with path.open() as file:
lines = file.readlines()
for i in range(0, len(lines)):
lines[i] = lines[i].strip()
testlist = pav_cfg['env_setup']
self.assertTrue(set(testlist).issubset(lines))
self.assertTrue(re.match(r'pav _run.*', lines[-1]))
def test_check_job(self):
"""Make sure we can get the test's scheduler status."""
cfg = self._quick_test_cfg()
cfg['run']['cmds'] = ['sleep 2']
test = self._quick_test(cfg=cfg)
test.status.set('SCHEDULED', 'but not really')
with Path('/proc/sys/kernel/pid_max').open() as pid_max_file:
max_pid = int(pid_max_file.read())
hostname = socket.gethostname()
raw = schedulers.get_scheduler_plugin('raw')
# Make a test from another host.
test.job_id = 'garbledhostnameasldfkjasd_{}'.format(os.getpid())
status = raw.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHEDULED)
# Make a test with a non-existent pid.
test.job_id = '{}_{}'.format(hostname, max_pid + 1)
status = raw.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHED_ERROR)
# Check the 'race condition' case of check_job
test.status.set(STATES.COMPLETE, 'not really this either.')
status = raw.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.COMPLETE)
test.status.set(STATES.SCHEDULED, "reseting.")
# Make a test with a re-used pid.
test.job_id = '{}_{}'.format(hostname, os.getpid())
status = raw.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHED_ERROR)
raw.schedule_test(self.pav_cfg, test)
status = raw.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHEDULED)
def test_schedule_test(self):
"""Try to schedule a test. It doesn't have to run (but it can!) """
slurm = schedulers.get_scheduler_plugin('slurm')
cfg = TestConfigLoader().validate({
'scheduler': 'slurm',
'run': {
'cmds': ['echo "Hello World."']
},
})
cfg['name'] = 'slurm_test'
test = PavTest(self.pav_cfg, cfg, {})
slurm.schedule_test(self.pav_cfg, test)
status = slurm.job_status(self.pav_cfg, test)
self.assertEqual(status.state, STATES.SCHEDULED)
status = slurm.cancel_job(test)
self.assertEqual(status.state, STATES.SCHED_CANCELLED)
def _scheduler_cmd(_, args, outfile=sys.stdout):
"""
:param argparse.Namespace args:
"""
sched = None # type : schedulers.SchedulerPlugin
if args.vars is not None or args.config is not None:
sched_name = args.vars if args.vars is not None else args.config
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except schedulers.SchedulerPluginError:
utils.fprint(
"Invalid scheduler plugin '{}'.".format(sched_name),
color=utils.RED,
)
return errno.EINVAL
if args.vars is not None:
sched_vars = []
svars = sched.get_vars(None)
for key in sorted(list(svars.keys())):
sched_vars.append(svars.info(key))
utils.draw_table(
outfile,
field_info={},
fields=['name', 'deferred', 'help'],
rows=sched_vars,
title="Variables for the {} scheduler plugin.".format(args.vars)
)
elif args.config is not None:
sched_config = sched.get_conf()
class Loader(yaml_config.YamlConfigLoader):
ELEMENTS = [sched_config]
defaults = Loader().load_empty()
Loader().dump(sys.stdout, values=defaults)
else:
# Assuming --list was given
scheds = []
for sched_name in schedulers.list_scheduler_plugins():
sched = schedulers.get_scheduler_plugin(sched_name)
scheds.append({
'name': sched_name,
'description': sched.description,
'path': sched.path
})
fields = ['name', 'description']
if args.verbose:
fields.append('path')
utils.draw_table(
outfile,
field_info={},
fields=fields,
rows=scheds,
title="Available Scheduler Plugins"
)
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
"""Resolve the test configurations into individual tests and assign to
schedulers. Have those schedulers kick off jobs to run the individual
tests themselves.
:param pav_cfg: The pavilion configuration.
:param args: The parsed command line argument object.
:param out_file: The file object to output to (stdout)
:param err_file: The file object to output errors to (stderr)
"""
# 1. Resolve the test configs
# - Get sched vars from scheduler.
# - Compile variables.
#
overrides = {}
for ovr in args.overrides:
if '=' not in ovr:
fprint(
"Invalid override value. Must be in the form: "
"<key>=<value>. Ex. -c run.modules=['gcc'] ",
file=self.errfile)
return errno.EINVAL
key, value = ovr.split('=', 1)
overrides[key] = value
sys_vars = system_variables.get_vars(True)
try:
configs_by_sched = self._get_tests(
pav_cfg=pav_cfg,
host=args.host,
test_files=args.files,
tests=args.tests,
modes=args.modes,
overrides=overrides,
sys_vars=sys_vars,
)
tests_by_sched = self._configs_to_tests(
pav_cfg=pav_cfg,
sys_vars=sys_vars,
configs_by_sched=configs_by_sched,
)
except commands.CommandError as err:
fprint(err, file=self.errfile)
return errno.EINVAL
all_tests = sum(tests_by_sched.values(), [])
if not all_tests:
fprint("You must specify at least one test.", file=self.errfile)
return errno.EINVAL
series = TestSeries(pav_cfg, all_tests)
rp_errors = []
for test in all_tests:
# Make sure the result parsers have reasonable arguments.
try:
result_parsers.check_args(test.config['results'])
except PavTestError as err:
rp_errors.append(str(err))
if rp_errors:
fprint("Result Parser configurations had errors:",
file=self.errfile,
color=utils.RED)
for msg in rp_errors:
fprint(msg, bullet=' - ', file=self.errfile)
return errno.EINVAL
# Building any tests that specify that they should be built before
for test in all_tests:
if test.config['build']['on_nodes'] not in ['true', 'True']:
if not test.build():
for oth_test in all_tests:
if oth_test.build_hash != test.build_hash:
oth_test.status.set(
STATES.BUILD_ERROR,
"Build cancelled because build {} failed.".
format(test.id))
fprint("Error building test: ",
file=self.errfile,
color=utils.RED)
fprint("status {status.state} - {status.note}".format(
status=test.status.current()),
file=self.errfile)
fprint(
"For more information, run 'pav log build {}'".format(
test.id),
file=self.errfile)
return errno.EINVAL
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
try:
sched.schedule_tests(pav_cfg, tests)
except schedulers.SchedulerPluginError as err:
fprint('Error scheduling tests:',
file=self.errfile,
color=utils.RED)
fprint(err, bullet=' ', file=self.errfile)
fprint('Cancelling already kicked off tests.',
file=self.errfile)
self._cancel_all(tests_by_sched)
# Tests should all be scheduled now, and have the SCHEDULED state
# (at some point, at least). Wait until something isn't scheduled
# anymore (either running or dead), or our timeout expires.
wait_result = None
if args.wait is not None:
end_time = time.time() + args.wait
while time.time() < end_time and wait_result is None:
last_time = time.time()
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
for test in tests:
status = test.status.current()
if status == STATES.SCHEDULED:
status = sched.job_status(pav_cfg, test)
if status != STATES.SCHEDULED:
# The test has moved past the scheduled state.
wait_result = None
break
break
if wait_result is None:
# Sleep at most SLEEP INTERVAL seconds, minus the time
# we spent checking our jobs.
time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time))
fprint("{} test{} started as test series {}.".format(
len(all_tests), 's' if len(all_tests) > 1 else '', series.id),
file=self.outfile,
color=utils.GREEN)
if args.status:
tests = list(series.tests.keys())
tests, _ = test_obj_from_id(pav_cfg, tests)
return print_from_test_obj(pav_cfg, tests, self.outfile, args.json)
return 0
def get_tests(self, pav_config, args):
"""Translate a general set of pavilion test configs into the final,
resolved configuration objects. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:returns: A dictionary (by scheduler type name) of lists of test
objects
"""
self.logger.DEBUG("Finding Configs")
# Use the sys_host if a host isn't specified.
if args.host is None:
host = pav_config.sys_vars.get('sys_host')
else:
host = args.host
tests = args.tests
for file in args.files:
try:
with open(file) as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
raw_tests = config_utils.get_tests(pav_config, host, args.mode, tests)
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
config_utils.apply_overrides(test_cfg, args.overrides)
except config_utils.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
for p_cfg, p_var_man in config_utils.resolve_permutations(
test_cfg, pav_config.pav_vars, pav_config.sys_vars):
sched = p_cfg['scheduler']
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except config_utils.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_scheduler_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the echeduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
test_var_man.add_var_set('sched', sched)
# Resolve all variables for the test.
try:
resolved_config = config_utils.resolve_all_vars(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = 'Error resolving variables in config: {}'.format(err)
self.logger.error(msg)
raise commands.CommandError(msg)
test = PavTest(pav_config, resolved_config)
tests_by_scheduler[sched.name].append(test)
return tests_by_scheduler
def _get_tests(self, pav_cfg, host, test_files, tests, modes, overrides,
sys_vars):
"""Translate a general set of pavilion test configs into the final,
resolved configurations. These objects will be organized in a
dictionary by scheduler, and have a scheduler object instantiated and
attached.
:param pav_cfg: The pavilion config
:param str host: The host config to target these tests with
:param list(str) modes: The mode configs to use.
:param list(Path) test_files: Files containing a newline separated
list of tests.
:param list(str) tests: The tests to run.
:param list(str) overrides: Overrides to apply to the configurations.
:param system_variables.SysVarDict sys_vars: The system variables dict.
:returns: A dictionary (by scheduler type name) of lists of test
configs.
"""
self.logger.debug("Finding Configs")
# Use the sys_host if a host isn't specified.
if host is None:
host = sys_vars.get('sys_name')
tests = list(tests)
for file in test_files:
try:
with file.open() as test_file:
for line in test_file.readlines():
line = line.strip()
if line and not line.startswith('#'):
tests.append(line)
except (OSError, IOError) as err:
msg = "Could not read test file {}: {}".format(file, err)
self.logger.error(msg)
raise commands.CommandError(msg)
try:
raw_tests = test_config.load_test_configs(pav_cfg, host, modes,
tests)
except test_config.TestConfigError as err:
self.logger.error(str(err))
raise commands.CommandError(str(err))
raw_tests_by_sched = defaultdict(lambda: [])
tests_by_scheduler = defaultdict(lambda: [])
# Apply config overrides.
for test_cfg in raw_tests:
# Apply the overrides to each of the config values.
try:
test_config.apply_overrides(test_cfg, overrides)
except test_config.TestConfigError as err:
msg = 'Error applying overrides to test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Resolve all configuration permutations.
try:
p_cfg, permutes = test_config.resolve_permutations(
test_cfg, pav_cfg.pav_vars, sys_vars)
for p_var_man in permutes:
sched = p_cfg['scheduler'].resolve(p_var_man)
raw_tests_by_sched[sched].append((p_cfg, p_var_man))
except test_config.TestConfigError as err:
msg = 'Error resolving permutations for test {} from {}: {}'\
.format(test_cfg['name'], test_cfg['suite_path'], err)
self.logger.error(msg)
raise commands.CommandError(msg)
# Get the schedulers for the tests, and the scheduler variables.
# The scheduler variables are based on all of the
for sched_name in raw_tests_by_sched.keys():
try:
sched = schedulers.get_scheduler_plugin(sched_name)
except KeyError:
msg = "Could not find scheduler '{}'.".format(sched_name)
self.logger.error(msg)
raise commands.CommandError(msg)
nondeferred_cfg_sctns = schedulers.list_scheduler_plugins()
# Builds must have the values of all their variables now.
nondeferred_cfg_sctns.append('build')
# Set the scheduler variables for each test.
for test_cfg, test_var_man in raw_tests_by_sched[sched_name]:
test_var_man.add_var_set('sched', sched.get_vars(test_cfg))
# Resolve all variables for the test.
try:
resolved_config = test_config.resolve_all_vars(
test_cfg,
test_var_man,
no_deferred_allowed=nondeferred_cfg_sctns)
except (ResolveError, KeyError) as err:
msg = "Error resolving variables in config at '{}': {}"\
.format(test_cfg['suite_path'].resolve(test_var_man),
err)
self.logger.error(msg)
raise commands.CommandError(msg)
tests_by_scheduler[sched.name].append(resolved_config)
return tests_by_scheduler
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
if not args.tests:
# Get the last series ran by this user.
series_id = series.TestSeries.load_user_series_id(pav_cfg)
if series_id is not None:
args.tests.append(series_id)
test_list = []
for test_id in args.tests:
if test_id.startswith('s'):
try:
test_list.extend(series.TestSeries.from_id(pav_cfg,
int(test_id[1:]))
.tests)
except series.TestSeriesError as err:
utils.fprint(
"Series {} could not be found.\n{}".format(test_id[1:],
err),
file=self.errfile,
color=utils.RED
)
return errno.EINVAL
except ValueError as err:
utils.fprint(
"Series {} is not a valid series.\n{}"
.format(test_id[1:], err), file=self.errfile,
color=utils.RED
)
return errno.EINVAL
else:
try:
test_list.append(int(test_id))
except ValueError as err:
utils.fprint(
"Test {} is not a valid test.\n{}".format(test_id,
err),
file=self.errfile, color=utils.RED
)
return errno.EINVAL
test_object_list = []
for test_id in test_list:
try:
test = PavTest.load(pav_cfg, test_id)
sched = schedulers.get_scheduler_plugin(test.scheduler)
test_object_list.append(test)
status = test.status.current()
# Won't try to cancel a completed job or a job that was
# previously cancelled.
if status.state != STATES.COMPLETE and \
status.state != STATES.SCHED_CANCELLED:
# Sets status based on the result of sched.cancel_job.
# Ran into trouble when 'cancelling' jobs that never
# actually started, ie. build errors/created job states.
test.status.set(sched.cancel_job(test).state,
sched.cancel_job(test).note)
utils.fprint("test {} cancelled."
.format(test_id), file=self.outfile,
color=utils.GREEN)
else:
utils.fprint("test {} could not be cancelled has state: {}."
.format(test_id, status.state),
file=self.outfile,
color=utils.RED)
except PavTestError as err:
utils.fprint("Test {} could not be cancelled, cannot be" \
" found. \n{}".format(test_id, err),
file=self.errfile,
color=utils.RED)
return errno.EINVAL
# Only prints statuses of tests if option is selected
# and test_list is not empty
if args.status and test_object_list:
return print_from_test_obj(pav_cfg, test_object_list, self.outfile,
args.json)
return 0
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
"""Load and run an already prepped test in the current environment.
:param out_file:
:param err_file:
"""
try:
test = PavTest.load(pav_cfg, args.test_id)
except PavTestError as err:
self.logger.error("Error loading test '%s': %s", args.test_id, err)
raise
try:
if test.config['build']['on_nodes'] in ['true', 'True']:
if not test.build():
self.logger.warning("Test {t.id} failed to build:")
except Exception:
test.status.set(STATES.BUILD_ERROR,
"Unknown build error. Refer to the kickoff log.")
raise
try:
sched = schedulers.get_scheduler_plugin(test.scheduler)
except Exception:
test.status.set(
STATES.BUILD_ERROR,
"Unknown error getting the scheduler. Refer to "
"the kickoff log.")
raise
# Optionally wait on other tests running under the same scheduler.
# This depends on the scheduler and the test configuration.
lock = sched.lock_concurrency(pav_cfg, test)
try:
run_result = test.run(sched.get_vars(test),
system_variables.get_vars(defer=False))
except PavTestError as err:
test.status.set(STATES.RUN_ERROR, err)
test.set_run_complete()
return 1
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error while running test. Refer to the kickoff log.")
raise
finally:
sched.unlock_concurrency(lock)
# The test.run() method should have already logged the error and
# set an appropriate status.
if run_result in (STATES.RUN_ERROR, STATES.RUN_TIMEOUT):
return 1
try:
rp_errors = []
# Make sure the result parsers have reasonable arguments.
# We check here because the parser code itself will likely assume
# the args are valid form _check_args, but those might not be
# checkable before kickoff due to deferred variables.
try:
result_parsers.check_args(test.config['results'])
except PavTestError as err:
rp_errors.append(str(err))
if rp_errors:
for msg in rp_errors:
test.status.set(STATES.RESULTS_ERROR, msg)
test.set_run_complete()
return 1
results = test.gather_results(run_result)
except result_parsers.ResultParserError as err:
self.logger.error("Unexpected error gathering results: %s", err)
test.status.set(STATES.RESULTS_ERROR,
"Error parsing results: {}".format(err))
test.set_run_complete()
return 1
try:
test.save_results(results)
result_logger = logging.getLogger('results')
result_logger.info(utils.json_dumps(results))
except Exception:
test.status.set(
STATES.RESULTS_ERROR,
"Unknown error while saving results. Refer to the kickoff log."
)
raise
try:
test.status.set(
STATES.COMPLETE, "The test completed with result: {}".format(
results.get('result', '<unknown>')))
test.set_run_complete()
except Exception:
test.status.set(
STATES.UNKNOWN,
"Unknown error while setting test completion. Refer to the "
"kickoff log.")
raise