def run(self, pav_cfg, args):
"""Cancel the given tests."""
user_id = os.geteuid() # gets unique user id
if not args.tests:
# user wants to cancel all current tests
if args.all:
tests_dir = pav_cfg.working_dir / 'test_runs'
# iterate through all the tests in the tests directory
for test in tests_dir.iterdir():
test_owner_id = test.stat().st_uid
if test_owner_id == user_id:
if not (test / 'RUN_COMPLETE').exists():
test_id = test.name
args.tests.append(test_id)
else:
# Get the last series ran by this user.
series_id = series.TestSeries.load_user_series_id(pav_cfg)
if series_id is not None:
args.tests.append(series_id)
test_list = []
for test_id in args.tests:
if test_id.startswith('s'):
try:
test_list.extend(
series.TestSeries.from_id(pav_cfg, test_id).tests)
except series.TestSeriesError as err:
output.fprint("Series {} could not be found.\n{}".format(
test_id, err),
file=self.errfile,
color=output.RED)
return errno.EINVAL
except ValueError as err:
output.fprint(
"Series {} is not a valid series.\n{}".format(
test_id, err),
color=output.RED,
file=self.errfile)
return errno.EINVAL
else:
try:
test_list.append(int(test_id))
except ValueError as err:
output.fprint("Test {} is not a valid test.\n{}".format(
test_id, err),
file=self.errfile,
color=output.RED)
return errno.EINVAL
cancel_failed = False
test_object_list = []
for test_id in test_list:
try:
test = TestRun.load(pav_cfg, test_id)
sched = schedulers.get_plugin(test.scheduler)
test_object_list.append(test)
status = test.status.current()
# Won't try to cancel a completed job or a job that was
# previously cancelled.
if status.state not in (STATES.COMPLETE,
STATES.SCHED_CANCELLED):
# Sets status based on the result of sched.cancel_job.
# Ran into trouble when 'cancelling' jobs that never
# actually started, ie. build errors/created job states.
cancel_status = sched.cancel_job(test)
test.status.set(cancel_status.state, cancel_status.note)
test.set_run_complete()
output.fprint("Test {} cancelled.".format(test_id),
file=self.outfile,
color=output.GREEN)
else:
output.fprint(
"Test {} could not be cancelled has state: {}.".format(
test_id, status.state),
file=self.outfile,
color=output.RED)
except TestRunError as err:
output.fprint(
"Test {} could not be cancelled, cannot be found. \n{}".
format(test_id, err),
file=self.errfile,
color=output.RED)
return errno.EINVAL
# Only prints statuses of tests if option is selected
# and test_list is not empty
if args.status and test_object_list:
print_from_test_obj(pav_cfg, test_object_list, self.outfile,
args.json)
return cancel_failed
return cancel_failed
def run_tests(self, pav_cfg, tests_by_sched, series, wait, report_status):
"""
:param pav_cfg:
:param dict[str,[TestRun]] tests_by_sched: A dict by scheduler name
of the tests (in a list).
:param series: The test series.
:param int wait: Wait this long for a test to start before exiting.
:param bool report_status: Do a 'pav status' after tests have started.
on nodes, and kick them off in build only mode.
:return:
"""
all_tests = sum(tests_by_sched.values(), [])
for sched_name in tests_by_sched.keys():
sched = schedulers.get_plugin(sched_name)
if not sched.available():
fprint("{} tests started with the {} scheduler, but "
"that scheduler isn't available on this system."
.format(len(tests_by_sched[sched_name]), sched_name),
file=self.errfile, color=output.RED)
return errno.EINVAL
for sched_name, tests in tests_by_sched.items():
tests = [test for test in tests if not test.skipped]
sched = schedulers.get_plugin(sched_name)
# Filter out any 'build_only' tests (it should be all or none)
# that shouldn't be scheduled.
tests = [test for test in tests if
# The non-build only tests
(not test.build_only) or
# The build only tests that are built on nodes
(not test.build_local and
# As long they need to be built.
(test.rebuild or not test.builder.exists()))]
# Skip this scheduler if it doesn't have tests that need to run.
if not tests:
continue
try:
sched.schedule_tests(pav_cfg, tests)
except schedulers.SchedulerPluginError as err:
fprint('Error scheduling tests:', file=self.errfile,
color=output.RED)
fprint(err, bullet=' ', file=self.errfile)
fprint('Cancelling already kicked off tests.',
file=self.errfile)
self._cancel_all(tests_by_sched)
# return so the rest of the tests don't actually run
return errno.EINVAL
# Tests should all be scheduled now, and have the SCHEDULED state
# (at some point, at least). Wait until something isn't scheduled
# anymore (either running or dead), or our timeout expires.
wait_result = None
if wait is not None:
end_time = time.time() + wait
while time.time() < end_time and wait_result is None:
last_time = time.time()
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_plugin(sched_name)
for test in tests:
status = test.status.current()
if status == STATES.SCHEDULED:
status = sched.job_status(pav_cfg, test)
if status != STATES.SCHEDULED:
# The test has moved past the scheduled state.
wait_result = None
break
break
if wait_result is None:
# Sleep at most SLEEP INTERVAL seconds, minus the time
# we spent checking our jobs.
time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time))
fprint("{} test{} started as test series {}."
.format(len(all_tests),
's' if len(all_tests) > 1 else '',
series.id),
file=self.outfile,
color=output.GREEN)
if report_status:
tests = list(series.tests.keys())
tests, _ = test_obj_from_id(pav_cfg, tests)
return print_from_test_obj(
pav_cfg=pav_cfg,
test_obj=tests,
outfile=self.outfile,
json=False)
return 0
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
"""Resolve the test configurations into individual tests and assign to
schedulers. Have those schedulers kick off jobs to run the individual
tests themselves.
:param pav_cfg: The pavilion configuration.
:param args: The parsed command line argument object.
:param out_file: The file object to output to (stdout)
:param err_file: The file object to output errors to (stderr)
"""
# 1. Resolve the test configs
# - Get sched vars from scheduler.
# - Compile variables.
#
overrides = {}
for ovr in args.overrides:
if '=' not in ovr:
fprint(
"Invalid override value. Must be in the form: "
"<key>=<value>. Ex. -c run.modules=['gcc'] ",
file=self.errfile)
return errno.EINVAL
key, value = ovr.split('=', 1)
overrides[key] = value
sys_vars = system_variables.get_vars(True)
try:
configs_by_sched = self._get_tests(
pav_cfg=pav_cfg,
host=args.host,
test_files=args.files,
tests=args.tests,
modes=args.modes,
overrides=overrides,
sys_vars=sys_vars,
)
tests_by_sched = self._configs_to_tests(
pav_cfg=pav_cfg,
sys_vars=sys_vars,
configs_by_sched=configs_by_sched,
)
except commands.CommandError as err:
fprint(err, file=self.errfile)
return errno.EINVAL
all_tests = sum(tests_by_sched.values(), [])
if not all_tests:
fprint("You must specify at least one test.", file=self.errfile)
return errno.EINVAL
series = TestSeries(pav_cfg, all_tests)
rp_errors = []
for test in all_tests:
# Make sure the result parsers have reasonable arguments.
try:
result_parsers.check_args(test.config['results'])
except PavTestError as err:
rp_errors.append(str(err))
if rp_errors:
fprint("Result Parser configurations had errors:",
file=self.errfile,
color=utils.RED)
for msg in rp_errors:
fprint(msg, bullet=' - ', file=self.errfile)
return errno.EINVAL
# Building any tests that specify that they should be built before
for test in all_tests:
if test.config['build']['on_nodes'] not in ['true', 'True']:
if not test.build():
for oth_test in all_tests:
if oth_test.build_hash != test.build_hash:
oth_test.status.set(
STATES.BUILD_ERROR,
"Build cancelled because build {} failed.".
format(test.id))
fprint("Error building test: ",
file=self.errfile,
color=utils.RED)
fprint("status {status.state} - {status.note}".format(
status=test.status.current()),
file=self.errfile)
fprint(
"For more information, run 'pav log build {}'".format(
test.id),
file=self.errfile)
return errno.EINVAL
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
try:
sched.schedule_tests(pav_cfg, tests)
except schedulers.SchedulerPluginError as err:
fprint('Error scheduling tests:',
file=self.errfile,
color=utils.RED)
fprint(err, bullet=' ', file=self.errfile)
fprint('Cancelling already kicked off tests.',
file=self.errfile)
self._cancel_all(tests_by_sched)
# Tests should all be scheduled now, and have the SCHEDULED state
# (at some point, at least). Wait until something isn't scheduled
# anymore (either running or dead), or our timeout expires.
wait_result = None
if args.wait is not None:
end_time = time.time() + args.wait
while time.time() < end_time and wait_result is None:
last_time = time.time()
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
for test in tests:
status = test.status.current()
if status == STATES.SCHEDULED:
status = sched.job_status(pav_cfg, test)
if status != STATES.SCHEDULED:
# The test has moved past the scheduled state.
wait_result = None
break
break
if wait_result is None:
# Sleep at most SLEEP INTERVAL seconds, minus the time
# we spent checking our jobs.
time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time))
fprint("{} test{} started as test series {}.".format(
len(all_tests), 's' if len(all_tests) > 1 else '', series.id),
file=self.outfile,
color=utils.GREEN)
if args.status:
tests = list(series.tests.keys())
tests, _ = test_obj_from_id(pav_cfg, tests)
return print_from_test_obj(pav_cfg, tests, self.outfile, args.json)
return 0
def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
if not args.tests:
# Get the last series ran by this user.
series_id = series.TestSeries.load_user_series_id(pav_cfg)
if series_id is not None:
args.tests.append(series_id)
test_list = []
for test_id in args.tests:
if test_id.startswith('s'):
try:
test_list.extend(series.TestSeries.from_id(pav_cfg,
int(test_id[1:]))
.tests)
except series.TestSeriesError as err:
utils.fprint(
"Series {} could not be found.\n{}".format(test_id[1:],
err),
file=self.errfile,
color=utils.RED
)
return errno.EINVAL
except ValueError as err:
utils.fprint(
"Series {} is not a valid series.\n{}"
.format(test_id[1:], err), file=self.errfile,
color=utils.RED
)
return errno.EINVAL
else:
try:
test_list.append(int(test_id))
except ValueError as err:
utils.fprint(
"Test {} is not a valid test.\n{}".format(test_id,
err),
file=self.errfile, color=utils.RED
)
return errno.EINVAL
test_object_list = []
for test_id in test_list:
try:
test = PavTest.load(pav_cfg, test_id)
sched = schedulers.get_scheduler_plugin(test.scheduler)
test_object_list.append(test)
status = test.status.current()
# Won't try to cancel a completed job or a job that was
# previously cancelled.
if status.state != STATES.COMPLETE and \
status.state != STATES.SCHED_CANCELLED:
# Sets status based on the result of sched.cancel_job.
# Ran into trouble when 'cancelling' jobs that never
# actually started, ie. build errors/created job states.
test.status.set(sched.cancel_job(test).state,
sched.cancel_job(test).note)
utils.fprint("test {} cancelled."
.format(test_id), file=self.outfile,
color=utils.GREEN)
else:
utils.fprint("test {} could not be cancelled has state: {}."
.format(test_id, status.state),
file=self.outfile,
color=utils.RED)
except PavTestError as err:
utils.fprint("Test {} could not be cancelled, cannot be" \
" found. \n{}".format(test_id, err),
file=self.errfile,
color=utils.RED)
return errno.EINVAL
# Only prints statuses of tests if option is selected
# and test_list is not empty
if args.status and test_object_list:
return print_from_test_obj(pav_cfg, test_object_list, self.outfile,
args.json)
return 0
