def run(self, pav_cfg, args, out_file=sys.stdout, err_file=sys.stderr):
"""Resolve the test configurations into individual tests and assign to
schedulers. Have those schedulers kick off jobs to run the individual
tests themselves.
:param pav_cfg: The pavilion configuration.
:param args: The parsed command line argument object.
:param out_file: The file object to output to (stdout)
:param err_file: The file object to output errors to (stderr)
"""
# 1. Resolve the test configs
# - Get sched vars from scheduler.
# - Compile variables.
#
overrides = {}
for ovr in args.overrides:
if '=' not in ovr:
fprint(
"Invalid override value. Must be in the form: "
"<key>=<value>. Ex. -c run.modules=['gcc'] ",
file=self.errfile)
return errno.EINVAL
key, value = ovr.split('=', 1)
overrides[key] = value
sys_vars = system_variables.get_vars(True)
try:
configs_by_sched = self._get_tests(
pav_cfg=pav_cfg,
host=args.host,
test_files=args.files,
tests=args.tests,
modes=args.modes,
overrides=overrides,
sys_vars=sys_vars,
)
tests_by_sched = self._configs_to_tests(
pav_cfg=pav_cfg,
sys_vars=sys_vars,
configs_by_sched=configs_by_sched,
)
except commands.CommandError as err:
fprint(err, file=self.errfile)
return errno.EINVAL
all_tests = sum(tests_by_sched.values(), [])
if not all_tests:
fprint("You must specify at least one test.", file=self.errfile)
return errno.EINVAL
series = TestSeries(pav_cfg, all_tests)
rp_errors = []
for test in all_tests:
# Make sure the result parsers have reasonable arguments.
try:
result_parsers.check_args(test.config['results'])
except PavTestError as err:
rp_errors.append(str(err))
if rp_errors:
fprint("Result Parser configurations had errors:",
file=self.errfile,
color=utils.RED)
for msg in rp_errors:
fprint(msg, bullet=' - ', file=self.errfile)
return errno.EINVAL
# Building any tests that specify that they should be built before
for test in all_tests:
if test.config['build']['on_nodes'] not in ['true', 'True']:
if not test.build():
for oth_test in all_tests:
if oth_test.build_hash != test.build_hash:
oth_test.status.set(
STATES.BUILD_ERROR,
"Build cancelled because build {} failed.".
format(test.id))
fprint("Error building test: ",
file=self.errfile,
color=utils.RED)
fprint("status {status.state} - {status.note}".format(
status=test.status.current()),
file=self.errfile)
fprint(
"For more information, run 'pav log build {}'".format(
test.id),
file=self.errfile)
return errno.EINVAL
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
try:
sched.schedule_tests(pav_cfg, tests)
except schedulers.SchedulerPluginError as err:
fprint('Error scheduling tests:',
file=self.errfile,
color=utils.RED)
fprint(err, bullet=' ', file=self.errfile)
fprint('Cancelling already kicked off tests.',
file=self.errfile)
self._cancel_all(tests_by_sched)
# Tests should all be scheduled now, and have the SCHEDULED state
# (at some point, at least). Wait until something isn't scheduled
# anymore (either running or dead), or our timeout expires.
wait_result = None
if args.wait is not None:
end_time = time.time() + args.wait
while time.time() < end_time and wait_result is None:
last_time = time.time()
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_scheduler_plugin(sched_name)
for test in tests:
status = test.status.current()
if status == STATES.SCHEDULED:
status = sched.job_status(pav_cfg, test)
if status != STATES.SCHEDULED:
# The test has moved past the scheduled state.
wait_result = None
break
break
if wait_result is None:
# Sleep at most SLEEP INTERVAL seconds, minus the time
# we spent checking our jobs.
time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time))
fprint("{} test{} started as test series {}.".format(
len(all_tests), 's' if len(all_tests) > 1 else '', series.id),
file=self.outfile,
color=utils.GREEN)
if args.status:
tests = list(series.tests.keys())
tests, _ = test_obj_from_id(pav_cfg, tests)
return print_from_test_obj(pav_cfg, tests, self.outfile, args.json)
return 0
def run_tests(self, pav_cfg, tests_by_sched, series, wait, report_status):
"""
:param pav_cfg:
:param dict[str,[TestRun]] tests_by_sched: A dict by scheduler name
of the tests (in a list).
:param series: The test series.
:param int wait: Wait this long for a test to start before exiting.
:param bool report_status: Do a 'pav status' after tests have started.
on nodes, and kick them off in build only mode.
:return:
"""
all_tests = sum(tests_by_sched.values(), [])
for sched_name in tests_by_sched.keys():
sched = schedulers.get_plugin(sched_name)
if not sched.available():
fprint("{} tests started with the {} scheduler, but "
"that scheduler isn't available on this system."
.format(len(tests_by_sched[sched_name]), sched_name),
file=self.errfile, color=output.RED)
return errno.EINVAL
for sched_name, tests in tests_by_sched.items():
tests = [test for test in tests if not test.skipped]
sched = schedulers.get_plugin(sched_name)
# Filter out any 'build_only' tests (it should be all or none)
# that shouldn't be scheduled.
tests = [test for test in tests if
# The non-build only tests
(not test.build_only) or
# The build only tests that are built on nodes
(not test.build_local and
# As long they need to be built.
(test.rebuild or not test.builder.exists()))]
# Skip this scheduler if it doesn't have tests that need to run.
if not tests:
continue
try:
sched.schedule_tests(pav_cfg, tests)
except schedulers.SchedulerPluginError as err:
fprint('Error scheduling tests:', file=self.errfile,
color=output.RED)
fprint(err, bullet=' ', file=self.errfile)
fprint('Cancelling already kicked off tests.',
file=self.errfile)
self._cancel_all(tests_by_sched)
# return so the rest of the tests don't actually run
return errno.EINVAL
# Tests should all be scheduled now, and have the SCHEDULED state
# (at some point, at least). Wait until something isn't scheduled
# anymore (either running or dead), or our timeout expires.
wait_result = None
if wait is not None:
end_time = time.time() + wait
while time.time() < end_time and wait_result is None:
last_time = time.time()
for sched_name, tests in tests_by_sched.items():
sched = schedulers.get_plugin(sched_name)
for test in tests:
status = test.status.current()
if status == STATES.SCHEDULED:
status = sched.job_status(pav_cfg, test)
if status != STATES.SCHEDULED:
# The test has moved past the scheduled state.
wait_result = None
break
break
if wait_result is None:
# Sleep at most SLEEP INTERVAL seconds, minus the time
# we spent checking our jobs.
time.sleep(self.SLEEP_INTERVAL - (time.time() - last_time))
fprint("{} test{} started as test series {}."
.format(len(all_tests),
's' if len(all_tests) > 1 else '',
series.id),
file=self.outfile,
color=output.GREEN)
if report_status:
tests = list(series.tests.keys())
tests, _ = test_obj_from_id(pav_cfg, tests)
return print_from_test_obj(
pav_cfg=pav_cfg,
test_obj=tests,
outfile=self.outfile,
json=False)
return 0