def runcase(self, case):
super().runcase(case)
check, partition, environ = case
self.printer.status(
'RUN',
f'{check.name} @{partition.fullname}+{environ.name}'
)
task = RegressionTask(case, self.task_listeners)
self._task_index[case] = task
self.stats.add_task(task)
try:
# Do not run test if any of its dependencies has failed
# NOTE: Restored dependencies are not in the task_index
if any(self._task_index[c].failed
for c in case.deps if c in self._task_index):
raise TaskDependencyError('dependencies failed')
if any(self._task_index[c].skipped
for c in case.deps if c in self._task_index):
# We raise the SkipTestError here and catch it immediately in
# order for `skip()` to get the correct exception context.
try:
raise SkipTestError('skipped due to skipped dependencies')
except SkipTestError as e:
task.skip()
raise TaskExit from e
def runcase(self, case):
super().runcase(case)
check, partition, environ = case
self.printer.status(
'RUN', '%s on %s using %s' %
(check.name, partition.fullname, environ.name)
)
task = RegressionTask(case, self.task_listeners)
self._task_index[case] = task
self.stats.add_task(task)
try:
# Do not run test if any of its dependencies has failed
# NOTE: Restored dependencies are not in the task_index
if any(self._task_index[c].failed
for c in case.deps if c in self._task_index):
raise TaskDependencyError('dependencies failed')
partname = task.testcase.partition.fullname
task.setup(task.testcase.partition,
task.testcase.environ,
sched_flex_alloc_nodes=self.sched_flex_alloc_nodes,
sched_options=self.sched_options)
task.compile()
task.compile_wait()
task.run()
# Pick the right scheduler
if task.check.local:
sched = self.local_scheduler
else:
sched = partition.scheduler
while True:
sched.poll(task.check.job)
if task.run_complete():
break
self._pollctl.running_tasks(1).snooze()
task.run_wait()
if not self.skip_sanity_check:
task.sanity()
if not self.skip_performance_check:
task.performance()
self._retired_tasks.append(task)
task.finalize()
def _setup_task(self, task):
if self.deps_succeeded(task):
try:
task.setup(task.testcase.partition,
task.testcase.environ,
sched_flex_alloc_nodes=self.sched_flex_alloc_nodes,
sched_options=self.sched_options)
except TaskExit:
return False
else:
return True
elif self.deps_failed(task):
exc = TaskDependencyError('dependencies failed')
task.fail((type(exc), exc, None))
return False
else:
# Not all dependencies have finished yet
return False
def _setup_all(self):
still_waiting = []
for task in self._waiting_tasks:
if self.deps_failed(task):
exc = TaskDependencyError('dependencies failed')
task.fail((type(exc), exc, None))
elif self.deps_succeeded(task):
task.setup(task.testcase.partition,
task.testcase.environ,
sched_flex_alloc_tasks=self.sched_flex_alloc_tasks,
sched_account=self.sched_account,
sched_partition=self.sched_partition,
sched_reservation=self.sched_reservation,
sched_nodelist=self.sched_nodelist,
sched_exclude_nodelist=self.sched_exclude_nodelist,
sched_options=self.sched_options)
else:
still_waiting.append(task)
self._waiting_tasks[:] = still_waiting
def _advance_startup(self, task):
if self.deps_skipped(task):
try:
raise SkipTestError('skipped due to skipped dependencies')
except SkipTestError as e:
task.skip()
self._current_tasks.remove(task)
return 1
elif self.deps_succeeded(task):
try:
part = task.testcase.partition
env = task.testcase.environ.name
self.printer.status('RUN', task.info())
task.setup(task.testcase.partition,
task.testcase.environ,
sched_flex_alloc_nodes=self.sched_flex_alloc_nodes,
sched_options=self.sched_options)
except TaskExit:
self._current_tasks.remove(task)
return 1
if isinstance(task.check, RunOnlyRegressionTest):
# All tests should execute all the pipeline stages, even if
# they are no-ops
self._exec_stage(
task,
[task.compile, task.compile_complete, task.compile_wait])
return 1
elif self.deps_failed(task):
exc = TaskDependencyError('dependencies failed')
task.fail((type(exc), exc, None))
self._current_tasks.remove(task)
return 1
else:
# Not all dependencies have finished yet
getlogger().debug2(f'{task.check.info()} waiting for dependencies')
return 0
def runcase(self, case):
super().runcase(case)
check, partition, environ = case
self.printer.status(
'RUN', '%s on %s using %s' %
(check.name, partition.fullname, environ.name))
task = RegressionTask(case, self.task_listeners)
self._task_index[case] = task
self.stats.add_task(task)
try:
# Do not run test if any of its dependencies has failed
if any(self._task_index[c].failed for c in case.deps):
raise TaskDependencyError('dependencies failed')
task.setup(partition,
environ,
sched_flex_alloc_tasks=self.sched_flex_alloc_tasks,
sched_account=self.sched_account,
sched_partition=self.sched_partition,
sched_reservation=self.sched_reservation,
sched_nodelist=self.sched_nodelist,
sched_exclude_nodelist=self.sched_exclude_nodelist,
sched_options=self.sched_options)
task.compile()
task.compile_wait()
task.run()
task.wait()
if not self.skip_sanity_check:
task.sanity()
if not self.skip_performance_check:
task.performance()
self._retired_tasks.append(task)
task.finalize()
def runcase(self, case):
super().runcase(case)
check, partition, environ = case
# Set partition-based counters, if not set already
self._running_tasks_counts.setdefault(partition.fullname, 0)
self._ready_tasks.setdefault(partition.fullname, [])
self._max_jobs.setdefault(partition.fullname, partition.max_jobs)
task = RegressionTask(case, self.task_listeners)
self._task_index[case] = task
self.stats.add_task(task)
self.printer.status(
'RUN', '%s on %s using %s' %
(check.name, partition.fullname, environ.name))
try:
partname = partition.fullname
if self.deps_failed(task):
exc = TaskDependencyError('dependencies failed')
task.fail((type(exc), exc, None))
return
if not self.deps_succeeded(task):
self.printer.status('DEP',
'%s on %s using %s' %
(check.name, partname, environ.name),
just='right')
self._waiting_tasks.append(task)
return
task.setup(partition,
environ,
sched_flex_alloc_tasks=self.sched_flex_alloc_tasks,
sched_account=self.sched_account,
sched_partition=self.sched_partition,
sched_reservation=self.sched_reservation,
sched_nodelist=self.sched_nodelist,
sched_exclude_nodelist=self.sched_exclude_nodelist,
sched_options=self.sched_options)
if self._running_tasks_counts[partname] >= partition.max_jobs:
# Make sure that we still exceeded the job limit
getlogger().debug('reached job limit (%s) for partition %s' %
(partition.max_jobs, partname))
self._poll_tasks()
if self._running_tasks_counts[partname] < partition.max_jobs:
# Task was put in _ready_tasks during setup
self._ready_tasks[partname].pop()
self._reschedule(task)
else:
self.printer.status('HOLD', task.check.info(), just='right')
except TaskExit:
if not task.failed:
self._reschedule(task)
return
except ABORT_REASONS as e:
if not task.failed:
# Abort was caused due to failure elsewhere, abort current
# task as well
task.abort(e)
self._failall(e)
raise