def runcase(self, case): super().runcase(case) check, partition, environ = case self.printer.status( 'RUN', f'{check.name} @{partition.fullname}+{environ.name}' ) task = RegressionTask(case, self.task_listeners) self._task_index[case] = task self.stats.add_task(task) try: # Do not run test if any of its dependencies has failed # NOTE: Restored dependencies are not in the task_index if any(self._task_index[c].failed for c in case.deps if c in self._task_index): raise TaskDependencyError('dependencies failed') if any(self._task_index[c].skipped for c in case.deps if c in self._task_index): # We raise the SkipTestError here and catch it immediately in # order for `skip()` to get the correct exception context. try: raise SkipTestError('skipped due to skipped dependencies') except SkipTestError as e: task.skip() raise TaskExit from e
def runcase(self, case): super().runcase(case) check, partition, environ = case self.printer.status( 'RUN', '%s on %s using %s' % (check.name, partition.fullname, environ.name) ) task = RegressionTask(case, self.task_listeners) self._task_index[case] = task self.stats.add_task(task) try: # Do not run test if any of its dependencies has failed # NOTE: Restored dependencies are not in the task_index if any(self._task_index[c].failed for c in case.deps if c in self._task_index): raise TaskDependencyError('dependencies failed') partname = task.testcase.partition.fullname task.setup(task.testcase.partition, task.testcase.environ, sched_flex_alloc_nodes=self.sched_flex_alloc_nodes, sched_options=self.sched_options) task.compile() task.compile_wait() task.run() # Pick the right scheduler if task.check.local: sched = self.local_scheduler else: sched = partition.scheduler while True: sched.poll(task.check.job) if task.run_complete(): break self._pollctl.running_tasks(1).snooze() task.run_wait() if not self.skip_sanity_check: task.sanity() if not self.skip_performance_check: task.performance() self._retired_tasks.append(task) task.finalize()
def _setup_task(self, task): if self.deps_succeeded(task): try: task.setup(task.testcase.partition, task.testcase.environ, sched_flex_alloc_nodes=self.sched_flex_alloc_nodes, sched_options=self.sched_options) except TaskExit: return False else: return True elif self.deps_failed(task): exc = TaskDependencyError('dependencies failed') task.fail((type(exc), exc, None)) return False else: # Not all dependencies have finished yet return False
def _setup_all(self): still_waiting = [] for task in self._waiting_tasks: if self.deps_failed(task): exc = TaskDependencyError('dependencies failed') task.fail((type(exc), exc, None)) elif self.deps_succeeded(task): task.setup(task.testcase.partition, task.testcase.environ, sched_flex_alloc_tasks=self.sched_flex_alloc_tasks, sched_account=self.sched_account, sched_partition=self.sched_partition, sched_reservation=self.sched_reservation, sched_nodelist=self.sched_nodelist, sched_exclude_nodelist=self.sched_exclude_nodelist, sched_options=self.sched_options) else: still_waiting.append(task) self._waiting_tasks[:] = still_waiting
def _advance_startup(self, task): if self.deps_skipped(task): try: raise SkipTestError('skipped due to skipped dependencies') except SkipTestError as e: task.skip() self._current_tasks.remove(task) return 1 elif self.deps_succeeded(task): try: part = task.testcase.partition env = task.testcase.environ.name self.printer.status('RUN', task.info()) task.setup(task.testcase.partition, task.testcase.environ, sched_flex_alloc_nodes=self.sched_flex_alloc_nodes, sched_options=self.sched_options) except TaskExit: self._current_tasks.remove(task) return 1 if isinstance(task.check, RunOnlyRegressionTest): # All tests should execute all the pipeline stages, even if # they are no-ops self._exec_stage( task, [task.compile, task.compile_complete, task.compile_wait]) return 1 elif self.deps_failed(task): exc = TaskDependencyError('dependencies failed') task.fail((type(exc), exc, None)) self._current_tasks.remove(task) return 1 else: # Not all dependencies have finished yet getlogger().debug2(f'{task.check.info()} waiting for dependencies') return 0
def runcase(self, case): super().runcase(case) check, partition, environ = case self.printer.status( 'RUN', '%s on %s using %s' % (check.name, partition.fullname, environ.name)) task = RegressionTask(case, self.task_listeners) self._task_index[case] = task self.stats.add_task(task) try: # Do not run test if any of its dependencies has failed if any(self._task_index[c].failed for c in case.deps): raise TaskDependencyError('dependencies failed') task.setup(partition, environ, sched_flex_alloc_tasks=self.sched_flex_alloc_tasks, sched_account=self.sched_account, sched_partition=self.sched_partition, sched_reservation=self.sched_reservation, sched_nodelist=self.sched_nodelist, sched_exclude_nodelist=self.sched_exclude_nodelist, sched_options=self.sched_options) task.compile() task.compile_wait() task.run() task.wait() if not self.skip_sanity_check: task.sanity() if not self.skip_performance_check: task.performance() self._retired_tasks.append(task) task.finalize()
def runcase(self, case): super().runcase(case) check, partition, environ = case # Set partition-based counters, if not set already self._running_tasks_counts.setdefault(partition.fullname, 0) self._ready_tasks.setdefault(partition.fullname, []) self._max_jobs.setdefault(partition.fullname, partition.max_jobs) task = RegressionTask(case, self.task_listeners) self._task_index[case] = task self.stats.add_task(task) self.printer.status( 'RUN', '%s on %s using %s' % (check.name, partition.fullname, environ.name)) try: partname = partition.fullname if self.deps_failed(task): exc = TaskDependencyError('dependencies failed') task.fail((type(exc), exc, None)) return if not self.deps_succeeded(task): self.printer.status('DEP', '%s on %s using %s' % (check.name, partname, environ.name), just='right') self._waiting_tasks.append(task) return task.setup(partition, environ, sched_flex_alloc_tasks=self.sched_flex_alloc_tasks, sched_account=self.sched_account, sched_partition=self.sched_partition, sched_reservation=self.sched_reservation, sched_nodelist=self.sched_nodelist, sched_exclude_nodelist=self.sched_exclude_nodelist, sched_options=self.sched_options) if self._running_tasks_counts[partname] >= partition.max_jobs: # Make sure that we still exceeded the job limit getlogger().debug('reached job limit (%s) for partition %s' % (partition.max_jobs, partname)) self._poll_tasks() if self._running_tasks_counts[partname] < partition.max_jobs: # Task was put in _ready_tasks during setup self._ready_tasks[partname].pop() self._reschedule(task) else: self.printer.status('HOLD', task.check.info(), just='right') except TaskExit: if not task.failed: self._reschedule(task) return except ABORT_REASONS as e: if not task.failed: # Abort was caused due to failure elsewhere, abort current # task as well task.abort(e) self._failall(e) raise