def cleanup_full(self, trial_runner: "trial_runner.TrialRunner"): """Cleans up bracket after bracket is completely finished. Lets the last trial continue to run until termination condition kicks in.""" for trial in self.current_trials(): if (trial.status == Trial.PAUSED): trial_runner.stop_trial(trial)
def _process_bracket( self, trial_runner: "trial_runner.TrialRunner", bracket: "_Bracket" ) -> str: """This is called whenever a trial makes progress. When all live trials in the bracket have no more iterations left, Trials will be successively halved. If bracket is done, all non-running trials will be stopped and cleaned up, and during each halving phase, bad trials will be stopped while good trials will return to "PENDING".""" action = TrialScheduler.PAUSE if bracket.cur_iter_done(): if bracket.finished(): bracket.cleanup_full(trial_runner) return TrialScheduler.STOP good, bad = bracket.successive_halving(self._metric, self._metric_op) # kill bad trials self._num_stopped += len(bad) for t in bad: if t.status == Trial.PAUSED: trial_runner.stop_trial(t) elif t.status == Trial.RUNNING: bracket.cleanup_trial(t) action = TrialScheduler.STOP else: raise TuneError( f"Trial with unexpected bad status " f"encountered: {t.status}" ) # ready the good trials - if trial is too far ahead, don't continue for t in good: if t.status not in [Trial.PAUSED, Trial.RUNNING]: raise TuneError( f"Trial with unexpected good status " f"encountered: {t.status}" ) if bracket.continue_trial(t): if t.status == Trial.PAUSED: t.status = Trial.PENDING elif t.status == Trial.RUNNING: action = TrialScheduler.CONTINUE return action