def cleanup_full(self, trial_runner: "trial_runner.TrialRunner"):
"""Cleans up bracket after bracket is completely finished.
Lets the last trial continue to run until termination condition
kicks in."""
for trial in self.current_trials():
if (trial.status == Trial.PAUSED):
trial_runner.stop_trial(trial)
def _process_bracket(
self, trial_runner: "trial_runner.TrialRunner", bracket: "_Bracket"
) -> str:
"""This is called whenever a trial makes progress.
When all live trials in the bracket have no more iterations left,
Trials will be successively halved. If bracket is done, all
non-running trials will be stopped and cleaned up,
and during each halving phase, bad trials will be stopped while good
trials will return to "PENDING"."""
action = TrialScheduler.PAUSE
if bracket.cur_iter_done():
if bracket.finished():
bracket.cleanup_full(trial_runner)
return TrialScheduler.STOP
good, bad = bracket.successive_halving(self._metric, self._metric_op)
# kill bad trials
self._num_stopped += len(bad)
for t in bad:
if t.status == Trial.PAUSED:
trial_runner.stop_trial(t)
elif t.status == Trial.RUNNING:
bracket.cleanup_trial(t)
action = TrialScheduler.STOP
else:
raise TuneError(
f"Trial with unexpected bad status " f"encountered: {t.status}"
)
# ready the good trials - if trial is too far ahead, don't continue
for t in good:
if t.status not in [Trial.PAUSED, Trial.RUNNING]:
raise TuneError(
f"Trial with unexpected good status " f"encountered: {t.status}"
)
if bracket.continue_trial(t):
if t.status == Trial.PAUSED:
t.status = Trial.PENDING
elif t.status == Trial.RUNNING:
action = TrialScheduler.CONTINUE
return action
