def _stop_trial(self, trial: Trial, error=False, error_msg=None):
"""Stops this trial.
Stops this trial, releasing all allocating resources. If stopping the
trial fails, the run will be marked as terminated in error, but no
exception will be thrown.
Args:
error (bool): Whether to mark this trial as terminated in error.
error_msg (str): Optional error message.
"""
self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED)
self._trial_just_finished = True
trial.set_location(Location())
try:
trial.write_error_log(error_msg)
if hasattr(trial, "runner") and trial.runner:
if (not error and self._reuse_actors
and (len(self._cached_actor_pg) <
(self._cached_actor_pg.maxlen or float("inf")))):
logger.debug("Reusing actor for %s", trial.runner)
# Move PG into cache (disassociate from trial)
pg = self._pg_manager.cache_trial_pg(trial)
if pg:
# True if a placement group was replaced
self._cached_actor_pg.append((trial.runner, pg))
should_destroy_actor = False
else:
# False if no placement group was replaced. This should
# only be the case if there are no more trials with
# this placement group factory to run
logger.debug(
"Could not cache of trial {trial} actor for "
"reuse, as there are no pending trials "
"requiring its resources.")
should_destroy_actor = True
else:
should_destroy_actor = True
if should_destroy_actor:
logger.debug("Trial %s: Destroying actor.", trial)
with self._change_working_directory(trial):
future = trial.runner.stop.remote()
pg = self._pg_manager.remove_from_in_use(trial)
self._futures[future] = (ExecutorEventType.STOP_RESULT, pg)
if self._trial_cleanup: # force trial cleanup within a deadline
self._trial_cleanup.add(future)
if trial in self._staged_trials:
self._staged_trials.remove(trial)
except Exception:
logger.exception("Trial %s: Error stopping runner.", trial)
self.set_status(trial, Trial.ERROR)
finally:
trial.set_runner(None)
def _setup_remote_runner(self, trial: Trial, res: Resources,
reuse_allowed: bool) -> Any:
trial.init_logger()
# We checkpoint metadata here to try mitigating logdir duplication
self.try_checkpoint_metadata(trial)
remote_logdir = trial.logdir
cls = ray.remote(
num_cpus=res.cpu,
num_gpus=0 if self._fake_gpus else res.gpu,
memory=res.memory,
object_store_memory=res.object_store_memory,
resources=res.custom_resources,
)(trial.get_trainable_cls())
def logger_creator(config):
# Set the working dir in the remote process, for user file writes
os.makedirs(remote_logdir, exist_ok=True)
if not ray.worker._mode() == ray.worker.LOCAL_MODE:
os.chdir(remote_logdir)
return NoopLogger(config, remote_logdir)
# Clear the Trial's location (to be updated later on result)
# since we don't know where the remote runner is placed.
trial.set_location(Location())
logger.debug("Trial %s: Setting up new remote runner.", trial)
# Logging for trials is handled centrally by TrialRunner, so
# configure the remote runner to use a noop-logger.
trial_config = copy.deepcopy(trial.config)
trial_config[TRIAL_INFO] = TrialInfo(trial)
kwargs = {
"config": trial_config,
"logger_creator": logger_creator,
}
if issubclass(trial.get_trainable_cls(), DurableTrainable):
kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir
with _change_working_directory(trial):
return cls.remote(**kwargs)
def _stop_trial(self,
trial: Trial,
error=False,
error_msg=None,
destroy_pg_if_cannot_replace=True):
"""Stops this trial.
Stops this trial, releasing all allocating resources. If stopping the
trial fails, the run will be marked as terminated in error, but no
exception will be thrown.
If the placement group will be used right away
(destroy_pg_if_cannot_replace=False), we do not remove its placement
group (or a surrogate placement group).
Args:
error (bool): Whether to mark this trial as terminated in error.
error_msg (str): Optional error message.
"""
self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED)
self._trial_just_finished = True
trial.set_location(Location())
try:
trial.write_error_log(error_msg)
if hasattr(trial, "runner") and trial.runner:
if (not error and self._reuse_actors
and (len(self._cached_actor_pg) <
(self._cached_actor_pg.maxlen or float("inf")))):
logger.debug("Reusing actor for %s", trial.runner)
# Move PG into cache (disassociate from trial)
pg = self._pg_manager.cache_trial_pg(trial)
if pg or not trial.uses_placement_groups:
# True if a placement group was replaced
self._cached_actor_pg.append((trial.runner, pg))
should_destroy_actor = False
else:
# False if no placement group was replaced. This should
# only be the case if there are no more trials with
# this placement group factory to run
logger.debug(
"Could not cache of trial {trial} actor for "
"reuse, as there are no pending trials "
"requiring its resources.")
should_destroy_actor = True
else:
should_destroy_actor = True
if should_destroy_actor:
logger.debug("Trial %s: Destroying actor.", trial)
# Try to return the placement group for other trials to use
self._pg_manager.return_pg(trial,
destroy_pg_if_cannot_replace)
with self._change_working_directory(trial):
self._trial_cleanup.add(trial, actor=trial.runner)
if trial in self._staged_trials:
self._staged_trials.remove(trial)
except Exception:
logger.exception("Trial %s: Error stopping runner.", trial)
self.set_status(trial, Trial.ERROR)
finally:
trial.set_runner(None)
