def _stop_trial(self, trial: Trial, error=False, error_msg=None): """Stops this trial. Stops this trial, releasing all allocating resources. If stopping the trial fails, the run will be marked as terminated in error, but no exception will be thrown. Args: error (bool): Whether to mark this trial as terminated in error. error_msg (str): Optional error message. """ self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) self._trial_just_finished = True trial.set_location(Location()) try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: if (not error and self._reuse_actors and (len(self._cached_actor_pg) < (self._cached_actor_pg.maxlen or float("inf")))): logger.debug("Reusing actor for %s", trial.runner) # Move PG into cache (disassociate from trial) pg = self._pg_manager.cache_trial_pg(trial) if pg: # True if a placement group was replaced self._cached_actor_pg.append((trial.runner, pg)) should_destroy_actor = False else: # False if no placement group was replaced. This should # only be the case if there are no more trials with # this placement group factory to run logger.debug( "Could not cache of trial {trial} actor for " "reuse, as there are no pending trials " "requiring its resources.") should_destroy_actor = True else: should_destroy_actor = True if should_destroy_actor: logger.debug("Trial %s: Destroying actor.", trial) with self._change_working_directory(trial): future = trial.runner.stop.remote() pg = self._pg_manager.remove_from_in_use(trial) self._futures[future] = (ExecutorEventType.STOP_RESULT, pg) if self._trial_cleanup: # force trial cleanup within a deadline self._trial_cleanup.add(future) if trial in self._staged_trials: self._staged_trials.remove(trial) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None)
def _setup_remote_runner(self, trial: Trial, res: Resources, reuse_allowed: bool) -> Any: trial.init_logger() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) remote_logdir = trial.logdir cls = ray.remote( num_cpus=res.cpu, num_gpus=0 if self._fake_gpus else res.gpu, memory=res.memory, object_store_memory=res.object_store_memory, resources=res.custom_resources, )(trial.get_trainable_cls()) def logger_creator(config): # Set the working dir in the remote process, for user file writes os.makedirs(remote_logdir, exist_ok=True) if not ray.worker._mode() == ray.worker.LOCAL_MODE: os.chdir(remote_logdir) return NoopLogger(config, remote_logdir) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) kwargs = { "config": trial_config, "logger_creator": logger_creator, } if issubclass(trial.get_trainable_cls(), DurableTrainable): kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir with _change_working_directory(trial): return cls.remote(**kwargs)
def _stop_trial(self, trial: Trial, error=False, error_msg=None, destroy_pg_if_cannot_replace=True): """Stops this trial. Stops this trial, releasing all allocating resources. If stopping the trial fails, the run will be marked as terminated in error, but no exception will be thrown. If the placement group will be used right away (destroy_pg_if_cannot_replace=False), we do not remove its placement group (or a surrogate placement group). Args: error (bool): Whether to mark this trial as terminated in error. error_msg (str): Optional error message. """ self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) self._trial_just_finished = True trial.set_location(Location()) try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: if (not error and self._reuse_actors and (len(self._cached_actor_pg) < (self._cached_actor_pg.maxlen or float("inf")))): logger.debug("Reusing actor for %s", trial.runner) # Move PG into cache (disassociate from trial) pg = self._pg_manager.cache_trial_pg(trial) if pg or not trial.uses_placement_groups: # True if a placement group was replaced self._cached_actor_pg.append((trial.runner, pg)) should_destroy_actor = False else: # False if no placement group was replaced. This should # only be the case if there are no more trials with # this placement group factory to run logger.debug( "Could not cache of trial {trial} actor for " "reuse, as there are no pending trials " "requiring its resources.") should_destroy_actor = True else: should_destroy_actor = True if should_destroy_actor: logger.debug("Trial %s: Destroying actor.", trial) # Try to return the placement group for other trials to use self._pg_manager.return_pg(trial, destroy_pg_if_cannot_replace) with self._change_working_directory(trial): self._trial_cleanup.add(trial, actor=trial.runner) if trial in self._staged_trials: self._staged_trials.remove(trial) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None)