def _stop_trial(self, trial, error=False, error_msg=None, stop_logger=True): """Stops this trial. Stops this trial, releasing all allocating resources. If stopping the trial fails, the run will be marked as terminated in error, but no exception will be thrown. Args: error (bool): Whether to mark this trial as terminated in error. error_msg (str): Optional error message. stop_logger (bool): Whether to shut down the trial logger. """ self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) trial.set_location(Location()) try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: if (not error and self._reuse_actors and self._cached_actor is None): logger.debug("Reusing actor for %s", trial.runner) self._cached_actor = trial.runner else: logger.debug("Trial %s: Destroying actor.", trial) with self._change_working_directory(trial): self._trial_cleanup.add(trial, actor=trial.runner) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None) if stop_logger: trial.close_logger()
def _stop_trial(self, trial: Trial, error=False, error_msg=None): """Stops this trial. Stops this trial, releasing all allocating resources. If stopping the trial fails, the run will be marked as terminated in error, but no exception will be thrown. Args: error (bool): Whether to mark this trial as terminated in error. error_msg (str): Optional error message. """ self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) self._trial_just_finished = True trial.set_location(Location()) try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: if (not error and self._reuse_actors and (len(self._cached_actor_pg) < (self._cached_actor_pg.maxlen or float("inf")))): logger.debug("Reusing actor for %s", trial.runner) # Move PG into cache (disassociate from trial) pg = self._pg_manager.cache_trial_pg(trial) if pg: # True if a placement group was replaced self._cached_actor_pg.append((trial.runner, pg)) should_destroy_actor = False else: # False if no placement group was replaced. This should # only be the case if there are no more trials with # this placement group factory to run logger.debug( "Could not cache of trial {trial} actor for " "reuse, as there are no pending trials " "requiring its resources.") should_destroy_actor = True else: should_destroy_actor = True if should_destroy_actor: logger.debug("Trial %s: Destroying actor.", trial) with self._change_working_directory(trial): future = trial.runner.stop.remote() pg = self._pg_manager.remove_from_in_use(trial) self._futures[future] = (ExecutorEventType.STOP_RESULT, pg) if self._trial_cleanup: # force trial cleanup within a deadline self._trial_cleanup.add(future) if trial in self._staged_trials: self._staged_trials.remove(trial) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None)
def _stop_trial(self, trial, error=False, error_msg=None): """Stops this trial. Stops this trial, releasing all allocating resources. If stopping the trial fails, the run will be marked as terminated in error, but no exception will be thrown. If the trial should be paused (``pause=True``), we do not remove its placement group (or a surrogate placement group). Args: error (bool): Whether to mark this trial as terminated in error. error_msg (str): Optional error message. """ self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) self._trial_just_finished = True trial.set_location(Location()) try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: if (not error and self._reuse_actors and self._cached_actor_pg[0] is None): logger.debug("Reusing actor for %s", trial.runner) # Move PG into cache (disassociate from trial) pg = self._pg_manager.cache_trial_pg(trial) if pg or not trial.uses_placement_groups: # True if a placement group was replaced self._cached_actor_pg = (trial.runner, pg) should_destroy_actor = False else: # False if no placement group was replaced. This should # only be the case if there are no more trials with # this placement group factory to run logger.debug( "Could not cache of trial {trial} actor for " "reuse, as there are no pending trials " "requiring its resources.") should_destroy_actor = True else: should_destroy_actor = True if should_destroy_actor: logger.debug("Trial %s: Destroying actor.", trial) # Try to return the placement group for other trials to use self._pg_manager.return_pg(trial) with self._change_working_directory(trial): self._trial_cleanup.add(trial, actor=trial.runner) if trial in self._staged_trials: self._staged_trials.remove(trial) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None)
def _setup_remote_runner(self, trial, reuse_allowed): trial.init_logger() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) remote_logdir = trial.logdir if (self._reuse_actors and reuse_allowed and self._cached_actor is not None): logger.debug("Trial %s: Reusing cached runner %s", trial, self._cached_actor) existing_runner = self._cached_actor self._cached_actor = None trial.set_runner(existing_runner) if not self.reset_trial(trial, trial.config, trial.experiment_tag): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner if self._cached_actor: logger.debug("Cannot reuse cached runner {} for new trial".format( self._cached_actor)) with self._change_working_directory(trial): self._cached_actor.stop.remote() self._cached_actor.__ray_terminate__.remote() self._cached_actor = None cls = ray.remote( num_cpus=trial.resources.cpu, num_gpus=trial.resources.gpu, memory=trial.resources.memory, object_store_memory=trial.resources.object_store_memory, resources=trial.resources.custom_resources)( trial.get_trainable_cls()) def logger_creator(config): # Set the working dir in the remote process, for user file writes os.makedirs(remote_logdir, exist_ok=True) if not ray.worker._mode() == ray.worker.LOCAL_MODE: os.chdir(remote_logdir) return NoopLogger(config, remote_logdir) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) kwargs = { "config": trial_config, "logger_creator": logger_creator, } if issubclass(trial.get_trainable_cls(), DurableTrainable): kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir with self._change_working_directory(trial): return cls.remote(**kwargs)
def _get_trial_location(trial: Trial, result: dict) -> Location: # we get the location from the result, as the one in trial will be # reset when trial terminates node_ip, pid = result.get(NODE_IP, None), result.get(PID, None) if node_ip and pid: location = Location(node_ip, pid) else: # fallback to trial location if there hasn't been a report yet location = trial.location return location
def _setup_remote_runner(self, trial, reuse_allowed): trial.init_logger() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) logger_creator = partial(noop_logger_creator, logdir=trial.logdir) if (self._reuse_actors and reuse_allowed and self._cached_actor is not None): logger.debug("Trial %s: Reusing cached runner %s", trial, self._cached_actor) existing_runner = self._cached_actor self._cached_actor = None trial.set_runner(existing_runner) if not self.reset_trial(trial, trial.config, trial.experiment_tag, logger_creator): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner if self._cached_actor: logger.debug("Cannot reuse cached runner {} for new trial".format( self._cached_actor)) with self._change_working_directory(trial): self._trial_cleanup.add(trial, actor=self._cached_actor) self._cached_actor = None _actor_cls = _class_cache.get(trial.get_trainable_cls()) full_actor_class = _actor_cls.options( num_cpus=trial.resources.cpu, num_gpus=trial.resources.gpu, memory=trial.resources.memory or None, object_store_memory=trial.resources.object_store_memory or None, resources=trial.resources.custom_resources) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) stdout_file, stderr_file = trial.log_to_file trial_config[STDOUT_FILE] = stdout_file trial_config[STDERR_FILE] = stderr_file kwargs = { "config": trial_config, "logger_creator": logger_creator, } if issubclass(trial.get_trainable_cls(), DurableTrainable): kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir with self._change_working_directory(trial): return full_actor_class.remote(**kwargs)
def _setup_remote_runner(self, trial, reuse_allowed): trial.init_logger() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) remote_logdir = trial.logdir if (self._reuse_actors and reuse_allowed and self._cached_actor is not None): logger.debug("Reusing cached runner {} for {}".format( self._cached_actor, trial.trial_id)) existing_runner = self._cached_actor self._cached_actor = None trial.runner = existing_runner if not self.reset_trial(trial, trial.config, trial.experiment_tag): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner if self._cached_actor: logger.debug("Cannot reuse cached runner {} for new trial".format( self._cached_actor)) self._cached_actor.stop.remote() self._cached_actor.__ray_terminate__.remote() self._cached_actor = None cls = ray.remote( num_cpus=trial.resources.cpu, num_gpus=trial.resources.gpu, memory=trial.resources.memory, object_store_memory=trial.resources.object_store_memory, resources=trial.resources.custom_resources)( trial.get_trainable_cls()) def logger_creator(config): # Set the working dir in the remote process, for user file writes if not os.path.exists(remote_logdir): os.makedirs(remote_logdir) if not ray.worker._mode() == ray.worker.LOCAL_MODE: os.chdir(remote_logdir) return NoopLogger(config, remote_logdir) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.info("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. return cls.remote(config=trial.config, logger_creator=logger_creator)
def _ensure_stop( self, trial, error=False, error_msg="", stop_logger=True, release_resources=True, update_status=False, ): """Stops the trial and its logger Handles any error """ logger.debug(f"_ensure_stop: trial.resources={trial.resources}") if stop_logger: trial.close_logger() prior_status = trial.status trial.set_location(Location()) if update_status: self.set_status(trial, Trial.ERROR if error else Trial.TERMINATED) # remove from running in_flight = [ j for _, j in self.jobs_running.items() if j.trial == trial ] for j in in_flight: self.jobs_running.pop(j.in_flight_future) if in_flight: if prior_status not in [Trial.RUNNING, Trial.ERROR]: assert False, "trial status invalid" # release resources if release_resources: self._return_resources(trial) # remove from trial group # del self.trial_groups[trial.trial_id] try: trial.write_error_log(error_msg) if hasattr(trial, "runner") and trial.runner: logger.debug("Trial %s: Destroying actor.", trial) with _change_working_directory(trial): self._trial_cleanup.add(trial, actor=trial.runner) except Exception: logger.exception("Trial %s: Error stopping runner.", trial) self.set_status(trial, Trial.ERROR) finally: trial.set_runner(None)
def _setup_remote_runner(self, trial: Trial, res: Resources, reuse_allowed: bool) -> Any: trial.init_logger() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) remote_logdir = trial.logdir cls = ray.remote( num_cpus=res.cpu, num_gpus=0 if self._fake_gpus else res.gpu, memory=res.memory, object_store_memory=res.object_store_memory, resources=res.custom_resources, )(trial.get_trainable_cls()) def logger_creator(config): # Set the working dir in the remote process, for user file writes os.makedirs(remote_logdir, exist_ok=True) if not ray.worker._mode() == ray.worker.LOCAL_MODE: os.chdir(remote_logdir) return NoopLogger(config, remote_logdir) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) kwargs = { "config": trial_config, "logger_creator": logger_creator, } if issubclass(trial.get_trainable_cls(), DurableTrainable): kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir with _change_working_directory(trial): return cls.remote(**kwargs)
def _setup_remote_runner(self, trial): trial.init_logdir() # We checkpoint metadata here to try mitigating logdir duplication self._trials_to_cache.add(trial) logger_creator = partial(noop_logger_creator, logdir=trial.logdir) if len(self._cached_actor_pg) > 0: assert self._reuse_actors existing_runner, pg = self._cached_actor_pg.popleft() logger.debug(f"Trial {trial}: Reusing cached runner " f"{existing_runner}") trial.set_runner(existing_runner) if pg: self._pg_manager.assign_cached_pg(pg, trial) if not self.reset_trial(trial, trial.config, trial.experiment_tag, logger_creator): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner trainable_cls = trial.get_trainable_cls() if not trainable_cls: raise AbortTrialExecution( f"Invalid trainable: {trial.trainable_name}. If you passed " f"a string, make sure the trainable was registered before.") _actor_cls = _class_cache.get(trainable_cls) if not self._pg_manager.has_ready(trial): return None full_actor_class = self._pg_manager.get_full_actor_cls( trial, _actor_cls) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) stdout_file, stderr_file = trial.log_to_file trial_config[STDOUT_FILE] = stdout_file trial_config[STDERR_FILE] = stderr_file kwargs = { "config": trial_config, "logger_creator": logger_creator, } if trial.uses_cloud_checkpointing: # We keep these kwargs separate for backwards compatibility # with trainables that don't provide these keyword arguments kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir kwargs["sync_function_tpl"] = trial.sync_function_tpl # Throw a meaningful error if trainable does not use the # new API sig = inspect.signature(trial.get_trainable_cls()) try: sig.bind_partial(**kwargs) except Exception as e: raise RuntimeError( "Your trainable class does not accept a " "`remote_checkpoint_dir` or `sync_function_tpl` argument " "in its constructor, but you've passed a " "`upload_dir` to your SyncConfig. Without accepting " "these parameters and passing them to the base trainable " "constructor in the init call, cloud checkpointing is " "effectively disabled. To resolve this issue, add the " "parameters to your trainable class constructor or " "disable cloud checkpointing by setting `upload_dir=None`." ) from e with self._change_working_directory(trial): return full_actor_class.remote(**kwargs)
def _setup_remote_runner(self, trial): trial.init_logdir() # We checkpoint metadata here to try mitigating logdir duplication self.try_checkpoint_metadata(trial) logger_creator = partial(noop_logger_creator, logdir=trial.logdir) if self._reuse_actors and self._cached_actor_pg[0] is not None: logger.debug(f"Trial {trial}: Reusing cached runner " f"{self._cached_actor_pg[0]}") existing_runner, pg = self._cached_actor_pg self._cached_actor_pg = (None, None) trial.set_runner(existing_runner) if pg and trial.uses_placement_groups: self._pg_manager.assign_cached_pg(pg, trial) if not self.reset_trial(trial, trial.config, trial.experiment_tag, logger_creator): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner if self._cached_actor_pg[0]: logger.debug("Cannot reuse cached runner {} for new trial".format( self._cached_actor_pg[0])) existing_runner, pg = self._cached_actor_pg if pg: self._pg_manager.return_or_clean_cached_pg(pg) with self._change_working_directory(trial): self._trial_cleanup.add(trial, actor=existing_runner) self._cached_actor_pg = (None, None) trainable_cls = trial.get_trainable_cls() if not trainable_cls: raise AbortTrialExecution( f"Invalid trainable: {trial.trainable_name}. If you passed " f"a string, make sure the trainable was registered before.") _actor_cls = _class_cache.get(trainable_cls) if trial.uses_placement_groups: if not self._pg_manager.has_ready(trial, update=True): if trial not in self._staged_trials: if self._pg_manager.stage_trial_pg(trial): self._staged_trials.add(trial) self._just_staged_trials.add(trial) just_staged = trial in self._just_staged_trials # This part of the code is mostly here for testing # purposes. If self._wait_for_pg is set, we will wait here # for that many seconds until the placement group is ready. # This ensures that the trial can be started right away and # not just in the next step() of the trial runner. # We only do this if we have reason to believe that resources # will be ready, soon, i.e. when a) we just staged the PG, # b) another trial just exited, freeing resources, or c) # when there are no currently running trials. if self._wait_for_pg is not None and ( just_staged or self._trial_just_finished_before or not self.get_running_trials()): logger.debug( f"Waiting up to {self._wait_for_pg} seconds for " f"placement group of trial {trial} to become ready.") wait_end = time.monotonic() + self._wait_for_pg while time.monotonic() < wait_end: self._pg_manager.update_status() if self._pg_manager.has_ready(trial): break time.sleep(0.1) else: return None if not self._pg_manager.has_ready(trial): # PG may have become ready during waiting period return None full_actor_class = self._pg_manager.get_full_actor_cls( trial, _actor_cls) else: full_actor_class = _actor_cls.options( num_cpus=trial.resources.cpu, num_gpus=trial.resources.gpu, memory=trial.resources.memory or None, object_store_memory=trial.resources.object_store_memory or None, resources=trial.resources.custom_resources) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) stdout_file, stderr_file = trial.log_to_file trial_config[STDOUT_FILE] = stdout_file trial_config[STDERR_FILE] = stderr_file kwargs = { "config": trial_config, "logger_creator": logger_creator, } if issubclass(trial.get_trainable_cls(), DurableTrainable): kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir with self._change_working_directory(trial): return full_actor_class.remote(**kwargs)
def _setup_remote_runner(self, trial): trial.init_logdir() # We checkpoint metadata here to try mitigating logdir duplication self._trials_to_cache.add(trial) logger_creator = partial(noop_logger_creator, logdir=trial.logdir) if len(self._cached_actor_pg) > 0: assert self._reuse_actors existing_runner, pg = self._cached_actor_pg.popleft() logger.debug(f"Trial {trial}: Reusing cached runner " f"{existing_runner}") trial.set_runner(existing_runner) if pg: self._pg_manager.assign_cached_pg(pg, trial) if not self.reset_trial(trial, trial.config, trial.experiment_tag, logger_creator): raise AbortTrialExecution( "Trainable runner reuse requires reset_config() to be " "implemented and return True.") return existing_runner trainable_cls = trial.get_trainable_cls() if not trainable_cls: raise AbortTrialExecution( f"Invalid trainable: {trial.trainable_name}. If you passed " f"a string, make sure the trainable was registered before.") _actor_cls = _class_cache.get(trainable_cls) if not self._pg_manager.has_ready(trial, update=True): if trial not in self._staged_trials: if self._pg_manager.stage_trial_pg(trial): self._staged_trials.add(trial) self._just_staged_trials.add(trial) just_staged = trial in self._just_staged_trials # This part of the code is mostly here for testing # purposes. If self._wait_for_pg is set, we will wait here # for that many seconds until the placement group is ready. # This ensures that the trial can be started right away and # not just in the next step() of the trial runner. # We only do this if we have reason to believe that resources # will be ready, soon, i.e. when a) we just staged the PG, # b) another trial just exited, freeing resources, or c) # when there are no currently running trials. if self._wait_for_pg is not None and ( just_staged or self._trial_just_finished_before or not self.get_running_trials()): logger.debug( f"Waiting up to {self._wait_for_pg} seconds for " f"placement group of trial {trial} to become ready.") wait_end = time.monotonic() + self._wait_for_pg while time.monotonic() < wait_end: self._pg_manager.update_status() if self._pg_manager.has_ready(trial): break time.sleep(0.1) else: return None if not self._pg_manager.has_ready(trial): # PG may have become ready during waiting period return None full_actor_class = self._pg_manager.get_full_actor_cls( trial, _actor_cls) # Clear the Trial's location (to be updated later on result) # since we don't know where the remote runner is placed. trial.set_location(Location()) logger.debug("Trial %s: Setting up new remote runner.", trial) # Logging for trials is handled centrally by TrialRunner, so # configure the remote runner to use a noop-logger. trial_config = copy.deepcopy(trial.config) trial_config[TRIAL_INFO] = TrialInfo(trial) stdout_file, stderr_file = trial.log_to_file trial_config[STDOUT_FILE] = stdout_file trial_config[STDERR_FILE] = stderr_file kwargs = { "config": trial_config, "logger_creator": logger_creator, } if trial.uses_cloud_checkpointing: # We keep these kwargs separate for backwards compatibility # with trainables that don't provide these keyword arguments kwargs["remote_checkpoint_dir"] = trial.remote_checkpoint_dir kwargs["sync_function_tpl"] = trial.sync_function_tpl # Throw a meaningful error if trainable does not use the # new API sig = inspect.signature(trial.get_trainable_cls()) try: sig.bind_partial(**kwargs) except Exception as e: raise RuntimeError( "Your trainable class does not accept a " "`remote_checkpoint_dir` or `sync_function_tpl` argument " "in its constructor, but you've passed a " "`upload_dir` to your SyncConfig. Without accepting " "these parameters and passing them to the base trainable " "constructor in the init call, cloud checkpointing is " "effectively disabled. To resolve this issue, add the " "parameters to your trainable class constructor or " "disable cloud checkpointing by setting `upload_dir=None`." ) from e with self._change_working_directory(trial): return full_actor_class.remote(**kwargs)