def wait_or_retry(self, max_retries: int = 3, backoff_s: int = 5): assert max_retries > 0 for _ in range(max_retries - 1): try: self.wait() except TuneError as e: logger.error( f"Caught sync error: {e}. " f"Retrying after sleeping for {backoff_s} seconds...") time.sleep(backoff_s) self.cmd_process = self._start_process(self._last_cmd) continue return self.cmd_process = None raise TuneError(f"Failed sync even after {max_retries} retries.")
def register(self, category, key, value): """Registers the value with the global registry. Raises: PicklingError if unable to pickle to provided file. """ if category not in KNOWN_CATEGORIES: from ray.tune import TuneError raise TuneError( "Unknown category {} not among {}".format(category, KNOWN_CATEGORIES) ) self._to_flush[(category, key)] = pickle.dumps_debug(value) if _internal_kv_initialized(): self.flush_values()
def register_if_needed(cls, run_object): """Registers Trainable or Function at runtime. Assumes already registered if run_object is a string. Also, does not inspect interface of given run_object. Arguments: run_object (str|function|class): Trainable to run. If string, assumes it is an ID and does not modify it. Otherwise, returns a string corresponding to the run_object name. Returns: A string representing the trainable identifier. """ if isinstance(run_object, str): return run_object elif isinstance(run_object, Domain): logger.warning("Not registering trainable. Resolving as variant.") return run_object elif isinstance(run_object, type) or callable(run_object): name = "DEFAULT" if hasattr(run_object, "__name__"): name = run_object.__name__ else: logger.warning( "No name detected on trainable. Using {}.".format(name)) try: register_trainable(name, run_object) except (TypeError, PicklingError) as e: msg = ( f"{str(e)}. The trainable ({str(run_object)}) could not " "be serialized, which is needed for parallel execution. " "To diagnose the issue, try the following:\n\n" "\t- Run `tune.utils.diagnose_serialization(trainable)` " "to check if non-serializable variables are captured " "in scope.\n" "\t- Try reproducing the issue by calling " "`pickle.dumps(trainable)`.\n" "\t- If the error is typing-related, try removing " "the type annotations and try again.\n\n" "If you have any suggestions on how to improve " "this error message, please reach out to the " "Ray developers on github.com/ray-project/ray/issues/") raise type(e)(msg) from None return name else: raise TuneError("Improper 'run' - not string nor trainable.")
def __init__(self, experiment_checkpoint_path, trials=None): experiment_checkpoint_path = os.path.expanduser( experiment_checkpoint_path) if not os.path.isfile(experiment_checkpoint_path): raise ValueError( "{} is not a valid file.".format(experiment_checkpoint_path)) with open(experiment_checkpoint_path) as f: _experiment_state = json.load(f) self._experiment_state = _experiment_state if "checkpoints" not in _experiment_state: raise TuneError("Experiment state invalid; no checkpoints found.") self._checkpoints = _experiment_state["checkpoints"] self.trials = trials super(ExperimentAnalysis, self).__init__(os.path.dirname(experiment_checkpoint_path))
def register_if_needed(cls, run_object): """Registers Trainable or Function at runtime. Assumes already registered if run_object is a string. Also, does not inspect interface of given run_object. Arguments: run_object (str|function|class): Trainable to run. If string, assumes it is an ID and does not modify it. Otherwise, returns a string corresponding to the run_object name. Returns: A string representing the trainable identifier. """ if isinstance(run_object, str): return run_object elif isinstance(run_object, Domain): logger.warning("Not registering trainable. Resolving as variant.") return run_object elif isinstance(run_object, type) or callable(run_object): name = "DEFAULT" if hasattr(run_object, "_name"): name = run_object._name elif hasattr(run_object, "__name__"): fn_name = run_object.__name__ if fn_name == "<lambda>": name = "lambda" elif fn_name.startswith("<"): name = "DEFAULT" else: name = fn_name else: logger.warning( "No name detected on trainable. Using {}.".format(name)) try: register_trainable(name, run_object) except (TypeError, PicklingError) as e: extra_msg = ("Other options: " "\n-Try reproducing the issue by calling " "`pickle.dumps(trainable)`. " "\n-If the error is typing-related, try removing " "the type annotations and try again.") raise type(e)(str(e) + " " + extra_msg) from None return name else: raise TuneError("Improper 'run' - not string nor trainable.")
def __init__(self, experiment_checkpoint_path, trials=None): """Initializer. Args: experiment_path (str): Path to where experiment is located. trials (list|None): List of trials that can be accessed via `analysis.trials`. """ with open(experiment_checkpoint_path) as f: _experiment_state = json.load(f) if "checkpoints" not in _experiment_state: raise TuneError("Experiment state invalid; no checkpoints found.") self._checkpoints = _experiment_state["checkpoints"] self.trials = trials super(ExperimentAnalysis, self).__init__( os.path.dirname(experiment_checkpoint_path))
def add_configurations(self, experiments): """Registers experiment specifications. Arguments: experiments (Experiment | list | dict): Experiments to run. """ logger.debug("added configurations") experiment_list = convert_to_experiment_list(experiments) assert len(experiment_list) == 1, ( "SearchAlgorithms can only support 1 experiment at a time.") self._experiment = experiment_list[0] experiment_spec = self._experiment.spec self._total_samples = experiment_spec.get("num_samples", 1) _warn_on_repeater(self.searcher, self._total_samples) if "run" not in experiment_spec: raise TuneError("Must specify `run` in {}".format(experiment_spec))
def _load_checkpoints_from_latest(self, latest_checkpoint: List[str]) -> None: # Collect all checkpoints and their directory paths. for path in latest_checkpoint: with open(path) as f: experiment_state = json.load(f, cls=TuneFunctionDecoder) self._experiment_states.append(experiment_state) if "checkpoints" not in experiment_state: raise TuneError( "Experiment state invalid; no checkpoints found.") self._checkpoints_and_paths += [ (_decode_checkpoint_from_experiment_state(cp), Path(path).parent) for cp in experiment_state["checkpoints"] ] self._checkpoints_and_paths = sorted( self._checkpoints_and_paths, key=lambda tup: tup[0]["trial_id"])
def __next__(self): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Returns: Trial object """ if "run" not in self.unresolved_spec: raise TuneError("Must specify `run` in {}".format( self.unresolved_spec)) if self.variants and self.variants.has_next(): # This block will be skipped upon instantiation. # `variants` will be set later after the first loop. resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) if self.points_to_evaluate: config = self.points_to_evaluate.pop(0) self.num_samples_left -= 1 self.variants = _VariantIterator(get_preset_variants( self.unresolved_spec, config, constant_grid_search=self.constant_grid_search, random_state=self.random_state), lazy_eval=self.lazy_eval) resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) elif self.num_samples_left > 0: self.variants = _VariantIterator(generate_variants( self.unresolved_spec, constant_grid_search=self.constant_grid_search, random_state=self.random_state), lazy_eval=self.lazy_eval) self.num_samples_left -= 1 resolved_vars, spec = next(self.variants) return self.create_trial(resolved_vars, spec) else: raise StopIteration
def __init__(self, time_attr="time_total_s", reward_attr=None, metric="episode_reward_mean", mode="max", perturbation_interval=60.0, hyperparam_mutations={}, resample_probability=0.25, custom_explore_fn=None, log_config=True): if not hyperparam_mutations and not custom_explore_fn: raise TuneError( "You must specify at least one of `hyperparam_mutations` or " "`custom_explore_fn` to use PBT.") assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" if reward_attr is not None: mode = "max" metric = reward_attr logger.warning( "`reward_attr` is deprecated and will be removed in a future " "version of Tune. " "Setting `metric={}` and `mode=max`.".format(reward_attr)) FIFOScheduler.__init__(self) self._metric = metric if mode == "max": self._metric_op = 1. elif mode == "min": self._metric_op = -1. self._time_attr = time_attr self._perturbation_interval = perturbation_interval self._hyperparam_mutations = hyperparam_mutations self._resample_probability = resample_probability self._trial_state = {} self._custom_explore_fn = custom_explore_fn self._log_config = log_config # Metrics self._num_checkpoints = 0 self._num_perturbations = 0
def _update_avail_resources(self, num_retries=5): for i in range(num_retries): resources = ray.global_state.cluster_resources() if not resources: logger.warning("Cluster resources not detected. Retrying...") time.sleep(0.5) if not resources or "CPU" not in resources: raise TuneError("Cluster resources cannot be detected. " "You can resume this experiment by passing in " "`resume=True` to `run_experiments`.") resources = resources.copy() num_cpus = resources.pop("CPU") num_gpus = resources.pop("GPU") custom_resources = resources self._avail_resources = Resources( int(num_cpus), int(num_gpus), custom_resources=custom_resources) self._resources_initialized = True
def _generate_trials(self, experiment_spec, output_path=""): """Generates trials with configurations from `_suggest`. Creates a trial_id that is passed into `_suggest`. Yields: Trial objects constructed according to `spec` """ if "run" not in experiment_spec: raise TuneError("Must specify `run` in {}".format(experiment_spec)) for _ in range(experiment_spec.get("num_samples", 1)): trial_id = Trial.generate_id() while True: suggested_config = self._suggest(trial_id) if suggested_config is None: yield None else: break # spec = copy.deepcopy(experiment_spec) # spec["config"] = suggested_config self._counter += 1 def resolve(spec): res = {} for k, v in spec.items(): if isinstance(v, dict): for k_, v_ in resolve(v).items(): res[(k, ) + k_] = v_ else: res[(k, )] = v return res resolved_config = resolve({"config": suggested_config}) tag = "{0}_{1}".format(str(self._counter), format_vars(resolved_config)) spec = merge_dicts(experiment_spec, {"config": suggested_config}) yield create_trial_from_spec(spec, output_path, self._parser, experiment_tag=tag, trial_id=trial_id)
def get_trainable_name(cls, run_object: Union[str, Callable, Type]): """Get Trainable name. Args: run_object: Trainable to run. If string, assumes it is an ID and does not modify it. Otherwise, returns a string corresponding to the run_object name. Returns: A string representing the trainable identifier. Raises: TuneError: if ``run_object`` passed in is invalid. """ from ray.tune.search.sample import Domain if isinstance(run_object, str) or isinstance(run_object, Domain): return run_object elif isinstance(run_object, type) or callable(run_object): name = "DEFAULT" if hasattr(run_object, "_name"): name = run_object._name elif hasattr(run_object, "__name__"): fn_name = run_object.__name__ if fn_name == "<lambda>": name = "lambda" elif fn_name.startswith("<"): name = "DEFAULT" else: name = fn_name elif (isinstance(run_object, partial) and hasattr(run_object, "func") and hasattr(run_object.func, "__name__")): name = run_object.func.__name__ else: logger.warning( "No name detected on trainable. Using {}.".format(name)) return name else: raise TuneError("Improper 'run' - not string nor trainable.")
def _train(self, trial): """Start one iteration of training and save remote id.""" if self._find_item(self._paused, trial): raise TuneError( "Should not call `train` on PAUSED trial {}. " "This is an internal error - please file an issue " "on https://github.com/ray-project/ray/issues/.".format( str(trial))) if self._find_item(self._running, trial): logging.debug( "Trial {} already has a queued future. Skipping this " "`train` call. This may occur if a trial has " "been unpaused within a scheduler callback.".format( str(trial))) return assert trial.status == Trial.RUNNING, trial.status buffer_time_s = max( TUNE_RESULT_BUFFER_MIN_TIME_S, min(TUNE_RESULT_BUFFER_MAX_TIME_S, len(self._running) // 10)) with self._change_working_directory(trial): if TUNE_RESULT_BUFFER_LENGTH > 1: buffer_length = TUNE_RESULT_BUFFER_LENGTH if trial.checkpoint_freq > 0: buffer_length = min(buffer_length, trial.checkpoint_freq) remote = trial.runner.train_buffered.remote( buffer_time_s, buffer_length) else: remote = trial.runner.train.remote() # Local Mode if isinstance(remote, dict): remote = _LocalWrapper(remote) self._running[remote] = trial trial_item = self._find_item(self._running, trial) assert len(trial_item) < 2, trial_item
def wait_or_retry(self, max_retries: int = 3, backoff_s: int = 5): assert max_retries > 0 for _ in range(max_retries - 1): try: self.wait() except TuneError as e: logger.error( f"Caught sync error: {e}. " f"Retrying after sleeping for {backoff_s} seconds...") time.sleep(backoff_s) self._execute_sync( self._last_source_tuple, self._last_target_tuple, ) continue return self._sync_future = None self._stored_pack_actor_ref = None self._stored_files_stats_future = None raise TuneError(f"Failed sync even after {max_retries} retries.")
def convert_to_experiment_list(experiments: Union[Experiment, List[Experiment], Dict]): """Produces a list of Experiment objects. Converts input from dict, single experiment, or list of experiments to list of experiments. If input is None, will return an empty list. Arguments: experiments: Experiments to run. Returns: List of experiments. """ exp_list = experiments # Transform list if necessary if experiments is None: exp_list = [] elif isinstance(experiments, Experiment): exp_list = [experiments] elif type(experiments) is dict: exp_list = [ Experiment.from_json(name, spec) for name, spec in experiments.items() ] # Validate exp_list if type(exp_list) is list and all( isinstance(exp, Experiment) for exp in exp_list): if len(exp_list) > 1: logger.info( "Running with multiple concurrent experiments. " "All experiments will be using the same SearchAlgorithm.") else: raise TuneError("Invalid argument: {}".format(experiments)) return exp_list
def __init__(self, time_attr="time_total_s", reward_attr="episode_reward_mean", perturbation_interval=60.0, hyperparam_mutations={}, resample_probability=0.25, custom_explore_fn=None): if not hyperparam_mutations and not custom_explore_fn: raise TuneError( "You must specify at least one of `hyperparam_mutations` or " "`custom_explore_fn` to use PBT.") FIFOScheduler.__init__(self) self._reward_attr = reward_attr self._time_attr = time_attr self._perturbation_interval = perturbation_interval self._hyperparam_mutations = hyperparam_mutations self._resample_probability = resample_probability self._trial_state = {} self._custom_explore_fn = custom_explore_fn # Metrics self._num_checkpoints = 0 self._num_perturbations = 0
def _may_warn_insufficient_resources(self, all_trials): # This is approximately saying we are not making progress. if len(all_trials) == self._all_trials_size: if self._no_running_trials_since == -1: self._no_running_trials_since = time.monotonic() elif (time.monotonic() - self._no_running_trials_since > _get_insufficient_resources_warning_threshold()): if not is_ray_cluster(): # autoscaler not enabled # If any of the pending trial cannot be fulfilled, # that's a good enough hint of trial resources not enough. for trial in all_trials: if (trial.status is Trial.PENDING and not _can_fulfill_no_autoscaler(trial)): raise TuneError( _get_insufficient_resources_error_msg(trial)) else: # TODO(xwjiang): Output a more helpful msg for autoscaler. # https://github.com/ray-project/ray/issues/17799 logger.warning(_get_insufficient_resources_warning_msg()) self._no_running_trials_since = time.monotonic() else: self._no_running_trials_since = -1 self._all_trials_size = len(all_trials)
def _generate_trials(self, num_samples, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(num_samples): for resolved_vars, spec in generate_variants(unresolved_spec): while True: if self._num_live_trials() >= self._max_concurrent: yield None else: break trial_id = "%05d" % self._counter experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(format_vars(resolved_vars)) self._counter += 1 self._live_trials.add(trial_id) yield create_trial_from_spec( spec, output_path, self._parser, evaluated_params=flatten_resolved_vars(resolved_vars), trial_id=trial_id, experiment_tag=experiment_tag, )
def convert_to_experiment_list(experiments): """Produces a list of Experiment objects. Converts input from dict, single experiment, or list of experiments to list of experiments. If input is None, will return an empty list. Arguments: experiments (Experiment | list | dict): Experiments to run. Returns: List of experiments. """ exp_list = experiments # Transform list if necessary if experiments is None: exp_list = [] elif isinstance(experiments, Experiment): exp_list = [experiments] elif type(experiments) is dict: exp_list = [ Experiment.from_json(name, spec) for name, spec in experiments.items() ] # Validate exp_list if (type(exp_list) is list and all(isinstance(exp, Experiment) for exp in exp_list)): if len(exp_list) > 1: print("Warning: All experiments will be" " using the same Search Algorithm.") else: raise TuneError("Invalid argument: {}".format(experiments)) return exp_list
def _generate_trials(self, unresolved_spec, output_path=""): """Generates Trial objects with the variant generation process. Uses a fixed point iteration to resolve variants. All trials should be able to be generated at once. See also: `ray.tune.suggest.variant_generator`. Yields: Trial object """ if "run" not in unresolved_spec: raise TuneError("Must specify `run` in {}".format(unresolved_spec)) for _ in range(unresolved_spec.get("repeat", 1)): for resolved_vars, spec in generate_variants(unresolved_spec): experiment_tag = str(self._counter) if resolved_vars: experiment_tag += "_{}".format(resolved_vars) self._counter += 1 yield create_trial_from_spec(spec, output_path, self._parser, experiment_tag=experiment_tag)
def _populate_exception(trial: Trial) -> Optional[TuneError]: if trial.error_file: with open(trial.error_file, "r") as f: return TuneError(f.read()) return None
def __init__( self, time_attr: str = "time_total_s", metric: Optional[str] = None, mode: Optional[str] = None, perturbation_interval: float = 60.0, burn_in_period: float = 0.0, hyperparam_mutations: Dict = None, quantile_fraction: float = 0.25, resample_probability: float = 0.25, custom_explore_fn: Optional[Callable] = None, log_config: bool = True, require_attrs: bool = True, synch: bool = False, ): hyperparam_mutations = hyperparam_mutations or {} for value in hyperparam_mutations.values(): if not (isinstance(value, (list, dict, Domain)) or callable(value)): raise TypeError("`hyperparam_mutation` values must be either " "a List, Dict, a tune search space object, or " "a callable.") if isinstance(value, Function): raise ValueError("arbitrary tune.sample_from objects are not " "supported for `hyperparam_mutation` values." "You must use other built in primitives like" "tune.uniform, tune.loguniform, etc.") if not hyperparam_mutations and not custom_explore_fn: raise TuneError( "You must specify at least one of `hyperparam_mutations` " "or `custom_explore_fn` to use PBT.") if quantile_fraction > 0.5 or quantile_fraction < 0: raise ValueError( "You must set `quantile_fraction` to a value between 0 and" "0.5. Current value: '{}'".format(quantile_fraction)) if perturbation_interval <= 0: raise ValueError( "perturbation_interval must be a positive number greater " "than 0. Current value: '{}'".format(perturbation_interval)) if mode: assert mode in ["min", "max"], "`mode` must be 'min' or 'max'." FIFOScheduler.__init__(self) self._metric = metric self._mode = mode self._metric_op = None if self._mode == "max": self._metric_op = 1.0 elif self._mode == "min": self._metric_op = -1.0 self._time_attr = time_attr self._perturbation_interval = perturbation_interval self._burn_in_period = burn_in_period self._hyperparam_mutations = hyperparam_mutations self._quantile_fraction = quantile_fraction self._resample_probability = resample_probability self._trial_state = {} self._custom_explore_fn = custom_explore_fn self._log_config = log_config self._require_attrs = require_attrs self._synch = synch self._next_perturbation_sync = max( self._perturbation_interval, self._burn_in_period, ) # Metrics self._num_checkpoints = 0 self._num_perturbations = 0
def _make_scheduler(args): if args.scheduler in _SCHEDULERS: return _SCHEDULERS[args.scheduler](**args.scheduler_config) else: raise TuneError("Unknown scheduler: {}, should be one of {}".format( args.scheduler, _SCHEDULERS.keys()))
def run_experiments(experiments, search_alg=None, scheduler=None, with_server=False, server_port=TuneServer.DEFAULT_PORT, verbose=2, resume=False, queue_trials=False, trial_executor=None, raise_on_failed_trial=True): """Runs and blocks until all trials finish. Args: experiments (Experiment | list | dict): Experiments to run. Will be passed to `search_alg` via `add_configurations`. search_alg (SearchAlgorithm): Search Algorithm. Defaults to BasicVariantGenerator. scheduler (TrialScheduler): Scheduler for executing the experiment. Choose among FIFO (default), MedianStopping, AsyncHyperBand, and HyperBand. with_server (bool): Starts a background Tune server. Needed for using the Client API. server_port (int): Port number for launching TuneServer. verbose (int): 0, 1, or 2. Verbosity mode. 0 = silent, 1 = only status updates, 2 = status and trial results. resume (bool|"prompt"): If checkpoint exists, the experiment will resume from there. If resume is "prompt", Tune will prompt if checkpoint detected. queue_trials (bool): Whether to queue trials when the cluster does not currently have enough resources to launch one. This should be set to True when running on an autoscaling cluster to enable automatic scale-up. trial_executor (TrialExecutor): Manage the execution of trials. raise_on_failed_trial (bool): Raise TuneError if there exists failed trial (of ERROR state) when the experiments complete. Examples: >>> experiment_spec = Experiment("experiment", my_func) >>> run_experiments(experiments=experiment_spec) >>> experiment_spec = {"experiment": {"run": my_func}} >>> run_experiments(experiments=experiment_spec) >>> run_experiments( >>> experiments=experiment_spec, >>> scheduler=MedianStoppingRule(...)) >>> run_experiments( >>> experiments=experiment_spec, >>> search_alg=SearchAlgorithm(), >>> scheduler=MedianStoppingRule(...)) Returns: List of Trial objects, holding data for each executed trial. """ # This is important to do this here # because it schematize the experiments # and it conducts the implicit registration. experiments = convert_to_experiment_list(experiments) checkpoint_dir = _find_checkpoint_dir(experiments) runner = None restore = False if TrialRunner.checkpoint_exists(checkpoint_dir): if resume == "prompt": msg = ("Found incomplete experiment at {}. " "Would you like to resume it?".format(checkpoint_dir)) restore = click.confirm(msg, default=False) if restore: logger.info("Tip: to always resume, " "pass resume=True to run_experiments()") else: logger.info("Tip: to always start a new experiment, " "pass resume=False to run_experiments()") elif resume: restore = True else: logger.info( "Tip: to resume incomplete experiments, " "pass resume='prompt' or resume=True to run_experiments()") else: logger.info( "Did not find checkpoint file in {}.".format(checkpoint_dir)) if restore: runner = try_restore_runner(checkpoint_dir, search_alg, scheduler, trial_executor) else: logger.info("Starting a new experiment.") if not runner: if scheduler is None: scheduler = FIFOScheduler() if search_alg is None: search_alg = BasicVariantGenerator() search_alg.add_configurations(experiments) runner = TrialRunner(search_alg, scheduler=scheduler, metadata_checkpoint_dir=checkpoint_dir, launch_web_server=with_server, server_port=server_port, verbose=bool(verbose > 1), queue_trials=queue_trials, trial_executor=trial_executor) if verbose: print(runner.debug_string(max_debug=99999)) last_debug = 0 while not runner.is_finished(): runner.step() if time.time() - last_debug > DEBUG_PRINT_INTERVAL: if verbose: print(runner.debug_string()) last_debug = time.time() if verbose: print(runner.debug_string(max_debug=99999)) wait_for_log_sync() errored_trials = [] for trial in runner.get_trials(): if trial.status != Trial.TERMINATED: errored_trials += [trial] if errored_trials: if raise_on_failed_trial: raise TuneError("Trials did not complete", errored_trials) else: logger.error("Trials did not complete: %s", errored_trials) return runner.get_trials()
def run(run_or_experiment, name=None, stop=None, config=None, resources_per_trial=None, num_samples=1, local_dir=None, upload_dir=None, trial_name_creator=None, loggers=None, sync_to_cloud=None, sync_to_driver=None, checkpoint_freq=0, checkpoint_at_end=False, export_formats=None, max_failures=3, restore=None, search_alg=None, scheduler=None, with_server=False, server_port=TuneServer.DEFAULT_PORT, verbose=2, resume=False, queue_trials=False, reuse_actors=True, trial_executor=None, raise_on_failed_trial=True, return_trials=True, ray_auto_init=True, sync_function=None): """Executes training. Args: run_or_experiment (function|class|str|Experiment): If function|class|str, this is the algorithm or model to train. This may refer to the name of a built-on algorithm (e.g. RLLib's DQN or PPO), a user-defined trainable function or class, or the string identifier of a trainable function or class registered in the tune registry. If Experiment, then Tune will execute training based on Experiment.spec. name (str): Name of experiment. stop (dict): The stopping criteria. The keys may be any field in the return result of 'train()', whichever is reached first. Defaults to empty dict. config (dict): Algorithm-specific configuration for Tune variant generation (e.g. env, hyperparams). Defaults to empty dict. Custom search algorithms may ignore this. resources_per_trial (dict): Machine resources to allocate per trial, e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be assigned unless you specify them here. Defaults to 1 CPU and 0 GPUs in ``Trainable.default_resource_request()``. num_samples (int): Number of times to sample from the hyperparameter space. Defaults to 1. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times. local_dir (str): Local dir to save training results to. Defaults to ``~/ray_results``. upload_dir (str): Optional URI to sync training results to (e.g. ``s3://bucket``). trial_name_creator (func): Optional function for generating the trial string representation. loggers (list): List of logger creators to be used with each Trial. If None, defaults to ray.tune.logger.DEFAULT_LOGGERS. See `ray/tune/logger.py`. sync_to_cloud (func|str): Function for syncing the local_dir to and from upload_dir. If string, then it must be a string template that includes `{source}` and `{target}` for the syncer to run. If not provided, the sync command defaults to standard S3 or gsutil sync comamnds. sync_to_driver (func|str): Function for syncing trial logdir from remote node to local. If string, then it must be a string template that includes `{source}` and `{target}` for the syncer to run. If not provided, defaults to using rsync. checkpoint_freq (int): How many training iterations between checkpoints. A value of 0 (default) disables checkpointing. checkpoint_at_end (bool): Whether to checkpoint at the end of the experiment regardless of the checkpoint_freq. Default is False. export_formats (list): List of formats that exported at the end of the experiment. Default is None. max_failures (int): Try to recover a trial from its last checkpoint at least this many times. Only applies if checkpointing is enabled. Setting to -1 will lead to infinite recovery retries. Defaults to 3. restore (str): Path to checkpoint. Only makes sense to set if running 1 trial. Defaults to None. search_alg (SearchAlgorithm): Search Algorithm. Defaults to BasicVariantGenerator. scheduler (TrialScheduler): Scheduler for executing the experiment. Choose among FIFO (default), MedianStopping, AsyncHyperBand, and HyperBand. with_server (bool): Starts a background Tune server. Needed for using the Client API. server_port (int): Port number for launching TuneServer. verbose (int): 0, 1, or 2. Verbosity mode. 0 = silent, 1 = only status updates, 2 = status and trial results. resume (str|bool): One of "LOCAL", "REMOTE", "PROMPT", or bool. LOCAL/True restores the checkpoint from the local_checkpoint_dir. REMOTE restores the checkpoint from remote_checkpoint_dir. PROMPT provides CLI feedback. False forces a new experiment. If resume is set but checkpoint does not exist, ValueError will be thrown. queue_trials (bool): Whether to queue trials when the cluster does not currently have enough resources to launch one. This should be set to True when running on an autoscaling cluster to enable automatic scale-up. reuse_actors (bool): Whether to reuse actors between different trials when possible. This can drastically speed up experiments that start and stop actors often (e.g., PBT in time-multiplexing mode). This requires trials to have the same resource requirements. trial_executor (TrialExecutor): Manage the execution of trials. raise_on_failed_trial (bool): Raise TuneError if there exists failed trial (of ERROR state) when the experiments complete. ray_auto_init (bool): Automatically starts a local Ray cluster if using a RayTrialExecutor (which is the default) and if Ray is not initialized. Defaults to True. sync_function: Deprecated. See `sync_to_cloud` and `sync_to_driver`. Returns: List of Trial objects. Raises: TuneError if any trials failed and `raise_on_failed_trial` is True. Examples: >>> tune.run(mytrainable, scheduler=PopulationBasedTraining()) >>> tune.run(mytrainable, num_samples=5, reuse_actors=True) >>> tune.run( "PG", num_samples=5, config={ "env": "CartPole-v0", "lr": tune.sample_from(lambda _: np.random.rand()) } ) """ trial_executor = trial_executor or RayTrialExecutor( queue_trials=queue_trials, reuse_actors=reuse_actors, ray_auto_init=ray_auto_init) experiment = run_or_experiment if not isinstance(run_or_experiment, Experiment): run_identifier = Experiment._register_if_needed(run_or_experiment) experiment = Experiment( name=name, run=run_identifier, stop=stop, config=config, resources_per_trial=resources_per_trial, num_samples=num_samples, local_dir=local_dir, upload_dir=upload_dir, sync_to_driver=sync_to_driver, trial_name_creator=trial_name_creator, loggers=loggers, checkpoint_freq=checkpoint_freq, checkpoint_at_end=checkpoint_at_end, export_formats=export_formats, max_failures=max_failures, restore=restore, sync_function=sync_function) else: logger.debug("Ignoring some parameters passed into tune.run.") if sync_to_cloud: assert experiment.remote_checkpoint_dir, ( "Need `upload_dir` if `sync_to_cloud` given.") runner = TrialRunner( search_alg=search_alg or BasicVariantGenerator(), scheduler=scheduler or FIFOScheduler(), local_checkpoint_dir=experiment.checkpoint_dir, remote_checkpoint_dir=experiment.remote_checkpoint_dir, sync_to_cloud=sync_to_cloud, resume=resume, launch_web_server=with_server, server_port=server_port, verbose=bool(verbose > 1), trial_executor=trial_executor) runner.add_experiment(experiment) if verbose: print(runner.debug_string(max_debug=99999)) last_debug = 0 while not runner.is_finished(): runner.step() if time.time() - last_debug > DEBUG_PRINT_INTERVAL: if verbose: print(runner.debug_string()) last_debug = time.time() if verbose: print(runner.debug_string(max_debug=99999)) wait_for_sync() errored_trials = [] for trial in runner.get_trials(): if trial.status != Trial.TERMINATED: errored_trials += [trial] if errored_trials: if raise_on_failed_trial: raise TuneError("Trials did not complete", errored_trials) else: logger.error("Trials did not complete: %s", errored_trials) if return_trials: return runner.get_trials() return ExperimentAnalysis(experiment.checkpoint_dir)
def _exploit(self, trial_executor: "trial_executor.TrialExecutor", trial: Trial, trial_to_clone: Trial): """Transfers perturbed state from trial_to_clone -> trial. If specified, also logs the updated hyperparam state. """ trial_state = self._trial_state[trial] new_state = self._trial_state[trial_to_clone] logger.info("[exploit] transferring weights from trial " "{} (score {}) -> {} (score {})".format( trial_to_clone, new_state.last_score, trial, trial_state.last_score)) new_config = self._get_new_config(trial, trial_to_clone) # Only log mutated hyperparameters and not entire config. old_hparams = { k: v for k, v in trial_to_clone.config.items() if k in self._hyperparam_mutations } new_hparams = { k: v for k, v in new_config.items() if k in self._hyperparam_mutations } logger.info("[explore] perturbed config from {} -> {}".format( old_hparams, new_hparams)) if self._log_config: self._log_config_on_step(trial_state, new_state, trial, trial_to_clone, new_config) new_tag = make_experiment_tag(trial_state.orig_tag, new_config, self._hyperparam_mutations) if trial.status == Trial.PAUSED: # If trial is paused we update it with a new checkpoint. # When the trial is started again, the new checkpoint is used. if not self._synch: raise TuneError("Trials should be paused here only if in " "synchronous mode. If you encounter this error" " please raise an issue on Ray Github.") trial.set_experiment_tag(new_tag) trial.set_config(new_config) trial.on_checkpoint(new_state.last_checkpoint) else: # If trial is running, we first try to reset it. # If that is unsuccessful, then we have to stop it and start it # again with a new checkpoint. reset_successful = trial_executor.reset_trial( trial, new_config, new_tag) # TODO(ujvl): Refactor Scheduler abstraction to abstract # mechanism for trial restart away. We block on restore # and suppress train on start as a stop-gap fix to # https://github.com/ray-project/ray/issues/7258. if reset_successful: trial_executor.restore(trial, new_state.last_checkpoint, block=True) else: trial_executor.stop_trial(trial) trial.set_experiment_tag(new_tag) trial.set_config(new_config) trial_executor.start_trial(trial, new_state.last_checkpoint, train=False) self._num_perturbations += 1 # Transfer over the last perturbation time as well trial_state.last_perturbation_time = new_state.last_perturbation_time trial_state.last_train_time = new_state.last_train_time
def run( run_or_experiment: Union[str, Callable, Type], name: Optional[str] = None, metric: Optional[str] = None, mode: Optional[str] = None, stop: Union[None, Mapping, Stopper, Callable[[str, Mapping], bool]] = None, time_budget_s: Union[None, int, float, datetime.timedelta] = None, config: Optional[Dict[str, Any]] = None, resources_per_trial: Union[None, Mapping[str, Union[ float, int, Mapping]], PlacementGroupFactory] = None, num_samples: int = 1, local_dir: Optional[str] = None, search_alg: Optional[Union[Searcher, SearchAlgorithm, str]] = None, scheduler: Optional[Union[TrialScheduler, str]] = None, keep_checkpoints_num: Optional[int] = None, checkpoint_score_attr: Optional[str] = None, checkpoint_freq: int = 0, checkpoint_at_end: bool = False, verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS, progress_reporter: Optional[ProgressReporter] = None, log_to_file: bool = False, trial_name_creator: Optional[Callable[[Trial], str]] = None, trial_dirname_creator: Optional[Callable[[Trial], str]] = None, sync_config: Optional[SyncConfig] = None, export_formats: Optional[Sequence] = None, max_failures: int = 0, fail_fast: bool = False, restore: Optional[str] = None, server_port: Optional[int] = None, resume: bool = False, reuse_actors: bool = False, trial_executor: Optional[RayTrialExecutor] = None, raise_on_failed_trial: bool = True, callbacks: Optional[Sequence[Callback]] = None, max_concurrent_trials: Optional[int] = None, # Deprecated args queue_trials: Optional[bool] = None, loggers: Optional[Sequence[Type[Logger]]] = None, _remote: Optional[bool] = None, ) -> ExperimentAnalysis: """Executes training. When a SIGINT signal is received (e.g. through Ctrl+C), the tuning run will gracefully shut down and checkpoint the latest experiment state. Sending SIGINT again (or SIGKILL/SIGTERM instead) will skip this step. Many aspects of Tune, such as the frequency of global checkpointing, maximum pending placement group trials and the path of the result directory be configured through environment variables. Refer to :ref:`tune-env-vars` for a list of environment variables available. Examples: .. code-block:: python # Run 10 trials (each trial is one instance of a Trainable). Tune runs # in parallel and automatically determines concurrency. tune.run(trainable, num_samples=10) # Run 1 trial, stop when trial has reached 10 iterations tune.run(my_trainable, stop={"training_iteration": 10}) # automatically retry failed trials up to 3 times tune.run(my_trainable, stop={"training_iteration": 10}, max_failures=3) # Run 1 trial, search over hyperparameters, stop after 10 iterations. space = {"lr": tune.uniform(0, 1), "momentum": tune.uniform(0, 1)} tune.run(my_trainable, config=space, stop={"training_iteration": 10}) # Resumes training if a previous machine crashed tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume=True) # Rerun ONLY failed trials after an experiment is finished. tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume="ERRORED_ONLY") Args: run_or_experiment (function | class | str | :class:`Experiment`): If function|class|str, this is the algorithm or model to train. This may refer to the name of a built-on algorithm (e.g. RLLib's DQN or PPO), a user-defined trainable function or class, or the string identifier of a trainable function or class registered in the tune registry. If Experiment, then Tune will execute training based on Experiment.spec. If you want to pass in a Python lambda, you will need to first register the function: ``tune.register_trainable("lambda_id", lambda x: ...)``. You can then use ``tune.run("lambda_id")``. metric (str): Metric to optimize. This metric should be reported with `tune.report()`. If set, will be passed to the search algorithm and scheduler. mode (str): Must be one of [min, max]. Determines whether objective is minimizing or maximizing the metric attribute. If set, will be passed to the search algorithm and scheduler. name (str): Name of experiment. stop (dict | callable | :class:`Stopper`): Stopping criteria. If dict, the keys may be any field in the return result of 'train()', whichever is reached first. If function, it must take (trial_id, result) as arguments and return a boolean (True if trial should be stopped, False otherwise). This can also be a subclass of ``ray.tune.Stopper``, which allows users to implement custom experiment-wide stopping (i.e., stopping an entire Tune run based on some time constraint). time_budget_s (int|float|datetime.timedelta): Global time budget in seconds after which all trials are stopped. Can also be a ``datetime.timedelta`` object. config (dict): Algorithm-specific configuration for Tune variant generation (e.g. env, hyperparams). Defaults to empty dict. Custom search algorithms may ignore this. resources_per_trial (dict|PlacementGroupFactory): Machine resources to allocate per trial, e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be assigned unless you specify them here. Defaults to 1 CPU and 0 GPUs in ``Trainable.default_resource_request()``. This can also be a PlacementGroupFactory object wrapping arguments to create a per-trial placement group. num_samples (int): Number of times to sample from the hyperparameter space. Defaults to 1. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times. If this is -1, (virtually) infinite samples are generated until a stopping condition is met. local_dir (str): Local dir to save training results to. Defaults to ``~/ray_results``. search_alg (Searcher|SearchAlgorithm|str): Search algorithm for optimization. You can also use the name of the algorithm. scheduler (TrialScheduler|str): Scheduler for executing the experiment. Choose among FIFO (default), MedianStopping, AsyncHyperBand, HyperBand and PopulationBasedTraining. Refer to ray.tune.schedulers for more options. You can also use the name of the scheduler. keep_checkpoints_num (int): Number of checkpoints to keep. A value of `None` keeps all checkpoints. Defaults to `None`. If set, need to provide `checkpoint_score_attr`. checkpoint_score_attr (str): Specifies by which attribute to rank the best checkpoint. Default is increasing order. If attribute starts with `min-` it will rank attribute in decreasing order, i.e. `min-validation_loss`. checkpoint_freq (int): How many training iterations between checkpoints. A value of 0 (default) disables checkpointing. This has no effect when using the Functional Training API. checkpoint_at_end (bool): Whether to checkpoint at the end of the experiment regardless of the checkpoint_freq. Default is False. This has no effect when using the Functional Training API. verbose (Union[int, Verbosity]): 0, 1, 2, or 3. Verbosity mode. 0 = silent, 1 = only status updates, 2 = status and brief trial results, 3 = status and detailed trial results. Defaults to 3. progress_reporter (ProgressReporter): Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in a Jupyter notebook. log_to_file (bool|str|Sequence): Log stdout and stderr to files in Tune's trial directories. If this is `False` (default), no files are written. If `true`, outputs are written to `trialdir/stdout` and `trialdir/stderr`, respectively. If this is a single string, this is interpreted as a file relative to the trialdir, to which both streams are written. If this is a Sequence (e.g. a Tuple), it has to have length 2 and the elements indicate the files to which stdout and stderr are written, respectively. trial_name_creator (Callable[[Trial], str]): Optional function for generating the trial string representation. trial_dirname_creator (Callable[[Trial], str]): Function for generating the trial dirname. This function should take in a Trial object and return a string representing the name of the directory. The return value cannot be a path. sync_config (SyncConfig): Configuration object for syncing. See tune.SyncConfig. export_formats (list): List of formats that exported at the end of the experiment. Default is None. max_failures (int): Try to recover a trial at least this many times. Ray will recover from the latest checkpoint if present. Setting to -1 will lead to infinite recovery retries. Setting to 0 will disable retries. Defaults to 0. fail_fast (bool | str): Whether to fail upon the first error. If fail_fast='raise' provided, Tune will automatically raise the exception received by the Trainable. fail_fast='raise' can easily leak resources and should be used with caution (it is best used with `ray.init(local_mode=True)`). restore (str): Path to checkpoint. Only makes sense to set if running 1 trial. Defaults to None. server_port (int): Port number for launching TuneServer. resume (str|bool): One of "LOCAL", "REMOTE", "PROMPT", "ERRORED_ONLY", or bool. LOCAL/True restores the checkpoint from the local experiment directory, determined by ``name`` and ``local_dir``. REMOTE restores the checkpoint from ``upload_dir`` (as passed to ``sync_config``). PROMPT provides CLI feedback. False forces a new experiment. ERRORED_ONLY resets and reruns ERRORED trials upon resume - previous trial artifacts will be left untouched. If resume is set but checkpoint does not exist, ValueError will be thrown. reuse_actors (bool): Whether to reuse actors between different trials when possible. This can drastically speed up experiments that start and stop actors often (e.g., PBT in time-multiplexing mode). This requires trials to have the same resource requirements. trial_executor (TrialExecutor): Manage the execution of trials. raise_on_failed_trial (bool): Raise TuneError if there exists failed trial (of ERROR state) when the experiments complete. callbacks (list): List of callbacks that will be called at different times in the training loop. Must be instances of the ``ray.tune.callback.Callback`` class. If not passed, `LoggerCallback` and `SyncerCallback` callbacks are automatically added. max_concurrent_trials (int): Maximum number of trials to run concurrently. Must be non-negative. If None or 0, no limit will be applied. This is achieved by wrapping the ``search_alg`` in a :class:`ConcurrencyLimiter`, and thus setting this argument will raise an exception if the ``search_alg`` is already a :class:`ConcurrencyLimiter`. Defaults to None. _remote (bool): Whether to run the Tune driver in a remote function. This is disabled automatically if a custom trial executor is passed in. This is enabled by default in Ray client mode. Returns: ExperimentAnalysis: Object for experiment analysis. Raises: TuneError: Any trials failed and `raise_on_failed_trial` is True. """ # To be removed in 1.9. if queue_trials is not None: raise DeprecationWarning( "`queue_trials` has been deprecated and is replaced by " "the `TUNE_MAX_PENDING_TRIALS_PG` environment variable. " "Per default at least one Trial is queued at all times, " "so you likely don't need to change anything other than " "removing this argument from your call to `tune.run()`") # NO CODE IS TO BE ADDED ABOVE THIS COMMENT # remote_run_kwargs must be defined before any other # code is ran to ensure that at this point, # `locals()` is equal to args and kwargs remote_run_kwargs = locals().copy() remote_run_kwargs.pop("_remote") if _remote is None: _remote = ray.util.client.ray.is_connected() if _remote is True and trial_executor: raise ValueError("cannot use custom trial executor") if not trial_executor or isinstance(trial_executor, RayTrialExecutor): _ray_auto_init() if _remote: remote_run = ray.remote(num_cpus=0)(run) # Make sure tune.run is called on the sever node. remote_run = force_on_current_node(remote_run) # JupyterNotebooks don't work with remote tune runs out of the box # (e.g. via Ray client) as they don't have access to the main # process stdout. So we introduce a queue here that accepts # callables, which will then be executed on the driver side. if isinstance(progress_reporter, JupyterNotebookReporter): execute_queue = Queue(actor_options={ "num_cpus": 0, **force_on_current_node(None) }) progress_reporter.set_output_queue(execute_queue) def get_next_queue_item(): try: return execute_queue.get(block=False) except Empty: return None else: # If we don't need a queue, use this dummy get fn instead of # scheduling an unneeded actor def get_next_queue_item(): return None def _handle_execute_queue(): execute_item = get_next_queue_item() while execute_item: if isinstance(execute_item, Callable): execute_item() execute_item = get_next_queue_item() remote_future = remote_run.remote(_remote=False, **remote_run_kwargs) # ray.wait(...)[1] returns futures that are not ready, yet while ray.wait([remote_future], timeout=0.2)[1]: # Check if we have items to execute _handle_execute_queue() # Handle queue one last time _handle_execute_queue() return ray.get(remote_future) del remote_run_kwargs all_start = time.time() if loggers: # Raise DeprecationWarning in 1.9, remove in 1.10/1.11 warnings.warn( "The `loggers` argument is deprecated. Please pass the respective " "`LoggerCallback` classes to the `callbacks` argument instead. " "See https://docs.ray.io/en/latest/tune/api_docs/logging.html") if mode and mode not in ["min", "max"]: raise ValueError( "The `mode` parameter passed to `tune.run()` has to be one of " "['min', 'max']") set_verbosity(verbose) config = config or {} sync_config = sync_config or SyncConfig() set_sync_periods(sync_config) if num_samples == -1: num_samples = sys.maxsize result_buffer_length = None # Create scheduler here as we need access to some of its properties if isinstance(scheduler, str): # importing at top level causes a recursive dependency from ray.tune.schedulers import create_scheduler scheduler = create_scheduler(scheduler) scheduler = scheduler or FIFOScheduler() if not scheduler.supports_buffered_results: # Result buffering with e.g. a Hyperband scheduler is a bad idea, as # hyperband tries to stop trials when processing brackets. With result # buffering, we might trigger this multiple times when evaluating # a single trial, which leads to unexpected behavior. env_result_buffer_length = os.getenv("TUNE_RESULT_BUFFER_LENGTH", "") if env_result_buffer_length: warnings.warn( f"You are using a {type(scheduler)} scheduler, but " f"TUNE_RESULT_BUFFER_LENGTH is set " f"({env_result_buffer_length}). This can lead to undesired " f"and faulty behavior, so the buffer length was forcibly set " f"to 1 instead.") result_buffer_length = 1 if isinstance(scheduler, (PopulationBasedTraining, PopulationBasedTrainingReplay)) and not reuse_actors: warnings.warn( "Consider boosting PBT performance by enabling `reuse_actors` as " "well as implementing `reset_config` for Trainable.") trial_executor = trial_executor or RayTrialExecutor( reuse_actors=reuse_actors, result_buffer_length=result_buffer_length) if isinstance(run_or_experiment, list): experiments = run_or_experiment else: experiments = [run_or_experiment] for i, exp in enumerate(experiments): if not isinstance(exp, Experiment): experiments[i] = Experiment( name=name, run=exp, stop=stop, time_budget_s=time_budget_s, config=config, resources_per_trial=resources_per_trial, num_samples=num_samples, local_dir=local_dir, sync_config=sync_config, trial_name_creator=trial_name_creator, trial_dirname_creator=trial_dirname_creator, log_to_file=log_to_file, checkpoint_freq=checkpoint_freq, checkpoint_at_end=checkpoint_at_end, keep_checkpoints_num=keep_checkpoints_num, checkpoint_score_attr=checkpoint_score_attr, export_formats=export_formats, max_failures=max_failures, restore=restore) else: logger.debug("Ignoring some parameters passed into tune.run.") if fail_fast and max_failures != 0: raise ValueError("max_failures must be 0 if fail_fast=True.") if isinstance(search_alg, str): # importing at top level causes a recursive dependency from ray.tune.suggest import create_searcher search_alg = create_searcher(search_alg) # if local_mode=True is set during ray.init(). is_local_mode = ray.worker._mode() == ray.worker.LOCAL_MODE if is_local_mode: max_concurrent_trials = 1 if not search_alg: search_alg = BasicVariantGenerator( max_concurrent=max_concurrent_trials or 0) elif max_concurrent_trials: if isinstance(search_alg, ConcurrencyLimiter): if search_alg.max_concurrent != max_concurrent_trials: raise ValueError( "You have specified `max_concurrent_trials=" f"{max_concurrent_trials}`, but the `search_alg` is " "already a `ConcurrencyLimiter` with `max_concurrent=" f"{search_alg.max_concurrent}. FIX THIS by setting " "`max_concurrent_trials=None`.") else: logger.warning( "You have specified `max_concurrent_trials=" f"{max_concurrent_trials}`, but the `search_alg` is " "already a `ConcurrencyLimiter`. `max_concurrent_trials` " "will be ignored.") else: if max_concurrent_trials < 1: raise ValueError( "`max_concurrent_trials` must be greater or equal than 1, " f"got {max_concurrent_trials}.") if isinstance(search_alg, Searcher): search_alg = ConcurrencyLimiter( search_alg, max_concurrent=max_concurrent_trials) elif not is_local_mode: logger.warning( "You have passed a `SearchGenerator` instance as the " "`search_alg`, but `max_concurrent_trials` requires a " "`Searcher` instance`. `max_concurrent_trials` " "will be ignored.") if isinstance(search_alg, Searcher): search_alg = SearchGenerator(search_alg) if config and not set_search_properties_backwards_compatible( search_alg.set_search_properties, metric, mode, config, ** experiments[0].public_spec): if has_unresolved_values(config): raise ValueError( "You passed a `config` parameter to `tune.run()` with " "unresolved parameters, but the search algorithm was already " "instantiated with a search space. Make sure that `config` " "does not contain any more parameter definitions - include " "them in the search algorithm's search space if necessary.") if not scheduler.set_search_properties(metric, mode): raise ValueError( "You passed a `metric` or `mode` argument to `tune.run()`, but " "the scheduler you are using was already instantiated with their " "own `metric` and `mode` parameters. Either remove the arguments " "from your scheduler or from your call to `tune.run()`") # Create syncer callbacks callbacks = create_default_callbacks( callbacks, sync_config, metric=metric, loggers=loggers) runner = TrialRunner( search_alg=search_alg, scheduler=scheduler, local_checkpoint_dir=experiments[0].checkpoint_dir, remote_checkpoint_dir=experiments[0].remote_checkpoint_dir, sync_config=sync_config, stopper=experiments[0].stopper, resume=resume, server_port=server_port, fail_fast=fail_fast, trial_executor=trial_executor, callbacks=callbacks, metric=metric, # Driver should only sync trial checkpoints if # checkpoints are not synced to cloud driver_sync_trial_checkpoints=not bool(sync_config.upload_dir)) if not runner.resumed: for exp in experiments: search_alg.add_configurations([exp]) else: logger.info("TrialRunner resumed, ignoring new add_experiment but " "updating trial resources.") if resources_per_trial: runner.update_pending_trial_resources(resources_per_trial) progress_reporter = progress_reporter or detect_reporter() if not progress_reporter.set_search_properties(metric, mode): raise ValueError( "You passed a `metric` or `mode` argument to `tune.run()`, but " "the reporter you are using was already instantiated with their " "own `metric` and `mode` parameters. Either remove the arguments " "from your reporter or from your call to `tune.run()`") progress_reporter.set_total_samples(search_alg.total_samples) # Calls setup on callbacks runner.setup_experiments( experiments=experiments, total_num_samples=search_alg.total_samples) # User Warning for GPUs if trial_executor.has_gpus(): if isinstance(resources_per_trial, dict) and "gpu" in resources_per_trial: # "gpu" is manually set. pass elif _check_default_resources_override(experiments[0].run_identifier): # "default_resources" is manually overridden. pass else: logger.warning("Tune detects GPUs, but no trials are using GPUs. " "To enable trials to use GPUs, set " "tune.run(resources_per_trial={'gpu': 1}...) " "which allows Tune to expose 1 GPU to each trial. " "You can also override " "`Trainable.default_resource_request` if using the " "Trainable API.") original_handler = signal.getsignal(signal.SIGINT) state = {signal.SIGINT: False} def sigint_handler(sig, frame): logger.warning( "SIGINT received (e.g. via Ctrl+C), ending Ray Tune run. " "This will try to checkpoint the experiment state one last time. " "Press CTRL+C one more time (or send SIGINT/SIGKILL/SIGTERM) " "to skip. ") state[signal.SIGINT] = True # Restore original signal handler to react to future SIGINT signals signal.signal(signal.SIGINT, original_handler) if not int(os.getenv("TUNE_DISABLE_SIGINT_HANDLER", "0")): signal.signal(signal.SIGINT, sigint_handler) tune_start = time.time() progress_reporter.set_start_time(tune_start) while not runner.is_finished() and not state[signal.SIGINT]: runner.step() if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter) tune_taken = time.time() - tune_start try: runner.checkpoint(force=True) except Exception as e: logger.warning(f"Trial Runner checkpointing failed: {str(e)}") if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter, done=True) wait_for_sync() runner.cleanup() incomplete_trials = [] for trial in runner.get_trials(): if trial.status != Trial.TERMINATED: incomplete_trials += [trial] if incomplete_trials: if raise_on_failed_trial and not state[signal.SIGINT]: raise TuneError("Trials did not complete", incomplete_trials) else: logger.error("Trials did not complete: %s", incomplete_trials) all_taken = time.time() - all_start if has_verbosity(Verbosity.V1_EXPERIMENT): logger.info(f"Total run time: {all_taken:.2f} seconds " f"({tune_taken:.2f} seconds for the tuning loop).") if state[signal.SIGINT]: logger.warning( "Experiment has been interrupted, but the most recent state was " "saved. You can continue running this experiment by passing " "`resume=True` to `tune.run()`") trials = runner.get_trials() return ExperimentAnalysis( runner.checkpoint_file, trials=trials, default_metric=metric, default_mode=mode, sync_config=sync_config)
def run( run_or_experiment, name=None, metric=None, mode=None, stop=None, time_budget_s=None, config=None, resources_per_trial=None, num_samples=1, local_dir=None, search_alg=None, scheduler=None, keep_checkpoints_num=None, checkpoint_score_attr=None, checkpoint_freq=0, checkpoint_at_end=False, verbose=2, progress_reporter=None, loggers=None, log_to_file=False, trial_name_creator=None, trial_dirname_creator=None, sync_config=None, export_formats=None, max_failures=0, fail_fast=False, restore=None, server_port=None, resume=False, reuse_actors=False, trial_executor=None, raise_on_failed_trial=True, # Deprecated args ray_auto_init=None, run_errored_only=None, queue_trials=None, global_checkpoint_period=None, with_server=None, upload_dir=None, sync_to_cloud=None, sync_to_driver=None, sync_on_checkpoint=None, ): """Executes training. Examples: .. code-block:: python # Run 10 trials (each trial is one instance of a Trainable). Tune runs # in parallel and automatically determines concurrency. tune.run(trainable, num_samples=10) # Run 1 trial, stop when trial has reached 10 iterations tune.run(my_trainable, stop={"training_iteration": 10}) # automatically retry failed trials up to 3 times tune.run(my_trainable, stop={"training_iteration": 10}, max_failures=3) # Run 1 trial, search over hyperparameters, stop after 10 iterations. space = {"lr": tune.uniform(0, 1), "momentum": tune.uniform(0, 1)} tune.run(my_trainable, config=space, stop={"training_iteration": 10}) # Resumes training if a previous machine crashed tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume=True) # Rerun ONLY failed trials after an experiment is finished. tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume="ERRORED_ONLY") Args: run_or_experiment (function | class | str | :class:`Experiment`): If function|class|str, this is the algorithm or model to train. This may refer to the name of a built-on algorithm (e.g. RLLib's DQN or PPO), a user-defined trainable function or class, or the string identifier of a trainable function or class registered in the tune registry. If Experiment, then Tune will execute training based on Experiment.spec. If you want to pass in a Python lambda, you will need to first register the function: ``tune.register_trainable("lambda_id", lambda x: ...)``. You can then use ``tune.run("lambda_id")``. metric (str): Metric to optimize. This metric should be reported with `tune.report()`. If set, will be passed to the search algorithm and scheduler. mode (str): Must be one of [min, max]. Determines whether objective is minimizing or maximizing the metric attribute. If set, will be passed to the search algorithm and scheduler. name (str): Name of experiment. stop (dict | callable | :class:`Stopper`): Stopping criteria. If dict, the keys may be any field in the return result of 'train()', whichever is reached first. If function, it must take (trial_id, result) as arguments and return a boolean (True if trial should be stopped, False otherwise). This can also be a subclass of ``ray.tune.Stopper``, which allows users to implement custom experiment-wide stopping (i.e., stopping an entire Tune run based on some time constraint). time_budget_s (int|float|datetime.timedelta): Global time budget in seconds after which all trials are stopped. Can also be a ``datetime.timedelta`` object. config (dict): Algorithm-specific configuration for Tune variant generation (e.g. env, hyperparams). Defaults to empty dict. Custom search algorithms may ignore this. resources_per_trial (dict): Machine resources to allocate per trial, e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be assigned unless you specify them here. Defaults to 1 CPU and 0 GPUs in ``Trainable.default_resource_request()``. num_samples (int): Number of times to sample from the hyperparameter space. Defaults to 1. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times. local_dir (str): Local dir to save training results to. Defaults to ``~/ray_results``. search_alg (Searcher): Search algorithm for optimization. scheduler (TrialScheduler): Scheduler for executing the experiment. Choose among FIFO (default), MedianStopping, AsyncHyperBand, HyperBand and PopulationBasedTraining. Refer to ray.tune.schedulers for more options. keep_checkpoints_num (int): Number of checkpoints to keep. A value of `None` keeps all checkpoints. Defaults to `None`. If set, need to provide `checkpoint_score_attr`. checkpoint_score_attr (str): Specifies by which attribute to rank the best checkpoint. Default is increasing order. If attribute starts with `min-` it will rank attribute in decreasing order, i.e. `min-validation_loss`. checkpoint_freq (int): How many training iterations between checkpoints. A value of 0 (default) disables checkpointing. This has no effect when using the Functional Training API. checkpoint_at_end (bool): Whether to checkpoint at the end of the experiment regardless of the checkpoint_freq. Default is False. This has no effect when using the Functional Training API. verbose (int): 0, 1, or 2. Verbosity mode. 0 = silent, 1 = only status updates, 2 = status and trial results. progress_reporter (ProgressReporter): Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in a Jupyter notebook. loggers (list): List of logger creators to be used with each Trial. If None, defaults to ray.tune.logger.DEFAULT_LOGGERS. See `ray/tune/logger.py`. log_to_file (bool|str|Sequence): Log stdout and stderr to files in Tune's trial directories. If this is `False` (default), no files are written. If `true`, outputs are written to `trialdir/stdout` and `trialdir/stderr`, respectively. If this is a single string, this is interpreted as a file relative to the trialdir, to which both streams are written. If this is a Sequence (e.g. a Tuple), it has to have length 2 and the elements indicate the files to which stdout and stderr are written, respectively. trial_name_creator (Callable[[Trial], str]): Optional function for generating the trial string representation. trial_dirname_creator (Callable[[Trial], str]): Function for generating the trial dirname. This function should take in a Trial object and return a string representing the name of the directory. The return value cannot be a path. sync_config (SyncConfig): Configuration object for syncing. See tune.SyncConfig. export_formats (list): List of formats that exported at the end of the experiment. Default is None. max_failures (int): Try to recover a trial at least this many times. Ray will recover from the latest checkpoint if present. Setting to -1 will lead to infinite recovery retries. Setting to 0 will disable retries. Defaults to 3. fail_fast (bool | str): Whether to fail upon the first error. If fail_fast='raise' provided, Tune will automatically raise the exception received by the Trainable. fail_fast='raise' can easily leak resources and should be used with caution (it is best used with `ray.init(local_mode=True)`). restore (str): Path to checkpoint. Only makes sense to set if running 1 trial. Defaults to None. server_port (int): Port number for launching TuneServer. resume (str|bool): One of "LOCAL", "REMOTE", "PROMPT", "ERRORED_ONLY", or bool. LOCAL/True restores the checkpoint from the local_checkpoint_dir, determined by `name` and `local_dir`. REMOTE restores the checkpoint from remote_checkpoint_dir. PROMPT provides CLI feedback. False forces a new experiment. ERRORED_ONLY resets and reruns ERRORED trials upon resume - previous trial artifacts will be left untouched. If resume is set but checkpoint does not exist, ValueError will be thrown. reuse_actors (bool): Whether to reuse actors between different trials when possible. This can drastically speed up experiments that start and stop actors often (e.g., PBT in time-multiplexing mode). This requires trials to have the same resource requirements. trial_executor (TrialExecutor): Manage the execution of trials. raise_on_failed_trial (bool): Raise TuneError if there exists failed trial (of ERROR state) when the experiments complete. Returns: ExperimentAnalysis: Object for experiment analysis. Raises: TuneError: Any trials failed and `raise_on_failed_trial` is True. """ if global_checkpoint_period: raise ValueError("global_checkpoint_period is deprecated. Set env var " "'TUNE_GLOBAL_CHECKPOINT_S' instead.") if queue_trials: raise ValueError( "queue_trials is deprecated. " "Set env var 'TUNE_DISABLE_QUEUE_TRIALS=1' instead to " "disable queuing behavior.") if ray_auto_init: raise ValueError("ray_auto_init is deprecated. " "Set env var 'TUNE_DISABLE_AUTO_INIT=1' instead or " "call 'ray.init' before calling 'tune.run'.") if with_server: raise ValueError( "with_server is deprecated. It is now enabled by default " "if 'server_port' is not None.") if sync_on_checkpoint or sync_to_cloud or sync_to_driver or upload_dir: raise ValueError( "sync_on_checkpoint / sync_to_cloud / sync_to_driver / " "upload_dir must now be set via `tune.run(" "sync_config=SyncConfig(...)`. See `ray.tune.SyncConfig` for " "more details.") if mode and mode not in ["min", "max"]: raise ValueError( "The `mode` parameter passed to `tune.run()` has to be one of " "['min', 'max']") config = config or {} sync_config = sync_config or SyncConfig() set_sync_periods(sync_config) trial_executor = trial_executor or RayTrialExecutor( reuse_actors=reuse_actors) if isinstance(run_or_experiment, list): experiments = run_or_experiment else: experiments = [run_or_experiment] for i, exp in enumerate(experiments): if not isinstance(exp, Experiment): experiments[i] = Experiment( name=name, run=exp, stop=stop, time_budget_s=time_budget_s, config=config, resources_per_trial=resources_per_trial, num_samples=num_samples, local_dir=local_dir, upload_dir=sync_config.upload_dir, sync_to_driver=sync_config.sync_to_driver, trial_name_creator=trial_name_creator, trial_dirname_creator=trial_dirname_creator, loggers=loggers, log_to_file=log_to_file, checkpoint_freq=checkpoint_freq, checkpoint_at_end=checkpoint_at_end, sync_on_checkpoint=sync_config.sync_on_checkpoint, keep_checkpoints_num=keep_checkpoints_num, checkpoint_score_attr=checkpoint_score_attr, export_formats=export_formats, max_failures=max_failures, restore=restore) else: logger.debug("Ignoring some parameters passed into tune.run.") if sync_config.sync_to_cloud: for exp in experiments: assert exp.remote_checkpoint_dir, ( "Need `upload_dir` if `sync_to_cloud` given.") if fail_fast and max_failures != 0: raise ValueError("max_failures must be 0 if fail_fast=True.") if issubclass(type(search_alg), Searcher): search_alg = SearchGenerator(search_alg) if not search_alg: search_alg = BasicVariantGenerator() if config and not search_alg.set_search_properties(metric, mode, config): if has_unresolved_values(config): raise ValueError( "You passed a `config` parameter to `tune.run()` with " "unresolved parameters, but the search algorithm was already " "instantiated with a search space. Make sure that `config` " "does not contain any more parameter definitions - include " "them in the search algorithm's search space if necessary.") scheduler = scheduler or FIFOScheduler() if not scheduler.set_search_properties(metric, mode): raise ValueError( "You passed a `metric` or `mode` argument to `tune.run()`, but " "the scheduler you are using was already instantiated with their " "own `metric` and `mode` parameters. Either remove the arguments " "from your scheduler or from your call to `tune.run()`") runner = TrialRunner( search_alg=search_alg, scheduler=scheduler, local_checkpoint_dir=experiments[0].checkpoint_dir, remote_checkpoint_dir=experiments[0].remote_checkpoint_dir, sync_to_cloud=sync_config.sync_to_cloud, stopper=experiments[0].stopper, resume=resume, server_port=server_port, verbose=bool(verbose > 1), fail_fast=fail_fast, trial_executor=trial_executor) if not runner.resumed: for exp in experiments: search_alg.add_configurations([exp]) else: logger.info("TrialRunner resumed, ignoring new add_experiment.") if progress_reporter is None: if IS_NOTEBOOK: progress_reporter = JupyterNotebookReporter(overwrite=verbose < 2) else: progress_reporter = CLIReporter() # User Warning for GPUs if trial_executor.has_gpus(): if isinstance(resources_per_trial, dict) and "gpu" in resources_per_trial: # "gpu" is manually set. pass elif _check_default_resources_override(experiments[0].run_identifier): # "default_resources" is manually overriden. pass else: logger.warning("Tune detects GPUs, but no trials are using GPUs. " "To enable trials to use GPUs, set " "tune.run(resources_per_trial={'gpu': 1}...) " "which allows Tune to expose 1 GPU to each trial. " "You can also override " "`Trainable.default_resource_request` if using the " "Trainable API.") while not runner.is_finished(): runner.step() if verbose: _report_progress(runner, progress_reporter) try: runner.checkpoint(force=True) except Exception as e: logger.warning(f"Trial Runner checkpointing failed: {str(e)}") if verbose: _report_progress(runner, progress_reporter, done=True) wait_for_sync() runner.cleanup_trials() incomplete_trials = [] for trial in runner.get_trials(): if trial.status != Trial.TERMINATED: incomplete_trials += [trial] if incomplete_trials: if raise_on_failed_trial: raise TuneError("Trials did not complete", incomplete_trials) else: logger.error("Trials did not complete: %s", incomplete_trials) trials = runner.get_trials() return ExperimentAnalysis(runner.checkpoint_file, trials=trials, default_metric=metric, default_mode=mode)
def get_next_executor_event(self, live_trials: Set[Trial], next_trial_exists: bool) -> ExecutorEvent: """Get the next executor event to be processed in TrialRunner. In case there are multiple events available for handling, the next event is determined by the following priority: 1. if there is `next_trial_exists`, and if there is cached resources to use, PG_READY is emitted. 2. if there is `next_trial_exists` and there is no cached resources to use, wait on pg future and randomized other futures. If multiple futures are ready, pg future will take priority to be handled first. 3. if there is no `next_trial_exists`, wait on just randomized other futures. An example of #3 would be synchronous hyperband. Although there are pgs ready, the scheduler is holding back scheduling new trials since the whole band of trials is waiting for the slowest trial to finish. In this case, we prioritize handling training result to avoid deadlock situation. This is a blocking wait with a timeout (specified with env var). The reason for the timeout is we still want to print status info periodically in TrialRunner for better user experience. The handle of `ExecutorEvent.STOP_RESULT` is purely internal to RayTrialExecutor itself. All the other future results are handled by TrialRunner. In the future we may want to do most of the handle of `ExecutorEvent.RESTORE_RESULT` and `SAVING_RESULT` in RayTrialExecutor itself and only notify TrialRunner to invoke corresponding callbacks. This view is more consistent with our goal of TrialRunner responsible for external facing Trial state transition, while RayTrialExecutor responsible for internal facing transitions, namely, `is_saving`, `is_restoring` etc. Also you may notice that the boundary between RayTrialExecutor and PlacementGroupManager right now is really blurry. This will be improved once we move to an ActorPool abstraction. `next_trial_exists` means that there is a trial to run - prioritize returning PG_READY in this case. """ # First update status of staged placement groups self._stage_and_update_status(live_trials) while True: ################################################################### # when next_trial_exists and there are cached resources ################################################################### # There could be existing PGs from either `self._cached_actor_pg` # or from `self._pg_manager._ready`. If so and if there is indeed # a next trial to run, we return `PG_READY` future for trial # runner. The next trial can then be scheduled on this PG. if next_trial_exists: if len(self._cached_actor_pg) > 0: return ExecutorEvent(ExecutorEventType.PG_READY) # TODO(xwjiang): Expose proper API when we decide to do # ActorPool abstraction. if any(len(r) > 0 for r in self._pg_manager._ready.values()): return ExecutorEvent(ExecutorEventType.PG_READY) ################################################################### # Prepare for futures to wait ################################################################### futures_to_wait = list(self._futures.keys()) random.shuffle(futures_to_wait) if next_trial_exists: # Only wait for pg explicitly if there is next trial to run. # In which case, handling PG_READY triumphs handling other events. # Since we want to place pending trial ASAP. futures_to_wait = (self._pg_manager.get_staging_future_list() + futures_to_wait) logger.debug(f"get_next_executor_event before wait with futures " f"{futures_to_wait} and " f"next_trial_exists={next_trial_exists}") ready_futures, _ = ray.wait(futures_to_wait, num_returns=1, timeout=self._get_next_event_wait) ################################################################### # Dealing with no future returned case. ################################################################### if len(ready_futures) == 0: if len(self._futures) == 0: # No running trial and timing out with wait, could be we may # have insufficient cluster resources that makes tune run # infeasible. # TODO: Move InsufficientResourceManager's logic # to TrialExecutor. It is not Runner's responsibility! return ExecutorEvent( ExecutorEventType.NO_RUNNING_TRIAL_TIMEOUT) else: # Training simply takes long time, yield the control back to main # event loop to print progress info etc. return ExecutorEvent(ExecutorEventType.YIELD) ################################################################### # If there is future returned. ################################################################### assert len(ready_futures) == 1 ready_future = ready_futures[0] ################################################################### # If it is a PG_READY event. ################################################################### if ready_future not in self._futures.keys(): self._pg_manager.handle_ready_future(ready_future) return ExecutorEvent(ExecutorEventType.PG_READY) ################################################################### # non PG_READY event ################################################################### result_type, trial_or_pg = self._futures.pop(ready_future) if result_type == ExecutorEventType.STOP_RESULT: pg = trial_or_pg post_stop_cleanup(ready_future, pg) else: trial = trial_or_pg assert isinstance(trial, Trial) try: future_result = ray.get(ready_future) # For local mode if isinstance(future_result, _LocalWrapper): future_result = future_result.unwrap() if result_type in ( ExecutorEventType.TRAINING_RESULT, ExecutorEventType.SAVING_RESULT, ExecutorEventType.RESTORING_RESULT, ): logger.debug( f"Returning [{result_type}] for trial {trial}") return ExecutorEvent( result_type, trial, result={ ExecutorEvent.KEY_FUTURE_RESULT: future_result }, ) else: raise TuneError( f"Unexpected future type - [{result_type}]") except RayTaskError as e: return ExecutorEvent( ExecutorEventType.ERROR, trial, result={ ExecutorEvent.KEY_EXCEPTION: e.as_instanceof_cause() }, ) except Exception: return ExecutorEvent( ExecutorEventType.ERROR, trial, result={ ExecutorEvent.KEY_EXCEPTION: TuneGetNextExecutorEventError( traceback.format_exc()) }, )