def testExperimentTagTruncation(self): ray.init(num_cpus=2) def train(config, reporter): reporter(timesteps_total=1) trial_executor = RayTrialExecutor() register_trainable("f1", train) experiments = { "foo": { "run": "f1", "config": { "a" * 50: tune.sample_from(lambda spec: 5.0 / 7), "b" * 50: tune.sample_from(lambda spec: "long" * 40), }, } } for name, spec in experiments.items(): trial_generator = BasicVariantGenerator() trial_generator.add_configurations({name: spec}) while not trial_generator.is_finished(): trial = trial_generator.next_trial() if not trial: break trial_executor.start_trial(trial) self.assertLessEqual(len(os.path.basename(trial.logdir)), 200) trial_executor.stop_trial(trial)
def testQueueFilling(self): os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "1" ray.init(num_cpus=4) def f1(config): for i in range(10): yield i tune.register_trainable("f1", f1) search_alg = BasicVariantGenerator() search_alg.add_configurations( { "foo": { "run": "f1", "num_samples": 100, "config": { "a": tune.sample_from(lambda spec: 5.0 / 7), "b": tune.sample_from(lambda spec: "long" * 40), }, "resources_per_trial": {"cpu": 2}, } } ) runner = TrialRunner(search_alg=search_alg) runner.step() runner.step() runner.step() self.assertEqual(len(runner._trials), 3) runner.step() self.assertEqual(len(runner._trials), 3) self.assertEqual(runner._trials[0].status, Trial.RUNNING) self.assertEqual(runner._trials[1].status, Trial.RUNNING) self.assertEqual(runner._trials[2].status, Trial.PENDING)
def test_migration_checkpoint_removal(start_connected_emptyhead_cluster, tmpdir, durable): """Test checks that trial restarts if checkpoint is lost w/ node fail.""" cluster = start_connected_emptyhead_cluster node = cluster.add_node(num_cpus=1) cluster.wait_for_nodes() if durable: upload_dir = "file://" + str(tmpdir) syncer_callback = SyncerCallback() else: upload_dir = None syncer_callback = custom_driver_logdir_callback(str(tmpdir)) runner = TrialRunner(BasicVariantGenerator(), callbacks=[syncer_callback]) kwargs = { "stopping_criterion": { "training_iteration": 4 }, "checkpoint_freq": 2, "max_failures": 2, "remote_checkpoint_dir": upload_dir, } # Test recovery of trial that has been checkpointed t1 = Trial("__fake", **kwargs) runner.add_trial(t1) # Start trial, process result (x2), process save while not t1.has_checkpoint(): runner.step() cluster.add_node(num_cpus=1) cluster.remove_node(node) cluster.wait_for_nodes() # Remove checkpoint on "remote" node shutil.rmtree(os.path.dirname(t1.checkpoint.dir_or_data)) if not durable: # Recover from driver file t1.checkpoint.dir_or_data = os.path.join( tmpdir, t1.relative_logdir, os.path.relpath(t1.checkpoint.dir_or_data, t1.logdir), ) while not runner.is_finished(): runner.step() assert t1.status == Trial.TERMINATED, runner.debug_string()
def generate_trials(spec, name): suggester = BasicVariantGenerator() suggester.add_configurations({name: spec}) trials = [] while not suggester.is_finished(): trial = suggester.next_trial() if trial: trials.append(trial) else: break return trials
def test_trial_requeue(start_connected_emptyhead_cluster, tmpdir, durable): """Removing a node in full cluster causes Trial to be requeued.""" os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "1" cluster = start_connected_emptyhead_cluster node = cluster.add_node(num_cpus=1) cluster.wait_for_nodes() if durable: upload_dir = "file://" + str(tmpdir) syncer_callback = SyncerCallback() else: upload_dir = None syncer_callback = custom_driver_logdir_callback(str(tmpdir)) runner = TrialRunner(BasicVariantGenerator(), callbacks=[syncer_callback]) # noqa kwargs = { "stopping_criterion": { "training_iteration": 5 }, "checkpoint_freq": 1, "max_failures": 1, "remote_checkpoint_dir": upload_dir, } trials = [Trial("__fake", **kwargs), Trial("__fake", **kwargs)] for t in trials: runner.add_trial(t) runner.step() # Start trial runner.step() # Process result, dispatch save runner.step() # Process save running_trials = _get_running_trials(runner) assert len(running_trials) == 1 assert _check_trial_running(running_trials[0]) cluster.remove_node(node) cluster.wait_for_nodes() time.sleep(0.1) # Sleep so that next step() refreshes cluster resources runner.step() # Process result, dispatch save runner.step() # Process save (detect error), requeue trial assert all(t.status == Trial.PENDING for t in trials), runner.debug_string()
def _add_trials(self, name, spec): """Add trial by invoking TrialRunner.""" resource = {} resource["trials"] = [] trial_generator = BasicVariantGenerator() trial_generator.add_configurations({name: spec}) while not trial_generator.is_finished(): trial = trial_generator.next_trial() if not trial: break runner.add_trial(trial) resource["trials"].append(self._trial_info(trial)) return resource
def test_trial_processed_after_node_failure(start_connected_emptyhead_cluster): """Tests that Tune processes a trial as failed if its node died.""" cluster = start_connected_emptyhead_cluster node = cluster.add_node(num_cpus=1) cluster.wait_for_nodes() runner = TrialRunner(BasicVariantGenerator()) mock_process_failure = MagicMock(side_effect=runner._process_trial_failure) runner._process_trial_failure = mock_process_failure runner.add_trial(Trial("__fake")) runner.step() runner.step() assert not mock_process_failure.called cluster.remove_node(node) runner.step() if not mock_process_failure.called: runner.step() assert mock_process_failure.called
def test_remove_node_before_result(start_connected_emptyhead_cluster): """Tune continues when node is removed before trial returns.""" cluster = start_connected_emptyhead_cluster node = cluster.add_node(num_cpus=1) cluster.wait_for_nodes() runner = TrialRunner(BasicVariantGenerator()) kwargs = { "stopping_criterion": { "training_iteration": 3 }, "checkpoint_freq": 2, "max_failures": 2, } trial = Trial("__fake", **kwargs) runner.add_trial(trial) runner.step() # Start trial, call _train once running_trials = _get_running_trials(runner) assert len(running_trials) == 1 assert _check_trial_running(running_trials[0]) assert not trial.has_reported_at_least_once assert trial.status == Trial.RUNNING cluster.remove_node(node) cluster.add_node(num_cpus=1) cluster.wait_for_nodes() assert ray.cluster_resources()["CPU"] == 1 # Process result: fetch data, invoke _train again runner.step() assert trial.last_result.get("training_iteration") == 1 # Process result: discover failure, recover, _train (from scratch) while trial.status != Trial.TERMINATED: runner.step() assert trial.last_result.get("training_iteration") > 1 with pytest.raises(TuneError): runner.step()
def test_counting_resources(start_connected_cluster): """Tests that Tune accounting is consistent with actual cluster.""" cluster = start_connected_cluster nodes = [] assert ray.cluster_resources()["CPU"] == 1 runner = TrialRunner(BasicVariantGenerator()) kwargs = {"stopping_criterion": {"training_iteration": 10}} trials = [Trial("__fake", **kwargs), Trial("__fake", **kwargs)] for t in trials: runner.add_trial(t) runner.step() running_trials = _get_running_trials(runner) assert len(running_trials) == 1 assert _check_trial_running(running_trials[0]) assert ray.available_resources().get("CPU", 0) == 0 nodes += [cluster.add_node(num_cpus=1)] cluster.wait_for_nodes() assert ray.cluster_resources()["CPU"] == 2 cluster.remove_node(nodes.pop()) cluster.wait_for_nodes() assert ray.cluster_resources()["CPU"] == 1 runner.step() # Only 1 trial can be running due to resource limitation. assert sum(t.status == Trial.RUNNING for t in runner.get_trials()) == 1 for i in range(5): nodes += [cluster.add_node(num_cpus=1)] cluster.wait_for_nodes() assert ray.cluster_resources()["CPU"] == 6 # This is to make sure that pg is ready for the previous pending trial, # so that when runner.step() is called next, the trial can be started in # the same event loop. time.sleep(5) runner.step() assert sum(t.status == Trial.RUNNING for t in runner.get_trials()) == 2
def testBasicVariantLimiter(self): search_alg = BasicVariantGenerator(max_concurrent=2) experiment_spec = { "run": "__fake", "num_samples": 5, "stop": { "training_iteration": 1 }, } search_alg.add_configurations({"test": experiment_spec}) trial1 = search_alg.next_trial() self.assertTrue(trial1) trial2 = search_alg.next_trial() self.assertTrue(trial2) # Returns None because of limiting trial3 = search_alg.next_trial() self.assertFalse(trial3) # Finish trial, now trial 3 should be created search_alg.on_trial_complete(trial1.trial_id, None, False) trial3 = search_alg.next_trial() self.assertTrue(trial3) trial4 = search_alg.next_trial() self.assertFalse(trial4) search_alg.on_trial_complete(trial2.trial_id, None, False) search_alg.on_trial_complete(trial3.trial_id, None, False) trial4 = search_alg.next_trial() self.assertTrue(trial4) trial5 = search_alg.next_trial() self.assertTrue(trial5) search_alg.on_trial_complete(trial4.trial_id, None, False) # Should also be None because search is finished trial6 = search_alg.next_trial() self.assertFalse(trial6)
def test_trial_migration(start_connected_emptyhead_cluster, tmpdir, durable): """Removing a node while cluster has space should migrate trial. The trial state should also be consistent with the checkpoint. """ cluster = start_connected_emptyhead_cluster node = cluster.add_node(num_cpus=1) cluster.wait_for_nodes() if durable: upload_dir = "file://" + str(tmpdir) syncer_callback = SyncerCallback() else: upload_dir = None syncer_callback = custom_driver_logdir_callback(str(tmpdir)) runner = TrialRunner(BasicVariantGenerator(), callbacks=[syncer_callback]) kwargs = { "stopping_criterion": { "training_iteration": 4 }, "checkpoint_freq": 2, "max_failures": 2, "remote_checkpoint_dir": upload_dir, } # Test recovery of trial that hasn't been checkpointed t = Trial("__fake", **kwargs) runner.add_trial(t) runner.step() # Start trial runner.step() # Process result assert t.last_result node2 = cluster.add_node(num_cpus=1) cluster.remove_node(node) cluster.wait_for_nodes() # TODO(ujvl): Node failure does not propagate until a step after it # actually should. This is possibly a problem with `Cluster`. runner.step() runner.step() # Recovery step # TODO(rliaw): This assertion is not critical but will not pass # because checkpoint handling is messy and should be refactored # rather than hotfixed. # assert t.last_result is None, "Trial result not restored correctly." # Process result (x2), process save, process result (x2), process save while not runner.is_finished(): runner.step() assert t.status == Trial.TERMINATED, runner.debug_string() # Test recovery of trial that has been checkpointed t2 = Trial("__fake", **kwargs) runner.add_trial(t2) # Start trial, process result (x2), process save while not t2.has_checkpoint(): runner.step() node3 = cluster.add_node(num_cpus=1) cluster.remove_node(node2) cluster.wait_for_nodes() while not runner.is_finished(): runner.step() assert t2.status == Trial.TERMINATED, runner.debug_string() # Test recovery of trial that won't be checkpointed kwargs = { "stopping_criterion": { "training_iteration": 3 }, "remote_checkpoint_dir": upload_dir, } t3 = Trial("__fake", **kwargs) runner.add_trial(t3) runner.step() # Start trial runner.step() # Process result 1 cluster.add_node(num_cpus=1) cluster.remove_node(node3) cluster.wait_for_nodes() while not runner.is_finished(): runner.step() assert t3.status == Trial.ERROR, runner.debug_string() with pytest.raises(TuneError): runner.step()
def run( run_or_experiment: Union[str, Callable, Type], name: Optional[str] = None, metric: Optional[str] = None, mode: Optional[str] = None, stop: Optional[Union[Mapping, Stopper, Callable[[str, Mapping], bool]]] = None, time_budget_s: Optional[Union[int, float, datetime.timedelta]] = None, config: Optional[Dict[str, Any]] = None, resources_per_trial: Union[None, Mapping[str, Union[float, int, Mapping]], PlacementGroupFactory] = None, num_samples: int = 1, local_dir: Optional[str] = None, search_alg: Optional[Union[Searcher, SearchAlgorithm, str]] = None, scheduler: Optional[Union[TrialScheduler, str]] = None, keep_checkpoints_num: Optional[int] = None, checkpoint_score_attr: Optional[str] = None, checkpoint_freq: int = 0, checkpoint_at_end: bool = False, verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS, progress_reporter: Optional[ProgressReporter] = None, log_to_file: bool = False, trial_name_creator: Optional[Callable[[Trial], str]] = None, trial_dirname_creator: Optional[Callable[[Trial], str]] = None, sync_config: Optional[SyncConfig] = None, export_formats: Optional[Sequence] = None, max_failures: int = 0, fail_fast: bool = False, restore: Optional[str] = None, server_port: Optional[int] = None, resume: Union[bool, str] = False, reuse_actors: Optional[bool] = None, trial_executor: Optional[RayTrialExecutor] = None, raise_on_failed_trial: bool = True, callbacks: Optional[Sequence[Callback]] = None, max_concurrent_trials: Optional[int] = None, # == internal only == _experiment_checkpoint_dir: Optional[str] = None, _remote: Optional[bool] = None, ) -> ExperimentAnalysis: """Executes training. When a SIGINT signal is received (e.g. through Ctrl+C), the tuning run will gracefully shut down and checkpoint the latest experiment state. Sending SIGINT again (or SIGKILL/SIGTERM instead) will skip this step. Many aspects of Tune, such as the frequency of global checkpointing, maximum pending placement group trials and the path of the result directory be configured through environment variables. Refer to :ref:`tune-env-vars` for a list of environment variables available. Examples: .. code-block:: python # Run 10 trials (each trial is one instance of a Trainable). Tune runs # in parallel and automatically determines concurrency. tune.run(trainable, num_samples=10) # Run 1 trial, stop when trial has reached 10 iterations tune.run(my_trainable, stop={"training_iteration": 10}) # automatically retry failed trials up to 3 times tune.run(my_trainable, stop={"training_iteration": 10}, max_failures=3) # Run 1 trial, search over hyperparameters, stop after 10 iterations. space = {"lr": tune.uniform(0, 1), "momentum": tune.uniform(0, 1)} tune.run(my_trainable, config=space, stop={"training_iteration": 10}) # Resumes training if a previous machine crashed tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume=True) # Rerun ONLY failed trials after an experiment is finished. tune.run(my_trainable, config=space, local_dir=<path/to/dir>, resume="ERRORED_ONLY") Args: run_or_experiment: If function|class|str, this is the algorithm or model to train. This may refer to the name of a built-on algorithm (e.g. RLlib's DQN or PPO), a user-defined trainable function or class, or the string identifier of a trainable function or class registered in the tune registry. If Experiment, then Tune will execute training based on Experiment.spec. If you want to pass in a Python lambda, you will need to first register the function: ``tune.register_trainable("lambda_id", lambda x: ...)``. You can then use ``tune.run("lambda_id")``. metric: Metric to optimize. This metric should be reported with `tune.report()`. If set, will be passed to the search algorithm and scheduler. mode: Must be one of [min, max]. Determines whether objective is minimizing or maximizing the metric attribute. If set, will be passed to the search algorithm and scheduler. name: Name of experiment. stop: Stopping criteria. If dict, the keys may be any field in the return result of 'train()', whichever is reached first. If function, it must take (trial_id, result) as arguments and return a boolean (True if trial should be stopped, False otherwise). This can also be a subclass of ``ray.tune.Stopper``, which allows users to implement custom experiment-wide stopping (i.e., stopping an entire Tune run based on some time constraint). time_budget_s: Global time budget in seconds after which all trials are stopped. Can also be a ``datetime.timedelta`` object. config: Algorithm-specific configuration for Tune variant generation (e.g. env, hyperparams). Defaults to empty dict. Custom search algorithms may ignore this. resources_per_trial: Machine resources to allocate per trial, e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be assigned unless you specify them here. Defaults to 1 CPU and 0 GPUs in ``Trainable.default_resource_request()``. This can also be a PlacementGroupFactory object wrapping arguments to create a per-trial placement group. num_samples: Number of times to sample from the hyperparameter space. Defaults to 1. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times. If this is -1, (virtually) infinite samples are generated until a stopping condition is met. local_dir: Local dir to save training results to. Defaults to ``~/ray_results``. search_alg: Search algorithm for optimization. You can also use the name of the algorithm. scheduler: Scheduler for executing the experiment. Choose among FIFO (default), MedianStopping, AsyncHyperBand, HyperBand and PopulationBasedTraining. Refer to ray.tune.schedulers for more options. You can also use the name of the scheduler. keep_checkpoints_num: Number of checkpoints to keep. A value of `None` keeps all checkpoints. Defaults to `None`. If set, need to provide `checkpoint_score_attr`. checkpoint_score_attr: Specifies by which attribute to rank the best checkpoint. Default is increasing order. If attribute starts with `min-` it will rank attribute in decreasing order, i.e. `min-validation_loss`. checkpoint_freq: How many training iterations between checkpoints. A value of 0 (default) disables checkpointing. This has no effect when using the Functional Training API. checkpoint_at_end: Whether to checkpoint at the end of the experiment regardless of the checkpoint_freq. Default is False. This has no effect when using the Functional Training API. verbose: 0, 1, 2, or 3. Verbosity mode. 0 = silent, 1 = only status updates, 2 = status and brief trial results, 3 = status and detailed trial results. Defaults to 3. progress_reporter: Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in a Jupyter notebook. log_to_file: Log stdout and stderr to files in Tune's trial directories. If this is `False` (default), no files are written. If `true`, outputs are written to `trialdir/stdout` and `trialdir/stderr`, respectively. If this is a single string, this is interpreted as a file relative to the trialdir, to which both streams are written. If this is a Sequence (e.g. a Tuple), it has to have length 2 and the elements indicate the files to which stdout and stderr are written, respectively. trial_name_creator: Optional function for generating the trial string representation. trial_dirname_creator: Function for generating the trial dirname. This function should take in a Trial object and return a string representing the name of the directory. The return value cannot be a path. sync_config: Configuration object for syncing. See tune.SyncConfig. export_formats: List of formats that exported at the end of the experiment. Default is None. max_failures: Try to recover a trial at least this many times. Ray will recover from the latest checkpoint if present. Setting to -1 will lead to infinite recovery retries. Setting to 0 will disable retries. Defaults to 0. fail_fast: Whether to fail upon the first error. If fail_fast='raise' provided, Tune will automatically raise the exception received by the Trainable. fail_fast='raise' can easily leak resources and should be used with caution (it is best used with `ray.init(local_mode=True)`). restore: Path to checkpoint. Only makes sense to set if running 1 trial. Defaults to None. server_port: Port number for launching TuneServer. resume: One of "LOCAL", "REMOTE", "PROMPT", "ERRORED_ONLY", "AUTO", or bool. "LOCAL"/True restores the checkpoint from the local experiment directory, determined by ``name`` and ``local_dir``. "REMOTE" restores the checkpoint from ``upload_dir`` (as passed to ``sync_config``). "PROMPT" provides the CLI feedback. False forces a new experiment. "ERRORED_ONLY" resets and reruns errored trials upon resume - previous trial artifacts will be left untouched. "AUTO" will attempt to resume from a checkpoint and otherwise start a new experiment. If resume is set but checkpoint does not exist, ValueError will be thrown. reuse_actors: Whether to reuse actors between different trials when possible. This can drastically speed up experiments that start and stop actors often (e.g., PBT in time-multiplexing mode). This requires trials to have the same resource requirements. Defaults to ``True`` for function trainables and ``False`` for class and registered trainables. trial_executor: Manage the execution of trials. raise_on_failed_trial: Raise TuneError if there exists failed trial (of ERROR state) when the experiments complete. callbacks: List of callbacks that will be called at different times in the training loop. Must be instances of the ``ray.tune.callback.Callback`` class. If not passed, `LoggerCallback` and `SyncerCallback` callbacks are automatically added. max_concurrent_trials: Maximum number of trials to run concurrently. Must be non-negative. If None or 0, no limit will be applied. This is achieved by wrapping the ``search_alg`` in a :class:`ConcurrencyLimiter`, and thus setting this argument will raise an exception if the ``search_alg`` is already a :class:`ConcurrencyLimiter`. Defaults to None. _remote: Whether to run the Tune driver in a remote function. This is disabled automatically if a custom trial executor is passed in. This is enabled by default in Ray client mode. Returns: ExperimentAnalysis: Object for experiment analysis. Raises: TuneError: Any trials failed and `raise_on_failed_trial` is True. """ # NO CODE IS TO BE ADDED ABOVE THIS COMMENT # remote_run_kwargs must be defined before any other # code is ran to ensure that at this point, # `locals()` is equal to args and kwargs remote_run_kwargs = locals().copy() remote_run_kwargs.pop("_remote") if _remote is None: _remote = ray.util.client.ray.is_connected() if _remote is True and trial_executor: raise ValueError("cannot use custom trial executor") if not trial_executor or isinstance(trial_executor, RayTrialExecutor): _ray_auto_init() if _remote: remote_run = ray.remote(num_cpus=0)(run) # Make sure tune.run is called on the sever node. remote_run = force_on_current_node(remote_run) set_verbosity(verbose) progress_reporter = progress_reporter or detect_reporter() # JupyterNotebooks don't work with remote tune runs out of the box # (e.g. via Ray client) as they don't have access to the main # process stdout. So we introduce a queue here that accepts # strings, which will then be displayed on the driver side. if isinstance(progress_reporter, RemoteReporterMixin): string_queue = Queue(actor_options={ "num_cpus": 0, **force_on_current_node(None) }) progress_reporter.output_queue = string_queue def get_next_queue_item(): try: return string_queue.get(block=False) except Empty: return None else: # If we don't need a queue, use this dummy get fn instead of # scheduling an unneeded actor def get_next_queue_item(): return None def _handle_string_queue(): string_item = get_next_queue_item() while string_item is not None: # This happens on the driver side progress_reporter.display(string_item) string_item = get_next_queue_item() # Override with detected progress reporter remote_run_kwargs["progress_reporter"] = progress_reporter remote_future = remote_run.remote(_remote=False, **remote_run_kwargs) # ray.wait(...)[1] returns futures that are not ready, yet while ray.wait([remote_future], timeout=0.2)[1]: # Check if we have items to execute _handle_string_queue() # Handle queue one last time _handle_string_queue() return ray.get(remote_future) del remote_run_kwargs all_start = time.time() if mode and mode not in ["min", "max"]: raise ValueError( "The `mode` parameter passed to `tune.run()` has to be one of " "['min', 'max']") set_verbosity(verbose) config = config or {} sync_config = sync_config or SyncConfig() _validate_upload_dir(sync_config) if num_samples == -1: num_samples = sys.maxsize result_buffer_length = None # Create scheduler here as we need access to some of its properties if isinstance(scheduler, str): # importing at top level causes a recursive dependency from ray.tune.schedulers import create_scheduler scheduler = create_scheduler(scheduler) scheduler = scheduler or FIFOScheduler() if not scheduler.supports_buffered_results: # Result buffering with e.g. a Hyperband scheduler is a bad idea, as # hyperband tries to stop trials when processing brackets. With result # buffering, we might trigger this multiple times when evaluating # a single trial, which leads to unexpected behavior. env_result_buffer_length = os.getenv("TUNE_RESULT_BUFFER_LENGTH", "") if env_result_buffer_length: warnings.warn( f"You are using a {type(scheduler)} scheduler, but " f"TUNE_RESULT_BUFFER_LENGTH is set " f"({env_result_buffer_length}). This can lead to undesired " f"and faulty behavior, so the buffer length was forcibly set " f"to 1 instead.") result_buffer_length = 1 # If reuse_actors is unset, default to False for string and class trainables, # and default to True for everything else (i.e. function trainables) if reuse_actors is None: trainable = (run_or_experiment.run_identifier if isinstance( run_or_experiment, Experiment) else run_or_experiment) reuse_actors = ( # Only default to True for function trainables that meet certain conditions is_function_trainable(trainable) and not ( # Changing resources requires restarting actors scheduler and isinstance(scheduler, ResourceChangingScheduler)) and not ( # If GPUs are requested we could run into problems with device memory _check_gpus_in_resources(resources_per_trial)) and not ( # If the resource request is overridden, we don't know if GPUs # will be requested, yet, so default to False _check_default_resources_override(trainable))) if (isinstance(scheduler, (PopulationBasedTraining, PopulationBasedTrainingReplay)) and not reuse_actors): warnings.warn( "Consider boosting PBT performance by enabling `reuse_actors` as " "well as implementing `reset_config` for Trainable.") trial_executor = trial_executor or RayTrialExecutor( reuse_actors=reuse_actors, result_buffer_length=result_buffer_length) if isinstance(run_or_experiment, list): experiments = run_or_experiment else: experiments = [run_or_experiment] for i, exp in enumerate(experiments): if not isinstance(exp, Experiment): experiments[i] = Experiment( name=name, run=exp, stop=stop, time_budget_s=time_budget_s, config=config, resources_per_trial=resources_per_trial, num_samples=num_samples, local_dir=local_dir, _experiment_checkpoint_dir=_experiment_checkpoint_dir, sync_config=sync_config, trial_name_creator=trial_name_creator, trial_dirname_creator=trial_dirname_creator, log_to_file=log_to_file, checkpoint_freq=checkpoint_freq, checkpoint_at_end=checkpoint_at_end, keep_checkpoints_num=keep_checkpoints_num, checkpoint_score_attr=checkpoint_score_attr, export_formats=export_formats, max_failures=max_failures, restore=restore, ) else: logger.debug("Ignoring some parameters passed into tune.run.") if fail_fast and max_failures != 0: raise ValueError("max_failures must be 0 if fail_fast=True.") if isinstance(search_alg, str): search_alg = create_searcher(search_alg) # if local_mode=True is set during ray.init(). is_local_mode = ray._private.worker._mode( ) == ray._private.worker.LOCAL_MODE if is_local_mode: max_concurrent_trials = 1 if not search_alg: search_alg = BasicVariantGenerator( max_concurrent=max_concurrent_trials or 0) elif max_concurrent_trials or is_local_mode: if isinstance(search_alg, ConcurrencyLimiter): if not is_local_mode: if search_alg.max_concurrent != max_concurrent_trials: raise ValueError( "You have specified `max_concurrent_trials=" f"{max_concurrent_trials}`, but the `search_alg` is " "already a `ConcurrencyLimiter` with `max_concurrent=" f"{search_alg.max_concurrent}. FIX THIS by setting " "`max_concurrent_trials=None`.") else: logger.warning( "You have specified `max_concurrent_trials=" f"{max_concurrent_trials}`, but the `search_alg` is " "already a `ConcurrencyLimiter`. " "`max_concurrent_trials` will be ignored.") else: if max_concurrent_trials < 1: raise ValueError( "`max_concurrent_trials` must be greater or equal than 1, " f"got {max_concurrent_trials}.") if isinstance(search_alg, Searcher): search_alg = ConcurrencyLimiter( search_alg, max_concurrent=max_concurrent_trials) elif not is_local_mode: logger.warning( "You have passed a `SearchGenerator` instance as the " "`search_alg`, but `max_concurrent_trials` requires a " "`Searcher` instance`. `max_concurrent_trials` " "will be ignored.") if isinstance(search_alg, Searcher): search_alg = SearchGenerator(search_alg) if config and not searcher_set_search_props( search_alg.set_search_properties, metric, mode, config, **experiments[0].public_spec, ): if has_unresolved_values(config): raise ValueError( "You passed a `config` parameter to `tune.run()` with " "unresolved parameters, but the search algorithm was already " "instantiated with a search space. Make sure that `config` " "does not contain any more parameter definitions - include " "them in the search algorithm's search space if necessary.") if not scheduler_set_search_props(scheduler.set_search_properties, metric, mode, **experiments[0].public_spec): raise ValueError( "You passed a `metric` or `mode` argument to `tune.run()`, but " "the scheduler you are using was already instantiated with their " "own `metric` and `mode` parameters. Either remove the arguments " "from your scheduler or from your call to `tune.run()`") # Create syncer callbacks callbacks = create_default_callbacks(callbacks, sync_config, metric=metric) runner = TrialRunner( search_alg=search_alg, scheduler=scheduler, local_checkpoint_dir=experiments[0].checkpoint_dir, remote_checkpoint_dir=experiments[0].remote_checkpoint_dir, sync_config=sync_config, stopper=experiments[0].stopper, resume=resume, server_port=server_port, fail_fast=fail_fast, trial_executor=trial_executor, callbacks=callbacks, metric=metric, # Driver should only sync trial checkpoints if # checkpoints are not synced to cloud driver_sync_trial_checkpoints=not bool(sync_config.upload_dir), ) if not runner.resumed: for exp in experiments: search_alg.add_configurations([exp]) else: logger.info("TrialRunner resumed, ignoring new add_experiment but " "updating trial resources.") if resources_per_trial: runner.update_pending_trial_resources(resources_per_trial) # Calls setup on callbacks runner.setup_experiments(experiments=experiments, total_num_samples=search_alg.total_samples) # User Warning for GPUs if trial_executor.has_gpus(): if _check_gpus_in_resources(resources=resources_per_trial): # "gpu" is manually set. pass elif _check_default_resources_override(experiments[0].run_identifier): # "default_resources" is manually overridden. pass else: logger.warning("Tune detects GPUs, but no trials are using GPUs. " "To enable trials to use GPUs, set " "tune.run(resources_per_trial={'gpu': 1}...) " "which allows Tune to expose 1 GPU to each trial. " "You can also override " "`Trainable.default_resource_request` if using the " "Trainable API.") original_handler = signal.getsignal(signal.SIGINT) state = {"signal": None} def signal_interrupt_tune_run(sig: int, frame): logger.warning( "Stop signal received (e.g. via SIGINT/Ctrl+C), ending Ray Tune run. " "This will try to checkpoint the experiment state one last time. " "Press CTRL+C (or send SIGINT/SIGKILL/SIGTERM) " "to skip. ") state["signal"] = sig # Restore original signal handler to react to future SIGINT signals signal.signal(signal.SIGINT, original_handler) # We should only install the handler when it is safe to do so. # When tune.run() is called from worker thread, signal.signal will # fail. allow_signal_catching = True if threading.current_thread() != threading.main_thread(): allow_signal_catching = False if allow_signal_catching: if not int(os.getenv("TUNE_DISABLE_SIGINT_HANDLER", "0")): signal.signal(signal.SIGINT, signal_interrupt_tune_run) # Always register SIGUSR1 if available (not available e.g. on Windows) if hasattr(signal, "SIGUSR1"): signal.signal(signal.SIGUSR1, signal_interrupt_tune_run) progress_reporter = progress_reporter or detect_reporter() tune_start = time.time() progress_reporter.setup( start_time=tune_start, total_samples=search_alg.total_samples, metric=metric, mode=mode, ) while not runner.is_finished() and not state["signal"]: runner.step() if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter) tune_taken = time.time() - tune_start try: runner.checkpoint(force=True) except Exception as e: logger.warning(f"Trial Runner checkpointing failed: {str(e)}") if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter, done=True) # Wait for syncing to finish for callback in callbacks: if isinstance(callback, SyncerCallback): try: callback.wait_for_all() except TuneError as e: logger.error(e) runner.cleanup() incomplete_trials = [] for trial in runner.get_trials(): if trial.status != Trial.TERMINATED: incomplete_trials += [trial] if incomplete_trials: if raise_on_failed_trial and not state["signal"]: raise TuneError("Trials did not complete", incomplete_trials) else: logger.error("Trials did not complete: %s", incomplete_trials) all_taken = time.time() - all_start if has_verbosity(Verbosity.V1_EXPERIMENT): logger.info(f"Total run time: {all_taken:.2f} seconds " f"({tune_taken:.2f} seconds for the tuning loop).") if state["signal"]: logger.warning( "Experiment has been interrupted, but the most recent state was " "saved. You can continue running this experiment by passing " "`resume=True` to `tune.run()`") trials = runner.get_trials() return ExperimentAnalysis( runner.checkpoint_file, trials=trials, default_metric=metric, default_mode=mode, sync_config=sync_config, )