def _optimize( study: "optuna.Study", func: "optuna.study.ObjectiveFuncType", n_trials: Optional[int] = None, timeout: Optional[float] = None, n_jobs: int = 1, catch: Tuple[Type[Exception], ...] = (), callbacks: Optional[List[Callable[["optuna.Study", FrozenTrial], None]]] = None, gc_after_trial: bool = False, show_progress_bar: bool = False, ) -> None: if not isinstance(catch, tuple): raise TypeError( "The catch argument is of type '{}' but must be a tuple.".format(type(catch).__name__) ) if not study._optimize_lock.acquire(False): raise RuntimeError("Nested invocation of `Study.optimize` method isn't allowed.") # TODO(crcrpar): Make progress bar work when n_jobs != 1. progress_bar = pbar_module._ProgressBar(show_progress_bar and n_jobs == 1, n_trials, timeout) study._stop_flag = False try: if n_jobs == 1: _optimize_sequential( study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng=False, time_start=None, progress_bar=progress_bar, ) else: if show_progress_bar: warnings.warn("Progress bar only supports serial execution (`n_jobs=1`).") time_start = datetime.datetime.now() def _should_stop() -> bool: if study._stop_flag: return True if timeout is not None: # This is needed for mypy. t: float = timeout return (datetime.datetime.now() - time_start).total_seconds() > t return False if n_trials is not None: _iter = iter(range(n_trials)) else: _iter = iter(_should_stop, True) with Parallel(n_jobs=n_jobs, prefer="threads") as parallel: if not isinstance( parallel._backend, joblib.parallel.ThreadingBackend ) and isinstance(study._storage, storages.InMemoryStorage): warnings.warn( "The default storage cannot be shared by multiple processes. " "Please use an RDB (RDBStorage) when you use joblib for " "multi-processing. The usage of RDBStorage can be found in " "https://optuna.readthedocs.io/en/stable/tutorial/rdb.html.", UserWarning, ) parallel( delayed(_optimize_sequential)( study, func, 1, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng=True, time_start=time_start, progress_bar=None, ) for _ in _iter ) finally: study._optimize_lock.release() progress_bar.close()
def optimize( self, func, # type: ObjectiveFuncType n_trials=None, # type: Optional[int] timeout=None, # type: Optional[float] n_jobs=1, # type: int catch=(), # type: Union[Tuple[()], Tuple[Type[Exception]]] callbacks=None, # type: Optional[List[Callable[[Study, structs.FrozenTrial], None]]] gc_after_trial=True, # type: bool show_progress_bar=False, # type: bool ): # type: (...) -> None """Optimize an objective function. Optimization is done by choosing a suitable set of hyperparameter values from a given range. Uses a sampler which implements the task of value suggestion based on a specified distribution. The sampler is specified in :func:`~optuna.study.create_study` and the default choice for the sampler is TPE. See also :class:`~optuna.samplers.TPESampler` for more details on 'TPE'. Args: func: A callable that implements objective function. n_trials: The number of trials. If this argument is set to :obj:`None`, there is no limitation on the number of trials. If :obj:`timeout` is also set to :obj:`None`, the study continues to create trials until it receives a termination signal such as Ctrl+C or SIGTERM. timeout: Stop study after the given number of second(s). If this argument is set to :obj:`None`, the study is executed without time limitation. If :obj:`n_trials` is also set to :obj:`None`, the study continues to create trials until it receives a termination signal such as Ctrl+C or SIGTERM. n_jobs: The number of parallel jobs. If this argument is set to :obj:`-1`, the number is set to CPU count. catch: A study continues to run even when a trial raises one of the exceptions specified in this argument. Default is an empty tuple, i.e. the study will stop for any exception except for :class:`~optuna.exceptions.TrialPruned`. callbacks: List of callback functions that are invoked at the end of each trial. Each function must accept two parameters with the following types in this order: :class:`~optuna.study.Study` and :class:`~optuna.structs.FrozenTrial`. gc_after_trial: Flag to execute garbage collection at the end of each trial. By default, garbage collection is enabled, just in case. You can turn it off with this argument if memory is safely managed in your objective function. show_progress_bar: Flag to show progress bars or not. To disable progress bar, set this ``False``. Currently, progress bar is experimental feature and disabled when ``n_jobs`` :math:`\\ne 1`. """ if not isinstance(catch, tuple): raise TypeError( "The catch argument is of type '{}' but must be a tuple.". format(type(catch).__name__)) if not self._optimize_lock.acquire(False): raise RuntimeError( "Nested invocation of `Study.optimize` method isn't allowed.") # TODO(crcrpar): Make progress bar work when n_jobs != 1. self._progress_bar = pbar_module._ProgressBar( show_progress_bar and n_jobs == 1, n_trials, timeout) try: if n_jobs == 1: self._optimize_sequential(func, n_trials, timeout, catch, callbacks, gc_after_trial, None) else: if show_progress_bar: msg = "Progress bar only supports serial execution (`n_jobs=1`)." warnings.warn(msg) _logger.warning(msg) time_start = datetime.datetime.now() if n_trials is not None: _iter = iter(range(n_trials)) elif timeout is not None: # This is needed for mypy actual_timeout = timeout # type: float _iter = iter( lambda: (datetime.datetime.now() - time_start). total_seconds() > actual_timeout, True, ) else: # The following expression makes an iterator that never ends. _iter = iter(int, 1) with Parallel(n_jobs=n_jobs, prefer="threads") as parallel: if not isinstance( parallel._backend, joblib.parallel.ThreadingBackend) and isinstance( self._storage, storages.InMemoryStorage): msg = ( "The default storage cannot be shared by multiple processes. " "Please use an RDB (RDBStorage) when you use joblib for " "multi-processing. The usage of RDBStorage can be found in " "https://optuna.readthedocs.io/en/stable/tutorial/rdb.html." ) warnings.warn(msg, UserWarning) _logger.warning(msg) parallel( delayed(self._optimize_sequential) (func, 1, timeout, catch, callbacks, gc_after_trial, time_start) for _ in _iter) finally: self._optimize_lock.release() self._progress_bar.close() del self._progress_bar
def _optimize( study: "optuna.Study", func: "optuna.study.ObjectiveFuncType", n_trials: Optional[int] = None, timeout: Optional[float] = None, n_jobs: int = 1, catch: Tuple[Type[Exception], ...] = (), callbacks: Optional[List[Callable[["optuna.Study", FrozenTrial], None]]] = None, gc_after_trial: bool = False, show_progress_bar: bool = False, ) -> None: if not isinstance(catch, tuple): raise TypeError( "The catch argument is of type '{}' but must be a tuple.".format(type(catch).__name__) ) if not study._optimize_lock.acquire(False): raise RuntimeError("Nested invocation of `Study.optimize` method isn't allowed.") # TODO(crcrpar): Make progress bar work when n_jobs != 1. progress_bar = pbar_module._ProgressBar(show_progress_bar and n_jobs == 1, n_trials, timeout) study._stop_flag = False try: if n_jobs == 1: _optimize_sequential( study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng=False, time_start=None, progress_bar=progress_bar, ) else: if show_progress_bar: warnings.warn("Progress bar only supports serial execution (`n_jobs=1`).") if n_jobs == -1: n_jobs = os.cpu_count() or 1 time_start = datetime.datetime.now() futures: Set[Future] = set() with ThreadPoolExecutor(max_workers=n_jobs) as executor: for n_submitted_trials in itertools.count(): if study._stop_flag: break if ( timeout is not None and (datetime.datetime.now() - time_start).total_seconds() > timeout ): break if n_trials is not None and n_submitted_trials >= n_trials: break if len(futures) >= n_jobs: completed, futures = wait(futures, return_when=FIRST_COMPLETED) # Raise if exception occurred in executing the completed futures. for f in completed: f.result() futures.add( executor.submit( _optimize_sequential, study, func, 1, timeout, catch, callbacks, gc_after_trial, True, time_start, None, ) ) finally: study._optimize_lock.release() progress_bar.close()
def optimize( self, func: ObjectiveFuncType, n_trials: Optional[int] = None, timeout: Optional[float] = None, n_jobs: int = 1, catch: Tuple[Type[Exception], ...] = (), callbacks: Optional[List[Callable[["Study", FrozenTrial], None]]] = None, gc_after_trial: bool = False, show_progress_bar: bool = False, ) -> None: """Optimize an objective function. Optimization is done by choosing a suitable set of hyperparameter values from a given range. Uses a sampler which implements the task of value suggestion based on a specified distribution. The sampler is specified in :func:`~optuna.study.create_study` and the default choice for the sampler is TPE. See also :class:`~optuna.samplers.TPESampler` for more details on 'TPE'. Example: .. testcode:: import optuna def objective(trial): x = trial.suggest_uniform("x", -1, 1) return x ** 2 study = optuna.create_study() study.optimize(objective, n_trials=3) Args: func: A callable that implements objective function. n_trials: The number of trials. If this argument is set to :obj:`None`, there is no limitation on the number of trials. If :obj:`timeout` is also set to :obj:`None`, the study continues to create trials until it receives a termination signal such as Ctrl+C or SIGTERM. timeout: Stop study after the given number of second(s). If this argument is set to :obj:`None`, the study is executed without time limitation. If :obj:`n_trials` is also set to :obj:`None`, the study continues to create trials until it receives a termination signal such as Ctrl+C or SIGTERM. n_jobs: The number of parallel jobs. If this argument is set to :obj:`-1`, the number is set to CPU count. catch: A study continues to run even when a trial raises one of the exceptions specified in this argument. Default is an empty tuple, i.e. the study will stop for any exception except for :class:`~optuna.exceptions.TrialPruned`. callbacks: List of callback functions that are invoked at the end of each trial. Each function must accept two parameters with the following types in this order: :class:`~optuna.study.Study` and :class:`~optuna.FrozenTrial`. gc_after_trial: Flag to determine whether to automatically run garbage collection after each trial. Set to :obj:`True` to run the garbage collection, :obj:`False` otherwise. When it runs, it runs a full collection by internally calling :func:`gc.collect`. If you see an increase in memory consumption over several trials, try setting this flag to :obj:`True`. .. seealso:: :ref:`out-of-memory-gc-collect` show_progress_bar: Flag to show progress bars or not. To disable progress bar, set this ``False``. Currently, progress bar is experimental feature and disabled when ``n_jobs`` :math:`\\ne 1`. Raises: RuntimeError: If nested invocation of this method occurs. """ if not isinstance(catch, tuple): raise TypeError( "The catch argument is of type '{}' but must be a tuple.". format(type(catch).__name__)) if not self._optimize_lock.acquire(False): raise RuntimeError( "Nested invocation of `Study.optimize` method isn't allowed.") # TODO(crcrpar): Make progress bar work when n_jobs != 1. self._progress_bar = pbar_module._ProgressBar( show_progress_bar and n_jobs == 1, n_trials, timeout) self._stop_flag = False try: if n_jobs == 1: self._optimize_sequential(func, n_trials, timeout, catch, callbacks, gc_after_trial, None) else: if show_progress_bar: msg = "Progress bar only supports serial execution (`n_jobs=1`)." warnings.warn(msg) time_start = datetime.datetime.now() def _should_stop() -> bool: if self._stop_flag: return True if timeout is not None: # This is needed for mypy. t = timeout # type: float return (datetime.datetime.now() - time_start).total_seconds() > t return False if n_trials is not None: _iter = iter(range(n_trials)) else: _iter = iter(_should_stop, True) with Parallel(n_jobs=n_jobs, prefer="threads") as parallel: if not isinstance( parallel._backend, joblib.parallel.ThreadingBackend) and isinstance( self._storage, storages.InMemoryStorage): msg = ( "The default storage cannot be shared by multiple processes. " "Please use an RDB (RDBStorage) when you use joblib for " "multi-processing. The usage of RDBStorage can be found in " "https://optuna.readthedocs.io/en/stable/tutorial/rdb.html." ) warnings.warn(msg, UserWarning) parallel( delayed(self._reseed_and_optimize_sequential) (func, 1, timeout, catch, callbacks, gc_after_trial, time_start) for _ in _iter) finally: self._optimize_lock.release() self._progress_bar.close() del self._progress_bar