def _optimize(
study: "optuna.Study",
func: "optuna.study.ObjectiveFuncType",
n_trials: Optional[int] = None,
timeout: Optional[float] = None,
n_jobs: int = 1,
catch: Tuple[Type[Exception], ...] = (),
callbacks: Optional[List[Callable[["optuna.Study", FrozenTrial], None]]] = None,
gc_after_trial: bool = False,
show_progress_bar: bool = False,
) -> None:
if not isinstance(catch, tuple):
raise TypeError(
"The catch argument is of type '{}' but must be a tuple.".format(type(catch).__name__)
)
if not study._optimize_lock.acquire(False):
raise RuntimeError("Nested invocation of `Study.optimize` method isn't allowed.")
# TODO(crcrpar): Make progress bar work when n_jobs != 1.
progress_bar = pbar_module._ProgressBar(show_progress_bar and n_jobs == 1, n_trials, timeout)
study._stop_flag = False
try:
if n_jobs == 1:
_optimize_sequential(
study,
func,
n_trials,
timeout,
catch,
callbacks,
gc_after_trial,
reseed_sampler_rng=False,
time_start=None,
progress_bar=progress_bar,
)
else:
if show_progress_bar:
warnings.warn("Progress bar only supports serial execution (`n_jobs=1`).")
time_start = datetime.datetime.now()
def _should_stop() -> bool:
if study._stop_flag:
return True
if timeout is not None:
# This is needed for mypy.
t: float = timeout
return (datetime.datetime.now() - time_start).total_seconds() > t
return False
if n_trials is not None:
_iter = iter(range(n_trials))
else:
_iter = iter(_should_stop, True)
with Parallel(n_jobs=n_jobs, prefer="threads") as parallel:
if not isinstance(
parallel._backend, joblib.parallel.ThreadingBackend
) and isinstance(study._storage, storages.InMemoryStorage):
warnings.warn(
"The default storage cannot be shared by multiple processes. "
"Please use an RDB (RDBStorage) when you use joblib for "
"multi-processing. The usage of RDBStorage can be found in "
"https://optuna.readthedocs.io/en/stable/tutorial/rdb.html.",
UserWarning,
)
parallel(
delayed(_optimize_sequential)(
study,
func,
1,
timeout,
catch,
callbacks,
gc_after_trial,
reseed_sampler_rng=True,
time_start=time_start,
progress_bar=None,
)
for _ in _iter
)
finally:
study._optimize_lock.release()
progress_bar.close()
def optimize(
self,
func, # type: ObjectiveFuncType
n_trials=None, # type: Optional[int]
timeout=None, # type: Optional[float]
n_jobs=1, # type: int
catch=(), # type: Union[Tuple[()], Tuple[Type[Exception]]]
callbacks=None, # type: Optional[List[Callable[[Study, structs.FrozenTrial], None]]]
gc_after_trial=True, # type: bool
show_progress_bar=False, # type: bool
):
# type: (...) -> None
"""Optimize an objective function.
Optimization is done by choosing a suitable set of hyperparameter values from a given
range. Uses a sampler which implements the task of value suggestion based on a specified
distribution. The sampler is specified in :func:`~optuna.study.create_study` and the
default choice for the sampler is TPE.
See also :class:`~optuna.samplers.TPESampler` for more details on 'TPE'.
Args:
func:
A callable that implements objective function.
n_trials:
The number of trials. If this argument is set to :obj:`None`, there is no
limitation on the number of trials. If :obj:`timeout` is also set to :obj:`None`,
the study continues to create trials until it receives a termination signal such
as Ctrl+C or SIGTERM.
timeout:
Stop study after the given number of second(s). If this argument is set to
:obj:`None`, the study is executed without time limitation. If :obj:`n_trials` is
also set to :obj:`None`, the study continues to create trials until it receives a
termination signal such as Ctrl+C or SIGTERM.
n_jobs:
The number of parallel jobs. If this argument is set to :obj:`-1`, the number is
set to CPU count.
catch:
A study continues to run even when a trial raises one of the exceptions specified
in this argument. Default is an empty tuple, i.e. the study will stop for any
exception except for :class:`~optuna.exceptions.TrialPruned`.
callbacks:
List of callback functions that are invoked at the end of each trial. Each function
must accept two parameters with the following types in this order:
:class:`~optuna.study.Study` and :class:`~optuna.structs.FrozenTrial`.
gc_after_trial:
Flag to execute garbage collection at the end of each trial. By default, garbage
collection is enabled, just in case. You can turn it off with this argument if
memory is safely managed in your objective function.
show_progress_bar:
Flag to show progress bars or not. To disable progress bar, set this ``False``.
Currently, progress bar is experimental feature and disabled
when ``n_jobs`` :math:`\\ne 1`.
"""
if not isinstance(catch, tuple):
raise TypeError(
"The catch argument is of type '{}' but must be a tuple.".
format(type(catch).__name__))
if not self._optimize_lock.acquire(False):
raise RuntimeError(
"Nested invocation of `Study.optimize` method isn't allowed.")
# TODO(crcrpar): Make progress bar work when n_jobs != 1.
self._progress_bar = pbar_module._ProgressBar(
show_progress_bar and n_jobs == 1, n_trials, timeout)
try:
if n_jobs == 1:
self._optimize_sequential(func, n_trials, timeout, catch,
callbacks, gc_after_trial, None)
else:
if show_progress_bar:
msg = "Progress bar only supports serial execution (`n_jobs=1`)."
warnings.warn(msg)
_logger.warning(msg)
time_start = datetime.datetime.now()
if n_trials is not None:
_iter = iter(range(n_trials))
elif timeout is not None:
# This is needed for mypy
actual_timeout = timeout # type: float
_iter = iter(
lambda: (datetime.datetime.now() - time_start).
total_seconds() > actual_timeout,
True,
)
else:
# The following expression makes an iterator that never ends.
_iter = iter(int, 1)
with Parallel(n_jobs=n_jobs, prefer="threads") as parallel:
if not isinstance(
parallel._backend,
joblib.parallel.ThreadingBackend) and isinstance(
self._storage, storages.InMemoryStorage):
msg = (
"The default storage cannot be shared by multiple processes. "
"Please use an RDB (RDBStorage) when you use joblib for "
"multi-processing. The usage of RDBStorage can be found in "
"https://optuna.readthedocs.io/en/stable/tutorial/rdb.html."
)
warnings.warn(msg, UserWarning)
_logger.warning(msg)
parallel(
delayed(self._optimize_sequential)
(func, 1, timeout, catch, callbacks, gc_after_trial,
time_start) for _ in _iter)
finally:
self._optimize_lock.release()
self._progress_bar.close()
del self._progress_bar
def _optimize(
study: "optuna.Study",
func: "optuna.study.ObjectiveFuncType",
n_trials: Optional[int] = None,
timeout: Optional[float] = None,
n_jobs: int = 1,
catch: Tuple[Type[Exception], ...] = (),
callbacks: Optional[List[Callable[["optuna.Study", FrozenTrial], None]]] = None,
gc_after_trial: bool = False,
show_progress_bar: bool = False,
) -> None:
if not isinstance(catch, tuple):
raise TypeError(
"The catch argument is of type '{}' but must be a tuple.".format(type(catch).__name__)
)
if not study._optimize_lock.acquire(False):
raise RuntimeError("Nested invocation of `Study.optimize` method isn't allowed.")
# TODO(crcrpar): Make progress bar work when n_jobs != 1.
progress_bar = pbar_module._ProgressBar(show_progress_bar and n_jobs == 1, n_trials, timeout)
study._stop_flag = False
try:
if n_jobs == 1:
_optimize_sequential(
study,
func,
n_trials,
timeout,
catch,
callbacks,
gc_after_trial,
reseed_sampler_rng=False,
time_start=None,
progress_bar=progress_bar,
)
else:
if show_progress_bar:
warnings.warn("Progress bar only supports serial execution (`n_jobs=1`).")
if n_jobs == -1:
n_jobs = os.cpu_count() or 1
time_start = datetime.datetime.now()
futures: Set[Future] = set()
with ThreadPoolExecutor(max_workers=n_jobs) as executor:
for n_submitted_trials in itertools.count():
if study._stop_flag:
break
if (
timeout is not None
and (datetime.datetime.now() - time_start).total_seconds() > timeout
):
break
if n_trials is not None and n_submitted_trials >= n_trials:
break
if len(futures) >= n_jobs:
completed, futures = wait(futures, return_when=FIRST_COMPLETED)
# Raise if exception occurred in executing the completed futures.
for f in completed:
f.result()
futures.add(
executor.submit(
_optimize_sequential,
study,
func,
1,
timeout,
catch,
callbacks,
gc_after_trial,
True,
time_start,
None,
)
)
finally:
study._optimize_lock.release()
progress_bar.close()
def optimize(
self,
func: ObjectiveFuncType,
n_trials: Optional[int] = None,
timeout: Optional[float] = None,
n_jobs: int = 1,
catch: Tuple[Type[Exception], ...] = (),
callbacks: Optional[List[Callable[["Study", FrozenTrial],
None]]] = None,
gc_after_trial: bool = False,
show_progress_bar: bool = False,
) -> None:
"""Optimize an objective function.
Optimization is done by choosing a suitable set of hyperparameter values from a given
range. Uses a sampler which implements the task of value suggestion based on a specified
distribution. The sampler is specified in :func:`~optuna.study.create_study` and the
default choice for the sampler is TPE.
See also :class:`~optuna.samplers.TPESampler` for more details on 'TPE'.
Example:
.. testcode::
import optuna
def objective(trial):
x = trial.suggest_uniform("x", -1, 1)
return x ** 2
study = optuna.create_study()
study.optimize(objective, n_trials=3)
Args:
func:
A callable that implements objective function.
n_trials:
The number of trials. If this argument is set to :obj:`None`, there is no
limitation on the number of trials. If :obj:`timeout` is also set to :obj:`None`,
the study continues to create trials until it receives a termination signal such
as Ctrl+C or SIGTERM.
timeout:
Stop study after the given number of second(s). If this argument is set to
:obj:`None`, the study is executed without time limitation. If :obj:`n_trials` is
also set to :obj:`None`, the study continues to create trials until it receives a
termination signal such as Ctrl+C or SIGTERM.
n_jobs:
The number of parallel jobs. If this argument is set to :obj:`-1`, the number is
set to CPU count.
catch:
A study continues to run even when a trial raises one of the exceptions specified
in this argument. Default is an empty tuple, i.e. the study will stop for any
exception except for :class:`~optuna.exceptions.TrialPruned`.
callbacks:
List of callback functions that are invoked at the end of each trial. Each function
must accept two parameters with the following types in this order:
:class:`~optuna.study.Study` and :class:`~optuna.FrozenTrial`.
gc_after_trial:
Flag to determine whether to automatically run garbage collection after each trial.
Set to :obj:`True` to run the garbage collection, :obj:`False` otherwise.
When it runs, it runs a full collection by internally calling :func:`gc.collect`.
If you see an increase in memory consumption over several trials, try setting this
flag to :obj:`True`.
.. seealso::
:ref:`out-of-memory-gc-collect`
show_progress_bar:
Flag to show progress bars or not. To disable progress bar, set this ``False``.
Currently, progress bar is experimental feature and disabled
when ``n_jobs`` :math:`\\ne 1`.
Raises:
RuntimeError:
If nested invocation of this method occurs.
"""
if not isinstance(catch, tuple):
raise TypeError(
"The catch argument is of type '{}' but must be a tuple.".
format(type(catch).__name__))
if not self._optimize_lock.acquire(False):
raise RuntimeError(
"Nested invocation of `Study.optimize` method isn't allowed.")
# TODO(crcrpar): Make progress bar work when n_jobs != 1.
self._progress_bar = pbar_module._ProgressBar(
show_progress_bar and n_jobs == 1, n_trials, timeout)
self._stop_flag = False
try:
if n_jobs == 1:
self._optimize_sequential(func, n_trials, timeout, catch,
callbacks, gc_after_trial, None)
else:
if show_progress_bar:
msg = "Progress bar only supports serial execution (`n_jobs=1`)."
warnings.warn(msg)
time_start = datetime.datetime.now()
def _should_stop() -> bool:
if self._stop_flag:
return True
if timeout is not None:
# This is needed for mypy.
t = timeout # type: float
return (datetime.datetime.now() -
time_start).total_seconds() > t
return False
if n_trials is not None:
_iter = iter(range(n_trials))
else:
_iter = iter(_should_stop, True)
with Parallel(n_jobs=n_jobs, prefer="threads") as parallel:
if not isinstance(
parallel._backend,
joblib.parallel.ThreadingBackend) and isinstance(
self._storage, storages.InMemoryStorage):
msg = (
"The default storage cannot be shared by multiple processes. "
"Please use an RDB (RDBStorage) when you use joblib for "
"multi-processing. The usage of RDBStorage can be found in "
"https://optuna.readthedocs.io/en/stable/tutorial/rdb.html."
)
warnings.warn(msg, UserWarning)
parallel(
delayed(self._reseed_and_optimize_sequential)
(func, 1, timeout, catch, callbacks, gc_after_trial,
time_start) for _ in _iter)
finally:
self._optimize_lock.release()
self._progress_bar.close()
del self._progress_bar
