def test_space_rvs():
"""Test that calling `Space.rvs` returns expected values. This is specifically
aimed at ensuring `Categorical` instances containing strings produce the entire
string, rather than the first character, for example"""
space = Space([Integer(50, 100), Categorical(["glorot_normal", "orthogonal"])])
sample_0 = space.rvs(random_state=32)
sample_1 = space.rvs(n_samples=1, random_state=32)
sample_2 = space.rvs(n_samples=2, random_state=32)
sample_3 = space.rvs(n_samples=3, random_state=32)
assert sample_0 == [[73, "glorot_normal"]]
assert sample_1 == [[73, "glorot_normal"]]
assert sample_2 == [[73, "glorot_normal"], [93, "orthogonal"]]
assert sample_3 == [[73, "glorot_normal"], [93, "glorot_normal"], [55, "orthogonal"]]
def mini_spaces(self) -> Dict[str, Space]:
"""Separate :attr:`space` into subspaces based on :attr:`model_params` keys
Returns
-------
Dict[str, Space]
Dict of subspaces, wherein keys are all keys of :attr:`model_params`. Each key's
corresponding value is a filtered subspace, containing all the dimensions in
:attr:`space` whose name tuples start with that key. Keys will usually be one of the
core hyperparameter group names ("model_init_params", "model_extra_params",
"feature_engineer", "feature_selector")
Examples
--------
>>> from hyperparameter_hunter import Integer
>>> def es_0(all_inputs):
... return all_inputs
>>> def es_1(all_inputs):
... return all_inputs
>>> def es_2(all_inputs):
... return all_inputs
>>> s = Space([
... Integer(900, 1500, name=("model_init_params", "max_iter")),
... Categorical(["svd", "cholesky", "lsgr"], name=("model_init_params", "solver")),
... Categorical([es_1, es_2], name=("feature_engineer", "steps", 1)),
... ])
>>> rf = ResultFinder(
... "a", "b", "c", ("oof", "d"), space=s, leaderboard_path="e", descriptions_dir="f",
... model_params=dict(
... model_init_params=dict(
... max_iter=s.dimensions[0], normalize=True, solver=s.dimensions[1],
... ),
... feature_engineer=FeatureEngineer([es_0, s.dimensions[2]]),
... ),
... )
>>> rf.mini_spaces # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
{'model_init_params': Space([Integer(low=900, high=1500),
Categorical(categories=('svd', 'cholesky', 'lsgr'))]),
'feature_engineer': Space([Categorical(categories=(<function es_1 at ...>,
<function es_2 at ...>))])}
"""
if self._mini_spaces is None:
self._mini_spaces = {}
# Need to use `space.names` and `get_by_name` because the `location` attribute should
# be preferred to `name` if it exists, which is the case for Keras
names = self.space.names()
for param_group_name in self.model_params.keys():
# Use `space.get_by_name` to respect `location` (see comment above)
self._mini_spaces[param_group_name] = Space([
self.space.get_by_name(n) for n in names
if n[0] == param_group_name
])
return self._mini_spaces
def test_normalize_dimensions_consecutive_calls(dimensions):
"""Test that :func:`normalize_dimensions` can be safely invoked consecutively on the space each
invocation returns. This doesn't test that the result of :func:`normalize_dimensions` is
actually correct - Only that the result remains unchanged after multiple invocations"""
space_0 = normalize_dimensions(dimensions)
space_1 = normalize_dimensions(space_0)
space_2 = normalize_dimensions(space_1)
# Same as above, but starting with a `Space` instance to make sure nothing changes
space_3 = normalize_dimensions(Space(dimensions))
space_4 = normalize_dimensions(space_3)
space_5 = normalize_dimensions(space_4)
assert space_0 == space_1 == space_2 == space_3 == space_4 == space_5
def space_fixture():
dimensions = [
Real(0.1, 0.9),
Categorical(["foo", "bar", "baz"]),
Integer(12, 18)
]
locations = [
("model_init_params", "a"),
("model_init_params", "b", "c"),
("model_extra_params", "e"),
]
for i in range(len(dimensions)):
setattr(dimensions[i], "location", locations[i])
return Space(dimensions)
def test_space_from_space():
"""Test that a `Space` instance can be passed to a `Space` constructor"""
space_0 = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"),
(1.0, 5.0, "log-uniform"), ("e", "f")])
space_1 = Space(space_0)
assert_equal(space_0, space_1)
def test_space_api():
# TODO: Refactor - Use PyTest - Break this up into multiple tests
space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"),
(1.0, 5.0, "log-uniform"), ("e", "f")])
cat_space = Space([(1, "r"), (1.0, "r")])
assert isinstance(cat_space.dimensions[0], Categorical)
assert isinstance(cat_space.dimensions[1], Categorical)
assert_equal(len(space.dimensions), 5)
assert isinstance(space.dimensions[0], Real)
assert isinstance(space.dimensions[1], Integer)
assert isinstance(space.dimensions[2], Categorical)
assert isinstance(space.dimensions[3], Real)
assert isinstance(space.dimensions[4], Categorical)
samples = space.rvs(n_samples=10, random_state=0)
assert_equal(len(samples), 10)
assert_equal(len(samples[0]), 5)
assert isinstance(samples, list)
for n in range(4):
assert isinstance(samples[n], list)
assert isinstance(samples[0][0], numbers.Real)
assert isinstance(samples[0][1], numbers.Integral)
assert isinstance(samples[0][2], str)
assert isinstance(samples[0][3], numbers.Real)
assert isinstance(samples[0][4], str)
samples_transformed = space.transform(samples)
assert_equal(samples_transformed.shape[0], len(samples))
assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1 + 1)
# our space contains mixed types, this means we can't use
# `array_allclose` or similar to check points are close after a round-trip
# of transformations
for orig, round_trip in zip(samples,
space.inverse_transform(samples_transformed)):
assert space.distance(orig, round_trip) < 1.0e-8
samples = space.inverse_transform(samples_transformed)
assert isinstance(samples[0][0], numbers.Real)
assert isinstance(samples[0][1], numbers.Integral)
assert isinstance(samples[0][2], str)
assert isinstance(samples[0][3], numbers.Real)
assert isinstance(samples[0][4], str)
for b1, b2 in zip(
space.bounds,
[(0.0, 1.0), (-5, 5),
np.asarray(["a", "b", "c"]), (1.0, 5.0),
np.asarray(["e", "f"])],
):
assert_array_equal(b1, b2)
for b1, b2 in zip(
space.transformed_bounds,
[
(0.0, 1.0),
(-5, 5),
(0.0, 1.0),
(0.0, 1.0),
(0.0, 1.0),
(np.log10(1.0), np.log10(5.0)),
(0.0, 1.0),
],
):
assert_array_equal(b1, b2)
def test_space_consistency():
# TODO: Refactor - Use PyTest
# Reals (uniform)
s1 = Space([Real(0.0, 1.0)])
s2 = Space([Real(0.0, 1.0)])
s3 = Space([Real(0, 1)])
s4 = Space([(0.0, 1.0)])
s5 = Space([(0.0, 1.0, "uniform")])
s6 = Space([(0, 1.0)])
s7 = Space([(np.float64(0.0), 1.0)])
s8 = Space([(0, np.float64(1.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
a5 = s5.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_equal(s1, s6)
assert_equal(s1, s7)
assert_equal(s1, s8)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
assert_array_equal(a1, a5)
# Reals (log-uniform)
s1 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform")])
s2 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform")])
s3 = Space([Real(10**-3, 10**3, prior="log-uniform")])
s4 = Space([(10**-3.0, 10**3.0, "log-uniform")])
s5 = Space([(np.float64(10**-3.0), 10**3.0, "log-uniform")])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
# Integers
s1 = Space([Integer(1, 5)])
s2 = Space([Integer(1.0, 5.0)])
s3 = Space([(1, 5)])
s4 = Space([(np.int64(1.0), 5)])
s5 = Space([(1, np.int64(5.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
# Categoricals
s1 = Space([Categorical(["a", "b", "c"])])
s2 = Space([Categorical(["a", "b", "c"])])
s3 = Space([["a", "b", "c"]])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_array_equal(a1, a2)
assert_equal(s1, s3)
assert_array_equal(a1, a3)
s1 = Space([(True, False)])
s2 = Space([Categorical([True, False])])
s3 = Space([np.array([True, False])])
assert s1 == s2 == s3
def cook_estimator(base_estimator, space=None, **kwargs):
"""Cook a default estimator
For the special `base_estimator` called "DUMMY", the return value is None. This corresponds to
sampling points at random, hence there is no need for an estimator
Parameters
----------
base_estimator: {SKLearn Regressor, "GP", "RF", "ET", "GBRT", "DUMMY"}, default="GP"
If not string, should inherit from `sklearn.base.RegressorMixin`. In addition, the `predict`
method should have an optional `return_std` argument, which returns `std(Y | x)`,
along with `E[Y | x]`.
If `base_estimator` is a string in {"GP", "RF", "ET", "GBRT", "DUMMY"}, a surrogate model
corresponding to the relevant `X_minimize` function is created
space: `hyperparameter_hunter.space.space_core.Space`
Required only if the `base_estimator` is a Gaussian Process. Ignored otherwise
**kwargs: Dict
Extra parameters provided to the `base_estimator` at initialization time
Returns
-------
SKLearn Regressor
Regressor instance cooked up according to `base_estimator` and `kwargs`"""
#################### Validate `base_estimator` ####################
str_estimators = ["GP", "ET", "RF", "GBRT", "DUMMY"]
if isinstance(base_estimator, str):
if base_estimator.upper() not in str_estimators:
raise ValueError(
f"Expected `base_estimator` in {str_estimators}. Got {base_estimator}"
)
# Convert to upper after error check, so above error shows actual given `base_estimator`
base_estimator = base_estimator.upper()
elif not is_regressor(base_estimator):
raise ValueError("`base_estimator` must be a regressor")
#################### Get Cooking ####################
if base_estimator == "GP":
if space is not None:
space = Space(space)
# NOTE: Below `normalize_dimensions` is NOT an unnecessary duplicate of the call in
# `Optimizer` - `Optimizer` calls `cook_estimator` before its `dimensions` have been
# normalized, so `normalize_dimensions` must also be called here
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a `Space` instance, not None")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# Only special if *all* dimensions are `Categorical`
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2,
)
elif base_estimator == "RF":
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "ET":
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "GBRT":
gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile")
base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt)
elif base_estimator == "DUMMY":
return None
base_estimator.set_params(**kwargs)
return base_estimator
class Optimizer(object):
"""Run bayesian optimisation loop
An `Optimizer` represents the steps of a bayesian optimisation loop. To use it you need to
provide your own loop mechanism. The various optimisers provided by `skopt` use this class
under the hood. Use this class directly if you want to control the iterations of your bayesian
optimisation loop
Parameters
----------
dimensions: List
List of shape (n_dims,) containing search space dimensions. Each search dimension can be
defined as any of the following:
* Instance of a `Dimension` object (`Real`, `Integer` or `Categorical`)
* (<lower_bound>, <upper_bound>) tuple (for `Real` or `Integer` dimensions)
* (<lower_bound>, <upper_bound>, <prior>) tuple (for `Real` dimensions)
* List of categories (for `Categorical` dimensions)
base_estimator: {SKLearn Regressor, "GP", "RF", "ET", "GBRT", "DUMMY"}, default="GP"
If not string, should inherit from `sklearn.base.RegressorMixin`. In addition, the `predict`
method should have an optional `return_std` argument, which returns `std(Y | x)`,
along with `E[Y | x]`.
If `base_estimator` is a string in {"GP", "RF", "ET", "GBRT", "DUMMY"}, a surrogate model
corresponding to the relevant `X_minimize` function is created
n_initial_points: Int, default=10
Number of evaluations of `func` with initialization points before approximating it with
`base_estimator`. Points provided as `x0` count as initialization points.
If len(`x0`) < `n_initial_points`, additional points are sampled at random
acq_func: {"LCB", "EI", "PI", "gp_hedge", "EIps", "PIps"}, default="gp_hedge"
Function to minimize over the posterior distribution. Can be any of the following:
* "LCB": Lower confidence bound
* "EI": Negative expected improvement
* "PI": Negative probability of improvement
* "gp_hedge": Probabilistically choose one of the above three acquisition functions at
every iteration
* The gains `g_i` are initialized to zero
* At every iteration,
* Each acquisition function is optimised independently to propose a candidate point
`X_i`
* Out of all these candidate points, the next point `X_best` is chosen by
`softmax(eta g_i)`
* After fitting the surrogate model with `(X_best, y_best)`, the gains are updated
such that `g_i -= mu(X_i)`
* "EIps": Negated expected improvement per second to take into account the function compute
time. Then, the objective function is assumed to return two values, the first being the
objective value and the second being the time taken in seconds
* "PIps": Negated probability of improvement per second. The return type of the objective
function is identical to that of "EIps"
acq_optimizer: {"sampling", "lbfgs", "auto"}, default="auto"
Method to minimize the acquisition function. The fit model is updated with the optimal
value obtained by optimizing `acq_func` with `acq_optimizer`
* "sampling": `acq_func` is optimized by computing `acq_func` at `n_initial_points`
randomly sampled points.
* "lbfgs": `acq_func` is optimized by
* Randomly sampling `n_restarts_optimizer` (from `acq_optimizer_kwargs`) points
* "lbfgs" is run for 20 iterations with these initial points to find local minima
* The optimal of these local minima is used to update the prior
* "auto": `acq_optimizer` is configured on the basis of the `base_estimator` and the search
space. If the space is `Categorical` or if the provided estimator is based on tree-models,
then this is set to "sampling"
random_state: Int, or RandomState instance (optional)
Set random state to something other than None for reproducible results
acq_func_kwargs: Dict (optional)
Additional arguments to be passed to the acquisition function.
acq_optimizer_kwargs: Dict (optional)
Additional arguments to be passed to the acquisition optimizer
warn_on_re_ask: Boolean, default=False
If True, and the internal `optimizer` recommends a point that has already been evaluated
on invocation of `ask`, a warning is logged before recommending a random point. Either
way, a random point is used instead of already-evaluated recommendations. However,
logging the fact that this has taken place can be useful to indicate that the optimizer
may be stalling, especially if it repeatedly recommends the same point. In these cases,
if the suggested point is not optimal, it can be helpful to switch a different OptPro
(especially `DummyOptPro`), which will suggest points using different criteria
Attributes
----------
Xi: List
Points at which objective has been evaluated
yi: List
Values of objective at corresponding points in `Xi`
models: List
Regression models used to fit observations and compute acquisition function
space: `hyperparameter_hunter.space.space_core.Space`
Stores parameter search space used to sample points, bounds, and type of parameters
n_initial_points_: Int
Original value passed through the `n_initial_points` kwarg. The value of this attribute
remains unchanged along the lifespan of `Optimizer`, unlike :attr:`_n_initial_points`
_n_initial_points: Int
Number of remaining points that must be evaluated before fitting a surrogate estimator and
using it to recommend incumbent search points. Initially, :attr:`_n_initial_points` is set
to the value of the `n_initial_points` kwarg, like :attr:`n_initial_points_`. However,
:attr:`_n_initial_points` is decremented for each point `tell`-ed to `Optimizer`
"""
def __init__(
self,
dimensions,
base_estimator="gp",
n_initial_points=10,
acq_func="gp_hedge",
acq_optimizer="auto",
random_state=None,
acq_func_kwargs=None,
acq_optimizer_kwargs=None,
warn_on_re_ask=False,
):
self.rng = check_random_state(random_state)
self.space = Space(dimensions)
#################### Configure Acquisition Function ####################
self.acq_func = acq_func
self.acq_func_kwargs = acq_func_kwargs
allowed_acq_funcs = ["gp_hedge", "EI", "LCB", "PI", "EIps", "PIps"]
if self.acq_func not in allowed_acq_funcs:
raise ValueError(
f"Expected `acq_func` in {allowed_acq_funcs}. Got {self.acq_func}"
)
# Treat hedging method separately
if self.acq_func == "gp_hedge":
self.cand_acq_funcs_ = ["EI", "LCB", "PI"]
self.gains_ = np.zeros(3)
else:
self.cand_acq_funcs_ = [self.acq_func]
if acq_func_kwargs is None:
acq_func_kwargs = dict()
self.eta = acq_func_kwargs.get("eta", 1.0)
#################### Configure Point Counters ####################
if n_initial_points < 0:
raise ValueError(
f"Expected `n_initial_points` >= 0. Got {n_initial_points}")
self._n_initial_points = n_initial_points # TODO: Rename to `remaining_n_points`
self.n_initial_points_ = n_initial_points
#################### Configure Estimator ####################
self.base_estimator = base_estimator
#################### Configure Optimizer ####################
self.acq_optimizer = acq_optimizer
if acq_optimizer_kwargs is None:
acq_optimizer_kwargs = dict()
self.n_points = acq_optimizer_kwargs.get("n_points", 10000)
self.n_restarts_optimizer = acq_optimizer_kwargs.get(
"n_restarts_optimizer", 5)
n_jobs = acq_optimizer_kwargs.get("n_jobs", 1)
self.n_jobs = n_jobs
self.acq_optimizer_kwargs = acq_optimizer_kwargs
self.warn_on_re_ask = warn_on_re_ask
#################### Configure Search Space ####################
if isinstance(self.base_estimator, GaussianProcessRegressor):
self.space = normalize_dimensions(self.space)
#################### Initialize Optimization Storage ####################
self.models = []
self.Xi = []
self.yi = []
# Initialize cache for `ask` method responses. Ensures that multiple calls to `ask` with
# n_points set return same sets of points. Reset to {} at every call to `tell`
self.cache_ = {}
##################################################
# Properties
##################################################
@property
def base_estimator(self):
return self._base_estimator
@base_estimator.setter
def base_estimator(self, value):
# Build `base_estimator` if string given
if isinstance(value, str):
value = cook_estimator(value,
space=self.space,
random_state=self.rng.randint(
0,
np.iinfo(np.int32).max))
# Check if regressor
if not is_regressor(value) and value is not None:
raise ValueError(
f"`base_estimator` must be a regressor. Got {value}")
# Treat per second acquisition function specially
is_multi_regressor = isinstance(value, MultiOutputRegressor)
if self.acq_func.endswith("ps") and not is_multi_regressor:
value = MultiOutputRegressor(value)
self._base_estimator = value
@property
def acq_optimizer(self) -> str:
"""Method to minimize the acquisition function. See documentation for the `acq_optimizer`
kwarg in :meth:`Optimizer.__init__` for additional information
Returns
-------
{"lbfgs", "sampling"}
String in {"lbfgs", "sampling"}. If originally "auto", one of the two aforementioned
strings is selected based on :attr:`base_estimator`"""
return self._acq_optimizer
@acq_optimizer.setter
def acq_optimizer(self, value):
# Decide optimizer based on gradient information
if value == "auto":
if has_gradients(self.base_estimator):
value = "lbfgs"
else:
value = "sampling"
if value not in ["lbfgs", "sampling"]:
raise ValueError(
f"`acq_optimizer` must be 'lbfgs' or 'sampling'. Got {value}")
if not has_gradients(self.base_estimator) and value != "sampling":
raise ValueError(
f"Regressor {type(self.base_estimator)} requires `acq_optimizer`='sampling'"
)
self._acq_optimizer = value
##################################################
# Ask
##################################################
def ask(self,
n_points=None,
strategy="cl_min"): # TODO: Try `n_points` default=1
"""Request point (or points) at which objective should be evaluated next
Parameters
----------
n_points: Int (optional)
Number of points returned by the ask method. If `n_points` not given, a single point
to evaluate is returned. Otherwise, a list of points to evaluate is returned of size
`n_points`. This is useful if you can evaluate your objective in parallel, and thus
obtain more objective function evaluations per unit of time
strategy: {"cl_min", "cl_mean", "cl_max"}, default="cl_min"
Method used to sample multiple points if `n_points` is an integer. If `n_points` is not
given, `strategy` is ignored.
If set to "cl_min", then "Constant Liar" strategy (see reference) is used with lie
objective value being minimum of observed objective values. "cl_mean" and "cl_max"
correspond to the mean and max of values, respectively.
With this strategy, a copy of optimizer is created, which is then asked for a point,
and the point is told to the copy of optimizer with some fake objective (lie), the
next point is asked from copy, it is also told to the copy with fake objective and so
on. The type of lie defines different flavours of "cl..." strategies
Returns
-------
List
Point (or points) recommended to be evaluated next
References
----------
.. [1] Chevalier, C.; Ginsbourger, D.: "Fast Computation of the Multi-points Expected
Improvement with Applications in Batch Selection".
https://hal.archives-ouvertes.fr/hal-00732512/document"""
if n_points is None:
return self._ask()
#################### Validate Parameters ####################
if not (isinstance(n_points, int) and n_points > 0):
raise ValueError(f"`n_points` must be int > 0. Got {n_points}")
supported_strategies = ["cl_min", "cl_mean", "cl_max"]
if strategy not in supported_strategies:
raise ValueError(
f"Expected `strategy` in {supported_strategies}. Got {strategy}"
)
#################### Check Cache ####################
# If repeated parameters given to `ask`, return cached entry
if (n_points, strategy) in self.cache_:
return self.cache_[(n_points, strategy)]
#################### Constant Liar ####################
# Copy of the optimizer is made in order to manage the deletion of points with "lie"
# objective (the copy of optimizer is simply discarded)
opt = self.copy(random_state=self.rng.randint(0,
np.iinfo(np.int32).max))
points = []
for i in range(n_points):
x = opt.ask()
points.append(x)
# TODO: Put below section into `how_to_lie` helper function for easier testing
ti_available = self.acq_func.endswith("ps") and len(opt.yi) > 0
ti = [t for (_, t) in opt.yi] if ti_available else None
# TODO: Do below `y_lie` lines directly calculate min/max/mean on `opt.yi` when it could also contain times?
if strategy == "cl_min":
y_lie = np.min(opt.yi) if opt.yi else 0.0 # CL-min lie
t_lie = np.min(ti) if ti is not None else log(
sys.float_info.max)
elif strategy == "cl_mean":
y_lie = np.mean(opt.yi) if opt.yi else 0.0 # CL-mean lie
t_lie = np.mean(ti) if ti is not None else log(
sys.float_info.max)
else:
y_lie = np.max(opt.yi) if opt.yi else 0.0 # CL-max lie
t_lie = np.max(ti) if ti is not None else log(
sys.float_info.max)
#################### Lie to Optimizer ####################
# Use `_tell` (not `tell`) to prevent repeated log transformations of computation times
if self.acq_func.endswith("ps"):
opt._tell(x, (y_lie, t_lie))
else:
opt._tell(x, y_lie)
#################### Cache and Return Result ####################
self.cache_ = {(n_points, strategy): points}
return points
def _ask(self):
"""Suggest next point at which to evaluate the objective
Returns
-------
Some point in :attr:`space`, which is random while less than `n_initial_points` observations
have been `tell`-ed. After that, `base_estimator` is used to determine the next point
Notes
-----
If the suggested point has already been evaluated, a random point will be returned instead,
optionally accompanied by a warning message (depending on :attr:`warn_on_re_ask`)"""
if self._n_initial_points > 0 or self.base_estimator is None:
# Does not copy `self.rng` in order to keep advancing random state
return self.space.rvs(random_state=self.rng)[0]
else:
if not self.models:
raise RuntimeError(
"Random evaluations exhausted and no model has been fit")
#################### Check for Repeated Suggestion ####################
next_x = self._next_x
# Check distances between `next_x` and all evaluated points
min_delta_x = min(
[self.space.distance(next_x, xi) for xi in self.Xi])
if abs(min_delta_x) <= 1e-8: # `next_x` has already been evaluated
if self.warn_on_re_ask:
G.warn_("Repeated suggestion: {}".format(next_x))
# Set `_next_x` to random point, then re-invoke `_ask` to validate new point
self._next_x = self.space.rvs(random_state=self.rng)[0]
return self._ask()
# Return point computed from last call to `tell`
return next_x
##################################################
# Tell
##################################################
def tell(self, x, y, fit=True):
"""Record an observation (or several) of the objective function
Provide values of the objective function at points suggested by :meth:`ask`, or arbitrary
points. By default, a new model will be fit to all observations. The new model is used to
suggest the next point at which to evaluate the objective. This point can be retrieved by
calling :meth:`ask`.
To add multiple observations in a batch, pass a list-of-lists for `x`, and a list of
scalars for `y`
Parameters
----------
x: List, or list-of-lists
Point(s) at which objective was evaluated
y: Scalar, or list
Value(s) of objective at `x`
fit: Boolean, default=True
Whether to fit a model to observed evaluations of the objective. A model will only be
fitted after `n_initial_points` points have been `tell`-ed to the optimizer,
irrespective of the value of `fit`. To add observations without fitting a new model,
set `fit` to False"""
check_x_in_space(x, self.space)
self._check_y_is_valid(x, y)
# Take logarithm of the computation times
if self.acq_func.endswith("ps"):
if is_2d_list_like(x):
y = [[val, log(t)] for (val, t) in y]
elif is_list_like(x):
y = list(y)
y[1] = log(y[1])
return self._tell(x, y, fit=fit)
def _tell(self, x, y, fit=True):
"""Perform the actual work of incorporating one or more new points. See :meth:`tell` for
the full description. This method exists to give access to the internals of adding points
by side-stepping all input validation and transformation"""
#################### Collect Search Points and Evaluations ####################
# TODO: Clean up below - Looks like the 4 extend/append blocks may be duplicated
if "ps" in self.acq_func:
if is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# If `y` isn't a scalar, we have been handed a batch of points
elif is_list_like(y) and is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError(
f"Incompatible argument types: `x` ({type(x)}) and `y` ({type(y)})"
)
# Optimizer learned something new. Discard `cache_`
self.cache_ = {}
#################### Fit Surrogate Model ####################
# After being `tell`-ed `n_initial_points`, use surrogate model instead of random sampling
# TODO: Clean up and separate below. Pretty hard to follow the whole thing
if fit and self._n_initial_points <= 0 and self.base_estimator is not None:
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
# Even with BFGS optimizer, we want to sample a large number of points, and
# pick the best ones as starting points
X = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rng))
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
# TODO: Rename `values` - Maybe `utilities`?
values = _gaussian_acquisition(
X=X,
model=est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs,
)
#################### Find Acquisition Function Minimum ####################
# Find acquisition function minimum by randomly sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the minimum of the acquisition function, the minimization starts
# from `n_restarts_optimizer` different points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20,
) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
else:
# `acq_optimizer` should have already been checked, so this shouldn't be hit,
# but, it's here anyways to prevent complaints about `next_x` not existing in
# the absence of this `else` clause
raise RuntimeError(
f"Invalid `acq_optimizer` value: {self.acq_optimizer}")
# L-BFGS-B should handle this, but just in case of precision errors...
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# Note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
##################################################
# Helper Methods
##################################################
def copy(self, random_state=None):
"""Create a shallow copy of an instance of the optimizer
Parameters
----------
random_state: Int, or RandomState instance (optional)
Set random state of the copy
Returns
-------
Optimizer
Shallow copy of self"""
optimizer = Optimizer(
dimensions=self.space.dimensions,
base_estimator=self.base_estimator,
n_initial_points=self.n_initial_points_,
acq_func=self.acq_func,
acq_optimizer=self.acq_optimizer,
acq_func_kwargs=self.acq_func_kwargs,
acq_optimizer_kwargs=self.acq_optimizer_kwargs,
random_state=random_state,
)
if hasattr(self, "gains_"):
optimizer.gains_ = np.copy(self.gains_)
if self.Xi:
optimizer._tell(self.Xi, self.yi)
return optimizer
def _check_y_is_valid(self, x, y):
"""Check if the shapes and types of `x` and `y` are consistent. Complains if anything
is weird about `y`"""
#################### Per-Second Acquisition Function ####################
if self.acq_func.endswith("ps"):
if is_2d_list_like(x):
if not (np.ndim(y) == 2 and np.shape(y)[1] == 2):
raise TypeError(
"Expected `y` to be a list of (func_val, t)")
elif is_list_like(x):
if not (np.ndim(y) == 1 and len(y) == 2):
raise TypeError("Expected `y` to be (func_val, t)")
#################### Standard Acquisition Function ####################
# If `y` isn't a scalar, we have been handed a batch of points
elif is_list_like(y) and is_2d_list_like(x):
for y_value in y:
if not isinstance(y_value, Number):
raise ValueError("Expected `y` to be a list of scalars")
elif is_list_like(x):
if not isinstance(y, Number):
raise ValueError("`func` should return a scalar")
else:
raise ValueError(
f"Incompatible argument types: `x` ({type(x)}) and `y` ({type(y)})"
)
def run(self, func, n_iter=1):
"""Execute :meth:`ask` + :meth:`tell` loop for `n_iter` iterations
Parameters
----------
func: Callable
Function that returns the objective value `y`, when given a search point `x`
n_iter: Int, default=1
Number of `ask`/`tell` sequences to execute
Returns
-------
OptimizeResult
`scipy.optimize.OptimizeResult` instance"""
for _ in range(n_iter):
x = self.ask()
self.tell(x, func(x))
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
def __init__(
self,
dimensions,
base_estimator="gp",
n_initial_points=10,
acq_func="gp_hedge",
acq_optimizer="auto",
random_state=None,
acq_func_kwargs=None,
acq_optimizer_kwargs=None,
warn_on_re_ask=False,
):
self.rng = check_random_state(random_state)
self.space = Space(dimensions)
#################### Configure Acquisition Function ####################
self.acq_func = acq_func
self.acq_func_kwargs = acq_func_kwargs
allowed_acq_funcs = ["gp_hedge", "EI", "LCB", "PI", "EIps", "PIps"]
if self.acq_func not in allowed_acq_funcs:
raise ValueError(
f"Expected `acq_func` in {allowed_acq_funcs}. Got {self.acq_func}"
)
# Treat hedging method separately
if self.acq_func == "gp_hedge":
self.cand_acq_funcs_ = ["EI", "LCB", "PI"]
self.gains_ = np.zeros(3)
else:
self.cand_acq_funcs_ = [self.acq_func]
if acq_func_kwargs is None:
acq_func_kwargs = dict()
self.eta = acq_func_kwargs.get("eta", 1.0)
#################### Configure Point Counters ####################
if n_initial_points < 0:
raise ValueError(
f"Expected `n_initial_points` >= 0. Got {n_initial_points}")
self._n_initial_points = n_initial_points # TODO: Rename to `remaining_n_points`
self.n_initial_points_ = n_initial_points
#################### Configure Estimator ####################
self.base_estimator = base_estimator
#################### Configure Optimizer ####################
self.acq_optimizer = acq_optimizer
if acq_optimizer_kwargs is None:
acq_optimizer_kwargs = dict()
self.n_points = acq_optimizer_kwargs.get("n_points", 10000)
self.n_restarts_optimizer = acq_optimizer_kwargs.get(
"n_restarts_optimizer", 5)
n_jobs = acq_optimizer_kwargs.get("n_jobs", 1)
self.n_jobs = n_jobs
self.acq_optimizer_kwargs = acq_optimizer_kwargs
self.warn_on_re_ask = warn_on_re_ask
#################### Configure Search Space ####################
if isinstance(self.base_estimator, GaussianProcessRegressor):
self.space = normalize_dimensions(self.space)
#################### Initialize Optimization Storage ####################
self.models = []
self.Xi = []
self.yi = []
# Initialize cache for `ask` method responses. Ensures that multiple calls to `ask` with
# n_points set return same sets of points. Reset to {} at every call to `tell`
self.cache_ = {}
def space_gbn_1():
return Space([Real(0.1, 0.9, name=("i am", "foo")), Integer(3, 15, name=("i am", "bar"))])
def space_gbn_0():
return Space([Real(0.1, 0.9, name="foo"), Integer(3, 15, name="bar")])
# Dimension Contains Tests
##################################################
@pytest.mark.parametrize(
["value", "is_in"], [(1, True), (5, True), (10, True), (0, False), (11, False), ("x", False)]
)
def test_integer_contains(value, is_in):
assert (value in Integer(1, 10)) is is_in
##################################################
# Space Size Tests
##################################################
@pytest.mark.parametrize(
["space", "size"],
[
(Space([Categorical(["a", "b"]), Real(0.1, 0.7)]), maxsize),
(Space([Categorical(["a", "b"]), Integer(1, 5)]), 10),
],
)
def test_space_len(space, size):
assert len(space) == size
##################################################
# Dimension `get_params` Tests
##################################################
#################### `Real.get_params` ####################
@pytest.mark.parametrize(
["given_params", "expected_params"],
[
(
