def interpolate(thetas,
z_thetas,
xx,
yy,
method='linear',
z_uncertainties_thetas=None,
matern_exponent=0.5,
length_scale_min=0.001,
length_scale_default=1.,
length_scale_max=1000.,
noise_level=0.001,
subtract_min=False):
if method == 'cubic':
interpolator = CloughTocher2DInterpolator(thetas[:], z_thetas)
zz = interpolator(np.dstack((xx.flatten(), yy.flatten())))
zi = zz.reshape(xx.shape)
elif method == 'gp':
if z_uncertainties_thetas is not None:
gp = GaussianProcessRegressor(
normalize_y=True,
kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern(
length_scale=[length_scale_default],
length_scale_bounds=[(length_scale_min, length_scale_max)],
nu=matern_exponent) + WhiteKernel(noise_level),
n_restarts_optimizer=10,
alpha=z_uncertainties_thetas)
else:
gp = GaussianProcessRegressor(
normalize_y=True,
kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern(
length_scale=length_scale_default,
length_scale_bounds=(length_scale_min, length_scale_max),
nu=matern_exponent) + WhiteKernel(noise_level),
n_restarts_optimizer=10)
gp.fit(thetas[:], z_thetas[:])
zz, _ = gp.predict(np.c_[xx.ravel(), yy.ravel()], return_std=True)
zi = zz.reshape(xx.shape)
elif method == 'linear':
interpolator = LinearNDInterpolator(thetas[:], z_thetas)
zz = interpolator(np.dstack((xx.flatten(), yy.flatten())))
zi = zz.reshape(xx.shape)
else:
raise ValueError
mle = np.unravel_index(zi.argmin(), zi.shape)
if subtract_min:
zi -= zi[mle]
return zi, mle
def init_model(self):
"""initializes the surrogate model of the gaussian process
the model gets created with the right parameters, but is not fit with any data yet. the `base_model` will be
cloned in `update_model` and fit with observation data
"""
# n_dims == n_hparams
n_dims = len(self.searchspace.keys())
if self.interim_results:
n_dims += 1 # add one dim for augumented budget
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims,
nu=2.5,
)
base_model = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2,
)
self.base_model = base_model
def setup_tuner(self):
self.tunecfg = self.experiment["tuner"]
self.parameters = list(self.tunecfg["parameters"].keys())
self.dimensions = self.parse_dimensions(self.tunecfg["parameters"])
self.space = normalize_dimensions(self.dimensions)
self.priors = self.parse_priors(self.tunecfg["priors"])
self.kernel = ConstantKernel(
constant_value=self.tunecfg.get("variance_value", 0.1**2),
constant_value_bounds=tuple(
self.tunecfg.get("variance_bounds", (0.01**2, 0.5**2))),
) * Matern(
length_scale=self.tunecfg.get("length_scale_value", 0.3),
length_scale_bounds=tuple(
self.tunecfg.get("length_scale_bounds", (0.2, 0.8))),
nu=2.5,
)
self.opt = Optimizer(
dimensions=self.dimensions,
n_points=self.tunecfg.get("n_points", 1000),
n_initial_points=self.tunecfg.get("n_initial_points",
5 * len(self.dimensions)),
gp_kernel=self.kernel,
gp_kwargs=dict(normalize_y=True),
gp_priors=self.priors,
acq_func=self.tunecfg.get("acq_func", "ts"),
acq_func_kwargs=self.tunecfg.get(
"acq_func_kwargs",
None), # TODO: Check if this works for all parameters
random_state=self.rng.randint(0,
np.iinfo(np.int32).max),
)
def __init__(self, paramRanges, objectiveFunction, genNewSamples=True, paramFile=None,\
startingSamples=2):
'''
Constructor
@params
-------
paramRanges - list of tuples, list of 2-length lists of floats
the ranges in which the algorithm should search for best values; passed to the field
objectiveFunction - function
the function the algorithm will optimize; passed to the field
genNewSamples - bool - default:True
flag on whether to gen new samples (True) or not (False)
paramFile - String - default:None
the path from which new samples should be read if genNewSamples is False
'''
self.objectiveFunction = objectiveFunction
kernel = Matern(length_scale=1.0, nu=2.5)
self.model = GaussianProcessRegressor(normalize_y=True)
self.paramRanges = asarray(paramRanges)
self.sampleArray = None
self.sampleResults = None
if (genNewSamples):
self.sampleArray = asarray(\
[[rand.RandomInRange_Tuple(paramRanges[i]) for i in range(len(paramRanges))] \
for temp in range(startingSamples)])
self.sampleResults = asarray(\
self.pollObjectiveFunction(self.sampleArray))
else:
self.readSamplesFromFile(paramFile)
self.model.fit(self.sampleArray, self.sampleResults.ravel())
def cook_estimator(base_estimator, space=None, **kwargs):
"""Cook a default estimator.
For the special base_estimator called "DUMMY" the return value is None.
This corresponds to sampling points at random, hence there is no need
for an estimator.
Parameters
----------
base_estimator : "GP", "RF", "ET", "GBRT", "DUMMY" or sklearn regressor
Should inherit from `sklearn.base.RegressorMixin`.
In addition the `predict` method should have an optional `return_std`
argument, which returns `std(Y | x)`` along with `E[Y | x]`.
If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a
surrogate model corresponding to the relevant `X_minimize` function
is created.
space : Space instance
Has to be provided if the base_estimator is a gaussian process.
Ignored otherwise.
kwargs : dict
Extra parameters provided to the base_estimator at init time.
"""
if isinstance(base_estimator, str):
base_estimator = base_estimator.upper()
if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY"]:
raise ValueError("Valid strings for the base_estimator parameter "
" are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not "
"%s." % base_estimator)
elif not is_regressor(base_estimator):
raise ValueError("base_estimator has to be a regressor.")
if base_estimator == "GP":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(kernel=cov_amplitude *
other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
if ('n_jobs' in kwargs.keys()) and not hasattr(base_estimator, 'n_jobs'):
del kwargs['n_jobs']
base_estimator.set_params(**kwargs)
return base_estimator
def kriging_surrogates():
surrogate_model = GaussianProcessRegressor(normalize_y=True)
kernels = [
1.0 * Matern(nu=5 / 2),
1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0))
]
params = [kernels]
return surrogate_model, params
def gpbo_cycle(ndim,
space,
target_f,
n_iters=10,
acq_function=ei,
model=None,
n_multi_start=100,
show_progress=True):
xrange = (lambda title, n: tqdm_notebook(range(n), postfix=title)
) if show_progress else (lambda title, n: range(n))
space = np.array(space)
if model is None:
kernel = WhiteKernel(0.001, noise_level_bounds=[1.0e-5, 1.0e-3]) + \
Matern(1.0, nu=1.5, length_scale_bounds=[1.0e-3, 1.0e+3])
model = GaussianProcessRegressor(kernel=kernel,
normalize_y=False,
noise=None,
n_restarts_optimizer=2)
known_points = []
known_values = []
cost = []
for i in xrange('BO iteration', n_iters):
acq = acq_function(model, known_points, known_values)
candidates = []
for _ in xrange('acquisition', n_multi_start):
x0 = np.random.uniform(size=(ndim, ))
x, f, _ = fmin_l_bfgs_b(maxiter=1000,
func=acq,
x0=x0,
approx_grad=False,
bounds=[(0, 1)] * ndim)
candidates.append((x, f))
best = np.argmin([f for x, f in candidates])
suggestion, _ = candidates[best]
suggestion = reverse_transform(suggestion.reshape(1, -1), space)[0, :]
point_cost, observed = target_f(suggestion)
known_points.append(suggestion)
known_values.append(observed)
cost.append(point_cost)
model.fit(transform(np.array(known_points), space),
np.array(known_values))
yield model, acq, space, known_points, known_values, cost
def test_guess_priors():
"""Construct a complicated kernel and check if priors are constructed
correctly."""
kernel = Exponentiation(
ConstantKernel(constant_value_bounds="fixed") * Matern() +
WhiteKernel() + CompoundKernel([RBF(), Matern()]),
2.0,
)
priors = guess_priors(kernel)
assert len(priors) == 4
expected = [
-1.737085713764618,
-4.107091211892862,
-1.737085713764618,
-1.737085713764618,
]
for p, v in zip(priors, expected):
assert_almost_equal(p(0.0), v)
def test_acquisition_gradient():
rng = np.random.RandomState(0)
X = rng.randn(20, 5)
y = rng.randn(20)
X_new = rng.randn(5)
mat = Matern()
wk = WhiteKernel()
gpr = GaussianProcessRegressor(kernel=mat + wk)
gpr.fit(X, y)
for acq_func in ["LCB", "PI", "EI"]:
check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'):
self.__all_experiments = pd.DataFrame()
self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments)
self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN')))
self.__all_experiments['client'] = [""] * len(self.__all_experiments)
self.__lease_duration = pd.to_timedelta(lease_timout)
self.__leased_experiments = []
dims = self.__load_dimensions(dimensions_file)
self.__dimension_names = list(dims.keys())
self.__dimensions = list(dims.values())
self.__min_num_results_to_fit = min_num_results_to_fit
# Initialize
dim_types = [check_dimension(d) for d in self.__dimensions]
is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types])
if is_cat:
transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions]
else:
transformed_dims = []
for dim_type, dim in zip(dim_types, self.__dimensions):
if isinstance(dim_type, Categorical):
transformed_dims.append(check_dimension(dim, transform="onehot"))
# To make sure that GP operates in the [0, 1] space
else:
transformed_dims.append(check_dimension(dim, transform="normalize"))
space = Space(transformed_dims)
# Default GP
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims))
acq_optimizer = "lbfgs"
else:
other_kernel = Matern(
length_scale=np.ones(space.transformed_n_dims),
length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, random_state=None, alpha=0.0, noise='gaussian',
n_restarts_optimizer=2)
self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs",
n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
def _get_gp_regressor(length_scale=1., nu=2.5, noise=0.1):
"""Creates the GaussianProcessRegressor model
Args:
length_scale (Union[float, list]): Length scale of the GP kernel. If float, it is the
same for all dimensions, if array each element defines the length scale of the
dimension
nu (float): Controls the smoothness of the approximation.
see https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.kernels.Matern.html
Returns:
A skopt.learning.GaussianProcessRegressor with the given parameters
"""
kernel = ConstantKernel(1.0) * Matern(length_scale=length_scale, nu=nu)
return GaussianProcessRegressor(kernel=kernel, alpha=noise ** 2)
def cook_estimator(base_estimator, space=None, **kwargs):
if isinstance(base_estimator, str):
base_estimator = base_estimator.upper()
allowed_estimators = ['GP', 'ET', 'RF', 'GBRT', 'DUMMY']
if base_estimator not in allowed_estimators:
raise ValueError(
'invalid estimator, should be in {}, got {}'.format(
allowed_estimators, base_estimator))
elif not is_regressor(base_estimator):
raise ValueError('base estimator should be a regressor, got {}'.format(
base_estimator))
if base_estimator == 'GP':
if space is not None:
# space = Space(space)
space = Space(normalize_param_space(space))
n_params = space.transformed_n_params
is_cat = space.is_categorical
else:
raise ValueError('expected a space instance, got None')
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_params))
else:
other_kernel = Matern(length_scale=np.ones(n_params),
length_scale_bounds=[(0.01, 100)] * n_params,
nu=2.5)
base_estimator = GaussianProcessRegressor(kernel=cov_amplitude *
other_kernel,
normalize_y=True,
noise='gaussian',
n_restarts_optimizer=2)
elif base_estimator == 'RF':
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == 'ET':
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == 'GRBT':
grbt = GradientBoostingRegressor(n_estimators=30, loss='quantile')
base_estimator = GradientBoostingQuantileRegressor(base_estimator=grbt)
elif base_estimator == 'DUMMY':
return None
base_estimator.set_params(**kwargs)
return base_estimator
def __init__(self, num_issues, X, y):
BaseSurrogate.__init__(self, num_issues, X, y)
# Instantiate a Gaussian Process model.
# TODO. A question we need to investigate is what kernel we should be using?
# TODO. For now, to have a valid comparison, I am using the same kernel used by the bayesian optimization.
# TODO. Note that I am using the Kernels as given by the skopt library, with the same parameters.
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = Matern(length_scale=np.ones(num_issues),
length_scale_bounds=[(0.01, 100)] * num_issues,
nu=2.5)
self.gp = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True,
n_restarts_optimizer=2,
noise=0.000000001,
random_state=np.random.mtrand._rand.randint(
0,
np.iinfo(np.int32).max))
def test_guess_priors():
"""Construct a complicated kernel and check if priors are constructed
correctly."""
kernel = Exponentiation(
ConstantKernel(constant_value_bounds="fixed") * Matern() +
WhiteKernel() + RBF(length_scale=(1.0, 1.0)),
2.0,
)
priors = guess_priors(kernel)
assert len(priors) == 4
expected = [
-0.02116327824572739,
-2.112906921232193,
-0.02116327824572739,
-0.02116327824572739,
]
for p, v in zip(priors, expected):
assert_almost_equal(p(-0.9), v)
def test_bayes_opt_base_estimator():
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import Matern
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
matern = Matern(
length_scale=np.ones(2), length_scale_bounds=[(0.01, 100)] * 2, nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * matern,
normalize_y=True, random_state=0, alpha=0.0,
noise="gaussian", n_restarts_optimizer=2)
opt = SkOptOptimizer(
dimensions=[(-1.0, 1.0), (-1.0, 1.0)], base_estimator=base_estimator,
random_state=0)
opt.init(2)
params = np.empty(2)
for _ in range(10):
opt.get_next_parameters(params)
feedback = [-np.linalg.norm(params - 0.5384 * np.ones(2))]
opt.set_evaluation_feedback(feedback)
assert_greater(opt.get_best_fitness(), -0.3)
def construct_default_kernel(dimensions):
"""Construct a Matern kernel as default kernel to be used in the optimizer.
Parameters
----------
dimensions : list of dimensions
Elements are skopt.space.Dimension instances (Real, Integer
or Categorical) or any other valid value that defines skopt
dimension (see skopt.Optimizer docs)
Returns
-------
kernel : kernel object
The kernel specifying the covariance function of the GP used in the
optimization.
"""
n_parameters = len(dimensions)
kernel = ConstantKernel(constant_value=1.0,
constant_value_bounds=(0.1, 2.0)) * Matern(
length_scale=[0.3] * n_parameters,
length_scale_bounds=(0.05, 1.0),
nu=2.5)
return kernel
from skopt.learning.gaussian_process import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import DotProduct
from skopt.learning.gaussian_process.kernels import ExpSineSquared
from skopt.learning.gaussian_process.kernels import HammingKernel
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import RationalQuadratic
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import WhiteKernel
KERNELS = []
for length_scale in [np.arange(1, 6), [0.2, 0.3, 0.5, 0.6, 0.1]]:
KERNELS.extend([
RBF(length_scale=length_scale),
Matern(length_scale=length_scale, nu=0.5),
Matern(length_scale=length_scale, nu=1.5),
Matern(length_scale=length_scale, nu=2.5),
RationalQuadratic(alpha=2.0, length_scale=2.0),
ExpSineSquared(length_scale=2.0, periodicity=3.0),
ConstantKernel(constant_value=1.0),
WhiteKernel(noise_level=2.0),
Matern(length_scale=length_scale, nu=2.5)**3.0,
RBF(length_scale=length_scale) +
Matern(length_scale=length_scale, nu=1.5),
RBF(length_scale=length_scale) *
Matern(length_scale=length_scale, nu=1.5),
DotProduct(sigma_0=2.0)
])
from sklearn.utils.testing import assert_array_equal
import pytest
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import WhiteKernel
from skopt.learning.gaussian_process.gpr import _param_for_white_kernel_in_Sum
rng = np.random.RandomState(0)
X = rng.randn(5, 5)
y = rng.randn(5)
rbf = RBF()
wk = WhiteKernel()
mat = Matern()
kernel1 = rbf
kernel2 = mat + rbf
kernel3 = mat * rbf
kernel4 = wk * rbf
kernel5 = mat + rbf * wk
def predict_wrapper(X, gpr):
"""Predict that can handle 1-D input"""
X = np.expand_dims(X, axis=0)
return gpr.predict(X, return_std=True)
@pytest.mark.parametrize("kernel", [kernel1, kernel2, kernel3, kernel4])
def test_param_for_white_kernel_in_Sum(kernel):
def cook_estimator(base_estimator, space=None, **kwargs):
"""Cook a default estimator
For the special `base_estimator` called "DUMMY", the return value is None. This corresponds to
sampling points at random, hence there is no need for an estimator
Parameters
----------
base_estimator: {SKLearn Regressor, "GP", "RF", "ET", "GBRT", "DUMMY"}, default="GP"
If not string, should inherit from `sklearn.base.RegressorMixin`. In addition, the `predict`
method should have an optional `return_std` argument, which returns `std(Y | x)`,
along with `E[Y | x]`.
If `base_estimator` is a string in {"GP", "RF", "ET", "GBRT", "DUMMY"}, a surrogate model
corresponding to the relevant `X_minimize` function is created
space: `hyperparameter_hunter.space.space_core.Space`
Required only if the `base_estimator` is a Gaussian Process. Ignored otherwise
**kwargs: Dict
Extra parameters provided to the `base_estimator` at initialization time
Returns
-------
SKLearn Regressor
Regressor instance cooked up according to `base_estimator` and `kwargs`"""
#################### Validate `base_estimator` ####################
str_estimators = ["GP", "ET", "RF", "GBRT", "DUMMY"]
if isinstance(base_estimator, str):
if base_estimator.upper() not in str_estimators:
raise ValueError(
f"Expected `base_estimator` in {str_estimators}. Got {base_estimator}"
)
# Convert to upper after error check, so above error shows actual given `base_estimator`
base_estimator = base_estimator.upper()
elif not is_regressor(base_estimator):
raise ValueError("`base_estimator` must be a regressor")
#################### Get Cooking ####################
if base_estimator == "GP":
if space is not None:
space = Space(space)
# NOTE: Below `normalize_dimensions` is NOT an unnecessary duplicate of the call in
# `Optimizer` - `Optimizer` calls `cook_estimator` before its `dimensions` have been
# normalized, so `normalize_dimensions` must also be called here
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a `Space` instance, not None")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# Only special if *all* dimensions are `Categorical`
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2,
)
elif base_estimator == "RF":
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "ET":
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "GBRT":
gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile")
base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt)
elif base_estimator == "DUMMY":
return None
base_estimator.set_params(**kwargs)
return base_estimator
import numpy as np
from skopt import Optimizer
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import Matern, WhiteKernel
#cov_amplitude = ConstantKernel(1.0, (0.01, 5.0))
cov_amplitude = ConstantKernel(1.0, "fixed")
other_kernel = Matern(
length_scale=np.ones(1),
length_scale_bounds=[(0.3, 10)],
nu=2.5)
white_kernel = WhiteKernel()
gp = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel + white_kernel,
normalize_y=True, alpha=0.0, noise=10e-7,
n_restarts_optimizer=2)
def get_optimizer(range,nrandom):
return Optimizer(dimensions=[range],
base_estimator=gp,
n_random_starts=nrandom)
# Gaussian process with Matérn kernel as surrogate model
from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic,
ExpSineSquared, DotProduct,
ConstantKernel)
kernels = [
1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
1.0 * ExpSineSquared(length_scale=1.0,
periodicity=3.0,
length_scale_bounds=(0.1, 10.0),
periodicity_bounds=(1.0, 10.0)),
ConstantKernel(0.1, (0.01, 10.0)) *
(DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0))**2),
1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=2.5)
]
#############################################################################
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel,
alpha=noise_level**2,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
opt = Optimizer([(-2.0, 2.0)],
base_estimator=gpr,
n_initial_points=5,
acq_optimizer="sampling",
random_state=42)
def __init__(self,
hyper_param_conf,
command,
expdir,
exp_recipe_dir,
recipe,
computing,
exp_proposal_watch_dir=None):
base_estimator = 'GP'
self.hyper_param_conf = hyper_param_conf
self.command = command
self.expdir = expdir
self.exp_recipe_dir = exp_recipe_dir
self.recipe = recipe
self.computing = computing
# read the hyper parameter file
hyper_param_cfg = configparser.ConfigParser()
hyper_param_cfg.read(hyper_param_conf)
hyper_info = dict(hyper_param_cfg.items('info'))
self.hyper_param_names = hyper_info['hyper_params'].split(' ')
self.num_iters = int(hyper_info['num_iters'])
self.n_initial_points = int(hyper_info['n_initial_points'])
self.n_initial_points_to_start = int(
hyper_info['n_initial_points_to_start'])
self.max_parallel_jobs = int(hyper_info['max_parallel_jobs'])
self.selected_segment_length = hyper_info['segment_length']
self.selected_task = hyper_info['task']
if 'adapt_hyper_param' in hyper_info:
self.adapt_param = {
'param_name': hyper_info['adapt_hyper_param'],
'param_thr': int(hyper_info['param_thr']),
'par_cnt_scheme': hyper_info['par_cnt_scheme']
}
else:
self.adapt_param = None
hyper_param_dict = dict()
skopt_dims = []
for par_name in self.hyper_param_names:
par_dict = dict(hyper_param_cfg.items(par_name))
par_type = par_dict['type']
if par_type == 'Integer':
skopt_dim = skopt_space.Integer(low=int(par_dict['min']),
high=int(par_dict['max']),
name=par_name)
elif par_type == 'Real':
skopt_dim = skopt_space.Real(low=float(par_dict['min']),
high=float(par_dict['max']),
name=par_name)
elif par_type == 'Categorical':
skopt_dim = skopt_space.Categorical(
categories=par_dict['categories'].split(' '),
name=par_name)
else:
raise ValueError('Type %s is not a valid parameter type' %
par_type)
hyper_param_dict[par_name] = par_dict
skopt_dims.append(skopt_dim)
self.hyper_param_dict = hyper_param_dict
self.skopt_dims = skopt_dims
self.last_result = None
# self.all_results = []
self.start_new_run_flag = True
self.iter_ind = 0
self.watch_list = dict()
self.all_dim_values = []
self.all_losses = dict()
self.n_job_running = 0
self.n_initial_points_started = 0
self.n_unsuitable_points_for_estimator = 0
self.max_n_unsuitable_points_for_estimator = 10000
self.unsuitable_runs = []
self.lost_runs = []
self.exp_proposal_watch_dir = exp_proposal_watch_dir
self.use_proposal_run = False
self.proposed_loss_runs = []
# only 0.25% of the point sample in the hyper space are wanted (since they lead to rougly the wanted amount of
# trainable parameters)
self.acq_optimizer_kwargs = {'n_points': 4000000}
if 'debug' in expdir:
self.acq_optimizer_kwargs = {'n_points': 40000}
if base_estimator == 'boundedGP':
# Make own estimator based on Gaussian Process Regressor.
if skopt_dims is not None:
space = Space(skopt_dims)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] *
n_dims,
nu=2.5)
base_estimator = BoundedGaussianProcessRegressor(
space,
self.hyper_param_names,
self.adapt_param,
kernel=cov_amplitude * other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
super(HyperParamOptimizer,
self).__init__(skopt_dims,
base_estimator=base_estimator,
n_initial_points=self.n_initial_points,
acq_optimizer_kwargs=self.acq_optimizer_kwargs)
def __init__(self,
dimensions,
base_estimator="GP",
maximize=True,
n_random_starts=10,
acq_func="LCB",
acq_optimizer="lbfgs",
random_state=None,
n_points=10000,
n_restarts_optimizer=5,
xi=0.01,
kappa=1.96,
n_jobs=1):
if not skopt_available:
raise ImportError("skopt is not installed correctly")
self.maximize = maximize
self.n_params = len(dimensions)
rng = check_random_state(random_state)
if isinstance(base_estimator, str):
if base_estimator == "RF":
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3,
n_jobs=n_jobs,
random_state=rng)
elif base_estimator == "ET":
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3,
n_jobs=n_jobs,
random_state=rng)
elif base_estimator == "GP":
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
matern = Matern(length_scale=np.ones(len(dimensions)),
length_scale_bounds=[(0.01, 100)] *
len(dimensions),
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * matern,
normalize_y=True,
random_state=rng,
alpha=0.0,
noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "GBRT":
gbrt = GradientBoostingRegressor(n_estimators=30,
loss="quantile")
base_estimator = GradientBoostingQuantileRegressor(
base_estimator=gbrt, n_jobs=n_jobs, random_state=rng)
else:
raise ValueError(
"Valid strings for the base_estimator parameter"
" are: 'RF', 'ET', or 'GP', not '%s'" % base_estimator)
acq_func_kwargs = {"xi": xi, "kappa": kappa}
acq_optimizer_kwargs = {
"n_points": n_points,
"n_restarts_optimizer": n_restarts_optimizer,
"n_jobs": n_jobs
}
self.optimizer = _SkOptOptimizer(
dimensions=dimensions,
base_estimator=base_estimator,
n_initial_points=n_random_starts,
acq_func=acq_func,
acq_optimizer=acq_optimizer,
random_state=random_state,
acq_func_kwargs=acq_func_kwargs,
acq_optimizer_kwargs=acq_optimizer_kwargs)
def cook_estimator(base_estimator, space=None, **kwargs):
"""
Cook a default estimator.
For the special base_estimator called "DUMMY" the return value is None.
This corresponds to sampling points at random, hence there is no need
for an estimator.
Parameters
----------
* `base_estimator` ["GP", "RF", "ET", "GBRT", "DUMMY"
or sklearn regressor, default="GP"]:
Should inherit from `sklearn.base.RegressorMixin`.
In addition the `predict` method should have an optional `return_std`
argument, which returns `std(Y | x)`` along with `E[Y | x]`.
If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a
surrogate model corresponding to the relevant `X_minimize` function
is created.
* `space` [Space instance]:
Has to be provided if the base_estimator is a gaussian process.
Ignored otherwise.
* `kwargs` [dict]:
Extra parameters provided to the base_estimator at init time.
"""
if isinstance(base_estimator, str):
base_estimator = base_estimator.upper()
if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY", "GPM32", "GPM1", "RBF", "RQ"]:
raise ValueError("Valid strings for the base_estimator parameter "
" are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not "
"%s." % base_estimator)
elif not is_regressor(base_estimator):
raise ValueError("base_estimator has to be a regressor.")
if base_estimator == "GP":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "GPM32":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "GPM1":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RBF":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = RBF(length_scale=np.ones(n_dims))
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RQ":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = RationalQuadratic(length_scale=np.ones(n_dims), alpha=0.1)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RF":
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "ET":
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "GBRT":
gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile")
base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt)
elif base_estimator == "DUMMY":
return None
base_estimator.set_params(**kwargs)
return base_estimator
def bayesian(X, Y):
bds = [{
'name': 'learning_rate',
'type': 'continuous',
'domain': (0.0, 1.0)
}, {
'name': 'gamma',
'type': 'continuous',
'domain': (0.0, 5.0)
}, {
'name': 'max_depth',
'type': 'discrete',
'domain': (1, 50)
}, {
'name': 'n_estimators',
'type': 'discrete',
'domain': (1, 300)
}, {
'name': 'min_child_weight',
'type': 'continuous',
'domain': (1.0, 100.)
}, {
'name': 'colsample_bytree',
'type': 'continuous',
'domain': (0.1, 0.8)
}, {
'name': 'subsample',
'type': 'continuous',
'domain': (0.1, 0.8)
}]
noise = 0
m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2)
def XGBRegressorFake(learning_rate, gamma, max_depth, n_estimators,
min_child_weight, colsample_bytree, subsample):
# print("hei")
return learning_rate**2
def cv_score_fake(parameters):
# parameters = parameters[0]
return (parameters[0] - 0.5)**2
# Optimization objective
def cv_score(parameters):
# parameters = parameters[0]
score = cross_val_score(XGBRegressor(learning_rate=parameters[0],
gamma=int(parameters[1]),
max_depth=int(parameters[2]),
n_estimators=int(parameters[3]),
min_child_weight=parameters[4],
colsample_bytree=parameters[5],
subsample=parameters[6]),
X,
Y,
scoring='neg_mean_squared_error').mean()
print(score)
return score
# optimizer = BayesianOptimization(f=cv_score,
# domain=bds,
# model_type='GP',
# acquisition_type ='EI',
# acquisition_jitter = 0.05,
# exact_feval=True,
# maximize=True)
# On|ly 20 iterations because we have 5 initial random points
boundaries_of_parameters = list(map(lambda x: x['domain'], bds))
print(boundaries_of_parameters)
# sys.exit()
opti_obj = gp_minimize(
lambda x: cv_score(x),
boundaries_of_parameters,
base_estimator=gpr,
acq_func='EI', # expected improvement
xi=0.01, # exploitation-exploration trade-off
n_calls=10, # number of iterations
n_random_starts=4 # initial samples are provided
)
return opti_obj
from sklearn.base import clone
from skopt import gp_minimize
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import ConstantKernel, Matern
from bayesian_optimization_util import plot_approximation, plot_acquisition
noise = 0.2
bounds = np.array([[-1.0, 2.0]])
def f(X, noise=noise):
return -np.sin(3*X) - X**2 + 0.7*X + noise * np.random.randn(*X.shape)
X = np.arange(bounds[:, 0], bounds[:, 1], 0.01).reshape(-1, 1)
# Noise-free objective function values at X
Y = f(X,0)
# Use custom kernel and estimator to match previous example
m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2)
X_init = np.array([[-0.9], [1.1]])
Y_init = f(X_init)
r = gp_minimize(lambda x: -f(np.array(x))[0],
bounds.tolist(),
base_estimator=gpr,
acq_func='EI', # expected improvement
xi=0.01, # exploitation-exploration trade-off
n_calls=10, # number of iterations
n_random_starts=0, # initial samples are provided
x0=X_init.tolist(), # initial samples
y0=-Y_init.ravel())
from utils import black_box
from utils import plot_space
import numpy as np
from skopt.optimizer import Optimizer
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import Matern
from skopt.benchmarks import branin
# Search from 0.0 to 6.0
dimensions = ((0.0, 6.0), )
# Initialize estimator.
gpr = GaussianProcessRegressor(kernel=Matern(), noise=0.0)
optimizer = Optimizer(dimensions=dimensions,
base_estimator=gpr,
n_random_starts=0,
acq_func="LCB",
random_state=0)
# Tell some points to the optimizer and ask for the next point.
X = np.reshape(np.linspace(5.0, 6.0, 10), (-1, 1)).tolist()
y = [black_box(xi) for xi in X]
optimizer.tell(X, y)
x_cand = optimizer.ask()
y_cand = black_box(x_cand)
plot = plot_space(X, y, optimizer.models[-1], x_cand)
plot.show()
# Tell and ask again.
optimizer.tell(x_cand, y_cand)
# from skopt.learning.gaussian_process.kernels import ConstantKernel, RBF
from skopt.learning.gaussian_process.kernels import ConstantKernel, Matern
# from skopt.plots import plot_convergence
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from skopt.sampler import Lhs
from skopt.space import Space
from skopt import dump, load
from QTransport_cls import OpenQT, NCohQT
from QTdata import BO_data_write
job = 'w_config'
# Optimization Parameters
# rbf = ConstantKernel(1.0) * RBF(length_scale_bounds=(0.01, 0.5)) # .107
m52 = ConstantKernel(1.0) * Matern(nu=2.5, length_scale_bounds=(0.01, 0.5))
gpr = GaussianProcessRegressor(kernel=m52, n_restarts_optimizer=2)
num_init = 50
num_itr = 200
T = np.pi / (2 * 0.125)
dim = (7, 3) # dimension of the problem
s, d = dim
w_bound = [(0.125, 12.5)] * (s - 2) # for w as the unit of V=0.125
x_bound = [(-0.9999, 0.9999)] * ((s - 2) * d)
bound = x_bound + w_bound
inv_G_ls = [0.35938137 * T] #[2.58*10**(-3)*T]
repeat = 1
for G, inv_G in enumerate(inv_G_ls):
y = lambda site_w: OpenQT(s,
d,
from scipy import optimize
from sklearn.utils.testing import assert_array_almost_equal
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import DotProduct
from skopt.learning.gaussian_process.kernels import Exponentiation
from skopt.learning.gaussian_process.kernels import ExpSineSquared
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import RationalQuadratic
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import WhiteKernel
length_scale = np.arange(1, 6)
KERNELS = [
RBF(length_scale=length_scale),
Matern(length_scale=length_scale, nu=0.5),
Matern(length_scale=length_scale, nu=1.5),
Matern(length_scale=length_scale, nu=2.5),
RationalQuadratic(alpha=2.0, length_scale=2.0),
ExpSineSquared(length_scale=2.0, periodicity=3.0),
ConstantKernel(constant_value=1.0),
WhiteKernel(noise_level=2.0),
Matern(length_scale=length_scale, nu=2.5)**3.0,
RBF(length_scale=length_scale) + Matern(length_scale=length_scale, nu=1.5),
RBF(length_scale=length_scale) * Matern(length_scale=length_scale, nu=1.5),
DotProduct(sigma_0=2.0)
]
rng = np.random.RandomState(0)
X = rng.randn(5)
Y = rng.randn(10, 5)