def __init__(self, dimensions, n_random_starts=10, mutation_rate=0.05):
self.Xi = []
self.yi = []
self.space = Space(dimensions)
self.n_random_starts = n_random_starts
self.n_random_starts_ = n_random_starts
self.mutation_rate = mutation_rate
def test_space_from_yaml():
with NamedTemporaryFile(delete=False) as tmp:
tmp.write(b"""
Space:
- Real:
low: 0.0
high: 1.0
- Integer:
low: -5
high: 5
- Categorical:
categories:
- a
- b
- c
- Real:
low: 1.0
high: 5.0
prior: log-uniform
- Categorical:
categories:
- e
- f
""")
tmp.flush()
space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"),
(1.0, 5.0, "log-uniform"), ("e", "f")])
space2 = Space.from_yaml(tmp.name)
assert_equal(space, space2)
tmp.close()
os.unlink(tmp.name)
def test_space_api():
space = Space([(0.0, 1.0), (-5, 5),
("a", "b", "c"), (1.0, 5.0, "log-uniform")])
assert_equal(len(space.dimensions), 4)
assert_true(isinstance(space.dimensions[0], Real))
assert_true(isinstance(space.dimensions[1], Integer))
assert_true(isinstance(space.dimensions[2], Categorical))
assert_true(isinstance(space.dimensions[3], Real))
samples = space.rvs(n_samples=10, random_state=0)
assert_equal(len(samples), 10)
assert_equal(len(samples[0]), 4)
samples_transformed = space.transform(samples)
assert_equal(samples_transformed.shape[0], len(samples))
assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1)
assert_array_equal(samples, space.inverse_transform(samples_transformed))
for b1, b2 in zip(space.bounds,
[(0.0, 1.0), (-5, 5),
np.asarray(["a", "b", "c"]), (1.0, 5.0)]):
assert_array_equal(b1, b2)
for b1, b2 in zip(space.transformed_bounds,
[(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0),
(np.log10(1.0), np.log10(5.0))]):
assert_array_equal(b1, b2)
def cook_estimator(base_estimator, space=None, **kwargs):
"""Cook a default estimator.
For the special base_estimator called "DUMMY" the return value is None.
This corresponds to sampling points at random, hence there is no need
for an estimator.
Parameters
----------
base_estimator : "GP", "RF", "ET", "GBRT", "DUMMY" or sklearn regressor
Should inherit from `sklearn.base.RegressorMixin`.
In addition the `predict` method should have an optional `return_std`
argument, which returns `std(Y | x)`` along with `E[Y | x]`.
If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a
surrogate model corresponding to the relevant `X_minimize` function
is created.
space : Space instance
Has to be provided if the base_estimator is a gaussian process.
Ignored otherwise.
kwargs : dict
Extra parameters provided to the base_estimator at init time.
"""
if isinstance(base_estimator, str):
base_estimator = base_estimator.upper()
if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY"]:
raise ValueError("Valid strings for the base_estimator parameter "
" are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not "
"%s." % base_estimator)
elif not is_regressor(base_estimator):
raise ValueError("base_estimator has to be a regressor.")
if base_estimator == "GP":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(kernel=cov_amplitude *
other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
if ('n_jobs' in kwargs.keys()) and not hasattr(base_estimator, 'n_jobs'):
del kwargs['n_jobs']
base_estimator.set_params(**kwargs)
return base_estimator
def test_space_from_space():
# can you pass a Space instance to the Space constructor?
space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"),
(1.0, 5.0, "log-uniform"), ("e", "f")])
space2 = Space(space)
assert_equal(space, space2)
def test_partly_categorical_space():
dims = Space([Categorical(['a', 'b', 'c']), Categorical(['A', 'B', 'C'])])
assert dims.is_partly_categorical
dims = Space([Categorical(['a', 'b', 'c']), Integer(1, 2)])
assert dims.is_partly_categorical
assert not dims.is_categorical
dims = Space([Integer(1, 2), Integer(1, 2)])
assert not dims.is_partly_categorical
def hyperband(objective, space, max_iter=100, eta=3, random_state=0,
verbose=True, n_evaluations=None, rank=0):
"""
Hyperband algorithm as defined by Kevin Jamieson.
Parameters:
----------
* `objective`: [function]
Objective function to be minimized.
* `space`: [hyperspace.space]
Hyperparameter search space bounds.
* `max_iter`: [int]
Maximum number of iterations.
* `eta`: [int]
Returns:
-------
* `result` [`OptimizeResult`, scipy object]
Reference:
---------
http://people.eecs.berkeley.edu/~kjamieson/hyperband.html
"""
logeta = lambda x: log(x)/log(eta)
s_max = int(logeta(max_iter)) # number of unique executions of Successive Halving (minus one)
B = (s_max+1)*max_iter # total number of iterations (without reuse) per execution of Succesive Halving (n,r)
# If space is the original list of tuples, convert to Space()
if isinstance(space, list):
space = Space(space)
#### Begin Finite Horizon Hyperband outlerloop. Repeat indefinetely.
yi = []
Xi = []
for s in reversed(range(s_max+1)):
n = int(ceil(int(B/max_iter/(s+1))*eta**s)) # initial number of configurations
r = max_iter*eta**(-s) # initial number of iterations to run configurations for
# Begin Finite Horizon Successive Halving with (n,r)
for i in range(s+1):
# Run each of the n_i configs for r_i iterations and keep best n_i/eta
n_i = ceil(n*eta**(-i))
r_i = ceil(r*eta**(i))
T = space.rvs(ceil(n_i), random_state)
iter_result = [objective(t, r_i) for t in T]
yi.append(iter_result)
Xi.append(T)
# Get next hyperparameter configurations
T = [ T[i] for i in np.argsort(iter_result)[0:int( n_i/eta )] ]
if verbose and rank == 0:
print(f'Iteration number: {i}, Epochs per config: {r_i}, Num configs: {n_i}, Incumbent: {min(iter_result)}')
result = create_result(Xi, yi, n_evaluations=n_evaluations, space=space, rng=random_state)
# End Finite Horizon Successive Halving with (n,r)
return result
def __init__(self,
dimensions,
n_random_starts=10,
tournament_fraction=0.2,
mutation_rate=0.05):
self.Xi = []
self.yi = []
self.space = Space(dimensions)
self.n_random_starts = n_random_starts
self.n_random_starts_ = n_random_starts
self.tournament_fraction = tournament_fraction
self.mutation_rate = mutation_rate
def test_normalize_types():
# can you pass a Space instance to the Space constructor?
space = Space([(0.0, 1.0), Integer(-5, 5, dtype=int), (True, False)])
space.set_transformer("normalize")
X = [[0., -5, False]]
Xt = np.zeros((1, 3))
assert_array_equal(space.transform(X), Xt)
assert_array_equal(space.inverse_transform(Xt), X)
assert_array_equal(space.inverse_transform(space.transform(X)), X)
assert isinstance(space.inverse_transform(Xt)[0][0], float)
assert isinstance(space.inverse_transform(Xt)[0][1], int)
assert isinstance(space.inverse_transform(Xt)[0][2], (np.bool_, bool))
def normalize_dimensions(dimensions):
"""Create a ``Space`` where all dimensions are normalized to unit range.
This is particularly useful for Gaussian process based regressors and is
used internally by ``gp_minimize``.
Parameters
----------
* `dimensions` [list, shape=(n_dims,)]:
List of search space dimensions.
Each search dimension can be defined either as
- a `(lower_bound, upper_bound)` tuple (for `Real` or `Integer`
dimensions),
- a `(lower_bound, upper_bound, "prior")` tuple (for `Real`
dimensions),
- as a list of categories (for `Categorical` dimensions), or
- an instance of a `Dimension` object (`Real`, `Integer` or
`Categorical`).
NOTE: The upper and lower bounds are inclusive for `Integer`
dimensions.
"""
space = Space(dimensions)
transformed_dimensions = []
if space.is_categorical:
# recreate the space and explicitly set transform to "identity"
# this is a special case for GP based regressors
for dimension in space:
transformed_dimensions.append(Categorical(dimension.categories,
dimension.prior,
name=dimension.name,
transform="identity"))
else:
for dimension in space.dimensions:
if isinstance(dimension, Categorical):
transformed_dimensions.append(dimension)
# To make sure that GP operates in the [0, 1] space
elif isinstance(dimension, Real):
transformed_dimensions.append(
Real(dimension.low, dimension.high, dimension.prior,
name=dimension.name,
transform="normalize")
)
elif isinstance(dimension, Integer):
transformed_dimensions.append(
Integer(dimension.low, dimension.high,
name=dimension.name,
transform="normalize")
)
else:
raise RuntimeError("Unknown dimension type "
"(%s)" % type(dimension))
return Space(transformed_dimensions)
def test_create_hyperspace(
integer=(0, 10), real=(20.0, 30), cat=['a', 'b', 'c', 'd']):
"""
Tests for correctly formatted combinations of Scikit-Optimize spaces.
"""
hparams0 = [integer, real]
hparams1 = [real, cat]
hparams2 = [integer, cat]
# Integer hparam
hyper_int = HyperInteger(low=integer[0], high=integer[1])
int_low, int_high = hyper_int.get_hyperspace()
# Real hparam
hyper_real = HyperReal(low=real[0], high=real[1])
real_low, real_high = hyper_real.get_hyperspace()
# Categorical hparam
hyper_cat = HyperCategorical(cat)
cat_low, cat_high = hyper_cat.get_hyperspace()
# Case 0
hyperspace0 = create_hyperspace(hparams0)
test_combinations0 = [[int_high, real_high], [int_low, real_high],
[int_high, real_low], [int_low, real_low]]
test_hyperspace0 = []
for space in test_combinations0:
test_hyperspace0.append(Space(space))
# Case 1: [real, cat]
hyperspace1 = create_hyperspace(hparams1)
test_combinations1 = [[real_high, cat_high], [real_low, cat_high],
[real_high, cat_low], [real_low, cat_low]]
test_hyperspace1 = []
for space in test_combinations1:
test_hyperspace1.append(Space(space))
# Case 3: [int, cat]
hyperspace2 = create_hyperspace(hparams2)
test_combinations2 = [[int_high, cat_high], [int_low, cat_high],
[int_high, cat_low], [int_low, cat_low]]
test_hyperspace2 = []
for space in test_combinations2:
test_hyperspace2.append(Space(space))
assert_equal(hyperspace0, test_hyperspace0)
assert_equal(hyperspace1, test_hyperspace1)
assert_equal(hyperspace2, test_hyperspace2)
def lhs_sample(n_samples):
"""
Takes random n_samples with the lhs method.
Returns array x and y.
"""
x = np.array([])
y = np.array([])
# Makes the space of points which van be chosen from
space = Space([(-2., 1.), (-1.5, 1.5)])
# Chooses which kind oh lhs will be used
lhs = Lhs(lhs_type="classic", criterion=None)
coordinates = 0
# Generates n_samples withhi the chosen space
coordinates = lhs.generate(space.dimensions, n_samples)
# appends all x and y values to array
for coordinate in coordinates:
a = coordinate[0]
x = np.append(x, a)
b = coordinate[1]
y = np.append(y, b)
return x, y
def create_hyperspace(hyperparameters):
"""
Converts hyperparameter lists to Scikit-Optimize Space instances.
Parameters
----------
* `hyperparameters` [list, shape=(n_hyperparameters,)]
Returns
-------
* `hyperspace` [list of lists, shape(n_spaces, n_hyperparameters)]
- All combinations of hyperspaces. Each list within hyperspace
is a search space to be distributed across 2**n_spaces nodes.
"""
hparams_low = []
hparams_high = []
for hparam in hyperparameters:
low, high = check_dimension(hparam)
hparams_low.append(low)
hparams_high.append(high)
all_spaces = fold_spaces(hparams_low, hparams_high)
hyperspace = []
for space in all_spaces:
hyperspace.append(Space(space))
return hyperspace
def test_minimizer_with_space(minimizer):
# check we can pass a Space instance as dimensions argument and get same
# result
n_calls = 7
n_random_starts = 4
space = Space([(-5.0, 10.0), (0.0, 15.0)])
space_result = minimizer(branin,
space,
n_calls=n_calls,
n_random_starts=n_random_starts,
random_state=1)
check_minimizer_api(space_result, n_calls)
check_minimizer_bounds(space_result, n_calls)
dimensions = [(-5.0, 10.0), (0.0, 15.0)]
result = minimizer(branin,
dimensions,
n_calls=n_calls,
n_random_starts=n_random_starts,
random_state=1)
assert_array_almost_equal(space_result.x_iters, result.x_iters)
assert_array_almost_equal(space_result.func_vals, result.func_vals)
def solve(self):
"""
Wrapper function for scipy.optimize.differential_evolution
"""
progress = []
def cb(xk, convergence):
progress.append(self.fun(xk))
# initialize number of points = popsize
space = Space([(0.,1.)]*len(self.bounds))
lhs = Lhs()
pop = np.asarray(lhs.generate(space.dimensions, self.popsize))
min_b, max_b = np.asarray(self.bounds).T
diff = max_b - min_b
pop = min_b + pop * diff
progress.append(np.min(np.apply_along_axis(self.fun, 1, pop)))
result = scipy_de(self.fun, self.bounds, popsize=1, maxiter = self.maxiter, tol=0.0001, disp=self.disp, callback=cb, init=pop)
self.x = result.x
self.fx = result.fun
f_calls = (np.arange(1,len(progress)+1)) * self.popsize
self.converge_data = np.vstack((f_calls, np.asarray(progress)))
self.solved = True
def test_space_api():
space = Space([(0.0, 1.0), (-5, 5),
("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")])
cat_space = Space([(1, "r"), (1.0, "r")])
assert isinstance(cat_space.dimensions[0], Categorical)
assert isinstance(cat_space.dimensions[1], Categorical)
assert_equal(len(space.dimensions), 5)
assert_true(isinstance(space.dimensions[0], Real))
assert_true(isinstance(space.dimensions[1], Integer))
assert_true(isinstance(space.dimensions[2], Categorical))
assert_true(isinstance(space.dimensions[3], Real))
assert_true(isinstance(space.dimensions[4], Categorical))
samples = space.rvs(n_samples=10, random_state=0)
assert_equal(len(samples), 10)
assert_equal(len(samples[0]), 5)
assert_true(isinstance(samples, list))
for n in range(4):
assert_true(isinstance(samples[n], list))
assert_true(isinstance(samples[0][0], numbers.Real))
assert_true(isinstance(samples[0][1], numbers.Integral))
assert_true(isinstance(samples[0][2], str))
assert_true(isinstance(samples[0][3], numbers.Real))
assert_true(isinstance(samples[0][4], str))
samples_transformed = space.transform(samples)
assert_equal(samples_transformed.shape[0], len(samples))
assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1 + 1)
# our space contains mixed types, this means we can't use
# `array_allclose` or similar to check points are close after a round-trip
# of transformations
for orig, round_trip in zip(samples,
space.inverse_transform(samples_transformed)):
assert space.distance(orig, round_trip) < 1.e-8
samples = space.inverse_transform(samples_transformed)
assert_true(isinstance(samples[0][0], numbers.Real))
assert_true(isinstance(samples[0][1], numbers.Integral))
assert_true(isinstance(samples[0][2], str))
assert_true(isinstance(samples[0][3], numbers.Real))
assert_true(isinstance(samples[0][4], str))
for b1, b2 in zip(space.bounds,
[(0.0, 1.0), (-5, 5),
np.asarray(["a", "b", "c"]), (1.0, 5.0),
np.asarray(["e", "f"])]):
assert_array_equal(b1, b2)
for b1, b2 in zip(space.transformed_bounds,
[(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0),
(np.log10(1.0), np.log10(5.0)), (0.0, 1.0)]):
assert_array_equal(b1, b2)
def test_fold_spaces(
integer=(0, 10), real=(20.0, 30), cat=['a', 'b', 'c', 'd']):
"""
Test all combinations of hyperparameter spaces.
"""
hparams = [integer, real, cat]
hparams_low = []
hparams_high = []
for hparam in hparams:
low, high = check_dimension(hparam)
hparams_low.append(low)
hparams_high.append(high)
all_spaces = fold_spaces(hparams_low, hparams_high)
# Integer hparam
hyper_int = HyperInteger(low=integer[0], high=integer[1])
int_low, int_high = hyper_int.get_hyperspace()
# Real hparam
hyper_real = HyperReal(low=real[0], high=real[1])
real_low, real_high = hyper_real.get_hyperspace()
# Categorical hparam
hyper_cat = HyperCategorical(cat)
cat_low, cat_high = hyper_cat.get_hyperspace()
# All combinations:
test_combinations = [[int_high, real_high, cat_high],
[int_low, real_high, cat_high],
[int_high, real_low, cat_high],
[int_low, real_low, cat_high],
[int_high, real_high, cat_low],
[int_low, real_high, cat_low],
[int_high, real_low, cat_low],
[int_low, real_low, cat_low]]
hyperspace = []
for space in all_spaces:
hyperspace.append(Space(space))
test_case = []
for space in test_combinations:
test_case.append(Space(space))
assert_equal(hyperspace, test_case)
def test_constant_property():
space = Space([(0.0, 1.0), (1,),
("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e",)])
assert space.n_constant_dimensions == 2
for i in [1, 4]:
assert space.dimensions[i].is_constant
for i in [0, 2, 3]:
assert not space.dimensions[i].is_constant
def test_normalize():
# can you pass a Space instance to the Space constructor?
space = Space([(0.0, 1.0), (-5, 5),
("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")])
space.set_transformer("normalize")
X = [[0., -5, 'a', 1., 'e']]
Xt = np.zeros((1, 5))
assert_array_equal(space.transform(X), Xt)
assert_array_equal(space.inverse_transform(Xt), X)
assert_array_equal(space.inverse_transform(space.transform(X)), X)
def main():
logging.basicConfig(format="[%(asctime)s] %(levelname)s %(threadName)-23s: %(message)s",
level=logging.INFO)
cli_parser = ArgumentParser()
cli_parser.add_argument("-c", "--config", type=str, required=True)
cli_parser.add_argument("-d", "--dataset", type=str, required=True)
cli_parser.add_argument("-r", "--repeats", type=int, default=15)
cli_parser.add_argument("-a", "--attempts", type=int, default=30)
cli_parser.add_argument("-p", "--parallel", type=int, default=1)
cli_parser.add_argument("--cache", type=str, default="cache")
cli_parser.add_argument("--iterations", type=int, default=1_000_000)
args = cli_parser.parse_args()
# Search parameters
parser = ArgumentParser()
parser.add_argument("--strategy", type=str, default='epsgreedy')
parser.add_argument("--cold", action='store_true')
parser.add_argument("--x0_min", type=float, default=1e-10)
parser.add_argument("--x0_max", type=float, default=10.0)
parser.add_argument("--x0_prior", type=str, default='log-uniform')
parser.add_argument("--x1_min", type=float, default=1e-10)
parser.add_argument("--x1_max", type=float, default=1000.0)
parser.add_argument("--x1_prior", type=str, default='log-uniform')
parser.add_argument("--lr", type=float, default=0.01)
parser.add_argument("--l2", type=float, default=1.0)
parser.add_argument("--eps", type=float, default=0.1)
parser.add_argument("--tau", type=float, default=1.0)
parser.add_argument("--alpha", type=float, default=1.0)
parser.add_argument("--cap", type=float, default=0.1)
# Read experiment configuration
with open(args.config, 'rt') as f:
lines = f.readlines()
configs = [parser.parse_args(line.strip().split(" ")) for line in lines]
# Run experiments in task executor
with MultiThreadScheduler(args.parallel) as scheduler:
targets = []
for config in configs:
space = Space([
Real(low=config.x0_min, high=config.x0_max, prior=config.x0_prior),
Real(low=config.x1_min, high=config.x1_max, prior=config.x1_prior)
])
maximize = True if config.strategy == 'ips' else False
hyperopt = LogGridOptimizer(target_fn, space, maximize=maximize, max_parallel=5, kwargs={
"config": config,
"data": args.dataset,
"repeats": args.repeats,
"iterations": args.iterations,
"seed_base": 4200
}, bases=[1, 3])
targets.append(hyperopt)
results = [target.optimize(args.attempts if args.attempts != -1 else hyperopt.nr_max_attempts) for target in targets]
scheduler.block_until_tasks_finish()
results = [r.result.value for r in results]
for target, result in zip(targets, results):
logging.info(f"{target.kwargs['config'].strategy} == LR:{result[0][0]}, L2:{result[0][1]} -> performance (95% CI) = {result[1]}")
def test_skopt():
space = Space([(0.0, 1.0)])
for i in range(TESTS):
prime_idx = randint(0, MAX_BASE)
hammer = Hammersley(prime_idx)
skhammer = SkoptHammersley(0, 0, [PRIME_VECTOR[prime_idx]])
h1 = hammer.get_array(SIZE)
h2 = np.squeeze(skhammer.generate(space.dimensions, SIZE))
assert np.allclose(h1, h2)
def generate(self, dimensions, n_samples, random_state=None):
"""Creates latin hypercube samples with maxpro criterion.
Parameters
----------
dimensions : list, shape (n_dims,)
List of search space dimensions.
Each search dimension can be defined either as
- a `(lower_bound, upper_bound)` tuple (for `Real` or `Integer`
dimensions),
- a `(lower_bound, upper_bound, "prior")` tuple (for `Real`
dimensions),
- as a list of categories (for `Categorical` dimensions), or
- an instance of a `Dimension` object (`Real`, `Integer` or
`Categorical`).
n_samples : int
The order of the LHS sequence. Defines the number of samples.
random_state : int, RandomState instance, or None (default)
Set random state to something other than None for reproducible
results.
Returns
-------
np.array, shape=(n_dim, n_samples)
LHS set
"""
rng = check_random_state(random_state)
space = Space(dimensions)
transformer = space.get_transformer()
n_dim = space.n_dims
space.set_transformer("normalize")
h = self._lhs_normalized(n_dim, n_samples, rng)
self.num_pts = n_samples
self.dim = n_dim
if self.use_gradient:
print('Using gradient descent')
bounds = [(0, 1)] * len(dimensions) * self.num_pts
h_opt = minimize(self.maxpro_criter,
h,
jac=self.maxpro_grad,
bounds=bounds)
h_opt = h_opt['x'].reshape(n_samples, n_dim)
else:
print('Using naive method')
best = 1e+6
for i in range(self.iterations):
h = self._lhs_normalized(n_dim, n_samples, i * rng)
criter = self.maxpro_criter(h)
if best > criter:
best = criter
h_opt = h.copy()
h_opt = space.inverse_transform(h_opt)
space.set_transformer(transformer)
return h_opt
def test_acquisition_gradient_cookbook():
rng = np.random.RandomState(0)
X = rng.randn(20, 5)
y = rng.randn(20)
X_new = rng.randn(5)
gpr = cook_estimator("GP", Space(((-5.0, 5.0), )), random_state=0)
gpr.fit(X, y)
for acq_func in ["LCB", "PI", "EI"]:
check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def test_acquisition_per_second_gradient(acq_func):
rng = np.random.RandomState(0)
X = rng.randn(20, 10)
# Make the second component large, so that mean_grad and std_grad
# do not become zero.
y = np.vstack((X[:, 0], np.abs(X[:, 0])**3)).T
for X_new in [rng.randn(10), rng.randn(10)]:
gpr = cook_estimator("GP", Space(((-5.0, 5.0), )), random_state=0)
mor = MultiOutputRegressor(gpr)
mor.fit(X, y)
check_gradient_correctness(X_new, mor, acq_func, 1.5)
def optimize(self,
num_vars,
objective_function,
gradient_function=None,
variable_bounds=None,
initial_point=None):
callbacks = []
def alt_obj_fn(pars):
fn = objective_function(pars)
callbacks.append({'point': pars, 'fn': fn})
return fn
result = super().optimize(num_vars, alt_obj_fn, gradient_function,
variable_bounds, initial_point)
if self._num_restarts is not None:
for i in range(self._num_restarts - 1):
if variable_bounds is not None:
init_pt = [
np.random.uniform(dn, up) for dn, up in variable_bounds
]
else:
init_pt = [
np.random.uniform(-np.pi, +np.pi)
for _ in range(num_vars)
]
result_new = super().optimize(num_vars, alt_obj_fn,
gradient_function,
variable_bounds, init_pt)
if result_new[1] < result[1]:
result = result_new
X = [step['point'] for step in callbacks]
y = [step['fn'] for step in callbacks]
if self._make_model and (len(callbacks) < self._max_model_points):
model = GaussianProcessRegressor()
model.fit(X, y)
else:
model = None
if variable_bounds is not None:
space = Space([Real(low, high) for low, high in variable_bounds])
else:
space = None
self.optimization_result = create_result(
X, y, space=space, models=[model] if model is not None else None)
return result
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'):
self.__all_experiments = pd.DataFrame()
self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments)
self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN')))
self.__all_experiments['client'] = [""] * len(self.__all_experiments)
self.__lease_duration = pd.to_timedelta(lease_timout)
self.__leased_experiments = []
dims = self.__load_dimensions(dimensions_file)
self.__dimension_names = list(dims.keys())
self.__dimensions = list(dims.values())
self.__min_num_results_to_fit = min_num_results_to_fit
# Initialize
dim_types = [check_dimension(d) for d in self.__dimensions]
is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types])
if is_cat:
transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions]
else:
transformed_dims = []
for dim_type, dim in zip(dim_types, self.__dimensions):
if isinstance(dim_type, Categorical):
transformed_dims.append(check_dimension(dim, transform="onehot"))
# To make sure that GP operates in the [0, 1] space
else:
transformed_dims.append(check_dimension(dim, transform="normalize"))
space = Space(transformed_dims)
# Default GP
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims))
acq_optimizer = "lbfgs"
else:
other_kernel = Matern(
length_scale=np.ones(space.transformed_n_dims),
length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims,
nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, random_state=None, alpha=0.0, noise='gaussian',
n_restarts_optimizer=2)
self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs",
n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
def test_space_api():
space = Space([(0.0, 1.0), (-5, 5),
("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")])
cat_space = Space([(1, "r"), (1.0, "r")])
assert isinstance(cat_space.dimensions[0], Categorical)
assert isinstance(cat_space.dimensions[1], Categorical)
assert_equal(len(space.dimensions), 5)
assert_true(isinstance(space.dimensions[0], Real))
assert_true(isinstance(space.dimensions[1], Integer))
assert_true(isinstance(space.dimensions[2], Categorical))
assert_true(isinstance(space.dimensions[3], Real))
assert_true(isinstance(space.dimensions[4], Categorical))
samples = space.rvs(n_samples=10, random_state=0)
assert_equal(len(samples), 10)
assert_equal(len(samples[0]), 5)
assert_true(isinstance(samples, list))
for n in range(4):
assert_true(isinstance(samples[n], list))
assert_true(isinstance(samples[0][0], numbers.Real))
assert_true(isinstance(samples[0][1], numbers.Integral))
assert_true(isinstance(samples[0][2], str))
assert_true(isinstance(samples[0][3], numbers.Real))
assert_true(isinstance(samples[0][4], str))
samples_transformed = space.transform(samples)
assert_equal(samples_transformed.shape[0], len(samples))
assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1 + 1)
assert_array_equal(samples, space.inverse_transform(samples_transformed))
samples = space.inverse_transform(samples_transformed)
assert_true(isinstance(samples[0][0], numbers.Real))
assert_true(isinstance(samples[0][1], numbers.Integral))
assert_true(isinstance(samples[0][2], str))
assert_true(isinstance(samples[0][3], numbers.Real))
assert_true(isinstance(samples[0][4], str))
for b1, b2 in zip(space.bounds,
[(0.0, 1.0), (-5, 5),
np.asarray(["a", "b", "c"]), (1.0, 5.0),
np.asarray(["e", "f"])]):
assert_array_equal(b1, b2)
for b1, b2 in zip(space.transformed_bounds,
[(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0),
(np.log10(1.0), np.log10(5.0)), (0.0, 1.0)]):
assert_array_equal(b1, b2)
def test_acquisition_per_second(acq_func):
X = np.reshape(np.linspace(4.0, 8.0, 10), (-1, 1))
y = np.vstack((np.ones(10), np.ravel(np.log(X)))).T
cgpr = ConstantGPRSurrogate(Space(((1.0, 9.0),)))
cgpr.fit(X, y)
X_pred = np.reshape(np.linspace(1.0, 11.0, 20), (-1, 1))
indices = np.arange(6)
vals = _gaussian_acquisition(X_pred, cgpr, y_opt=1.0, acq_func=acq_func)
for fast, slow in zip(indices[:-1], indices[1:]):
assert vals[slow] > vals[fast]
acq_wo_time = _gaussian_acquisition(
X, cgpr.estimators_[0], y_opt=1.2, acq_func=acq_func[:2])
acq_with_time = _gaussian_acquisition(
X, cgpr, y_opt=1.2, acq_func=acq_func)
assert_array_almost_equal(acq_wo_time / acq_with_time, np.ravel(X), 2)
def test_dimension_name():
notnames = [1, 1., True]
for n in notnames:
with pytest.raises(ValueError) as exc:
real = Real(1, 2, name=n)
assert("Dimension's name must be either string or"
"None." == exc.value.args[0])
s = Space([Real(1, 2, name="a"),
Integer(1, 100, name="b"),
Categorical(["red, blue"], name="c")])
assert s["a"] == (0, s.dimensions[0])
assert s["a", "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[["a", "c"]] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[("a", "c")] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[0] == (0, s.dimensions[0])
assert s[0, "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[0, 2] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
def test_normalize_bounds():
bounds = [(-999, 189000), Categorical((True, False))]
space = Space(normalize_dimensions(bounds))
for a in np.linspace(1e-9, 0.4999, 1000):
x = space.inverse_transform([[a, a]])
check_limits(x[0][0], -999, 189000)
y = space.transform(x)
check_limits(y, 0., 1.)
for a in np.linspace(0.50001, 1e-9 + 1., 1000):
x = space.inverse_transform([[a, a]])
check_limits(x[0][0], -999, 189000)
y = space.transform(x)
check_limits(y, 0., 1.)
def __init__(self, bounds, method='latin'):
self.dimensions = len(bounds)
self.position = np.empty(self.dimensions)
self.velocity = np.zeros(self.dimensions)
self.pbest_position = np.empty(self.dimensions)
self.pbest_value = None
self.lowerbounds, self.upperbounds = np.asarray(bounds).T
# initialize positions
if method == 'random':
position = np.random.rand(self.dimensions)
elif method == 'latin':
space = Space([(0.,1.)]*self.dimensions)
lhs = Lhs()
position = np.asarray(lhs.generate(space.dimensions,1))[0]
min_b, max_b = np.asarray(bounds).T
diff = max_b - min_b
self.position = min_b + position * diff
