def plot_evaluations(self):
res = create_result(Xi=self.opt.Xi,
yi=self.opt.yi,
space=self.opt.space,
rng=self.opt.rng,
models=self.opt.models)
plot_objective(res, dimensions=self.parameter_names)
plt.show()
def run(
self, func, n_iter=1, replace=False, n_samples=5, gp_samples=100, gp_burnin=10
):
"""Execute the ask/tell-loop on a given objective function.
Parameters
----------
func : function
The objective function to minimize. Should either return a scalar value,
or a tuple (value, noise) where the noise should be a variance.
n_iter : int, optional (default: 1)
Number of iterations to perform.
replace : bool, optional (default: False)
If True, the existing data points will be replaced with the ones collected
from now on. The existing model will be used as initialization.
n_samples : int, optional (default: 5)
Number of hyperposterior samples over which to average the acquisition
function.
gp_samples : int, optional (default: 100)
Number of hyperposterior samples to collect during inference. More samples
result in a more accurate representation of the hyperposterior, but
increase the running time.
Has to be a multiple of 100.
gp_burnin : int, optional (default: 10)
Number of inference iterations to discard before beginning collecting
hyperposterior samples. Only needs to be increased, if the hyperposterior
after burnin has not settled on the typical set. Drastically increases
running time.
Returns
-------
scipy.optimize.OptimizeResult object
Contains the points, the values of the objective function, the search space,
the random state and the list of models.
"""
for _ in range(n_iter):
x = self.ask()
out = func(x)
if hasattr(out, "__len__"):
val, noise = out
else:
val = out
noise = 0.0
self.tell(
x,
val,
noise_vector=noise,
n_samples=n_samples,
gp_samples=gp_samples,
gp_burnin=gp_burnin,
replace=replace,
)
replace = False
return create_result(self.Xi, self.yi, self.space, self.rng, models=[self.gp])
def optimize(self,
num_vars,
objective_function,
gradient_function=None,
variable_bounds=None,
initial_point=None):
callbacks = []
def alt_obj_fn(pars):
fn = objective_function(pars)
callbacks.append({'point': pars, 'fn': fn})
return fn
result = super().optimize(num_vars, alt_obj_fn, gradient_function,
variable_bounds, initial_point)
if self._num_restarts is not None:
for i in range(self._num_restarts - 1):
if variable_bounds is not None:
init_pt = [
np.random.uniform(dn, up) for dn, up in variable_bounds
]
else:
init_pt = [
np.random.uniform(-np.pi, +np.pi)
for _ in range(num_vars)
]
result_new = super().optimize(num_vars, alt_obj_fn,
gradient_function,
variable_bounds, init_pt)
if result_new[1] < result[1]:
result = result_new
X = [step['point'] for step in callbacks]
y = [step['fn'] for step in callbacks]
if self._make_model and (len(callbacks) < self._max_model_points):
model = GaussianProcessRegressor()
model.fit(X, y)
else:
model = None
if variable_bounds is not None:
space = Space([Real(low, high) for low, high in variable_bounds])
else:
space = None
self.optimization_result = create_result(
X, y, space=space, models=[model] if model is not None else None)
return result
def rs_minimize(func,
dimensions,
n_calls=64,
n_random_starts=13,
mutation_rate=0.1):
"""
Use the RandomStepOptimizer here
"""
optimizer = RandomStepOptimizer(dimensions=dimensions,
n_random_starts=n_random_starts,
mutation_rate=mutation_rate)
for i in range(n_calls):
x = optimizer.ask()
y = func(x)
optimizer.tell(x, y)
return create_result(optimizer.Xi, optimizer.yi, dimensions)
def run(self, func, n_iter=1):
"""Execute :meth:`ask` + :meth:`tell` loop for `n_iter` iterations
Parameters
----------
func: Callable
Function that returns the objective value `y`, when given a search point `x`
n_iter: Int, default=1
Number of `ask`/`tell` sequences to execute
Returns
-------
OptimizeResult
`scipy.optimize.OptimizeResult` instance"""
for _ in range(n_iter):
x = self.ask()
self.tell(x, func(x))
return create_result(self.Xi, self.yi, self.space, self.rng, models=self.models)
def ga_minimize(func,
dimensions,
n_calls=64,
n_random_starts=10,
tournament_fraction=0.2,
mutation_rate=0.05):
"""
Use the GeneticOptimizer here
"""
optimizer = GeneticOptimizer(dimensions=dimensions,
n_random_starts=n_random_starts,
tournament_fraction=tournament_fraction,
mutation_rate=mutation_rate)
for i in range(n_calls):
x = optimizer.ask()
y = func(x)
optimizer.tell(x, y)
return create_result(optimizer.Xi, optimizer.yi, dimensions)
def smac_minimize(func, dims, n_calls=64):
param_names = sorted([rnd_name() for d in dims])
search_space = dict(zip(param_names, dims))
cs = ConfigurationSpace()
for k, v in search_space.items():
if isinstance(v, Real):
cs.add_hyperparameter(
UniformFloatHyperparameter(name=k,
lower=v.bounds[0],
upper=v.bounds[1]))
elif isinstance(v, Integer):
cs.add_hyperparameter(
UniformIntegerHyperparameter(name=k,
lower=v.bounds[0],
upper=v.bounds[1]))
elif isinstance(v, Categorical):
cs.add_hyperparameter(
CategoricalHyperparameter(name=k, choices=v.categories))
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": n_calls, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
with tempfile.NamedTemporaryFile() as save_to:
tape_recorder = SMACBenchWrapper(func, save_to.name)
taf = ExecuteTAFuncDict(tape_recorder)
smac = SMAC(scenario=scenario, tae_runner=taf)
incumbent = smac.optimize()
Xi, yi = pickle.load(open(save_to.name, 'rb'))
#x = point_aslist(search_space, parameters)
#y = value
return create_result(Xi, yi, dims)
def best_params_(self):
check_is_fitted(self, "cv_results_")
if self.return_policy == "best_setting" or len(self.optimizers_) > 1:
if len(self.optimizers_) > 1:
logging.warning(
"Return policy 'best_mean' is incompatible with multiple search"
"spaces. Reverting to 'best_setting'."
)
return self.cv_results_["params"][self.best_index_]
if self.return_policy == "best_mean":
random_state = self.optimizer_kwargs_["random_state"]
# We construct a result object manually here, since in skopt versions up to
# 0.7.4 they were not saved yet:
opt = self.optimizers_[0]
result_object = create_result(
opt.Xi, opt.yi, space=opt.space, rng=random_state, models=[opt.gp]
)
point, _ = expected_minimum(
res=result_object, n_random_starts=100, random_state=random_state,
)
dict = point_asdict(self.search_spaces, point)
return dict
def gpyopt_minimize(func, dims, n_calls=64):
param_names = sorted([rnd_name() for d in dims])
bounds = []
transforms = []
for v, n in zip(dims, param_names):
if isinstance(v, Real):
bounds.append({
'name': n,
'type': 'continuous',
'domain': v.bounds
})
transforms.append(None)
elif isinstance(v, Integer):
bounds.append({
'name': n,
'type': 'continuous',
'domain': v.bounds
})
transforms.append(lambda x: int(np.round(x)))
elif isinstance(v, Categorical):
bounds.append({
'name': n,
'type': 'discrete',
'domain': v.categories
})
transforms.append(None)
with tempfile.NamedTemporaryFile() as save_to:
tape_recorder = BenchWrapper(func, save_to.name, transforms)
myProblem = GPyOpt.methods.BayesianOptimization(tape_recorder, bounds)
myProblem.run_optimization(n_calls)
Xi, yi = pickle.load(open(save_to.name, 'rb'))
# somehow gpyopt seems to evaluate more than asked
Xi = Xi[:n_calls]
yi = yi[:n_calls]
return create_result(Xi, yi, dims)
def hyperopt_minimize(func, dims, n_calls=64):
param_names = sorted([rnd_name() for d in dims])
search_space = dict(zip(param_names, dims))
for k, v in search_space.items():
if isinstance(v, Real):
search_space[k] = hp.uniform(k, v.bounds[0], v.bounds[1])
elif isinstance(v, Integer):
search_space[k] = hp.quniform(k, v.bounds[0], v.bounds[1], 1)
elif isinstance(v, Categorical):
search_space[k] = hp.choice(k, v.categories)
with tempfile.NamedTemporaryFile() as save_to:
tape_recorder = BenchWrapper(func, save_to.name, dims)
solution = fmin(fn=tape_recorder,
space=hp.choice('main', [search_space]),
algo=tpe.suggest,
max_evals=n_calls)
Xi, yi = pickle.load(open(save_to.name, 'rb'))
return create_result(Xi, yi, dims)
def _tell(self, x, y, fit=True):
# Copied from skopt
"""Perform the actual work of incorporating one or more new points.
See `tell()` for the full description.
This method exists to give access to the internals of adding points
by side stepping all input validation and transformation."""
if "ps" in self.acq_func:
if is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# if y isn't a scalar it means we have been handed a batch of points
elif is_listlike(y) and is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError(
"Type of arguments `x` (%s) and `y` (%s) not compatible." %
(type(x), type(y)))
# optimizer learned something new - discard cache
self.cache_ = {}
# after being "told" n_initial_points we switch from sampling
# random points to using a surrogate model
if fit and self._n_initial_points <= 0 and self.base_estimator_ is not None:
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator_)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
# We're gonna lie to the estimator by telling it a loss for the points that are still being evaluated,
# similar to what we do when we ask for multiple points in ask().
points_running = self.watch_list.keys()
num_points_running = len(points_running)
points_to_lie_about = [
self.all_dim_values[run_ind] for run_ind in points_running
]
strategy = "cl_mean"
if strategy == "cl_min":
y_lie = np.min(self.yi) if self.yi else 0.0 # CL-min lie
elif strategy == "cl_mean":
y_lie = np.mean(self.yi) if self.yi else 0.0 # CL-mean lie
else:
y_lie = np.max(self.yi) if self.yi else 0.0 # CL-max lie
# Lie to the fake optimizer.
fake_est = copy.deepcopy(est)
X_to_tell = self.Xi + points_to_lie_about
X_to_tell = self.space.transform(X_to_tell)
y_to_tell = self.yi + list(np.ones(num_points_running) * y_lie)
fake_est.fit(X_to_tell, y_to_tell)
# even with BFGS as optimizer we want to sample a large number
# of points and then pick the best ones as starting points
# X = self.space.transform(self.space.rvs(
# n_samples=self.n_points, random_state=self.rng))
Xspace = self.space.rvs(n_samples=self.n_points,
random_state=self.rng)
param_thr = self.adapt_param['param_thr']
par_cnt_scheme = self.adapt_param['par_cnt_scheme']
suitable_X, _ = check_parameter_count_for_sample(
Xspace, self.hyper_param_names, param_thr, par_cnt_scheme)
# for x in Xspace:
# vals_suitable, _ = check_parameter_count_for_sample(
# x, self.hyper_param_names, param_thr, par_cnt_scheme)
# suitable_X.append(vals_suitable)
Xspace = [
Xspace[ind] for ind, suit in enumerate(suitable_X) if suit
]
X = self.space.transform(Xspace)
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
values = _gaussian_acquisition(
X=X,
model=fake_est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs)
# Find the minimum of the acquisition function by randomly
# sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the mimimum of the acquisition function, the
# minimization starts from `n_restarts_optimizer` different
# points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(fake_est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
# lbfgs should handle this but just in case there are
# precision errors.
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
def probability_of_optimality(
self,
threshold,
n_space_samples=500,
n_gp_samples=200,
n_random_starts=100,
use_mean_gp=True,
normalized_scores=True,
random_state=None,
):
""" Compute the probability that the current expected optimum cannot be improved
by more than ``threshold`` points.
Parameters
----------
threshold : float or list-of-floats
Other points have to be better than the current optimum by at least a margin
of size ``threshold``. If a list is passed, this will return a list of
probabilities.
n_space_samples : int, default=500
Number of random samples used to cover the optimization space.
n_gp_samples : int, default=200
Number of functions to sample from the Gaussian process.
n_random_starts : int, default=100
Number of random positions to start the optimizer from in order to determine
the global optimum.
use_mean_gp : bool, default=True
If True, random functions will be sampled from the consensus GP, which is
usually faster, but could underestimate the variability. If False, the
posterior distribution over hyperparameters is used to sample different GPs
and then sample functions.
normalized_scores : bool, optional (default: True)
If True, normalize the optimality gaps by the function specific standard
deviation. This makes the optimality gaps more comparable, especially if
`use_mean_gp` is False.
random_state : int, RandomState instance, or None (default)
Set random state to something other than None for reproducible results.
Returns
-------
probabilities : float or list-of-floats
Probabilities of the current optimum to be optimal wrt the given thresholds.
"""
result = create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=[self.gp])
X_orig = [
expected_minimum(result,
random_state=random_state,
n_random_starts=n_random_starts)[0]
]
X_orig.extend(
self.space.rvs(n_samples=n_space_samples,
random_state=random_state))
X_trans = self.space.transform(X_orig)
score_samples = self.gp.sample_y(
X_trans,
n_samples=n_gp_samples,
sample_mean=use_mean_gp,
random_state=random_state,
)
if normalized_scores:
std = np.std(score_samples, axis=0)
if not is_listlike(threshold):
threshold = [threshold]
probabilities = []
for eps in threshold:
if normalized_scores:
diff = (score_samples[0][None, :] - score_samples) / std
else:
diff = score_samples[0][None, :] - score_samples
probabilities.append(((diff - eps).max(axis=0) < 0.0).mean())
if len(probabilities) == 1:
return probabilities[0]
return probabilities
def tell(
self,
x,
y,
noise_vector=None,
fit=True,
replace=False,
n_samples=0,
gp_samples=100,
gp_burnin=10,
progress=False,
):
"""Inform the optimizer about the objective function at discrete points.
Provide values of the objective function at points suggested by `ask()` or other
points. By default a new model will be fit to all observations.
The new model is used to suggest the next point at which to evaluate the
objective. This point can be retrieved by calling `ask()`.
To add observations without fitting a new model set `fit` to False.
To add multiple observations in a batch pass a list-of-lists for `x`
and a list of scalars for `y`.
Parameters
----------
x : list or list of lists
Point(s) at which the objective function was evaluated.
y : scalar or list
Value(s) of the objective function at `x`.
noise_vector : list, default=None
Variance(s) of the objective function at `x`.
fit : bool, optional (default: True)
If True, a model will be fitted to the points, if `n_initial_points` points
have been evaluated.
replace : bool, optional (default: False)
If True, the existing data points will be replaced with the one given in
`x` and `y`.
n_samples : int, optional (default: 0)
Number of hyperposterior samples over which to average the acquisition
function. More samples make the acquisition function more robust, but
increase the running time.
Can be set to 0 for `pvrs` and `vr`.
gp_samples : int, optional (default: 100)
Number of hyperposterior samples to collect during inference. More samples
result in a more accurate representation of the hyperposterior, but
increase the running time.
Has to be a multiple of 100.
gp_burnin : int, optional (default: 10)
Number of inference iterations to discard before beginning collecting
hyperposterior samples. Only needs to be increased, if the hyperposterior
after burnin has not settled on the typical set. Drastically increases
running time.
progress : bool, optional (default: False)
If True, show a progress bar during the inference phase.
Returns
-------
scipy.optimize.OptimizeResult object
Contains the points, the values of the objective function, the search space,
the random state and the list of models.
"""
if replace:
self.Xi = []
self.yi = []
self.noisei = []
self._n_initial_points = self.n_initial_points_
if is_listlike(y) and is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
if noise_vector is None:
noise_vector = [0.0] * len(y)
elif not is_listlike(noise_vector) or len(noise_vector) != len(y):
raise ValueError(
"Vector of noise variances needs to be of equal length as `y`."
)
self.noisei.extend(noise_vector)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
if noise_vector is None:
noise_vector = 0.0
elif is_listlike(noise_vector):
raise ValueError(
"Vector of noise variances is a list, while tell only received one"
"datapoint.")
self.noisei.append(noise_vector)
self._n_initial_points -= 1
else:
raise ValueError(
f"Type of arguments `x` ({type(x)}) and `y` ({type(y)}) "
"not compatible.")
if fit and self._n_initial_points <= 0:
if (self.gp_priors is not None and
len(self.gp_priors) != self.space.transformed_n_dims + 2):
raise ValueError(
"The number of priors does not match the number of dimensions + 2."
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if self.gp.pos_ is None or replace:
self.gp.fit(
self.space.transform(self.Xi),
self.yi,
noise_vector=np.array(self.noisei),
priors=self.gp_priors,
n_desired_samples=gp_samples,
n_burnin=gp_burnin,
progress=progress,
)
else:
self.gp.sample(
self.space.transform(self.Xi),
self.yi,
noise_vector=np.array(self.noisei),
priors=self.gp_priors,
n_desired_samples=gp_samples,
n_burnin=gp_burnin,
progress=progress,
)
if self.gp.warp_inputs:
X_warped = self.rng.uniform(
size=(self.n_points, self.space.transformed_n_dims))
X = self.gp.unwarp(X_warped)
else:
X = self.space.transform(
self.space.rvs(n_samples=self.n_points,
random_state=self.rng))
acq_values = evaluate_acquisitions(
X=X,
gpr=self.gp,
acquisition_functions=(self.acq_func, ),
n_samples=n_samples,
progress=False,
random_state=self.rng.randint(0,
np.iinfo(np.int32).max),
**self.acq_func_kwargs,
).flatten()
self._next_x = self.space.inverse_transform(
X[np.argmax(acq_values)].reshape((1, -1)))[0]
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=[self.gp])
def _tell(self, x, y, fit=True):
"""Perform the actual work of incorporating one or more new points. See :meth:`tell` for
the full description. This method exists to give access to the internals of adding points
by side-stepping all input validation and transformation"""
#################### Collect Search Points and Evaluations ####################
# TODO: Clean up below - Looks like the 4 extend/append blocks may be duplicated
if "ps" in self.acq_func:
if is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# If `y` isn't a scalar, we have been handed a batch of points
elif is_list_like(y) and is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError(
f"Incompatible argument types: `x` ({type(x)}) and `y` ({type(y)})"
)
# Optimizer learned something new. Discard `cache_`
self.cache_ = {}
#################### Fit Surrogate Model ####################
# After being `tell`-ed `n_initial_points`, use surrogate model instead of random sampling
# TODO: Clean up and separate below. Pretty hard to follow the whole thing
if fit and self._n_initial_points <= 0 and self.base_estimator is not None:
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
# Even with BFGS optimizer, we want to sample a large number of points, and
# pick the best ones as starting points
X = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rng))
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
# TODO: Rename `values` - Maybe `utilities`?
values = _gaussian_acquisition(
X=X,
model=est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs,
)
#################### Find Acquisition Function Minimum ####################
# Find acquisition function minimum by randomly sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the minimum of the acquisition function, the minimization starts
# from `n_restarts_optimizer` different points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20,
) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
else:
# `acq_optimizer` should have already been checked, so this shouldn't be hit,
# but, it's here anyways to prevent complaints about `next_x` not existing in
# the absence of this `else` clause
raise RuntimeError(
f"Invalid `acq_optimizer` value: {self.acq_optimizer}")
# L-BFGS-B should handle this, but just in case of precision errors...
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# Note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
def run(self):
# 0. Before we run the main loop, do we need to initialize or resume?
# * Resume from files (in experiment folder)
# * Create tune entry in db if it does not exist yet
if "tune_id" not in self.experiment:
with self.sessionmaker() as session:
tune = SqlTune(
weight=self.experiment.get("weight", 1.0),
description=self.experiment.get("description", None),
)
session.add(tune)
session.flush()
self.experiment["tune_id"] = tune.id
self.write_experiment_file()
new_x = self.opt.ask()
# Alter engine json using Initstrings
params = dict(zip(self.parameters, new_x))
self.change_engine_config(self.experiment["engine"], params)
self.insert_jobs(session, new_x)
self.logger.info("New jobs committed to database.")
while True:
self.logger.debug("Begin querying for new data...")
# Check if minimum sample size and minimum wait time are reached, then query
# data and update model:
with self.sessionmaker() as session:
X, y, variances, samplesize_reached = self.query_data(
session, include_active=True
)
self.logger.debug(
f"Queried the database for data and got (last 5):\n"
f"{X[-5:]}\n{y[-5:]}"
)
if len(X) == 0:
self.logger.info("There are no datapoints yet, start first job")
new_x = self.opt.ask()
# Alter engine json using Initstrings
params = dict(zip(self.parameters, new_x))
self.change_engine_config(self.experiment["engine"], params)
self.insert_jobs(session, new_x)
self.logger.info("New jobs committed to database.")
samplesize_reached = False
if not samplesize_reached:
sleep_seconds = self.experiment.get("sleep_time", 60)
self.logger.debug(
f"Required sample size not yet reached. Sleeping {sleep_seconds}"
f"seconds."
)
sleep(sleep_seconds)
continue
# Tell optimizer about the new results:
now = datetime.now()
self.opt.tell(
X.tolist(),
y.tolist(),
noise_vector=variances.tolist(),
fit=True,
replace=True,
n_samples=self.tunecfg["n_samples"],
gp_samples=self.tunecfg["gp_samples"],
gp_burnin=self.tunecfg["gp_burnin"],
progress=False,
)
later = datetime.now()
difference = (later - now).total_seconds()
self.logger.info(
f"Calculating GP posterior and acquisition function finished in "
f"{difference}s"
)
self.logger.info(f"Current GP kernel:\n{self.opt.gp.kernel_}")
if self.opt.gp.chain_ is not None:
self.logger.debug("Saving position and chain")
self.save_state()
# Ask optimizer for new configuration and insert jobs:
new_x = self.opt.ask()
# Alter engine json using Initstrings
params = dict(zip(self.parameters, new_x))
self.change_engine_config(self.experiment["engine"], params)
with self.sessionmaker() as session:
self.insert_jobs(session, new_x)
self.logger.info("New jobs committed to database.")
sleep(self.experiment.get("sleep_time", 60))
if self.opt.gp.chain_ is not None:
result_object = create_result(
Xi=X.tolist(),
yi=y.tolist(),
space=self.opt.space,
models=[self.opt.gp],
)
try:
opt_x, opt_y = expected_ucb(result_object)
self.logger.info(
f"Current optimum: "
f"{dict(zip(self.parameters, np.around(opt_x,4)))}"
)
except ValueError:
self.logger.info(
"Current optimum: None (optimizer errored out :( )"
)
def tell(
self,
x,
y,
noise_vector=None,
fit=True,
replace=False,
n_samples=0,
gp_samples=100,
gp_burnin=10,
progress=False,
):
# if y isn't a scalar it means we have been handed a batch of points
# TODO (noise vector):
# 1. Replace case should be easy
# 2. Add case should add noise values to list
# -> What if noise_vector is None? (have to set noise to 0)
if replace:
self.Xi = []
self.yi = []
self.noisei = []
self._n_initial_points = self.n_initial_points_
if is_listlike(y) and is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
if noise_vector is None:
noise_vector = [0.0] * len(y)
elif not is_listlike(noise_vector) or len(noise_vector) != len(y):
raise ValueError(
f"Vector of noise variances needs to be of equal length as `y`."
)
self.noisei.extend(noise_vector)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
if noise_vector is None:
noise_vector = 0.0
elif is_listlike(noise_vector):
raise ValueError(
f"Vector of noise variances is a list, while tell only received one datapoint."
)
self.noisei.append(noise_vector)
self._n_initial_points -= 1
else:
raise ValueError(
f"Type of arguments `x` ({type(x)}) and `y` ({type(y)}) "
"not compatible."
)
if fit and self._n_initial_points <= 0:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if self.gp.pos_ is None or replace:
self.gp.fit(
self.space.transform(self.Xi),
self.yi,
noise_vector=np.array(self.noisei),
priors=self.gp_priors,
n_desired_samples=gp_samples,
n_burnin=gp_burnin,
progress=progress,
)
else:
self.gp.sample(
self.space.transform(self.Xi),
self.yi,
noise_vector=np.array(self.noisei),
priors=self.gp_priors,
n_desired_samples=gp_samples,
n_burnin=gp_burnin,
progress=progress,
)
X = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rng)
)
acq_values = evaluate_acquisitions(
X=X,
gpr=self.gp,
acquisition_functions=(self.acq_func,),
n_samples=n_samples,
progress=False,
random_state=self.rng.randint(0, np.iinfo(np.int32).max),
**self.acq_func_kwargs,
).flatten()
self._next_x = self.space.inverse_transform(
X[np.argmax(acq_values)].reshape((1, -1))
)[0]
return create_result(self.Xi, self.yi, self.space, self.rng, models=[self.gp])
def run(self, func, n_iter=1, n_samples=5, gp_burnin=10):
for _ in range(n_iter):
x = self.ask()
self.tell(x, func(x), n_samples=n_samples, gp_burnin=gp_burnin)
return create_result(self.Xi, self.yi, self.space, self.rng, models=[self.gp])
def SkoptCMAoptimizer(
func,
dimensions,
n_calls,
verbose=False,
callback=(),
x0=None,
sigma0=.5,
normalize=True,
):
'''
Optmizer based on CMA-ES algorithm.
This is essentially a wrapper fuction for the cma library function
to align the interface with skopt library.
Args:
func (callable): function to optimize
dimensions: list of tuples. search dimensions
n_calls: the number of samples.
verbose: if this func should be verbose
callback: the list of callback functions.
x0: inital values
if None, random point will be sampled
sigma0: initial standard deviation
normalize: whether optimization domain should be normalized
Returns:
`res` skopt.OptimizeResult object
The optimization result returned as a dict object.
Important attributes are:
- `x` [list]: location of the minimum.
- `fun` [float]: function value at the minimum.
- `x_iters` [list of lists]: location of function evaluation for each
iteration.
- `func_vals` [array]: function value for each iteration.
- `space` [Space]: the optimization space.
'''
specs = {
'args': copy.copy(inspect.currentframe().f_locals),
'function': inspect.currentframe().f_code.co_name,
}
if normalize:
dimensions = list(
map(lambda x: check_dimension(x, 'normalize'), dimensions))
space = Space(dimensions)
if x0 is None: x0 = space.transform(space.rvs())[0]
tempdir = tempfile.mkdtemp()
xi, yi = [], []
options = {
'bounds': np.array(space.transformed_bounds).transpose().tolist(),
'verb_filenameprefix': tempdir,
}
def delete_tempdir(self, *args, **kargs):
os.removedirs(tempdir)
return
model = cma.CMAEvolutionStrategy(x0, sigma0, options)
model.logger.__del__ = delete_tempdir
for i in range(n_calls):
if model.stop(): break
new_xi = model.ask()
new_xi_denorm = space.inverse_transform(np.array(new_xi))
new_yi = [func(x) for x in new_xi_denorm]
model.tell(new_xi, new_yi)
model.logger.add()
if verbose: model.disp()
xi += new_xi_denorm
yi += new_yi
results = create_result(xi, yi)
for f in callback:
f(results)
results = create_result(xi, yi, space)
model.logger.load()
results.cma_logger = model.logger
results.specs = specs
return results
def spearmint_minimize(func, dims, n_calls=64):
param_names = sorted([rnd_name() for d in dims])
search_space = dict(zip(param_names, dims))
variables = {}
for k, v in search_space.items():
if isinstance(v, Real):
variables[k] = {
"type": "FLOAT",
"size": 1,
"min": v.bounds[0],
"max": v.bounds[1]
}
elif isinstance(v, Integer):
variables[k] = {
"type": "INT",
"size": 1,
"min": v.bounds[0],
"max": v.bounds[1]
}
elif isinstance(v, Categorical):
variables[k] = {
"type": "ENUM",
"size": 1,
"options": v.categories,
}
experiment_name = "exp_" + rnd_name()
cfg = {
"language": "PYTHON",
"main-file": "objective.py",
"experiment-name": experiment_name,
"polling-time": 0.1,
"max-finished-jobs": n_calls+1,
"variables": variables
}
obj_runner = """
import numpy as np
import math
import os
import pickle
loc = os.path.dirname(os.path.realpath(__file__))
loc = os.path.join(loc, 'objective.py.bin')
obj_func = pickle.load(open(loc, 'rb'))
# Write a function like this called 'main'
def main(job_id, params):
params = [params[k][0] for k in sorted(params.keys())]
return obj_func(params)
"""
exp_loc = tempfile.mkdtemp()
print(exp_loc)
obj_fnc =os.path.join(exp_loc, 'objective.py')
obj_pic = obj_fnc + ".bin"
import pickle
with open(obj_fnc, 'w') as f:
f.write(obj_runner)
with open(obj_pic, 'wb') as f:
pickle.dump(func, f)
import json
json.dump(cfg, open(os.path.join(exp_loc, 'config.json'), 'w'))
if sys.version[0] == '2':
python_executive = 'python'
elif sys.version[0] == '3':
python_executive = 'python3'
else:
raise EnvironmentError('Unsupported version of python: %s' % sys.version)
command = python_executive + " " + spearmint_home + " " + exp_loc
# https://stackoverflow.com/questions/4789837
proc = Popen(
[command],
stdout=subprocess.PIPE,
shell=True,
preexec_fn=os.setsid
)
from pymongo import MongoClient
col = MongoClient()['spearmint'][experiment_name + ".jobs"]
import time
while True:
current_n = len(list(col.find({})))
time.sleep(0.001)
if current_n > n_calls:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
break
all_jobs = list(col.find({}))
D = []
for job in all_jobs:
if job['status'] != 'complete':
continue
p = job['params']
t = job['start time']
y_value = job['values']['main']
p_dict = {k: decompress_array(v['values'])[0] for k, v in p.items()}
D.append((t, p_dict, y_value))
D.sort(key=lambda x: x[0])
D = D[:n_calls]
X = [point_aslist(search_space, d[1]) for d in D]
y = [d[2] for d in D]
shutil.rmtree(exp_loc)
return create_result(X, y, dims)
def local( # noqa: C901
tuning_config,
acq_function="mes",
acq_function_samples=1,
acq_function_lcb_alpha=1.96,
confidence=0.9,
data_path=None,
gp_burnin=5,
gp_samples=300,
gp_initial_burnin=100,
gp_initial_samples=300,
#kernel_lengthscale_prior_lower_bound=0.1,
#kernel_lengthscale_prior_upper_bound=0.5,
#kernel_lengthscale_prior_lower_steepness=2.0,
#kernel_lengthscale_prior_upper_steepness=1.0,
gp_signal_prior_scale=4.0,
gp_noise_prior_scale=0.0006,
gp_lengthscale_prior_lb=0.1,
gp_lengthscale_prior_ub=0.5,
normalize_y=True,
noise_scaling_coefficient=1,
logfile="log.txt",
n_initial_points=16,
n_points=500,
plot_every=1,
plot_path="plots",
plot_on_resume=False,
random_seed=0,
result_every=1,
resume=True,
fast_resume=True,
model_path="model.pkl",
point=None,
reset=False,
verbose=0,
warp_inputs=True,
rounds=10,
):
"""Run a local tune.
Parameters defined in the `tuning_config` file always take precedence.
"""
json_dict = json.load(tuning_config)
settings, commands, fixed_params, param_ranges = load_tuning_config(json_dict)
root_logger = setup_logger(
verbose=verbose, logfile=settings.get("logfile", logfile)
)
root_logger.debug(f"Got the following tuning settings:\n{json_dict}")
root_logger.debug(
f"Acquisition function: {acq_function}, Acquisition function samples: {acq_function_samples}, Acquisition function lcb alpha: {acq_function_lcb_alpha}, GP burnin: {gp_burnin}, GP samples: {gp_samples}, GP initial burnin: {gp_initial_burnin}, GP initial samples: {gp_initial_samples}, GP signal prior scale: {gp_signal_prior_scale}, GP noise prior scale: {gp_noise_prior_scale}, GP lengthscale prior lower bound: {gp_lengthscale_prior_lb}, GP lengthscale prior upper bound: {gp_lengthscale_prior_ub}, Warp inputs: {warp_inputs}, Normalize y: {normalize_y}, Noise scaling coefficient: {noise_scaling_coefficient}, Initial points: {n_initial_points}, Next points: {n_points}, Random seed: {random_seed}"
)
#root_logger.debug(
#f"Acquisition function: {acq_function}, Acquisition function samples: {acq_function_samples}, GP burnin: {gp_burnin}, GP samples: {gp_samples}, GP initial burnin: {gp_initial_burnin}, GP initial samples: {gp_initial_samples}, Kernel lengthscale prior lower bound: {kernel_lengthscale_prior_lower_bound}, Kernel lengthscale prior upper bound: {kernel_lengthscale_prior_upper_bound}, Kernel lengthscale prior lower steepness: {kernel_lengthscale_prior_lower_steepness}, Kernel lengthscale prior upper steepness: {kernel_lengthscale_prior_upper_steepness}, Warp inputs: {warp_inputs}, Normalize y: {normalize_y}, Noise scaling coefficient: {noise_scaling_coefficient}, Initial points: {n_initial_points}, Next points: {n_points}, Random seed: {random_seed}"
#)
root_logger.debug(
f"Chess Tuning Tools version: {importlib.metadata.version('chess-tuning-tools')}, Bayes-skopt version: {importlib.metadata.version('bask')}, Scikit-optimize version: {importlib.metadata.version('scikit-optimize')}, Scikit-learn version: {importlib.metadata.version('scikit-learn')}, SciPy version: {importlib.metadata.version('scipy')}"
)
#root_logger.debug(
#f"Chess Tuning Tools version: {pkg_resources.get_distribution('chess-tuning-tools').parsed_version}"
#)
# Initialize/import data structures:
if data_path is None:
data_path = "data.npz"
intermediate_data_path = data_path.replace(".", "_intermediate.", 1)
try:
X, y, noise, iteration, round, counts_array, point = initialize_data(
parameter_ranges=list(param_ranges.values()),
resume=resume,
data_path=data_path,
intermediate_data_path=intermediate_data_path,
)
except ValueError:
root_logger.error(
"The number of parameters are not matching the number of "
"dimensions. Rename the existing data file or ensure that the "
"parameter ranges are correct."
)
sys.exit(1)
# Initialize Optimizer object and if applicable, resume from existing
# data/optimizer:
gp_priors = create_priors(
n_parameters=len(param_ranges),
signal_scale=settings.get("gp_signal_prior_scale", gp_signal_prior_scale),
lengthscale_lower_bound=settings.get(
"gp_lengthscale_prior_lb", gp_lengthscale_prior_lb
),
lengthscale_upper_bound=settings.get(
"gp_lengthscale_prior_ub", gp_lengthscale_prior_ub
),
noise_scale=settings.get("gp_noise_prior_scale", gp_noise_prior_scale),
)
opt = initialize_optimizer(
X=X,
y=y,
noise=noise,
parameter_ranges=list(param_ranges.values()),
noise_scaling_coefficient=noise_scaling_coefficient,
random_seed=settings.get("random_seed", random_seed),
warp_inputs=settings.get("warp_inputs", warp_inputs),
normalize_y=settings.get("normalize_y", normalize_y),
#kernel_lengthscale_prior_lower_bound=settings.get("kernel_lengthscale_prior_lower_bound", kernel_lengthscale_prior_lower_bound),
#kernel_lengthscale_prior_upper_bound=settings.get("kernel_lengthscale_prior_upper_bound", kernel_lengthscale_prior_upper_bound),
#kernel_lengthscale_prior_lower_steepness=settings.get("kernel_lengthscale_prior_lower_steepness", kernel_lengthscale_prior_lower_steepness),
#kernel_lengthscale_prior_upper_steepness=settings.get("kernel_lengthscale_prior_upper_steepness", kernel_lengthscale_prior_upper_steepness),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
acq_function=settings.get("acq_function", acq_function),
acq_function_samples=settings.get("acq_function_samples", acq_function_samples),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
resume=resume,
fast_resume=fast_resume,
model_path=model_path,
gp_initial_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_initial_samples=settings.get("gp_initial_samples", gp_initial_samples),
gp_priors=gp_priors,
)
is_first_iteration_after_program_start = True
# Main optimization loop:
while True:
if round == 0:
root_logger.info("Starting iteration {}".format(iteration))
else:
root_logger.info("Resuming iteration {}".format(iteration))
# If a model has been fit, print/plot results so far:
if len(y) > 0 and opt.gp.chain_ is not None:
result_object = create_result(Xi=X, yi=y, space=opt.space, models=[opt.gp])
#root_logger.debug(f"result_object:\n{result_object}")
result_every_n = settings.get("result_every", result_every)
if result_every_n > 0 and iteration % result_every_n == 0:
print_results(
optimizer=opt,
result_object=result_object,
parameter_names=list(param_ranges.keys()),
confidence=settings.get("confidence", confidence),
)
plot_every_n = settings.get("plot_every", plot_every)
if (
plot_every_n > 0
and iteration % plot_every_n == 0
and (not is_first_iteration_after_program_start or plot_on_resume)
):
plot_results(
optimizer=opt,
result_object=result_object,
plot_path=settings.get("plot_path", plot_path),
parameter_names=list(param_ranges.keys()),
)
if point is None:
round = 0 # If previous tested point is not present, start over iteration.
counts_array = np.array([0, 0, 0, 0, 0])
if round == 0:
point = opt.ask() # Ask optimizer for next point.
point_dict = dict(zip(param_ranges.keys(), point))
root_logger.info("Testing {}".format(point_dict))
if len(y) > 0 and opt.gp.chain_ is not None:
testing_current_value = opt.gp.predict(opt.space.transform([point]))
with opt.gp.noise_set_to_zero():
_, testing_current_std = opt.gp.predict(
opt.space.transform([point]), return_std=True
)
root_logger.debug(
f"Predicted Elo: {np.around(-testing_current_value[0] * 100, 4)} +- "
f"{np.around(testing_current_std * 100, 4).item()}"
)
confidence_mult = erfinv(confidence) * np.sqrt(2)
lower_bound = np.around(
-testing_current_value * 100
- confidence_mult * testing_current_std * 100,
4,
).item()
upper_bound = np.around(
-testing_current_value * 100
+ confidence_mult * testing_current_std * 100,
4,
).item()
root_logger.debug(
f"{confidence * 100}% confidence interval of the Elo value: "
f"({lower_bound}, "
f"{upper_bound})"
)
root_logger.info("Start experiment")
else:
point_dict = dict(zip(param_ranges.keys(), point))
root_logger.info("Testing {}".format(point_dict))
if len(y) > 0 and opt.gp.chain_ is not None:
testing_current_value = opt.gp.predict(opt.space.transform([point]))
with opt.gp.noise_set_to_zero():
_, testing_current_std = opt.gp.predict(
opt.space.transform([point]), return_std=True
)
root_logger.debug(
f"Predicted Elo: {np.around(-testing_current_value[0] * 100, 4)} +- "
f"{np.around(testing_current_std * 100, 4).item()}"
)
confidence_mult = erfinv(confidence) * np.sqrt(2)
lower_bound = np.around(
-testing_current_value * 100
- confidence_mult * testing_current_std * 100,
4,
).item()
upper_bound = np.around(
-testing_current_value * 100
+ confidence_mult * testing_current_std * 100,
4,
).item()
root_logger.debug(
f"{confidence * 100}% confidence interval of the Elo value: "
f"({lower_bound}, "
f"{upper_bound})"
)
root_logger.info("Continue experiment")
# Run experiment:
now = datetime.now()
#settings["debug_mode"] = settings.get(
#"debug_mode", False if verbose <= 1 else True
#)
while round < settings.get("rounds", rounds):
round += 1
if round > 1:
root_logger.debug(
f"WW, WD, WL/DD, LD, LL experiment counts: {counts_array}"
)
score, error_variance = counts_to_penta(counts=counts_array)
root_logger.info(
"Experiment Elo so far: {} +- {}".format(
-score * 100, np.sqrt(error_variance) * 100
)
)
root_logger.debug(f"Round: {round}")
settings, commands, fixed_params, param_ranges = load_tuning_config(
json_dict
)
# Prepare engines.json file for cutechess-cli:
engine_json = prepare_engines_json(
commands=commands, fixed_params=fixed_params
)
root_logger.debug(f"engines.json is prepared:\n{engine_json}")
write_engines_json(engine_json, point_dict)
out_exp = []
out_all = []
for output_line in run_match(
**settings, tuning_config_name=tuning_config.name
):
line = output_line.rstrip()
is_debug = is_debug_log(line)
if is_debug and verbose > 2:
root_logger.debug(line)
if not is_debug:
out_exp.append(line)
out_all.append(line)
check_log_for_errors(cutechess_output=out_all)
out_exp = "\n".join(out_exp)
(
match_score,
match_error_variance,
match_counts_array,
) = parse_experiment_result(out_exp, **settings)
counts_array += match_counts_array
with AtomicWriter(
intermediate_data_path, mode="wb", overwrite=True
).open() as f:
np.savez_compressed(f, np.array(round), counts_array, point)
later = datetime.now()
difference = (later - now).total_seconds()
root_logger.info(f"Experiment finished ({difference}s elapsed).")
# Parse cutechess-cli output and report results (Elo and standard deviation):
root_logger.debug(f"WW, WD, WL/DD, LD, LL experiment counts: {counts_array}")
score, error_variance = counts_to_penta(counts=counts_array)
root_logger.info(
"Got Elo: {} +- {}".format(-score * 100, np.sqrt(error_variance) * 100)
)
X.append(point)
y.append(score)
noise.append(error_variance)
# Update data structures and persist to disk:
with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
with AtomicWriter(model_path, mode="wb", overwrite=True).open() as f:
dill.dump(opt, f)
round = 0
counts_array = np.array([0, 0, 0, 0, 0])
with AtomicWriter(
intermediate_data_path, mode="wb", overwrite=True
).open() as f:
np.savez_compressed(f, np.array(round), counts_array, point)
# Update model with the new data:
if reset:
root_logger.info("Deleting the model and generating a new one.")
# Reset optimizer.
del opt
if acq_function == "rand":
current_acq_func = random.choice(["mes", "pvrs", "ei", "lcb", "ts"])
root_logger.debug(
f"Current random acquisition function: {current_acq_func}"
)
else:
current_acq_func = acq_function
opt = initialize_optimizer(
X=X,
y=y,
noise=noise,
parameter_ranges=list(param_ranges.values()),
noise_scaling_coefficient=noise_scaling_coefficient,
random_seed=settings.get("random_seed", random_seed),
warp_inputs=settings.get("warp_inputs", warp_inputs),
normalize_y=settings.get("normalize_y", normalize_y),
#kernel_lengthscale_prior_lower_bound=settings.get("kernel_lengthscale_prior_lower_bound", kernel_lengthscale_prior_lower_bound),
#kernel_lengthscale_prior_upper_bound=settings.get("kernel_lengthscale_prior_upper_bound", kernel_lengthscale_prior_upper_bound),
#kernel_lengthscale_prior_lower_steepness=settings.get("kernel_lengthscale_prior_lower_steepness", kernel_lengthscale_prior_lower_steepness),
#kernel_lengthscale_prior_upper_steepness=settings.get("kernel_lengthscale_prior_upper_steepness", kernel_lengthscale_prior_upper_steepness),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
acq_function=current_acq_func,
acq_function_samples=settings.get(
"acq_function_samples", acq_function_samples
),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
resume=True,
fast_resume=False,
model_path=None,
gp_initial_burnin=settings.get("gp_burnin", gp_burnin),
gp_initial_samples=settings.get("gp_samples", gp_samples),
)
else:
root_logger.info("Updating model.")
if acq_function == "rand":
opt.acq_func = ACQUISITION_FUNC[
random.choice(["mes", "pvrs", "ei", "lcb", "ts"])
]
root_logger.debug(
f"Current random acquisition function: {opt.acq_func}"
)
update_model(
optimizer=opt,
point=point,
score=score,
variance=error_variance,
noise_scaling_coefficient=noise_scaling_coefficient,
acq_function_samples=settings.get(
"acq_function_samples", acq_function_samples
),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
gp_burnin=settings.get("gp_burnin", gp_burnin),
gp_samples=settings.get("gp_samples", gp_samples),
gp_initial_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_initial_samples=settings.get(
"gp_initial_samples", gp_initial_samples
),
)
iteration = len(X)
is_first_iteration_after_program_start = False
#with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
#np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
with AtomicWriter(model_path, mode="wb", overwrite=True).open() as f:
dill.dump(opt, f)
def local( # noqa: C901
tuning_config,
acq_function="mes",
acq_function_samples=1,
confidence=0.9,
data_path=None,
gp_burnin=5,
gp_samples=300,
gp_initial_burnin=100,
gp_initial_samples=300,
logfile="log.txt",
n_initial_points=30,
n_points=500,
plot_every=5,
plot_path="plots",
random_seed=0,
result_every=5,
resume=True,
verbose=False,
):
"""Run a local tune.
Parameters defined in the `tuning_config` file always take precedence.
"""
json_dict = json.load(tuning_config)
settings, commands, fixed_params, param_ranges = load_tuning_config(json_dict)
log_level = logging.DEBUG if verbose else logging.INFO
log_format = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
root_logger = logging.getLogger()
root_logger.setLevel(log_level)
file_logger = logging.FileHandler(settings.get("logfile", logfile))
file_logger.setFormatter(log_format)
root_logger.addHandler(file_logger)
console_logger = logging.StreamHandler(sys.stdout)
console_logger.setFormatter(log_format)
root_logger.addHandler(console_logger)
logging.debug(f"Got the following tuning settings:\n{json_dict}")
# 1. Create seed sequence
ss = np.random.SeedSequence(settings.get("random_seed", random_seed))
# 2. Create kernel
# 3. Create optimizer
random_state = np.random.RandomState(np.random.MT19937(ss.spawn(1)[0]))
opt = Optimizer(
dimensions=list(param_ranges.values()),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
# gp_kernel=kernel, # TODO: Let user pass in different kernels
gp_kwargs=dict(normalize_y=True),
# gp_priors=priors, # TODO: Let user pass in priors
acq_func=settings.get("acq_function", acq_function),
acq_func_kwargs=dict(alpha="inf", n_thompson=20),
random_state=random_state,
)
X = []
y = []
noise = []
iteration = 0
# 3.1 Resume from existing data:
if data_path is None:
data_path = "data.npz"
if resume:
path = pathlib.Path(data_path)
if path.exists():
with np.load(path) as importa:
X = importa["arr_0"].tolist()
y = importa["arr_1"].tolist()
noise = importa["arr_2"].tolist()
if len(X[0]) != opt.space.n_dims:
logging.error(
"The number of parameters are not matching the number of "
"dimensions. Rename the existing data file or ensure that the "
"parameter ranges are correct."
)
sys.exit(1)
reduction_needed, X_reduced, y_reduced, noise_reduced = reduce_ranges(
X, y, noise, opt.space
)
if reduction_needed:
backup_path = path.parent / (
path.stem + f"_backup_{int(time.time())}" + path.suffix
)
logging.warning(
f"The parameter ranges are smaller than the existing data. "
f"Some points will have to be discarded. "
f"The original {len(X)} data points will be saved to "
f"{backup_path}"
)
np.savez_compressed(
backup_path, np.array(X), np.array(y), np.array(noise)
)
X = X_reduced
y = y_reduced
noise = noise_reduced
iteration = len(X)
logging.info(
f"Importing {iteration} existing datapoints. This could take a while..."
)
opt.tell(
X,
y,
noise_vector=noise,
gp_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_samples=settings.get("gp_initial_samples", gp_initial_samples),
n_samples=settings.get("n_samples", 1),
progress=True,
)
logging.info("Importing finished.")
# 4. Main optimization loop:
while True:
logging.info("Starting iteration {}".format(iteration))
result_every_n = settings.get("result_every", result_every)
if (
result_every_n > 0
and iteration % result_every_n == 0
and opt.gp.chain_ is not None
):
result_object = create_result(Xi=X, yi=y, space=opt.space, models=[opt.gp])
try:
best_point, best_value = expected_ucb(result_object, alpha=0.0)
best_point_dict = dict(zip(param_ranges.keys(), best_point))
logging.info(f"Current optimum:\n{best_point_dict}")
logging.info(f"Estimated value: {best_value}")
confidence_val = settings.get("confidence", confidence)
confidence_out = confidence_intervals(
optimizer=opt,
param_names=list(param_ranges.keys()),
hdi_prob=confidence_val,
opt_samples=1000,
multimodal=False,
)
logging.info(
f"{confidence_val*100}% confidence intervals:\n{confidence_out}"
)
except ValueError:
logging.info(
"Computing current optimum was not successful. "
"This can happen in rare cases and running the "
"tuner again usually works."
)
plot_every_n = settings.get("plot_every", plot_every)
if (
plot_every_n > 0
and iteration % plot_every_n == 0
and opt.gp.chain_ is not None
):
logging.getLogger("matplotlib.font_manager").disabled = True
if opt.space.n_dims == 1:
logging.warning(
"Plotting for only 1 parameter is not supported yet."
)
else:
logging.debug("Starting to compute the next plot.")
result_object = create_result(
Xi=X, yi=y, space=opt.space, models=[opt.gp]
)
plt.style.use("dark_background")
fig, ax = plt.subplots(
nrows=opt.space.n_dims,
ncols=opt.space.n_dims,
figsize=(3 * opt.space.n_dims, 3 * opt.space.n_dims),
)
fig.patch.set_facecolor("#36393f")
for i in range(opt.space.n_dims):
for j in range(opt.space.n_dims):
ax[i, j].set_facecolor("#36393f")
timestr = time.strftime("%Y%m%d-%H%M%S")
plot_objective(
result_object, dimensions=list(param_ranges.keys()), fig=fig, ax=ax
)
plotpath = pathlib.Path(settings.get("plot_path", plot_path))
plotpath.mkdir(parents=True, exist_ok=True)
full_plotpath = plotpath / f"{timestr}-{iteration}.png"
plt.savefig(
full_plotpath,
pad_inches=0.1,
dpi=300,
bbox_inches="tight",
facecolor="#36393f",
)
logging.info(f"Saving a plot to {full_plotpath}.")
plt.close(fig)
point = opt.ask()
point_dict = dict(zip(param_ranges.keys(), point))
logging.info("Testing {}".format(point_dict))
engine_json = prepare_engines_json(commands=commands, fixed_params=fixed_params)
logging.debug(f"engines.json is prepared:\n{engine_json}")
write_engines_json(engine_json, point_dict)
logging.info("Start experiment")
now = datetime.now()
out_exp, out_exp_err = run_match(**settings)
later = datetime.now()
difference = (later - now).total_seconds()
logging.info(f"Experiment finished ({difference}s elapsed).")
logging.debug(f"Raw result:\n{out_exp}\n{out_exp_err}")
score, error = parse_experiment_result(out_exp, **settings)
logging.info("Got score: {} +- {}".format(score, error))
logging.info("Updating model")
while True:
try:
now = datetime.now()
# We fetch kwargs manually here to avoid collisions:
n_samples = settings.get("acq_function_samples", acq_function_samples)
gp_burnin = settings.get("gp_burnin", gp_burnin)
gp_samples = settings.get("gp_samples", gp_samples)
if opt.gp.chain_ is None:
gp_burnin = settings.get("gp_initial_burnin", gp_initial_burnin)
gp_samples = settings.get("gp_initial_samples", gp_initial_samples)
opt.tell(
point,
score,
n_samples=n_samples,
gp_samples=gp_samples,
gp_burnin=gp_burnin,
)
else:
opt.tell(
point,
score,
n_samples=n_samples,
gp_samples=gp_samples,
gp_burnin=gp_burnin,
)
later = datetime.now()
difference = (later - now).total_seconds()
logging.info(f"GP sampling finished ({difference}s)")
logging.debug(f"GP kernel: {opt.gp.kernel_}")
except ValueError:
logging.warning(
"Error encountered during fitting. Trying to sample chain a bit. "
"If this problem persists, restart the tuner to reinitialize."
)
opt.gp.sample(n_burnin=5, priors=opt.gp_priors)
else:
break
X.append(point)
y.append(score)
noise.append(error)
iteration = len(X)
with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
@author: ivan
"""
import numpy as np
import pickle
import matplotlib.pyplot as plt
from skopt import Optimizer
from skopt.learning import GaussianProcessRegressor
from skopt.space import Real
from skopt.utils import create_result
from skopt.plots import plot_objective, partial_dependence, plot_evaluations
with open('opt.pkl', 'rb') as f:
optimizer = pickle.load(f)
res = create_result(optimizer.Xi,
optimizer.yi,
optimizer.space,
optimizer.rng,
models=optimizer.models)
plot_evaluations(res)
plot_objective(res) #, levels=30, n_samples=500)
#f, ax = plt.subplots(1, 7, sharey=True)
#for plt_i, j in enumerate([0, 1, 2, 3, 4, 5, 7]):
# xi, yi, zi = partial_dependence(res.space, res.models[-1], i=6, j=j)
# ax[plt_i].contourf(xi, yi, zi, 10, alpha=.75, cmap='jet')
#plt.show()
