class steade(AbstractOptimizer):
primary_import = "pysot"
def __init__(self, api_config):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
"""
AbstractOptimizer.__init__(self, api_config)
self.search_space = JointSpace(api_config)
self.bounds = self.search_space.get_bounds()
self.iter = 0
# Sets up the optimization problem (needs self.bounds)
self.create_opt_prob()
self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int
self.batch_size = None
self.history = []
self.proposals = []
# Population-based parameters in DE
self.population = []
self.fitness = []
self.F = 0.7
self.Cr = 0.7
# For bayes opt
self.dim = len(self.search_space.param_list)
self.torch_bounds = torch.from_numpy(self.search_space.get_bounds().T)
self.min_max_bounds = torch.from_numpy(
np.stack([np.zeros(self.dim),
np.ones(self.dim)]))
self.archive = []
self.arc_fitness = []
def create_opt_prob(self):
"""Create an optimization problem object."""
opt = OptimizationProblem()
opt.lb = self.bounds[:, 0] # In warped space
opt.ub = self.bounds[:, 1] # In warped space
opt.dim = len(self.bounds)
opt.cont_var = np.arange(len(self.bounds))
opt.int_var = []
assert len(opt.cont_var) + len(opt.int_var) == opt.dim
opt.objfun = None
self.opt = opt
def start(self, max_evals):
"""Starts a new pySOT run."""
self.history = []
self.proposals = []
# Symmetric Latin hypercube design
des_pts = max([self.batch_size, 2 * (self.opt.dim + 1)])
slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts)
# Warped RBF interpolant
rbf = RBFInterpolant(dim=self.opt.dim,
lb=self.opt.lb,
ub=self.opt.ub,
kernel=CubicKernel(),
tail=LinearTail(self.opt.dim),
eta=1e-4)
# Optimization strategy
self.strategy = DYCORSStrategy(
max_evals=self.max_evals,
opt_prob=self.opt,
exp_design=slhd,
surrogate=rbf,
asynchronous=True,
batch_size=1,
use_restarts=True,
)
def _suggest(self, n_suggestions=1):
"""Get a suggestion from the optimizer.
Parameters
----------
n_suggestions : int
Desired number of parallel suggestions in the output
Returns
-------
next_guess : list of dict
List of `n_suggestions` suggestions to evaluate the objective
function. Each suggestion is a dictionary where each key
corresponds to a parameter being optimized.
"""
if self.batch_size is None: # First call to suggest
self.batch_size = n_suggestions
self.start(self.max_evals)
# Set the tolerances pretending like we are running batch
d, p = float(self.opt.dim), float(n_suggestions)
self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p)))
# Now we can make suggestions
x_w = []
self.proposals = []
for _ in range(n_suggestions):
proposal = self.strategy.propose_action()
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept() # This triggers all the callbacks
# It is possible that pySOT proposes a previously evaluated point
# when all variables are integers, so we just abort in this case
# since we have likely converged anyway. See PySOT issue #30.
x = list(proposal.record.params) # From tuple to list
x_unwarped, = self.search_space.unwarp(x)
if x_unwarped in self.history:
warnings.warn("pySOT proposed the same point twice")
self.start(self.max_evals)
return self.suggest(n_suggestions=n_suggestions)
# NOTE: Append unwarped to avoid rounding issues
self.history.append(copy(x_unwarped))
self.proposals.append(proposal)
x_w.append(copy(x_unwarped))
return x_w
@staticmethod
def make_model(train_x, train_y, state_dict=None):
"""
Define the models based on the observed data
:param train_x: The design points/ trial solutions
:param train_y: The objective functional value of the trial solutions used for model fitting
:param state_dict: Dictionary storing the parameters of the GP model
:return:
"""
try:
model = SingleTaskGP(train_x, train_y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
# load state dict if it is passed
if state_dict is not None:
model.load_state_dict(state_dict)
except Exception as e:
print('Exception: {} in make_model()'.format(e))
return model, mll
def get_bayes_pop(self, n_suggestions):
"""
Parameters
----------
n_suggestions: Number of new suggestions/trial solutions to generate using BO
Returns
The new set of trial solutions obtained by optimizing the acquisition function
-------
"""
try:
candidates, _ = optimize_acqf(
acq_function=self.acquisition,
bounds=self.min_max_bounds,
q=n_suggestions,
num_restarts=10,
raw_samples=512, # used for initialization heuristic
sequential=True)
bayes_pop = unnormalize(candidates, self.torch_bounds).numpy()
except Exception as e:
print('Error in get_bayes_pop(): {}'.format(e))
population = self.search_space.unwarp(
bayes_pop) # Translate the solution back to the original space
return population
def mutate(self, n_suggestions):
"""
Parameters
----------
n_suggestions
Returns
-------
"""
parents = self.search_space.warp(self.population)
surrogates = self.search_space.warp(self._suggest(n_suggestions))
# Pop out 'n_suggestions' number of solutions from the archives since they will be modified
for _ in range(n_suggestions):
self.history.pop()
# self.proposals.pop()
# Applying DE mutation, for more details refer to https://ieeexplore.ieee.org/abstract/document/5601760
a, b = 0, 0
while a == b:
a = random.randrange(1, n_suggestions - 1)
b = random.randrange(1, n_suggestions - 1)
rand1 = random.sample(range(0, n_suggestions), n_suggestions)
rand2 = [(r + a) % n_suggestions for r in rand1]
rand3 = [(r + b) % n_suggestions for r in rand1]
try:
bayes_pop = self.search_space.warp(
self.get_bayes_pop(n_suggestions))
# Bayesian mutation inspired from DE/rand/2 mutation
mutants = bayes_pop[rand1, :] + \
self.F * (surrogates[rand2, :] - surrogates[rand3, :]) + \
1 / self.iter * np.random.random(parents.shape) * (parents[rand2, :] - parents[rand3, :])
except Exception as e:
# DE/rand/2 mutation applied when decomposition error encountered in BO
mutants = parents[rand1, :] + \
self.F * (surrogates[rand2, :] - surrogates[rand3, :]) + \
1 / self.iter * np.random.random(parents.shape) * (parents[rand2, :] - parents[rand3, :])
# Check the bound constraints and do (binomial) crossover to generate offsprings/ donor vectors
offsprings = deepcopy(parents)
bounds = self.search_space.get_bounds()
dims = len(bounds)
for i in range(n_suggestions):
j_rand = random.randrange(dims)
for j in range(dims):
# Check if the bound-constraints are satisfied or not
if mutants[i, j] < bounds[j, 0]:
mutants[i, j] = bounds[j, 0] # surrogates[i, j]
if bounds[j, 1] < mutants[i, j]:
mutants[i, j] = bounds[j, 1] # surrogates[i, j]
if random.random() <= self.Cr or j == j_rand:
offsprings[i, j] = mutants[i, j]
# Translate the offspring back into the original space
population = self.search_space.unwarp(offsprings)
# Now insert the solutions back to the archive.
for i in range(n_suggestions):
self.history.append(population[i])
return population
def suggest(self, n_suggestions=1):
"""Get a suggestion from the optimizer.
Parameters
----------
n_suggestions : int
Desired number of parallel suggestions in the output
Returns
-------
next_guess : list of dict
List of `n_suggestions` suggestions to evaluate the objective
function. Each suggestion is a dictionary where each key
corresponds to a parameter being optimized.
"""
self.iter += 1
lamda = 10 # defines the transition point in the algorithm
if self.iter < lamda:
population = self._suggest(n_suggestions)
else:
population = self.mutate(n_suggestions)
return population
def _observe(self, x, y):
# Find the matching proposal and execute its callbacks
idx = [x == xx for xx in self.history]
i = np.argwhere(
idx)[0].item() # Pick the first index if there are ties
proposal = self.proposals[i]
proposal.record.complete(y)
self.proposals.pop(i)
self.history.pop(i)
def observe(self, X, y):
"""Send an observation of a suggestion back to the optimizer.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated
"""
try:
assert len(X) == len(y)
c = 0
for x_, y_ in zip(X, y):
# Archive stores all the solutions
self.archive.append(x_)
self.arc_fitness.append(
-y_) # As BoTorch solves a maximization problem
if self.iter == 1:
self.population.append(x_)
self.fitness.append(y_)
else:
if y_ <= self.fitness[c]:
self.population[c] = x_
self.fitness[c] = y_
c += 1
# Just ignore, any inf observations we got, unclear if right thing
if np.isfinite(y_):
self._observe(x_, y_)
# Transform the data (seen till now) into tensors and train the model
train_x = normalize(torch.from_numpy(
self.search_space.warp(self.archive)),
bounds=self.torch_bounds)
train_y = standardize(
torch.from_numpy(
np.array(self.arc_fitness).reshape(len(self.arc_fitness),
1)))
# Fit the GP based on the actual observed values
if self.iter == 1:
self.model, mll = self.make_model(train_x, train_y)
else:
self.model, mll = self.make_model(train_x, train_y,
self.model.state_dict())
# mll.train()
fit_gpytorch_model(mll)
# define the sampler
sampler = SobolQMCNormalSampler(num_samples=512)
# define the acquisition function
self.acquisition = qExpectedImprovement(model=self.model,
best_f=train_y.max(),
sampler=sampler)
except Exception as e:
print('Error: {} in observe()'.format(e))
class RandomSampler(HyperoptSampler):
num_samples = 10
def __init__(self,
goal: str,
parameters: Dict[str, Any],
num_samples=10,
**kwargs) -> None:
HyperoptSampler.__init__(self, goal, parameters)
params_for_join_space = copy.deepcopy(parameters)
cat_params_values_types = {}
for param_name, param_values in params_for_join_space.items():
if param_values[TYPE] == CATEGORY:
param_values[TYPE] = 'cat'
values_str = []
values_types = {}
for value in param_values['values']:
value_type = type(value)
if value_type == bool:
value_str = str(value)
value_type = str2bool
elif value_type == str or value_type == int or \
value_type == float:
value_str = str(value)
else:
value_str = json.dumps(value)
value_type = json.loads
values_str.append(value_str)
values_types[value_str] = value_type
param_values['values'] = values_str
cat_params_values_types[param_name] = values_types
if param_values[TYPE] == FLOAT:
param_values[TYPE] = 'real'
if param_values[TYPE] == INT or param_values[TYPE] == 'real':
if SPACE not in param_values:
param_values[SPACE] = 'linear'
param_values['range'] = (param_values['low'],
param_values['high'])
del param_values['low']
del param_values['high']
self.cat_params_values_types = cat_params_values_types
self.space = JointSpace(params_for_join_space)
self.num_samples = num_samples
self.samples = self._determine_samples()
self.sampled_so_far = 0
self.default_batch_size = self.num_samples
def _determine_samples(self):
samples = []
for _ in range(self.num_samples):
bnds = self.space.get_bounds()
x = bnds[:, 0] + (bnds[:, 1] - bnds[:, 0]) * np.random.rand(
1, len(self.space.get_bounds()))
sample = self.space.unwarp(x)[0]
samples.append(sample)
return samples
def sample(self) -> Dict[str, Any]:
if self.sampled_so_far >= len(self.samples):
raise IndexError()
sample = self.samples[self.sampled_so_far]
for key in sample:
if key in self.cat_params_values_types:
values_types = self.cat_params_values_types[key]
sample[key] = values_types[sample[key]](sample[key])
self.sampled_so_far += 1
return sample
def update(self, sampled_parameters: Dict[str, Any], metric_score: float):
pass
def finished(self) -> bool:
return self.sampled_so_far >= len(self.samples)
class PySOTOptimizer(AbstractOptimizer):
primary_import = "pysot"
def __init__(self, api_config):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
"""
AbstractOptimizer.__init__(self, api_config)
self.space_x = JointSpace(api_config)
self.bounds = self.space_x.get_bounds()
self.create_opt_prob(
) # Sets up the optimization problem (needs self.bounds)
self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int
self.batch_size = None
self.history = []
self.proposals = []
def create_opt_prob(self):
"""Create an optimization problem object."""
opt = OptimizationProblem()
opt.lb = self.bounds[:, 0] # In warped space
opt.ub = self.bounds[:, 1] # In warped space
opt.dim = len(self.bounds)
opt.cont_var = np.arange(len(self.bounds))
opt.int_var = []
assert len(opt.cont_var) + len(opt.int_var) == opt.dim
opt.objfun = None
self.opt = opt
def start(self, max_evals):
"""Starts a new pySOT run."""
self.history = []
self.proposals = []
# Symmetric Latin hypercube design
des_pts = max([self.batch_size, 2 * (self.opt.dim + 1)])
slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts)
# Warped RBF interpolant
rbf = RBFInterpolant(dim=self.opt.dim,
lb=self.opt.lb,
ub=self.opt.ub,
kernel=CubicKernel(),
tail=LinearTail(self.opt.dim),
eta=1e-4)
# Optimization strategy
'''
self.strategy = SRBFStrategy(
max_evals=self.max_evals,
opt_prob=self.opt,
exp_design=slhd,
surrogate=rbf,
asynchronous=True,
batch_size=1
, use_restarts=True,
)
'''
self.strategy = DYCORSStrategy(
max_evals=self.max_evals,
opt_prob=self.opt,
exp_design=slhd,
surrogate=rbf,
asynchronous=True,
batch_size=1,
use_restarts=True,
)
def suggest(self, n_suggestions=1):
"""Get a suggestion from the optimizer.
Parameters
----------
n_suggestions : int
Desired number of parallel suggestions in the output
Returns
-------
next_guess : list of dict
List of `n_suggestions` suggestions to evaluate the objective
function. Each suggestion is a dictionary where each key
corresponds to a parameter being optimized.
"""
if self.batch_size is None: # First call to suggest
self.batch_size = n_suggestions
self.start(self.max_evals)
# Set the tolerances pretending like we are running batch
d, p = float(self.opt.dim), float(n_suggestions)
self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p)))
# Now we can make suggestions
x_w = []
self.proposals = []
for _ in range(n_suggestions):
proposal = self.strategy.propose_action()
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept() # This triggers all the callbacks
# It is possible that pySOT proposes a previously evaluated point
# when all variables are integers, so we just abort in this case
# since we have likely converged anyway. See PySOT issue #30.
x = list(proposal.record.params) # From tuple to list
x_unwarped, = self.space_x.unwarp(x)
if x_unwarped in self.history:
warnings.warn("pySOT proposed the same point twice")
self.start(self.max_evals)
return self.suggest(n_suggestions=n_suggestions)
# NOTE: Append unwarped to avoid rounding issues
self.history.append(copy(x_unwarped))
self.proposals.append(proposal)
x_w.append(copy(x_unwarped))
return x_w
def _observe(self, x, y):
# Find the matching proposal and execute its callbacks
idx = [x == xx for xx in self.history]
i = np.argwhere(
idx)[0].item() # Pick the first index if there are ties
proposal = self.proposals[i]
proposal.record.complete(y)
self.proposals.pop(i)
self.history.pop(i)
def observe(self, X, y):
"""Send an observation of a suggestion back to the optimizer.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated
"""
assert len(X) == len(y)
for x_, y_ in zip(X, y):
# Just ignore, any inf observations we got, unclear if right thing
if np.isfinite(y_):
self._observe(x_, y_)
class tuSOTOptimizer(AbstractOptimizer):
primary_import = "pysot"
def __init__(self, api_config):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
"""
AbstractOptimizer.__init__(self, api_config)
self.space_x = JointSpace(api_config)
self.bounds = self.space_x.get_bounds()
self.create_opt_prob(
) # Sets up the optimization problem (needs self.bounds)
self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int
self.turbo_batch_size = None
self.pysot_batch_size = None
self.history = []
self.proposals = []
self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1]
self.dim = len(self.bounds)
self.turbo = Turbo1(
f=None,
lb=self.bounds[:, 0],
ub=self.bounds[:, 1],
n_init=2 * self.dim + 1,
max_evals=self.max_evals,
batch_size=4, # We need to update this later
verbose=False,
)
# hyperopt
self.random = np_random
space, self.round_to_values = tuSOTOptimizer.get_hyperopt_dimensions(
api_config)
self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None)
self.trials = Trials()
# Some book keeping like opentuner wrapper
self.trial_id_lookup = {}
# Store just for data validation
self.param_set_chk = frozenset(api_config.keys())
def restart(self):
self.turbo._restart()
self.turbo._X = np.zeros((0, self.turbo.dim))
self.turbo._fX = np.zeros((0, 1))
X_init = latin_hypercube(self.turbo.n_init, self.dim)
self.X_init = from_unit_cube(X_init, self.lb, self.ub)
def create_opt_prob(self):
"""Create an optimization problem object."""
opt = OptimizationProblem()
opt.lb = self.bounds[:, 0] # In warped space
opt.ub = self.bounds[:, 1] # In warped space
opt.dim = len(self.bounds)
opt.cont_var = np.arange(len(self.bounds))
opt.int_var = []
assert len(opt.cont_var) + len(opt.int_var) == opt.dim
opt.objfun = None
self.opt = opt
def start(self):
"""Starts a new pySOT run."""
self.history = []
self.proposals = []
# Symmetric Latin hypercube design
des_pts = max([self.pysot_batch_size, 2 * (self.opt.dim + 1)])
slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts)
# Warped RBF interpolant
rbf = RBFInterpolant(dim=self.opt.dim,
kernel=CubicKernel(),
tail=LinearTail(self.opt.dim),
eta=1e-4)
rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub)
# Optimization strategy
self.strategy = SRBFStrategy(
max_evals=self.max_evals,
opt_prob=self.opt,
exp_design=slhd,
surrogate=rbf,
asynchronous=True,
batch_size=1,
use_restarts=True,
)
@staticmethod
def hashable_dict(d):
hashable_object = frozenset(d.items())
return hashable_object
@staticmethod
def get_hyperopt_dimensions(api_config):
param_list = sorted(api_config.keys())
space = {}
round_to_values = {}
for param_name in param_list:
param_config = api_config[param_name]
param_type = param_config["type"]
param_space = param_config.get("space", None)
param_range = param_config.get("range", None)
param_values = param_config.get("values", None)
# Some setup for case that whitelist of values is provided:
values_only_type = param_type in ("cat", "ordinal")
if (param_values is not None) and (not values_only_type):
assert param_range is None
param_values = np.unique(param_values)
param_range = (param_values[0], param_values[-1])
round_to_values[param_name] = interp1d(
param_values,
param_values,
kind="nearest",
fill_value="extrapolate")
if param_type == "int":
low, high = param_range
if param_space in ("log", "logit"):
space[param_name] = hp.qloguniform(param_name, np.log(low),
np.log(high), 1)
else:
space[param_name] = hp.quniform(param_name, low, high, 1)
elif param_type == "bool":
assert param_range is None
assert param_values is None
space[param_name] = hp.choice(param_name, (False, True))
elif param_type in ("cat", "ordinal"):
assert param_range is None
space[param_name] = hp.choice(param_name, param_values)
elif param_type == "real":
low, high = param_range
if param_space in ("log", "logit"):
space[param_name] = hp.loguniform(param_name, np.log(low),
np.log(high))
else:
space[param_name] = hp.uniform(param_name, low, high)
else:
assert False, "type %s not handled in API" % param_type
return space, round_to_values
def get_trial(self, trial_id):
for trial in self.trials._dynamic_trials:
if trial["tid"] == trial_id:
assert isinstance(trial, dict)
# Make sure right kind of dict
assert "state" in trial and "result" in trial
assert trial["state"] == JOB_STATE_NEW
return trial
assert False, "No matching trial ID"
def cleanup_guess(self, x_guess):
assert isinstance(x_guess, dict)
# Also, check the keys are only the vars we are searching over:
assert frozenset(x_guess.keys()) == self.param_set_chk
# Do the rounding
# Make a copy to be safe, and also unpack singletons
# We may also need to consider clip_chk at some point like opentuner
x_guess = {k: only(x_guess[k]) for k in x_guess}
for param_name, round_f in self.round_to_values.items():
x_guess[param_name] = round_f(x_guess[param_name])
# Also ensure this is correct dtype so sklearn is happy
x_guess = {
k: DTYPE_MAP[self.api_config[k]["type"]](x_guess[k])
for k in x_guess
}
return x_guess
def pysot_suggest(self, n_suggestions=1):
if self.pysot_batch_size is None: # First call to suggest
self.pysot_batch_size = n_suggestions
self.start()
# Set the tolerances pretending like we are running batch
d, p = float(self.opt.dim), float(n_suggestions)
self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p)))
# Now we can make suggestions
x_w = []
self.proposals = []
for _ in range(n_suggestions):
proposal = self.strategy.propose_action()
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept() # This triggers all the callbacks
# It is possible that pySOT proposes a previously evaluated point
# when all variables are integers, so we just abort in this case
# since we have likely converged anyway. See PySOT issue #30.
x = list(proposal.record.params) # From tuple to list
x_unwarped, = self.space_x.unwarp(x)
if x_unwarped in self.history:
warnings.warn("pySOT proposed the same point twice")
self.start()
return self.suggest(n_suggestions=n_suggestions)
# NOTE: Append unwarped to avoid rounding issues
self.history.append(copy(x_unwarped))
self.proposals.append(proposal)
x_w.append(copy(x_unwarped))
return x_w
def pysot_get_suggest(self, suggests):
turbo_suggest_warps = self.space_x.warp(suggests)
for i, warps in enumerate(turbo_suggest_warps):
proposal = self.strategy.make_proposal(warps)
proposal.add_callback(self.strategy.on_initial_proposal)
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept()
self.history.append(copy(suggests[i]))
self.proposals.append(proposal)
def turbo_suggest(self, n_suggestions=1):
if self.turbo_batch_size is None: # Remember the batch size on the first call to suggest
self.turbo_batch_size = n_suggestions
self.turbo.batch_size = n_suggestions
self.turbo.failtol = np.ceil(
np.max([
4.0 / self.turbo_batch_size,
self.dim / self.turbo_batch_size
]))
self.turbo.n_init = max([self.turbo.n_init, self.turbo_batch_size])
self.restart()
X_next = np.zeros((n_suggestions, self.dim))
# Pick from the initial points
n_init = min(len(self.X_init), n_suggestions)
if n_init > 0:
X_next[:n_init] = deepcopy(self.X_init[:n_init, :])
self.X_init = self.X_init[
n_init:, :] # Remove these pending points
# Get remaining points from TuRBO
n_adapt = n_suggestions - n_init
if n_adapt > 0:
if len(self.turbo._X
) > 0: # Use random points if we can't fit a GP
X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub)
fX = copula_standardize(deepcopy(
self.turbo._fX).ravel()) # Use Copula
X_cand, y_cand, _ = self.turbo._create_candidates(
X,
fX,
length=self.turbo.length,
n_training_steps=100,
hypers={})
X_next[-n_adapt:, :] = self.turbo._select_candidates(
X_cand, y_cand)[:n_adapt, :]
X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :],
self.lb, self.ub)
# Unwarp the suggestions
suggestions = self.space_x.unwarp(X_next)
return suggestions
def _hyperopt_suggest(self):
new_ids = self.trials.new_trial_ids(1)
assert len(new_ids) == 1
self.trials.refresh()
seed = random_seed(self.random)
new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed)
assert len(new_trials) == 1
self.trials.insert_trial_docs(new_trials)
self.trials.refresh()
new_trial, = new_trials # extract singleton
return new_trial
def _hyperopt_transform(self, x):
new_id = self.trials.new_trial_ids(1)[0]
domain = self.domain
rng = np.random.RandomState(1)
idxs, vals = pyll.rec_eval(domain.s_idxs_vals,
memo={
domain.s_new_ids: [new_id],
domain.s_rng: rng,
})
rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)]
rval_results = domain.new_result()
for (k, _) in vals.items():
vals[k][0] = x[k]
miscs_update_idxs_vals(rval_miscs, idxs, vals)
rval_docs = self.trials.new_trial_docs([new_id], [None], rval_results,
rval_miscs)
return rval_docs[0]
def hyperopt_suggest(self, n_suggestions=1):
assert n_suggestions >= 1, "invalid value for n_suggestions"
# Get the new trials, it seems hyperopt either uses random search or
# guesses one at a time anyway, so we might as welll call serially.
new_trials = [self._hyperopt_suggest() for _ in range(n_suggestions)]
X = []
for trial in new_trials:
x_guess = self.cleanup_guess(trial["misc"]["vals"])
X.append(x_guess)
# Build lookup to get original trial object
x_guess_ = tuSOTOptimizer.hashable_dict(x_guess)
assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict"
self.trial_id_lookup[x_guess_] = trial["tid"]
assert len(X) == n_suggestions
return X
def hyperopt_get_suggest(self, suggests):
trials = [self._hyperopt_transform(x) for x in suggests]
for trial in trials:
x_guess = self.cleanup_guess(trial["misc"]["vals"])
x_guess_ = tuSOTOptimizer.hashable_dict(x_guess)
assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict"
self.trial_id_lookup[x_guess_] = trial["tid"]
self.trials.insert_trial_docs(trials)
self.trials.refresh()
def suggest(self, n_suggestions=1):
if n_suggestions == 1:
return self.turbo_suggest(n_suggestions)
else:
t_suggestion = n_suggestions // 2
# p_suggestion = int((n_suggestions - t_suggestion) * 3/4)
h_suggestion = n_suggestions - t_suggestion
turbo_suggest = self.turbo_suggest(t_suggestion)
# pysot_suggest = self.pysot_suggest(p_suggestion)
hyperopt_suggest = self.hyperopt_suggest(h_suggestion)
self.hyperopt_get_suggest(turbo_suggest)
# self.pysot_get_suggest(turbo_suggest + hyperopt_suggest)
return turbo_suggest + hyperopt_suggest
def _observe(self, x, y):
# Find the matching proposal and execute its callbacks
idx = [x == xx for xx in self.history]
if np.any(idx):
i = np.argwhere(
idx)[0].item() # Pick the first index if there are ties
proposal = self.proposals[i]
proposal.record.complete(y)
self.proposals.pop(i)
self.history.pop(i)
def observe(self, X, y):
"""Send an observation of a suggestion back to the optimizer.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated
"""
assert len(X) == len(y)
# # pysot observe
# for x_, y_ in zip(X, y):
# # Just ignore, any inf observations we got, unclear if right thing
# if np.isfinite(y_):
# self._observe(x_, y_)
# turbo observe
XX, yy = self.space_x.warp(X), np.array(y)[:, None]
if len(self.turbo._fX) >= self.turbo.n_init:
self.turbo._adjust_length(yy)
self.turbo.n_evals += self.turbo_batch_size
self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX)))
self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy)))
self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX)))
self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy)))
# Check for a restart
if self.turbo.length < self.turbo.length_min:
self.restart()
# hyperopt observe
for x_guess, y_ in zip(X, y):
x_guess_ = tuSOTOptimizer.hashable_dict(x_guess)
assert x_guess_ in self.trial_id_lookup, "Appears to be guess that did not originate from suggest"
assert x_guess_ in self.trial_id_lookup, "trial object not available in trial dict"
trial_id = self.trial_id_lookup.pop(x_guess_)
trial = self.get_trial(trial_id)
assert self.cleanup_guess(
trial["misc"]["vals"]
) == x_guess, "trial ID not consistent with x values stored"
# Cast to float to ensure native type
result = {"loss": float(y_), "status": STATUS_OK}
trial["state"] = JOB_STATE_DONE
trial["result"] = result
self.trials.refresh()
class TurboOptimizer(AbstractOptimizer):
primary_import = "Turbo"
def __init__(self, api_config, **kwargs):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
"""
AbstractOptimizer.__init__(self, api_config)
self.space_x = JointSpace(api_config)
self.bounds = self.space_x.get_bounds()
self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1]
self.dim = len(self.bounds)
self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int
self.batch_size = None
self.history = []
self.turbo = Turbo1(
f=None,
lb=self.bounds[:, 0],
ub=self.bounds[:, 1],
n_init=2 * self.dim + 1,
max_evals=self.max_evals,
batch_size=1, # We need to update this later
verbose=False,
)
def restart(self):
self.turbo._restart()
self.turbo._X = np.zeros((0, self.turbo.dim))
self.turbo._fX = np.zeros((0, 1))
X_init = latin_hypercube(self.turbo.n_init, self.dim)
self.X_init = from_unit_cube(X_init, self.lb, self.ub)
def suggest(self, n_suggestions=3):
if self.batch_size is None: # Remember the batch size on the first call to suggest
self.batch_size = n_suggestions
self.turbo.batch_size = n_suggestions
self.turbo.failtol = np.ceil(
np.max([4.0 / self.batch_size, self.dim / self.batch_size]))
self.turbo.n_init = max([self.turbo.n_init, self.batch_size])
self.restart()
X_next = np.zeros((n_suggestions, self.dim))
# Pick from the initial points
n_init = min(len(self.X_init), n_suggestions)
if n_init > 0:
X_next[:n_init] = deepcopy(self.X_init[:n_init, :])
self.X_init = self.X_init[
n_init:, :] # Remove these pending points
# Get remaining points from TuRBO
n_adapt = n_suggestions - n_init
if n_adapt > 0:
if len(self.turbo._X
) > 0: # Use random points if we can't fit a GP
X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub)
fX = copula_standardize(deepcopy(
self.turbo._fX).ravel()) # Use Copula
X_cand, y_cand, _ = self.turbo._create_candidates(
X,
fX,
length=self.turbo.length,
n_training_steps=100,
hypers={})
X_next[-n_adapt:, :] = self.turbo._select_candidates(
X_cand, y_cand)[:n_adapt, :]
X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :],
self.lb, self.ub)
# Unwarp the suggestions
suggestions = self.space_x.unwarp(X_next)
return suggestions
def observe(self, X, y):
"""Send an observation of a suggestion back to the optimizer.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated
"""
assert len(X) == len(y)
XX, yy = self.space_x.warp(X), np.array(y)[:, None]
if len(self.turbo._fX) >= self.turbo.n_init:
self.turbo._adjust_length(yy)
self.turbo.n_evals += self.batch_size
self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX)))
self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy)))
self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX)))
self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy)))
# Check for a restart
if self.turbo.length < self.turbo.length_min:
self.restart()
class SpacePartitioningOptimizer(AbstractOptimizer):
primary_import = 'scikit-learn'
def __init__(self, api_config, **kwargs):
AbstractOptimizer.__init__(self, api_config)
print('api_config:', api_config)
self.api_config = api_config
self.space_x = JointSpace(api_config)
self.bounds = self.space_x.get_bounds()
self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1]
self.dim = len(self.bounds)
self.X = np.zeros((0, self.dim))
self.y = np.zeros((0, 1))
self.X_init = None
self.batch_size = None
self.turbo = None
self.split_used = 0
self.node = None
self.best_values = []
self.config = self._read_config()
print('config:', self.config)
optimizer_seed = self.config.get('optimizer_seed')
fix_optimizer_seed(optimizer_seed)
self.sampler_seed = self.config.get('sampler_seed')
sampler.fix_sampler_seed(self.sampler_seed)
self.is_init_batch = False
self.init_batches = []
def _read_config(self):
return {'turbo_training_steps': 100, 'turbo_length_retries': 10, 'turbo_length_init_method': 'default', 'experimental_design': 'lhs_classic_ratio', 'n_init_points': 24, 'max_tree_depth': 5, 'kmeans_resplits': 10, 'split_model': {'type': 'SVC', 'args': {'kernel': 'poly', 'gamma': 'scale', 'C': 745.3227447730735}}, 'reset_no_improvement': 8, 'reset_split_after': 4, 'turbo': {'budget': 128, 'use_cylinder': 0, 'use_pull': 0, 'use_lcb': 0, 'kappa': 2.0, 'use_decay': 1, 'decay_alpha': 0.49937937259674076, 'decay_threshold': 0.5, 'length_min': 1e-06, 'length_max': 2.0, 'length_init': 0.8, 'length_multiplier': 2.0}, 'sampler_seed': 42, 'optimizer_seed': 578330}
def _init(self, n_suggestions):
self.batch_size = n_suggestions
n_init_points = self.config['n_init_points']
if n_init_points == -1:
# Special value to use the default 2*D+1 number.
n_init_points = 2 * self.dim + 1
self.n_init = max(self.batch_size, n_init_points)
exp_design = self.config['experimental_design']
if exp_design == 'latin_hypercube':
X_init = latin_hypercube(self.n_init, self.dim)
elif exp_design == 'halton':
halton_sampler = sampler.Sampler(method='halton', api_config=self.api_config, n_points=self.n_init)
X_init = halton_sampler.generate(random_state=self.sampler_seed)
X_init = self.space_x.warp(X_init)
X_init = to_unit_cube(X_init, self.lb, self.ub)
elif exp_design == 'lhs_classic_ratio':
lhs_sampler = sampler.Sampler(
method='lhs',
api_config=self.api_config,
n_points=self.n_init,
generator_kwargs={'lhs_type': 'classic', 'criterion': 'ratio'})
X_init = lhs_sampler.generate(random_state=self.sampler_seed)
X_init = self.space_x.warp(X_init)
X_init = to_unit_cube(X_init, self.lb, self.ub)
else:
raise ValueError(f'Unknown experimental design: {exp_design}.')
self.X_init = X_init
if DEBUG:
print(f'Initialized the method with {self.n_init} points by {exp_design}:')
print(X_init)
def _get_split_model(self, X, kmeans_labels):
split_model_config = self.config['split_model']
model_type = split_model_config['type']
args = split_model_config['args']
if model_type == 'SVC':
split_model = SVC(**args, max_iter=10**7)
elif model_type == 'KNeighborsClassifier':
split_model = KNeighborsClassifier(**args)
else:
raise ValueError(f'Unknown split model type in the config: {model_type}.')
split_model.fit(X, kmeans_labels)
split_model_predictions = split_model.predict(X)
split_model_matches = np.sum(split_model_predictions == kmeans_labels)
split_model_mismatches = np.sum(split_model_predictions != kmeans_labels)
print('Labels for the split model:', kmeans_labels)
print('Predictions of the split model:', split_model_predictions)
print(f'Split model matches {split_model_matches} and mismatches {split_model_mismatches}')
return split_model
def _find_split(self, X, y) -> Optional:
max_margin = None
max_margin_labels = None
for _ in range(self.config['kmeans_resplits']):
kmeans = KMeans(n_clusters=2).fit(y)
kmeans_labels = kmeans.labels_
if np.count_nonzero(kmeans_labels == 1) > 0 and np.count_nonzero(kmeans_labels == 0) > 0:
if np.mean(y[kmeans_labels == 1]) > np.mean(y[kmeans_labels == 0]):
# Reverse labels if the entries with 1s have a higher mean error, since 1s go to the left branch.
kmeans_labels = 1 - kmeans_labels
margin = -(np.mean(y[kmeans_labels == 1]) - np.mean(y[kmeans_labels == 0]))
if DEBUG:
print('MARGIN is', margin, np.count_nonzero(kmeans_labels == 1), np.count_nonzero(kmeans_labels == 0))
if max_margin is None or margin > max_margin:
max_margin = margin
max_margin_labels = kmeans_labels
if DEBUG:
print('MAX MARGIN is', max_margin)
if max_margin_labels is None:
return None
else:
return self._get_split_model(X, max_margin_labels)
def _build_tree(self, X, y, depth=0):
print('len(X) in _build_tree is', len(X))
if depth == self.config['max_tree_depth']:
return []
split = self._find_split(X, y)
if split is None:
return []
in_region_points = split.predict(X)
left_subtree_size = np.count_nonzero(in_region_points == 1)
right_subtree_size = np.count_nonzero(in_region_points == 0)
print(f'{len(X)} points would be split {left_subtree_size}/{right_subtree_size}.')
if left_subtree_size < self.n_init:
return []
idx = (in_region_points == 1)
splits = self._build_tree(X[idx], y[idx], depth + 1)
return [split] + splits
def _get_in_node_region(self, points, splits):
in_region = np.ones(len(points))
for split in splits:
split_in_region = split.predict(points)
in_region *= split_in_region
return in_region
def _suggest(self, n_suggestions):
X = to_unit_cube(deepcopy(self.X), self.lb, self.ub)
y = deepcopy(self.y)
if not self.node:
self.split_used = 0
self.node = self._build_tree(X, y)
used_budget = len(y)
idx = (self._get_in_node_region(X, self.node) == 1)
X = X[idx]
y = y[idx]
print(f'Rebuilt the tree of depth {len(self.node)}')
model_config = self.config['turbo']
#print('CONFIG!!!!!', model_config)
self.turbo = Turbo1(
f=None,
lb=self.bounds[:, 0],
ub=self.bounds[:, 1],
n_init=len(X),
max_evals=np.iinfo(np.int32).max,
batch_size=self.batch_size,
verbose=False,
use_cylinder=model_config['use_cylinder'],
budget=model_config['budget'],
use_decay=model_config['use_decay'],
decay_threshold=model_config['decay_threshold'],
decay_alpha=model_config['decay_alpha'],
use_pull=model_config['use_pull'],
use_lcb=model_config['use_lcb'],
kappa=model_config['kappa'],
length_min=model_config['length_min'],
length_max=model_config['length_max'],
length_init=model_config['length_init'],
length_multiplier=model_config['length_multiplier'],
used_budget=used_budget
)
self.turbo._X = np.array(X, copy=True)
self.turbo._fX = np.array(y, copy=True)
self.turbo.X = np.array(X, copy=True)
self.turbo.fX = np.array(y, copy=True)
print('Initialized TURBO')
else:
idx = (self._get_in_node_region(X, self.node) == 1)
X = X[idx]
y = y[idx]
self.split_used += 1
length_init_method = self.config['turbo_length_init_method']
if length_init_method == 'default':
length = self.turbo.length
elif length_init_method == 'length_init':
length = self.turbo.length_init
elif length_init_method == 'length_max':
length = self.turbo.length_max
elif length_init_method == 'infinity':
length = np.iinfo(np.int32).max
else:
raise ValueError(f'Unknown init method for turbo\'s length: {length_init_method}.')
length_reties = self.config['turbo_length_retries']
for retry in range(length_reties):
XX = X
yy = copula_standardize(y.ravel())
X_cand, y_cand, _ = self.turbo._create_candidates(
XX, yy, length=length, n_training_steps=self.config['turbo_training_steps'], hypers={})
in_region_predictions = self._get_in_node_region(X_cand, self.node)
in_region_idx = in_region_predictions == 1
if DEBUG:
print(f'In region: {np.sum(in_region_idx)} out of {len(X_cand)}')
if np.sum(in_region_idx) >= n_suggestions:
X_cand, y_cand = X_cand[in_region_idx], y_cand[in_region_idx]
self.turbo.f_var = self.turbo.f_var[in_region_idx]
if DEBUG:
print('Found a suitable set of candidates.')
break
else:
length /= 2
if DEBUG:
print(f'Retrying {retry + 1}/{length_reties} time')
X_cand = self.turbo._select_candidates(X_cand, y_cand)[:n_suggestions, :]
if DEBUG:
if X.shape[1] == 3:
tx = np.arange(0.0, 1.0 + 1e-6, 0.1)
ty = np.arange(0.0, 1.0 + 1e-6, 0.1)
tz = np.arange(0.0, 1.0 + 1e-6, 0.1)
p = np.array([[x, y, z] for x in tx for y in ty for z in tz])
elif X.shape[1] == 2:
tx = np.arange(0.0, 1.0 + 1e-6, 0.1)
ty = np.arange(0.0, 1.0 + 1e-6, 0.1)
p = np.array([[x, y] for x in tx for y in ty])
else:
raise ValueError('The points for the DEBUG should either be 2D or 3D.')
p_predictions = self._get_in_node_region(p, self.node)
in_turbo_bounds = np.logical_and(
np.all(self.turbo.cand_lb <= p, axis=1),
np.all(p <= self.turbo.cand_ub, axis=1))
pcds = []
_add_pcd(pcds, p[p_predictions == 0], (1.0, 0.0, 0.0))
_add_pcd(pcds, p[np.logical_and(p_predictions == 1, np.logical_not(in_turbo_bounds))], (0.0, 1.0, 0.0))
_add_pcd(pcds, p[np.logical_and(p_predictions == 1, in_turbo_bounds)], (0.0, 0.5, 0.0))
_add_pcd(pcds, X_cand, (0.0, 0.0, 0.0))
open3d.visualization.draw_geometries(pcds)
return X_cand
def suggest(self, n_suggestions=1):
X_suggestions = np.zeros((n_suggestions, self.dim))
# Initialize the design if it is the first call
if self.X_init is None:
self._init(n_suggestions)
if self.init_batches:
print('REUSING INITIALIZATION:')
for X, Y in self.init_batches:
print('Re-observing a batch!')
self.observe(X, Y)
self.X_init = []
# Pick from the experimental design
n_init = min(len(self.X_init), n_suggestions)
if n_init > 0:
X_suggestions[:n_init] = self.X_init[:n_init]
self.X_init = self.X_init[n_init:]
self.is_init_batch = True
else:
self.is_init_batch = False
# Pick from the model based on the already received observations
n_suggest = n_suggestions - n_init
if n_suggest > 0:
X_cand = self._suggest(n_suggest)
X_suggestions[-n_suggest:] = X_cand
# Map into the continuous space with the api bounds and unwarp the suggestions
X_min_bound = 0.0
X_max_bound = 1.0
X_suggestions_min = X_suggestions.min()
X_suggestions_max = X_suggestions.max()
if X_suggestions_min < X_min_bound or X_suggestions_max > X_max_bound:
print(f'Some suggestions are out of the bounds in suggest(): {X_suggestions_min}, {X_suggestions_max}')
print('Clipping everything...')
X_suggestions = np.clip(X_suggestions, X_min_bound, X_max_bound)
X_suggestions = from_unit_cube(X_suggestions, self.lb, self.ub)
X_suggestions = self.space_x.unwarp(X_suggestions)
return X_suggestions
def observe(self, X_observed, Y_observed):
if self.is_init_batch:
self.init_batches.append([X_observed, Y_observed])
X, Y = [], []
for x, y in zip(X_observed, Y_observed):
if np.isfinite(y):
X.append(x)
Y.append(y)
else:
# Ignore for now; could potentially substitute with an upper bound.
continue
if not X:
return
X, Y = self.space_x.warp(X), np.array(Y)[:, None]
self.X = np.vstack((self.X, deepcopy(X)))
self.y = np.vstack((self.y, deepcopy(Y)))
self.best_values.append(Y.min())
if self.turbo:
if len(self.turbo._X) >= self.turbo.n_init:
self.turbo._adjust_length(Y)
print('TURBO length:', self.turbo.length)
self.turbo._X = np.vstack((self.turbo._X, deepcopy(X)))
self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(Y)))
self.turbo.X = np.vstack((self.turbo.X, deepcopy(X)))
self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(Y)))
N = self.config['reset_no_improvement']
if len(self.best_values) > N and np.min(self.best_values[:-N]) <= np.min(self.best_values[-N:]):
print('########## RESETTING COMPLETELY! ##########')
self.X = np.zeros((0, self.dim))
self.y = np.zeros((0, 1))
self.best_values = []
self.X_init = None
self.node = None
self.turbo = None
self.split_used = 0
if self.split_used >= self.config['reset_split_after']:
print('########## REBUILDING THE SPLIT! ##########')
self.node = None
self.turbo = None
self.split_used = 0
class tuSOTOptimizer(AbstractOptimizer):
primary_import = "pysot"
def __init__(self, api_config):
"""Build wrapper class to use an optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
"""
AbstractOptimizer.__init__(self, api_config)
self.space_x = JointSpace(api_config)
self.bounds = self.space_x.get_bounds()
self.create_opt_prob() # Sets up the optimization problem (needs self.bounds)
self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int
self.turbo_batch_size = None
self.pysot_batch_size = None
self.history = []
self.proposals = []
self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1]
self.dim = len(self.bounds)
self.turbo = Turbo1(
f=None,
lb=self.bounds[:, 0],
ub=self.bounds[:, 1],
n_init=2 * self.dim + 1,
max_evals=self.max_evals,
batch_size=4, # We need to update this later
verbose=False,
)
def restart(self):
self.turbo._restart()
self.turbo._X = np.zeros((0, self.turbo.dim))
self.turbo._fX = np.zeros((0, 1))
X_init = latin_hypercube(self.turbo.n_init, self.dim)
self.X_init = from_unit_cube(X_init, self.lb, self.ub)
def create_opt_prob(self):
"""Create an optimization problem object."""
opt = OptimizationProblem()
opt.lb = self.bounds[:, 0] # In warped space
opt.ub = self.bounds[:, 1] # In warped space
opt.dim = len(self.bounds)
opt.cont_var = np.arange(len(self.bounds))
opt.int_var = []
assert len(opt.cont_var) + len(opt.int_var) == opt.dim
opt.objfun = None
self.opt = opt
def start(self):
"""Starts a new pySOT run."""
self.history = []
self.proposals = []
# Symmetric Latin hypercube design
des_pts = max([self.pysot_batch_size, 2 * (self.opt.dim + 1)])
slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts)
# Warped RBF interpolant
rbf = RBFInterpolant(dim=self.opt.dim, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4)
rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub)
# Optimization strategy
self.strategy = SRBFStrategy(
max_evals=self.max_evals,
opt_prob=self.opt,
exp_design=slhd,
surrogate=rbf,
asynchronous=True,
batch_size=1,
use_restarts=True,
)
def pysot_suggest(self, turbo_suggest, n_suggestions=1):
if self.pysot_batch_size is None: # First call to suggest
self.pysot_batch_size = n_suggestions
self.start()
# Set the tolerances pretending like we are running batch
d, p = float(self.opt.dim), float(n_suggestions)
self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p)))
# Now we can make suggestions
x_w = []
self.proposals = []
for _ in range(n_suggestions):
proposal = self.strategy.propose_action()
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept() # This triggers all the callbacks
# It is possible that pySOT proposes a previously evaluated point
# when all variables are integers, so we just abort in this case
# since we have likely converged anyway. See PySOT issue #30.
x = list(proposal.record.params) # From tuple to list
x_unwarped, = self.space_x.unwarp(x)
if x_unwarped in self.history:
warnings.warn("pySOT proposed the same point twice")
self.start()
return self.suggest(n_suggestions=n_suggestions)
# NOTE: Append unwarped to avoid rounding issues
self.history.append(copy(x_unwarped))
self.proposals.append(proposal)
x_w.append(copy(x_unwarped))
turbo_suggest_warps = self.space_x.warp(turbo_suggest)
for i, warps in enumerate(turbo_suggest_warps):
proposal = self.strategy.make_proposal(warps)
proposal.add_callback(self.strategy.on_initial_proposal)
record = EvalRecord(proposal.args, status="pending")
proposal.record = record
proposal.accept()
self.history.append(copy(turbo_suggest[i]))
self.proposals.append(proposal)
return x_w
def turbo_suggest(self, n_suggestions=1):
if self.turbo_batch_size is None: # Remember the batch size on the first call to suggest
self.turbo_batch_size = n_suggestions
self.turbo.batch_size = n_suggestions
self.turbo.failtol = np.ceil(np.max([4.0 / self.turbo_batch_size, self.dim / self.turbo_batch_size]))
self.turbo.n_init = max([self.turbo.n_init, self.turbo_batch_size])
self.restart()
X_next = np.zeros((n_suggestions, self.dim))
# Pick from the initial points
n_init = min(len(self.X_init), n_suggestions)
if n_init > 0:
X_next[:n_init] = deepcopy(self.X_init[:n_init, :])
self.X_init = self.X_init[n_init:, :] # Remove these pending points
# Get remaining points from TuRBO
n_adapt = n_suggestions - n_init
if n_adapt > 0:
if len(self.turbo._X) > 0: # Use random points if we can't fit a GP
X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub)
fX = copula_standardize(deepcopy(self.turbo._fX).ravel()) # Use Copula
X_cand, y_cand, _ = self.turbo._create_candidates(
X, fX, length=self.turbo.length, n_training_steps=100, hypers={}
)
X_next[-n_adapt:, :] = self.turbo._select_candidates(X_cand, y_cand)[:n_adapt, :]
X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :], self.lb, self.ub)
# Unwarp the suggestions
suggestions = self.space_x.unwarp(X_next)
return suggestions
def suggest(self, n_suggestions=1):
if n_suggestions == 1:
return self.turbo_suggest(n_suggestions)
else:
suggestion = n_suggestions // 2
turbo_suggest = self.turbo_suggest(suggestion)
pysot_suggest = self.pysot_suggest(turbo_suggest, n_suggestions - suggestion)
return turbo_suggest + pysot_suggest
def _observe(self, x, y):
# Find the matching proposal and execute its callbacks
idx = [x == xx for xx in self.history]
if np.any(idx):
i = np.argwhere(idx)[0].item() # Pick the first index if there are ties
proposal = self.proposals[i]
proposal.record.complete(y)
self.proposals.pop(i)
self.history.pop(i)
def observe(self, X, y):
"""Send an observation of a suggestion back to the optimizer.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated
"""
assert len(X) == len(y)
for x_, y_ in zip(X, y):
# Just ignore, any inf observations we got, unclear if right thing
if np.isfinite(y_):
self._observe(x_, y_)
XX, yy = self.space_x.warp(X), np.array(y)[:, None]
if len(self.turbo._fX) >= self.turbo.n_init:
self.turbo._adjust_length(yy)
self.turbo.n_evals += self.turbo_batch_size
self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX)))
self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy)))
self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX)))
self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy)))
# Check for a restart
if self.turbo.length < self.turbo.length_min:
self.restart()
