def setUp(self):
kwargs = {
'n_samples': 1000,
'n_burnin': 100,
'subsample_interval': 5,
'step_size': 1e-1,
'leapfrog_steps': 20,
'optimize_restarts': 10,
'num_inducing': 15,
'acquisition_transformation': 'softplus',
'acquisition_jitter': 0.02,
'acquisition_weight': 2.5,
'acquisition_transformation': 'softplus'
}
## --- Defaults for some of the tests
self.space = Design_space(space=[{
'name': 'var1',
'type': 'continuous',
'domain': (-10, 10),
'dimensionality': 2
}])
self.cost = CostModel(None)
self.arguments_manager = ArgumentsManager(kwargs)
self.model = self.arguments_manager.model_creator(model_type='GP',
exact_feval=True,
space=self.space)
self.acquisition_optimizer = AcquisitionOptimizer(self.space)
self.acquisition = self.arguments_manager.acquisition_creator(
'EI', self.model, self.space, self.acquisition_optimizer,
self.cost)
def setUp(self):
kwargs = {'n_samples':1000,
'n_burnin':100,
'subsample_interval':5,
'step_size': 1e-1,
'leapfrog_steps':20,
'optimize_restarts':10,
'num_inducing':15,
'acquisition_transformation':'softplus',
'acquisition_jitter':0.02,
'acquisition_weight':2.5,
'acquisition_transformation':'softplus'
}
## --- Defaults for some of the tests
self.space = Design_space(space =[{'name': 'var1', 'type': 'continuous', 'domain': (-10,10),'dimensionality': 2}])
self.cost = CostModel(None)
self.arguments_manager = ArgumentsManager(kwargs)
self.model = self.arguments_manager.model_creator(model_type = 'GP', exact_feval= True, space = self.space)
self.acquisition_optimizer = AcquisitionOptimizer(self.space)
self.acquisition = self.arguments_manager.acquisition_creator('EI', self.model, self.space, self.acquisition_optimizer, self.cost)
class TestArgumentsManager(unittest.TestCase):
'''
The test just checks that the arguments are passed and handled, not that the model
or the acquisition does the right thing with them.
'''
def setUp(self):
kwargs = {
'n_samples': 1000,
'n_burnin': 100,
'subsample_interval': 5,
'step_size': 1e-1,
'leapfrog_steps': 20,
'optimize_restarts': 10,
'num_inducing': 15,
'acquisition_transformation': 'softplus',
'acquisition_jitter': 0.02,
'acquisition_weight': 2.5,
'acquisition_transformation': 'softplus'
}
## --- Defaults for some of the tests
self.space = Design_space(space=[{
'name': 'var1',
'type': 'continuous',
'domain': (-10, 10),
'dimensionality': 2
}])
self.cost = CostModel(None)
self.arguments_manager = ArgumentsManager(kwargs)
self.model = self.arguments_manager.model_creator(model_type='GP',
exact_feval=True,
space=self.space)
self.acquisition_optimizer = AcquisitionOptimizer(self.space)
self.acquisition = self.arguments_manager.acquisition_creator(
'EI', self.model, self.space, self.acquisition_optimizer,
self.cost)
def test_model_gp_mcmc_arguments(self):
'''
Testing the arguments of the GP model with MCMC
'''
model_type = 'GP_MCMC'
exact_feval = True
created_model = self.arguments_manager.model_creator(
model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, GPModel_MCMC))
self.assertEquals(created_model.n_samples, 1000)
self.assertEquals(created_model.n_burnin, 100)
self.assertEquals(created_model.subsample_interval, 5)
self.assertEquals(created_model.step_size, 1e-1)
self.assertEquals(created_model.leapfrog_steps, 20)
def test_model_sparse_gp_arguments(self):
'''
Testing the arguments of the standard GP model
'''
model_type = 'sparseGP'
exact_feval = True
created_model = self.arguments_manager.model_creator(
model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, GPModel))
self.assertEquals(created_model.optimize_restarts, 10)
self.assertEquals(created_model.num_inducing, 15)
def test_model_rf_arguments(self):
'''
Testing the arguments of the Random Forrest
'''
model_type = 'RF'
exact_feval = True
created_model = self.arguments_manager.model_creator(
model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, RFModel))
def test_model_inputwarpedgp_arguments(self):
'''
Testing the arguments of the input warped GP
'''
model_type = 'input_warped_GP'
exact_feval = True
created_model = self.arguments_manager.model_creator(
model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, InputWarpedGPModel))
def test_model_warpedgo_arguments(self):
'''
Testing the arguments of the warped GP
'''
model_type = 'warpedGP'
exact_feval = True
created_model = self.arguments_manager.model_creator(
model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, WarpedGPModel))
def test_acquisition_ei_arguments(self):
'''
Testing the arguments of the Expected Improvement
'''
acquisition_type = 'EI'
created_acquisition = self.arguments_manager.acquisition_creator(
acquisition_type, self.model, self.space,
self.acquisition_optimizer, self.cost)
self.assertTrue(isinstance(created_acquisition, AcquisitionEI))
self.assertEquals(created_acquisition.jitter, 0.02)
def test_acquisition_lcb_arguments(self):
'''
Testing the arguments of the Lower Confidence Bound
'''
acquisition_type = 'LCB'
created_acquisition = self.arguments_manager.acquisition_creator(
acquisition_type, self.model, self.space,
self.acquisition_optimizer, self.cost)
self.assertTrue(isinstance(created_acquisition, AcquisitionLCB))
self.assertEquals(created_acquisition.exploration_weight, 2.5)
def test_evaluator_arguments(self):
'''
Testing the arguments of the local_penalization evaluator
'''
evaluator_type = 'local_penalization'
batch_size = 2
model_type = 'GP'
created_evaluator = self.arguments_manager.evaluator_creator(
evaluator_type, self.acquisition, batch_size, model_type,
self.model, self.space, self.acquisition_optimizer)
self.assertTrue(isinstance(created_evaluator, LocalPenalization))
self.assertEquals(created_evaluator.acquisition.transform, 'softplus')
def __init__(self,
estimator,
domain=None,
constraints=None,
cost_withGradients=None,
model_type="GP",
X=None,
Y=None,
initial_design_numdata=5,
initial_design_type="random",
acquisition_type="EI",
normalize_Y=True,
exact_feval=False,
acquisition_optimizer_type="lbfgs",
model_update_interval=1,
evaluator_type="sequential",
batch_size=1,
num_cores=1,
verbosity_model=False,
verbosity=False,
de_duplication=False,
max_iter=50,
refit=True,
cv=None,
scoring=None,
n_jobs=1,
verbose=False,
**kwargs):
"""Initialise the estimator."""
# super(BayesianOptimization, self).__init__(
# f=f, domain=domain, constraints=constraints,
# cost_withGradients=cost_withGradients, model_type=model_type,
# X=X, Y=Y, initial_design_numdata=initial_design_numdata,
# initial_design_type=initial_design_type,
# acquisition_type=acquisition_type, normalize_Y=normalize_Y,
# exact_feval=exact_feval,
# acquisition_optimizer_type=acquisition_optimizer_type,
# model_update_interval=model_update_interval,
# evaluator_type=evaluator_type, batch_size=batch_size,
# num_cores=num_cores,
# verbosity=verbosity, verbosity_model=verbosity_model,
# maximize=True, de_duplication=de_duplication, **kwargs)
self.modular_optimization = False
self.initial_iter = True
self.verbosity = verbosity
self.verbosity_model = verbosity_model
self.model_update_interval = model_update_interval
self.de_duplication = de_duplication
self.kwargs = kwargs
# --- Handle the arguments passed via kargs
self.problem_config = ArgumentsManager(kwargs)
# --- CHOOSE design space
self.constraints = constraints
self.domain = domain or {}
self.space = Design_space(self.domain, self.constraints)
# --- CHOOSE objective function
self.objective_name = kwargs.get("objective_name", "") or "no_name"
self.batch_size = batch_size
self.num_cores = num_cores
# self.maximize = True
# self.objective = None
# --- CHOOSE the cost model
self.cost = CostModel(cost_withGradients)
# --- CHOOSE initial design
self.X = X # parameters
self.Y = Y # evaluation
self.initial_design_type = initial_design_type
self.initial_design_numdata = initial_design_numdata
# --- CHOOSE the model type. If an instance of a GPyOpt model is passed (possibly user defined), it is used.
# note that this 2 options are not used with the predefined model
self.model_type = model_type
self.exact_feval = exact_feval
self.normalize_Y = normalize_Y
if "model" in self.kwargs and isinstance(kwargs["model"],
GPyOpt.models.base.BOModel):
self.model = kwargs["model"]
self.model_type = "User defined model used."
if self.verbose:
print("Using a model defined by the used.")
else:
self.model = self._model_chooser()
# --- CHOOSE the acquisition optimizer_type
# This states how the discrete variables are handled (exact search or rounding)
self.acquisition_optimizer_type = acquisition_optimizer_type
self.acquisition_optimizer = AcquisitionOptimizer(
self.space, self.acquisition_optimizer_type, model=self.model)
# --- CHOOSE acquisition function. If an instance of an acquisition is passed (possibly user defined), it is used.
self.acquisition_type = acquisition_type
if "acquisition" in self.kwargs and isinstance(
kwargs["acquisition"], GPyOpt.acquisitions.AcquisitionBase):
self.acquisition = kwargs["acquisition"]
self.acquisition_type = "User defined acquisition used."
if self.verbose:
print("Using an acquisition defined by the used.")
else:
self.acquisition = self._acquisition_chooser()
# --- CHOOSE evaluator method
self.evaluator_type = evaluator_type
self.evaluator = self._evaluator_chooser()
# --- Create optimization space
self.cost = CostModel(self.cost)
self.normalization_type = "stats" # not added in the API
self.estimator = estimator
self.cv = cv
self.max_iter = max_iter
self.refit = refit
self.scoring = scoring
self.verbose = verbose
self.n_jobs = n_jobs
def __init__(self,
fill_in_strategy,
f,
mix=0.5,
domain=None,
constraints=None,
cost_withGradients=None,
X=None,
Y=None,
subspace_dim_size=0,
model_type='GP',
initial_design_numdata=1,
initial_design_type='random',
acquisition_type='LCB',
normalize_Y=True,
exact_feval=False,
acquisition_optimizer_type='lbfgs',
model_update_interval=1,
evaluator_type='sequential',
batch_size=1,
maximize=False,
de_duplication=False):
if model_type == 'input_warped_GP':
raise NotImplementedError(
'input_warped_GP model is not implemented')
if acquisition_type in ['EI_MCMC', 'MPI_MCMC', 'LCB_MCMC']:
raise NotImplementedError('MCMC is not implemented')
if batch_size is not 1 or evaluator_type is not 'sequential':
raise NotImplementedError(
'only sequential evaluation is implemented')
if fill_in_strategy not in ['random', 'copy', 'mix']:
raise ValueError('fill_in_strategy has to be random, copy or mix')
# private field
self._arguments_mng = ArgumentsManager(kwargs=dict())
self.fill_in_strategy = fill_in_strategy
self.mix = mix
self.subspace_dim_size = subspace_dim_size
self.cost_withGradients = cost_withGradients
self.initial_design_numdata = initial_design_numdata
self.initial_design_type = initial_design_type
self.model_type = model_type
self.acquisition_type = acquisition_type
self.evaluator_type = evaluator_type
self.model_update_interval = model_update_interval
self.maximize = maximize
self.normalize_Y = normalize_Y
self.de_duplication = de_duplication
self.subspace_dim_size = subspace_dim_size
# --- property injected in other methods.
self.verbosity = False
self.subspace_idx = None
self.subspace = None
self.max_time = None
self.max_iter = None
self.cum_time = None
self.report_file = None
self.evaluations_file = None
self.models_file = None
self.eps = None
self.save_models_parameters = None
# --- unnecessary property
self.suggested_sample = None
self.Y_new = None
# --- BO class property in uncertain use
self.num_cores = 1
self.objective = SingleObjective(self._sign(f), batch_size,
f.get_function_name())
self.cost = CostModel(cost_withGradients=cost_withGradients)
self.space = initialize_space(domain=domain, constraints=constraints)
self.model = self._arguments_mng.model_creator(
model_type=self.model_type,
exact_feval=exact_feval,
space=self.space)
self.acquisition = self._arguments_mng.acquisition_creator(
acquisition_type=self.acquisition_type,
model=self.model,
space=self.space,
acquisition_optimizer=AcquisitionOptimizer(
space=self.space, optimizer=acquisition_optimizer_type),
cost_withGradients=self.cost_withGradients)
self._choose_evaluator()
self.X = X
self.Y = Y
self._set_initial_values()
super().__init__(model=self.model,
space=self.space,
objective=self.objective,
acquisition=self.acquisition,
evaluator=self.evaluator,
X_init=self.X,
Y_init=self.Y,
cost=self.cost,
normalize_Y=self.normalize_Y,
model_update_interval=self.model_update_interval,
de_duplication=self.de_duplication)
class Dropout(BO):
"""
Args:
f (function): function to optimize.
domain (list | None): the description of the inputs variables
(See GpyOpt.core.space.Design_space class for details)
constraints (list | None): the description of the problem constraints
(See GpyOpt.core.space.Design_space class for details)
Attributes:
initial_design_numdata (int):
initial_design_type (string):
domain (dict | None):
constraints (dict | None):
space (Design_space):
model (BOModel):
acquisition (AcquisitionBase):
cost (CostModel):
"""
def __init__(self,
fill_in_strategy,
f,
mix=0.5,
domain=None,
constraints=None,
cost_withGradients=None,
X=None,
Y=None,
subspace_dim_size=0,
model_type='GP',
initial_design_numdata=1,
initial_design_type='random',
acquisition_type='LCB',
normalize_Y=True,
exact_feval=False,
acquisition_optimizer_type='lbfgs',
model_update_interval=1,
evaluator_type='sequential',
batch_size=1,
maximize=False,
de_duplication=False):
if model_type == 'input_warped_GP':
raise NotImplementedError(
'input_warped_GP model is not implemented')
if acquisition_type in ['EI_MCMC', 'MPI_MCMC', 'LCB_MCMC']:
raise NotImplementedError('MCMC is not implemented')
if batch_size is not 1 or evaluator_type is not 'sequential':
raise NotImplementedError(
'only sequential evaluation is implemented')
if fill_in_strategy not in ['random', 'copy', 'mix']:
raise ValueError('fill_in_strategy has to be random, copy or mix')
# private field
self._arguments_mng = ArgumentsManager(kwargs=dict())
self.fill_in_strategy = fill_in_strategy
self.mix = mix
self.subspace_dim_size = subspace_dim_size
self.cost_withGradients = cost_withGradients
self.initial_design_numdata = initial_design_numdata
self.initial_design_type = initial_design_type
self.model_type = model_type
self.acquisition_type = acquisition_type
self.evaluator_type = evaluator_type
self.model_update_interval = model_update_interval
self.maximize = maximize
self.normalize_Y = normalize_Y
self.de_duplication = de_duplication
self.subspace_dim_size = subspace_dim_size
# --- property injected in other methods.
self.verbosity = False
self.subspace_idx = None
self.subspace = None
self.max_time = None
self.max_iter = None
self.cum_time = None
self.report_file = None
self.evaluations_file = None
self.models_file = None
self.eps = None
self.save_models_parameters = None
# --- unnecessary property
self.suggested_sample = None
self.Y_new = None
# --- BO class property in uncertain use
self.num_cores = 1
self.objective = SingleObjective(self._sign(f), batch_size,
f.get_function_name())
self.cost = CostModel(cost_withGradients=cost_withGradients)
self.space = initialize_space(domain=domain, constraints=constraints)
self.model = self._arguments_mng.model_creator(
model_type=self.model_type,
exact_feval=exact_feval,
space=self.space)
self.acquisition = self._arguments_mng.acquisition_creator(
acquisition_type=self.acquisition_type,
model=self.model,
space=self.space,
acquisition_optimizer=AcquisitionOptimizer(
space=self.space, optimizer=acquisition_optimizer_type),
cost_withGradients=self.cost_withGradients)
self._choose_evaluator()
self.X = X
self.Y = Y
self._set_initial_values()
super().__init__(model=self.model,
space=self.space,
objective=self.objective,
acquisition=self.acquisition,
evaluator=self.evaluator,
X_init=self.X,
Y_init=self.Y,
cost=self.cost,
normalize_Y=self.normalize_Y,
model_update_interval=self.model_update_interval,
de_duplication=self.de_duplication)
@property
def dimensionality(self):
return self.space.objective_dimensionality
@property
def subspace_domain(self):
return self.subspace.config_space
@property
def domain(self):
return self.space.config_space
@property
def constraints(self):
return self.space.constraints
@property
def f(self):
return self.objective.func
@property
def cost_type(self):
return self.cost.cost_withGradients
@property
def exact_feval(self):
return self.model.exact_feval
@property
def acquisition_optimizer_type(self):
return self.acquisition_optimizer.optimizer_name
@property
def acquisition_optimizer(self):
return self.acquisition.optimizer
@property
def batch_size(self):
return self.objective.n_procs
@property
def objective_name(self):
return self.objective.objective_name
def _choose_evaluator(self):
self.evaluator = self._arguments_mng.evaluator_creator(
evaluator_type=self.evaluator_type,
acquisition=self.acquisition,
batch_size=self.batch_size,
model_type=self.model_type,
model=self.model,
space=self.space,
acquisition_optimizer=self.acquisition_optimizer)
def run_optimization(self,
max_iter=0,
max_time=np.inf,
eps=1e-8,
context=None,
verbosity=False,
save_models_parameters=True,
report_file=None,
evaluations_file=None,
models_file=None):
if self.objective is None:
raise ValueError(
"Cannot run the optimization loop without the objective function"
)
# --- Save the options to print and save the results
self.verbosity = verbosity
self.save_models_parameters = save_models_parameters
self.report_file = report_file
self.evaluations_file = evaluations_file
self.models_file = models_file
self.model_parameters_iterations = None
self.context = context
# --- Check if we can save the model parameters in each iteration
if self.save_models_parameters:
if not (isinstance(self.model, GPModel)):
print(
'Models printout after each iteration is only available for GP and GP_MCMC models'
)
self.save_models_parameters = False
# --- Setting up stop conditions
self.eps = eps
if (max_iter is None) and (max_time is None):
self.max_iter = 0
self.max_time = np.inf
elif (max_iter is None) and (max_time is not None):
self.max_iter = np.inf
self.max_time = max_time
elif (max_iter is not None) and (max_time is None):
self.max_iter = max_iter
self.max_time = np.inf
else:
self.max_iter = max_iter
self.max_time = max_time
# --- Initialize iterations and running time
stopwatch = StopWatch()
self.num_acquisitions = self.initial_design_numdata
self.suggested_sample = self.X
self.Y_new = self.Y
self._compute_results()
self._run_optimization()
self.cum_time = stopwatch.passed_time()
# --- Stop messages and execution time
self._compute_results()
# --- Print the desired result in files
if self.report_file is not None:
self.save_report(self.report_file)
if self.evaluations_file is not None:
self.save_evaluations(self.evaluations_file)
if self.models_file is not None:
self.save_models(self.models_file)
self._save()
def _run_optimization(self):
while True:
print('.')
# --- update model
try:
self.update()
except np.linalg.LinAlgError:
print('np.linalg.LinAlgError')
break
if self.num_acquisitions >= self.max_iter:
break
self.next_point()
# --- Update current evaluation time and function evaluations
self.num_acquisitions += 1
def next_point(self):
self.suggested_sample = self._compute_next_evaluations()
# --- Augment X
self.X = np.vstack((self.X, self.suggested_sample))
# --- Evaluate *f* in X, augment Y and update cost function (if needed)
self.evaluate_objective()
def get_best_point(self):
self._compute_results()
return self.x_opt, self.fx_opt
def update(self):
self._update_model(self.normalization_type)
self._update_acquisition()
self._update_evaluator()
def _dropout_random(self, embedded_idx):
return initial_design(
'random', get_subspace(space=self.space,
subspace_idx=embedded_idx), 1)[0]
def _dropout_copy(self, embedded_idx):
x_opt, _ = self.get_best_point()
return x_opt[embedded_idx]
def _dropout_mix(self, embedded_idx):
if np.random.rand() < self.mix:
return self._dropout_random(embedded_idx=embedded_idx)
else:
return self._dropout_copy(embedded_idx=embedded_idx)
def _sign(self, f):
if self.maximize:
f_copy = f
def f(x):
return -f_copy(x)
return f
def _set_initial_values(self):
if self.X is None:
self.X = initial_design(self.initial_design_type, self.space,
self.initial_design_numdata)
self.Y, _ = self.objective.evaluate(self.X)
elif self.X is not None and self.Y is None:
self.Y, _ = self.objective.evaluate(self.X)
# save initial values
self.initial_X = deepcopy(self.X)
if self.maximize:
self.initial_Y = -deepcopy(self.Y)
else:
self.initial_Y = deepcopy(self.Y)
def _fill_in_strategy(self, embedded_idx):
if self.fill_in_strategy == 'random':
return self._dropout_random(embedded_idx=embedded_idx)
elif self.fill_in_strategy == 'copy':
return self._dropout_copy(embedded_idx=embedded_idx)
elif self.fill_in_strategy == 'mix':
return self._dropout_mix(embedded_idx=embedded_idx)
def _fill_in_dimensions(self, samples):
full_num = self.dimensionality
subspace_idx = self.subspace_idx
embedded_idx = [i for i in range(full_num) if i not in subspace_idx]
samples_ = list()
for sample in samples:
if len(sample) > len(subspace_idx):
raise ValueError(
'samples already have been full-dimensionality')
embedded_sample = self._fill_in_strategy(embedded_idx=embedded_idx)
sample_ = deepcopy(sample)
for emb_idx, insert_idx in enumerate(embedded_idx):
sample_ = np.insert(sample_, emb_idx, embedded_sample[emb_idx])
samples_.append(sample_)
return np.array(samples_)
def _update_acquisition(self):
self.acquisition = self._arguments_mng.acquisition_creator(
acquisition_type=self.acquisition_type,
model=self.model,
space=self.subspace,
acquisition_optimizer=AcquisitionOptimizer(
space=self.subspace,
optimizer=self.acquisition_optimizer_type),
cost_withGradients=self.cost_withGradients)
def _update_evaluator(self):
self.evaluator.acquisition = self.acquisition
def _update_model(self, normalization_type='stats'):
if self.num_acquisitions % self.model_update_interval == 0:
self.update_subspace()
self.model = self._arguments_mng.model_creator(
model_type=self.model_type,
exact_feval=self.exact_feval,
space=self.subspace)
X_inmodel, Y_inmodel = self._input_data(
normalization_type=normalization_type)
self.model.updateModel(X_inmodel, Y_inmodel, None, None)
self.X_inmodel = X_inmodel
self.Y_inmodel = Y_inmodel
# Save parameters of the model
self._save_model_parameter_values()
def _input_data(self, normalization_type):
# input that goes into the model (is unziped in case there are categorical variables)
X_inmodel = self.subspace.unzip_inputs(
np.array([xi[self.subspace_idx] for xi in self.X]))
# Y_inmodel is the output that goes into the model
if self.normalize_Y:
Y_inmodel = normalize(self.Y, normalization_type)
else:
Y_inmodel = self.Y
return X_inmodel, Y_inmodel
def update_subspace(self):
self.subspace_idx = np.sort(
np.random.choice(range(self.dimensionality),
self.subspace_dim_size,
replace=False))
self.subspace = get_subspace(space=self.space,
subspace_idx=self.subspace_idx)
def _compute_next_evaluations(self,
pending_zipped_X=None,
ignored_zipped_X=None):
context_manager, duplicate_manager = self._compute_setting(
pending_zipped_X=pending_zipped_X,
ignored_zipped_X=ignored_zipped_X)
# We zip the value in case there are categorical variables
suggested_ = self.subspace.zip_inputs(
self.evaluator.compute_batch(duplicate_manager=duplicate_manager,
context_manager=context_manager))
return self._fill_in_dimensions(samples=suggested_)
def _compute_setting(self, pending_zipped_X, ignored_zipped_X):
context_manager = ContextManager(self.subspace, self.context)
# --- Update the context if any
self.acquisition.optimizer.context_manager = context_manager
# --- Activate de_duplication
if self.de_duplication:
duplicate_manager = DuplicateManager(
space=self.subspace,
zipped_X=self.X,
pending_zipped_X=pending_zipped_X,
ignored_zipped_X=ignored_zipped_X)
else:
duplicate_manager = None
return context_manager, duplicate_manager
def _save(self):
mkdir_when_not_exist(abs_path=definitions.ROOT_DIR + '/storage/' +
self.objective_name)
dir_name = definitions.ROOT_DIR + '/storage/' + self.objective_name + '/' + now_str(
) + ' ' + str(self.dimensionality) + 'D ' + str(self.fill_in_strategy)
mkdir_when_not_exist(abs_path=dir_name)
self.save_report(report_file=dir_name + '/report.txt')
self.save_evaluations(evaluations_file=dir_name + '/evaluation.csv')
self.save_models(models_file=dir_name + '/model.csv')
def save_report(self, report_file=None):
with open(report_file, 'w') as file:
import GPyOpt
import time
file.write('-----------------------------' +
' GPyOpt Report file ' +
'-----------------------------------\n')
file.write('GPyOpt Version ' + str(GPyOpt.__version__) + '\n')
file.write('Date and time: ' + time.strftime("%c") +
'\n')
if self.num_acquisitions == self.max_iter:
file.write('Optimization completed: ' + 'YES, ' +
str(self.X.shape[0]).strip('[]') +
' samples collected.\n')
file.write('Number initial samples: ' +
str(self.initial_design_numdata) + ' \n')
else:
file.write('Optimization completed: ' + 'NO,' +
str(self.X.shape[0]).strip('[]') +
' samples collected.\n')
file.write('Number initial samples: ' +
str(self.initial_design_numdata) + ' \n')
file.write('Tolerance: ' + str(self.eps) + '.\n')
file.write('Optimization time: ' +
str(self.cum_time).strip('[]') + ' seconds.\n')
file.write('\n')
file.write('--------------------------------' +
' Problem set up ' +
'------------------------------------\n')
file.write('Problem name: ' + self.objective_name +
'\n')
file.write('Problem dimension: ' +
str(self.dimensionality) + '\n')
file.write('Number continuous variables ' +
str(len(self.space.get_continuous_dims())) + '\n')
file.write('Number discrete variables ' +
str(len(self.space.get_discrete_dims())) + '\n')
file.write('Number bandits ' +
str(self.space.get_bandit().shape[0]) + '\n')
file.write('Noiseless evaluations: ' +
str(self.exact_feval) + '\n')
file.write('Cost used: ' + self.cost.cost_type +
'\n')
file.write('Constraints: ' +
str(self.constraints == True) + '\n')
file.write('Subspace Dimension: ' +
str(self.subspace_dim_size) + '\n')
file.write('Fill in strategy: ' +
str(self.fill_in_strategy) + '\n')
file.write('\n')
file.write('------------------------------' +
' Optimization set up ' +
'---------------------------------\n')
file.write('Normalized outputs: ' +
str(self.normalize_Y) + '\n')
file.write('Model type: ' +
str(self.model_type).strip('[]') + '\n')
file.write('Model update interval: ' +
str(self.model_update_interval) + '\n')
file.write('Acquisition type: ' +
str(self.acquisition_type).strip('[]') + '\n')
file.write(
'Acquisition optimizer: ' +
str(self.acquisition_optimizer.optimizer_name).strip('[]') +
'\n')
file.write('Acquisition type: ' +
str(self.acquisition_type).strip('[]') + '\n')
if hasattr(self, 'acquisition_optimizer') and hasattr(
self.acquisition_optimizer, 'optimizer_name'):
file.write('Acquisition optimizer: ' + str(
self.acquisition_optimizer.optimizer_name).strip('[]') +
'\n')
else:
file.write('Acquisition optimizer: None\n')
file.write('Evaluator type (batch size): ' +
str(self.evaluator_type).strip('[]') + ' (' +
str(self.batch_size) + ')' + '\n')
file.write('Cores used: ' + str(self.num_cores) +
'\n')
file.write('\n')
file.write('---------------------------------' + ' Summary ' +
'------------------------------------------\n')
file.write('Initial X: ' +
str(self.initial_X) + '\n')
file.write('Initial Y: ' +
str(self.initial_Y) + '\n')
if self.maximize:
file.write('Value at maximum: ' +
str(format(-min(self.Y)[0], '.20f')).strip('[]') +
'\n')
file.write('Best found maximum location: ' +
str(self.X[np.argmin(self.Y), :]).strip('[]') +
'\n')
else:
file.write('Value at minimum: ' +
str(format(min(self.Y)[0], '.20f')).strip('[]') +
'\n')
file.write('Best found minimum location: ' +
str(self.X[np.argmin(self.Y), :]).strip('[]') +
'\n')
file.write(
'----------------------------------------------------------------------------------------------\n'
)
file.close()
class TestArgumentsManager(unittest.TestCase):
'''
The test just checks that the arguments are passed and handled, not that the model
or the acquisition does the right thing with them.
'''
def setUp(self):
kwargs = {'n_samples':1000,
'n_burnin':100,
'subsample_interval':5,
'step_size': 1e-1,
'leapfrog_steps':20,
'optimize_restarts':10,
'num_inducing':15,
'acquisition_transformation':'softplus',
'acquisition_jitter':0.02,
'acquisition_weight':2.5,
'acquisition_transformation':'softplus'
}
## --- Defaults for some of the tests
self.space = Design_space(space =[{'name': 'var1', 'type': 'continuous', 'domain': (-10,10),'dimensionality': 2}])
self.cost = CostModel(None)
self.arguments_manager = ArgumentsManager(kwargs)
self.model = self.arguments_manager.model_creator(model_type = 'GP', exact_feval= True, space = self.space)
self.acquisition_optimizer = AcquisitionOptimizer(self.space)
self.acquisition = self.arguments_manager.acquisition_creator('EI', self.model, self.space, self.acquisition_optimizer, self.cost)
def test_model_gp_mcmc_arguments(self):
'''
Testing the arguments of the GP model with MCMC
'''
model_type = 'GP_MCMC'
exact_feval = True
created_model = self.arguments_manager.model_creator(model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, GPModel_MCMC))
self.assertEquals(created_model.n_samples, 1000)
self.assertEquals(created_model.n_burnin, 100)
self.assertEquals(created_model.subsample_interval, 5)
self.assertEquals(created_model.step_size, 1e-1)
self.assertEquals(created_model.leapfrog_steps, 20)
def test_model_sparse_gp_arguments(self):
'''
Testing the arguments of the standard GP model
'''
model_type = 'sparseGP'
exact_feval = True
created_model = self.arguments_manager.model_creator(model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, GPModel))
self.assertEquals(created_model.optimize_restarts,10)
self.assertEquals(created_model.num_inducing,15)
def test_model_rf_arguments(self):
'''
Testing the arguments of the Random Forrest
'''
model_type = 'RF'
exact_feval = True
created_model = self.arguments_manager.model_creator(model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, RFModel))
def test_model_inputwarpedgp_arguments(self):
'''
Testing the arguments of the input warped GP
'''
model_type = 'input_warped_GP'
exact_feval = True
created_model = self.arguments_manager.model_creator(model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, InputWarpedGPModel))
def test_model_warpedgo_arguments(self):
'''
Testing the arguments of the warped GP
'''
model_type = 'warpedGP'
exact_feval = True
created_model = self.arguments_manager.model_creator(model_type, exact_feval, self.space)
self.assertTrue(isinstance(created_model, WarpedGPModel))
def test_acquisition_ei_arguments(self):
'''
Testing the arguments of the Expected Improvement
'''
acquisition_type = 'EI'
created_acquisition = self.arguments_manager.acquisition_creator(acquisition_type, self.model, self.space, self.acquisition_optimizer, self.cost)
self.assertTrue(isinstance(created_acquisition, AcquisitionEI))
self.assertEquals(created_acquisition.jitter,0.02)
def test_acquisition_lcb_arguments(self):
'''
Testing the arguments of the Lower Confidence Bound
'''
acquisition_type = 'LCB'
created_acquisition = self.arguments_manager.acquisition_creator(acquisition_type, self.model, self.space, self.acquisition_optimizer, self.cost)
self.assertTrue(isinstance(created_acquisition, AcquisitionLCB))
self.assertEquals(created_acquisition.exploration_weight,2.5)
def test_evaluator_arguments(self):
'''
Testing the arguments of the local_penalization evaluator
'''
evaluator_type = 'local_penalization'
batch_size = 2
model_type = 'GP'
created_evaluator = self.arguments_manager.evaluator_creator(evaluator_type, self.acquisition, batch_size, model_type, self.model, self.space, self.acquisition_optimizer)
self.assertTrue(isinstance(created_evaluator, LocalPenalization))
self.assertEquals(created_evaluator.acquisition.transform,'softplus')
def __init__(self,
domain=None,
constraints=None,
cost_withGradients=None,
model_type='GP',
X=None,
Y=None,
initial_design_numdata=7,
initial_design_type='random',
acquisition_type='EI',
normalize_Y=True,
exact_feval=False,
acquisition_optimizer_type='lbfgs',
model_update_interval=1,
evaluator_type='sequential',
batch_size=1,
num_cores=1,
verbosity=False,
verbosity_model=False,
maximize=False,
de_duplication=False,
**kwargs):
self.modular_optimization = False
self.initial_iter = True
self.verbosity = verbosity
self.verbosity_model = verbosity_model
self.model_update_interval = model_update_interval
self.de_duplication = de_duplication
self.kwargs = kwargs
# --- Handle the arguments passed via kargs
self.problem_config = ArgumentsManager(kwargs)
# --- CHOOSE design space
self.constraints = constraints
self.domain = domain
self.space = Design_space(self.domain, self.constraints)
# --- CHOOSE objective function
self.maximize = maximize
self.batch_size = batch_size
self.num_cores = num_cores
# --- CHOOSE the cost model
self.cost = CostModel(cost_withGradients)
# --- CHOOSE initial design
self.X = X
self.Y = Y
self.initial_design_type = initial_design_type
self.initial_design_numdata = initial_design_numdata
# --- CHOOSE the model type. If an instance of a GPyOpt model is passed (possibly user defined), it is used.
self.model_type = model_type
self.exact_feval = exact_feval # note that this 2 options are not used with the predefined model
self.normalize_Y = normalize_Y
self.model = self._model_chooser()
# --- CHOOSE the acquisition optimizer_type
# This states how the discrete variables are handled (exact search or rounding)
self.acquisition_optimizer_type = acquisition_optimizer_type
self.acquisition_optimizer = AcquisitionOptimizer(
self.space, self.acquisition_optimizer_type,
model=self.model) ## more arguments may come here
# --- CHOOSE acquisition function. If an instance of an acquisition is passed (possibly user defined), it is used.
self.acquisition_type = acquisition_type
self.acquisition = self.acquisition = self._acquisition_chooser()
# --- CHOOSE evaluator method
self.evaluator_type = evaluator_type
self.evaluator = self._evaluator_chooser()
# --- Create optimization space
super(RosBayesianOptimization, self).__init__(
model=self.model,
space=self.space,
acquisition=self.acquisition,
evaluator=self.evaluator,
X_init=self.X,
Y_init=self.Y,
cost=self.cost,
normalize_Y=self.normalize_Y,
model_update_interval=self.model_update_interval,
de_duplication=self.de_duplication,
initial_design_numdata=self.initial_design_numdata)
class RosBayesianOptimization(RosBO):
"""
Main class to initialize a Bayesian Optimization method.
:param f: function to optimize. It should take 2-dimensional numpy arrays as input and return 2-dimensional outputs (one evaluation per row).
:param domain: list of dictionaries containing the description of the inputs variables (See GPyOpt.core.space.Design_space class for details).
:param constraints: list of dictionaries containing the description of the problem constraints (See GPyOpt.core.space.Design_space class for details).
:cost_withGradients: cost function of the objective. The input can be:
- a function that returns the cost and the derivatives and any set of points in the domain.
- 'evaluation_time': a Gaussian process (mean) is used to handle the evaluation cost.
:model_type: type of model to use as surrogate:
- 'GP', standard Gaussian process.
- 'GP_MCMC', Gaussian process with prior in the hyper-parameters.
- 'sparseGP', sparse Gaussian process.
- 'warperdGP', warped Gaussian process.
- 'InputWarpedGP', input warped Gaussian process
- 'RF', random forest (scikit-learn).
:param X: 2d numpy array containing the initial inputs (one per row) of the model.
:param Y: 2d numpy array containing the initial outputs (one per row) of the model.
:initial_design_numdata: number of initial points that are collected jointly before start running the optimization.
:initial_design_type: type of initial design:
- 'random', to collect points in random locations.
- 'latin', to collect points in a Latin hypercube (discrete variables are sampled randomly.)
:acquisition_type: type of acquisition function to use.
- 'EI', expected improvement.
- 'EI_MCMC', integrated expected improvement (requires GP_MCMC model).
- 'MPI', maximum probability of improvement.
- 'MPI_MCMC', maximum probability of improvement (requires GP_MCMC model).
- 'LCB', GP-Lower confidence bound.
- 'LCB_MCMC', integrated GP-Lower confidence bound (requires GP_MCMC model).
:param normalize_Y: whether to normalize the outputs before performing any optimization (default, True).
:exact_feval: whether the outputs are exact (default, False).
:acquisition_optimizer_type: type of acquisition function to use.
- 'lbfgs': L-BFGS.
- 'DIRECT': Dividing Rectangles.
- 'CMA': covariance matrix adaptation.
:param model_update_interval: interval of collected observations after which the model is updated (default, 1).
:param evaluator_type: determines the way the objective is evaluated (all methods are equivalent if the batch size is one)
- 'sequential', sequential evaluations.
- 'random': synchronous batch that selects the first element as in a sequential policy and the rest randomly.
- 'local_penalization': batch method proposed in (Gonzalez et al. 2016).
- 'thompson_sampling': batch method using Thompson sampling.
:param batch_size: size of the batch in which the objective is evaluated (default, 1).
:param num_cores: number of cores used to evaluate the objective (default, 1).
:param verbosity: prints the models and other options during the optimization (default, False).
:param maximize: when True -f maximization of f is done by minimizing -f (default, False).
:param **kwargs: extra parameters. Can be used to tune the current optimization setup or to use deprecated options in this package release.
.. Note:: The parameters bounds, kernel, numdata_initial_design, type_initial_design, model_optimize_interval, acquisition, acquisition_par
model_optimize_restarts, sparseGP, num_inducing and normalize can still be used but will be deprecated in the next version.
"""
def __init__(self,
domain=None,
constraints=None,
cost_withGradients=None,
model_type='GP',
X=None,
Y=None,
initial_design_numdata=7,
initial_design_type='random',
acquisition_type='EI',
normalize_Y=True,
exact_feval=False,
acquisition_optimizer_type='lbfgs',
model_update_interval=1,
evaluator_type='sequential',
batch_size=1,
num_cores=1,
verbosity=False,
verbosity_model=False,
maximize=False,
de_duplication=False,
**kwargs):
self.modular_optimization = False
self.initial_iter = True
self.verbosity = verbosity
self.verbosity_model = verbosity_model
self.model_update_interval = model_update_interval
self.de_duplication = de_duplication
self.kwargs = kwargs
# --- Handle the arguments passed via kargs
self.problem_config = ArgumentsManager(kwargs)
# --- CHOOSE design space
self.constraints = constraints
self.domain = domain
self.space = Design_space(self.domain, self.constraints)
# --- CHOOSE objective function
self.maximize = maximize
self.batch_size = batch_size
self.num_cores = num_cores
# --- CHOOSE the cost model
self.cost = CostModel(cost_withGradients)
# --- CHOOSE initial design
self.X = X
self.Y = Y
self.initial_design_type = initial_design_type
self.initial_design_numdata = initial_design_numdata
# --- CHOOSE the model type. If an instance of a GPyOpt model is passed (possibly user defined), it is used.
self.model_type = model_type
self.exact_feval = exact_feval # note that this 2 options are not used with the predefined model
self.normalize_Y = normalize_Y
self.model = self._model_chooser()
# --- CHOOSE the acquisition optimizer_type
# This states how the discrete variables are handled (exact search or rounding)
self.acquisition_optimizer_type = acquisition_optimizer_type
self.acquisition_optimizer = AcquisitionOptimizer(
self.space, self.acquisition_optimizer_type,
model=self.model) ## more arguments may come here
# --- CHOOSE acquisition function. If an instance of an acquisition is passed (possibly user defined), it is used.
self.acquisition_type = acquisition_type
self.acquisition = self.acquisition = self._acquisition_chooser()
# --- CHOOSE evaluator method
self.evaluator_type = evaluator_type
self.evaluator = self._evaluator_chooser()
# --- Create optimization space
super(RosBayesianOptimization, self).__init__(
model=self.model,
space=self.space,
acquisition=self.acquisition,
evaluator=self.evaluator,
X_init=self.X,
Y_init=self.Y,
cost=self.cost,
normalize_Y=self.normalize_Y,
model_update_interval=self.model_update_interval,
de_duplication=self.de_duplication,
initial_design_numdata=self.initial_design_numdata)
def _model_chooser(self):
return self.problem_config.model_creator(self.model_type,
self.exact_feval, self.space)
def _acquisition_chooser(self):
return self.problem_config.acquisition_creator(
self.acquisition_type, self.model, self.space,
self.acquisition_optimizer, self.cost.cost_withGradients)
def _evaluator_chooser(self):
return self.problem_config.evaluator_creator(
self.evaluator_type, self.acquisition, self.batch_size,
self.model_type, self.model, self.space,
self.acquisition_optimizer)