def fit(self, X: list, y: list, n_jobs: int, max_iter: int = 50) -> None:
"""
Perform bayesian optimization on hyperparameter space.
Args:
x : Array containing training input data.
y : Array containing data labels.
n_jobs: Number of threads to run the algorithms.
max_iter : Number of iterations to perform bayesian optimization.
"""
self.X, self.y = shuffle(X, y)
self.n_jobs = n_jobs
num_features = X.shape[1]
for model_name in self.available_models:
self.current_model = model_name
print(self.current_model)
opt = BayesianOptimization(
f=self.perform_cv,
domain=self.grids[self.current_model])
opt.run_optimization(max_iter=max_iter)
def fit(self,
X: list,
y: list,
metric: str = None,
max_iter: int = 50,
silence=False) -> None:
"""
Perform bayesian optimization on hyperparameter space.
Args:
x : Array containing training input data.
y : Array containing data labels.
max_iter : Number of iterations to perform bayesian optimization.
"""
if metric not in self.metrics:
msg = "Unsupported metric '{}'".format(metric)
msg += "\nSupported metrics are: {}".format(self.metrics)
raise ValueError(msg)
self.X = X
self.y = y
self.core_metric = metric
opt = BayesianOptimization(f=self.run_iteration, domain=self.grid)
opt.run_optimization(max_iter=max_iter)
def run(_run=0):
true_theta = load_true_theta(_run)
true_y = load_true_y(_run)
theta_priors = {
'a': LogNormal(-11.86, 0.1),
'b': Normal(1.2e-4, 1e-6),
'k': LogNormal(-0.35, 0.01)
}
def f_of_(X, device=GLOBAL_DEVICE):
a = theta_priors['a'].icdf(tp.tensor(X[0,0]).to(device))
b = theta_priors['b'].icdf(tp.tensor(X[0,1]).to(device))
k = theta_priors['k'].icdf(tp.tensor(X[0,2]).to(device))
sample = simulate({'a':a, 'b':b, 'k':k}, device=device)
return loss(true_y, sample).cpu().item()
domain = [
{'name': 'a', 'type':'continuous','domain':(0.0001, 0.9999)},
{'name': 'b', 'type':'continuous','domain':(0.0001, 0.9999)},
{'name': 'k', 'type':'continuous','domain':(0.0001, 0.9999)}
]
experiment = BayesianOptimization(f = lambda x: f_of_(x), domain=domain)
experiment.run_optimization(max_iter=35)
return {'y':true_y, 'theta':true_theta}, experiment
def runOracle(self, budget):
self.data, self.result = self.initialise()
self.result_list = []
# Initialize wts and probs
print("Running ", self.policy, " with budget ", budget)
for i in tqdm(range(self.C)):
self.cat = i
print("Processing arm: ", self.cat)
myBopt = BayesianOptimization(f=self.my_func, domain=self.bounds)
myBopt.run_optimization(max_iter=budget)
self.result_list.append(myBopt.Y_best)
self.opt_vals = [self.result_list[ii][-1] for ii in range(self.C)]
self.opt_ht = np.argmin(self.opt_vals)
maxlen = 0
for i in range(self.C):
if len(self.result_list[i]) > maxlen:
maxlen = len(self.result_list[i])
vals = np.zeros((maxlen, self.C))
for i in range(self.C):
l = len(self.result_list[i])
vals[:, i][:l] = self.result_list[i][:l]
# for i in range(self.C):
# vals[:,i] = self.result_list[i]
bv = self.result_list[self.opt_ht] * -1
df = pd.DataFrame(data=vals, columns=np.arange(self.C))
# print("Size of best vals in loop: ", bv.shape)
return df, bv
def run_baeopt(opt_params):
# set up opt params
bounds = get_bounds(sm.run.ctl_str_ids, sm.run.n_ctl_steps)
max_iter = opt_params.get('max_iter', 15)
max_time = opt_params.get('max_time', 120)
initial_guess = opt_params.get('initial_guess', [])
if len(initial_guess) == 0:
initial_guess = None
else:
initial_guess = np.array([initial_guess])
eps = opt_params.get('eps', 0.01)
model_type = opt_params.get('model_type', 'GP')
acquisition_type = opt_params.get('acquisition_type', 'EI')
# instantiate object
bae_opt = BayOpt(
ev.evaluate,
domain=bounds,
model_type=model_type,
acquisition_type='EI',
X=initial_guess,
evaluator_type='local_penalization',
num_cores=opt_params['num_cores'],
batch_size=opt_params['num_cores'],
)
bae_opt.run_optimization(max_iter, max_time, eps)
return bae_opt.x_opt, bae_opt.fx_opt
def optimize_local(output=output):
#
if self.allocationMethod == 'variablePrecision':
# importing CMAES libarary
import cma
# setting up parameters and running the optimization
x0 = 50 + 15 * np.random.randn(12 * (self.depth - 1))
res = cma.fmin(obj_func, x0, 30, options={'bounds':[1,100],\
'tolfun':1, 'maxfevals': int(1e4)})
sigma_opt = res[0]
elif self.allocationMethod == 'equalPrecision':
# importing Bayesian optimization libraries
import GPy, GPyOpt
from GPyOpt.methods import BayesianOptimization
# setting up parameters and running the optimization
kernel = GPy.kern.Matern52(input_dim=1)
domain = [{
'name': 'var_1',
'type': 'continuous',
'domain': (0, 100)
}]
optimizer = BayesianOptimization(obj_func,
domain=domain,
kernel=kernel)
optimizer.run_optimization(max_iter=50)
sigma_opt = optimizer.X[optimizer.Y.argmin()]
# appending the result (scalar sigma) to output queue
output.put(sigma_opt)
def Bayesdefinition(self):
self.opt_search = BayesianOptimization(self.func.objective_function,
domain=self.domain,
model_type='GP',
acquisition_type='EI',
num_cores=-1,
verbosity=False)
def test_save_gp_2d(self):
k = GPy.kern.Matern52(input_dim=2)
m = GPModel(kernel=k)
myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d, model=m)
myBopt.run_optimization(max_iter=1, verbosity=False)
myBopt.save_models(self.outfile_path)
self.check_output_model_file(['Iteration'])
def text_cnn_train(args, train_path):
# load data
print("\nLoading data...")
text_field = data.Field(lower=True)
label_field = data.Field(sequential=False)
#train_iter, dev_iter = mr(text_field, label_field, device=-1, repeat=False)
train_iter, dev_iter = msw_text(text_field,
label_field,
train_path,
device=-1,
repeat=False)
# batch = next(iter(train_iter))
# print(type(batch))
# print(batch.text)
# train_iter, dev_iter, test_iter = sst(text_field, label_field, device=-1, repeat=False)
# update args and print
args.embed_num = len(
text_field.vocab) # .vocab을 해주면 단어 집합을 만들어 주는거 같다. 일단 추정
args.class_num = len(label_field.vocab) - 1
args.cuda = (not False) and torch.cuda.is_available()
kerns = '3,4,5'
args.kernel_sizes = [int(k) for k in kerns.split(',')]
re_train_path = train_path.split('/')[1][:-4]
save_path = os.path.join(args.save_dir, re_train_path)
print("\nParameters:")
for attr, value in sorted(args.__dict__.items()):
print("\t{}={}".format(attr.upper(), value))
# train or predict
if args.baye:
print('옵티마이즈 세팅')
myBopt = BayesianOptimization(f=baye,
domain=domain,
initial_design_numdata=5)
print('옵티마이즈 시작')
myBopt.run_optimization(max_iter=10)
print('옵티마이즈 결과 : ', myBopt.x_opt)
print('최적의 하이퍼 파라미터의 결과의 파라미터', args.lr, args.dropout)
else:
try:
cnn = model.CNN_Text(args)
if args.snapshot is not None:
print('\nLoading model from {}...'.format(args.snapshot))
cnn.load_state_dict(torch.load(args.snapshot))
if args.cuda:
torch.cuda.set_device(args.device)
cnn = cnn.cuda()
train.train(train_iter, dev_iter, cnn, args, save_path)
except KeyboardInterrupt:
print('\n' + '-' * 89)
print('Exiting from training early')
def break_grid_bo(w_, garden_index, grids):
# first compute convex hull of grids
# parametrize problem by just the center of the grid
num_poles = grids.shape[0]
bounds = []
for i in range(num_poles):
name_s = "p" + str(i)
lower_x = grids[i, 0] - 0.8
upper_x = grids[i, 0] + 0.8
lower_y = grids[i, 1] - 0.8
upper_y = grids[i, 1] + 0.8
domainx = (lower_x, upper_x)
domainy = (lower_y, upper_y)
dx = {'name': name_s + "x", 'type': 'continuous', 'domain': domainx}
dy = {'name': name_s + "y", 'type': 'continuous', 'domain': domainy}
bounds.append(dx)
bounds.append(dy)
Y_init, pole_platform, cc, _ = all_constraints(w_, garden_index, grids)
loss = 1000
current_iter = 0
X_step = np.zeros((1, num_poles * 2))
Y_step = np.empty((1, 1))
Y_step[0, 0] = Y_init
context = {}
while loss > 2:
if current_iter >= 200:
return grids
bo_step = BayesianOptimization(f=None,
model="GP",
domain=bounds,
X=X_step,
Y=Y_step,
maximize=False,
acquisition="LCB")
try:
x_next = bo_step.suggest_next_locations(context=context)
except:
x_next = bo_step.suggest_next_locations()
grids = x_next.reshape(num_poles, 2)
loss, pole_platform, cc, good_ind = all_constraints(
w_, garden_index, grids)
for i in good_ind:
context["p{}x".format(i)] = grids[i, 0]
context["p{}y".format(i)] = grids[i, 1]
print("Iteration {} has loss {}".format(current_iter, loss))
print("POLE2PLAT {}".format(pole_platform))
y_next = loss
y_tmp = np.empty((1, 1))
y_tmp[0, 0] = y_next
X_step = np.vstack((X_step, x_next))
Y_step = np.vstack((Y_step, y_tmp))
loss = y_next
current_iter += 1
return grids
def train(
self, bo_model=None, bo_space=None, bo_acquisition=None,
X_init=None, Y_init=None, BO_init_num=5, BO_max_num=20,
verbosity=False):
"""
Uses Bayesian optimisation to find the optimal design. The objective
function is the mutual information lower bound at a particular design,
obtained by training a MINE model.
Parameters
----------
bo_model:
bo_space:
bo_acquisition:
BO_init_num: int
The number of initial BO evaluations used to initialise the GP.
(default is 5)
BO_max_num: int
The maximum number of BO evaluations after the initialisation.
(default is 20)
verbosity: boolean
Turn off/on output to the command line.
(default is False)
"""
if verbosity:
print('Initialize Probabilistic Model')
if bo_model and bo_space and bo_acquisition:
raise NotImplementedError('Custom BO model not yet implemented.')
elif all(v is None for v in [bo_model, bo_space, bo_acquisition]):
pass
else:
raise ValueError(
'Either all BO arguments or none need to be specified.')
# Define GPyOpt Bayesian Optimization object
self.bo_obj = BayesianOptimization(
f=self._objective, domain=self.domain,
constraints=self.constraints, model_type='GP',
acquisition_type='EI', normalize_Y=False,
initial_design_numdata=BO_init_num, acquisition_jitter=0.01,
maximize=True, X=X_init, Y=Y_init)
# TODO: Implement a more modular approach with GPy model as input.
if verbosity:
print('Start Bayesian Optimisation')
# run the bayesian optimisation
self.bo_obj.run_optimization(
max_iter=BO_max_num, verbosity=verbosity, eps=1e-5)
# find optimal design from posterior GP model; stored as d_opt
self._compute_optimal_design(self.bo_obj)
def forward(self, output, target, bayes=True, best_params=None):
self.output = output
self.target = target
self.bayes = bayes
# vest params is None when it is first train.
if best_params is None:
var_args = [
0.5, 0.5, 0.5, 0.5, # threshold
0.0, 0.0, 0.0, 0.0, # minsizes
0.2,
]
else:
var_args = [
best_params['threshold_0'],
best_params['threshold_1'],
best_params['threshold_2'],
best_params['threshold_3'],
best_params['min_size_0'],
best_params['min_size_1'],
best_params['min_size_2'],
best_params['min_size_3'],
best_params['mask_threshold'],
]
if bayes is True:
pbounds = [
{'name': 'threshold_0', 'type': 'continuous', 'domain': (0, 1.0)},
{'name': 'threshold_1', 'type': 'continuous', 'domain': (0, 1.0)},
{'name': 'threshold_2', 'type': 'continuous', 'domain': (0, 1.0)},
{'name': 'threshold_3', 'type': 'continuous', 'domain': (0, 1.0)},
{'name': 'min_size_0', 'type': 'continuous', 'domain': (100, 10000)},
{'name': 'min_size_1', 'type': 'continuous', 'domain': (100, 10000)},
{'name': 'min_size_2', 'type': 'continuous', 'domain': (100, 10000)},
{'name': 'min_size_3', 'type': 'continuous', 'domain': (100, 10000)},
{'name': 'mask_threshold', 'type': 'continuous', 'domain': (0, 1.0)},
]
optimizer = BayesianOptimization(
f=self.calc_loss,
domain=pbounds,
num_cores=16,
maximize=True,
verbosity=False,
initial_design_numdata=2
)
optimizer.run_optimization(max_iter=18, verbosity=False)
dice_coefficient = -1 * (optimizer.fx_opt)
best_params = optimizer.x_opt
else:
dice_coefficient = self.calc_loss(var_args)
return dice_coefficient, best_params
def test_one_initial_data_point(self):
"""Make sure BO still works with only one initial data point."""
bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (-1, 1)}]
# Remove hyperparameter optimization, not needed here
model = GPyOpt.models.GPModel(max_iters=0)
opt = BayesianOptimization(lambda x: x, bounds, model=model, initial_design_numdata=1)
# Make sure run_optimization works
opt.run_optimization(max_iter=1)
assert len(opt.Y) > 1
def run_lf_opt(self):
def f_lf(X):
return(structure(self.lf_model, self.n, X[0], self.EI_req).score)
self.lf_opt = BayesianOptimization(f_lf,
domain=self.domain,
acquisition_type="EI",
model_type='GP',
exact_feval=True)
self.lf_opt.run_optimization(max_iter = self.lf_max_iter, eps=1e-6)
self.X_L, self.y_L = self.lf_opt.get_evaluations()
def test_save_gp_2d_ard(self):
"""
This was previously an edge-case, when some parameters were vectors, the naming of the columns was incorrect
"""
k = GPy.kern.Matern52(input_dim=2, ARD=True)
m = GPModel(kernel=k)
myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d, model=m)
myBopt.run_optimization(max_iter=1, verbosity=False)
myBopt.save_models(self.outfile_path)
self.check_output_model_file(['Iteration'])
def test_one_initial_data_point(self):
"""Make sure BO still works with only one initial data point."""
bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (-1, 1)}]
opt = BayesianOptimization(lambda x: x,
bounds,
initial_design_numdata=1)
# Make sure run_optimization works
opt.run_optimization(max_iter=1)
assert len(opt.Y) > 1
def bo_voronoi_directed(exp):
if not exp.initialised:
exp.setParameters()
G = getVoronoiDirectedGraph(occupancy_grid=exp.occupancy_grid,
nodes=exp.nodes,
edges=exp.edges_dir,
start_nodes=exp.start_nodes,
end_nodes=exp.end_nodes)
simulateObj = Simulator(G, exp.start_nodes, exp.end_nodes)
np.random.RandomState(42)
kern_eq = GPy.kern.RBF(input_dim=2, ARD=True) + GPy.kern.White(input_dim=2,
variance=1)
kern_bias = GPy.kern.Bias(input_dim=2)
kern = kern_eq + kern_bias
domain = []
k = 0
assigned = {}
for n in G.nodes:
for neighbor in G.neighbors(n):
if n != neighbor and frozenset(
(n, neighbor)) not in assigned.keys():
unused_percentage = {
'name': k,
'type': 'continuous',
'domain': (0, 1)
}
direction_percentage = {
'name': k + 1,
'type': 'continuous',
'domain': (0, 1)
}
domain.append(unused_percentage)
domain.append(direction_percentage)
assigned[frozenset(
(n, neighbor))] = 1 #tbc whether this will cause errors
k += 2
domain.append({'name': k + 1, 'type': 'continuous', 'domain': (0, 0.05)})
opt = BayesianOptimization(f = simulateObj.run_simulator, maximize=False, \
domain=domain, model_type='GP', \
initial_design_numdata = constant.BO_INITIAL_SAMPLES,\
kernel=kern, acquisition_type='EI')
out_name = "bo-results.txt"
opt.run_optimization(max_iter=constant.BO_OPT_SAMPLES,
report_file=out_name)
return opt, G
def bayesian_opt():
optimizer = BayesianOptimization(f=best_fitness_score,
domain=bounds(),
model_type='GP',
acquisition_type ='EI',
acquisition_jitter = 0.05,
exact_feval=True,
maximize=False)
# Only 20 iterations because we have 5 initial random points
optimizer.run_optimization(max_iter=20)
print(np.maximum.accumulate(-optimizer.Y).ravel())
def gpyopt(parallel, config_path, conversations_list, params):
from GPyOpt.methods import BayesianOptimization
global results
results = []
utility = make_utility_function_gpyopt(parallel, config_path,
conversations_list)
bo = BayesianOptimization(utility,
params,
verbosity=True,
initial_design_numdata=len(params) * 2)
bo.run_optimization(max_iter=200, verbosity=True)
print(f"done. best: {bo.fx_opt}")
return results
def test_one_initial_data_point(self):
"""Make sure BO still works with only one initial data point."""
bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (-1, 1)}]
# Remove hyperparameter optimization, not needed here
model = GPyOpt.models.GPModel(max_iters=0)
opt = BayesianOptimization(lambda x: x,
bounds,
model=model,
initial_design_numdata=1)
# Make sure run_optimization works
opt.run_optimization(max_iter=1)
assert len(opt.Y) > 1
def main_with_random_search():
bounds = [
{'name': 'batch_size', 'type': 'discrete',
'domain': (16, 32, 64)},
{'name': 'fc1_size', 'type': 'discrete',
'domain': (512, 1024, 2048, 4096)},
{'name': 'fc2_size', 'type': 'discrete',
'domain': (128, 256, 512, 1024)}]
optimizer = BayesianOptimization(f=function_to_optimize, domain=bounds)
optimizer.run_optimization(max_iter=10)
logger.info('optimized parameters: {}'.format(optimizer.x_opt))
logger.info('optimized accuracy: {}'.format(optimizer.fx_opt))
def propose_hypers(bayes_opt_results, bayes_opt_setup):
"""Proposes hyperparameters given the current results.
Args:
results: instance of BayesOptResults.
bayes_opt_setup: instance of BayesOptSetup
Returns:
x: the proposed point.
"""
bayes_opt = BayesianOptimization(f=None, domain=bayes_opt_setup.domain(),
X=bayes_opt_results.xs, Y=bayes_opt_results.ys)
proposed_x = bayes_opt.suggest_next_locations()
return proposed_x, bayes_opt
def test_next_locations_pending(self):
func = GPyOpt.objective_examples.experiments1d.forrester()
domain =[{'name': 'var1', 'type': 'continuous', 'domain': (0,1)}]
X_init = np.array([[0.0],[0.5],[1.0]])
Y_init = func.f(X_init)
np.random.seed(1)
bo_no_pending = BayesianOptimization(f = None, domain = domain, X = X_init, Y = Y_init)
x_no_pending = bo_no_pending.suggest_next_locations()
np.random.seed(1)
bo_pending = BayesianOptimization(f = None, domain = domain, X = X_init, Y = Y_init, de_duplication = True)
x_pending = bo_pending.suggest_next_locations(pending_X = x_no_pending)
self.assertFalse(np.isclose(x_pending, x_no_pending))
def _optimize(self, f, variables, X_init, Y_init, maxiter, maxeval,
iter_callback):
from GPyOpt.methods import BayesianOptimization
from GPyOpt.core.task.objective import SingleObjective
maxeval = get_maxeval_for_bo(maxeval, maxiter)
ndim = len(variables)
if len(Y_init) > 0:
x_best = X_init[0]
y_best = Y_init[0]
iter_callback(x_best, y_best, X_init, Y_init)
kernel = self.get_kernel(ndim)
gpy_domain = self.get_domain(variables)
Y_init = np.array(Y_init).reshape(len(Y_init), -1)
X_init = np.array(X_init, dtype=float)
bo = BayesianOptimization(f=f,
domain=gpy_domain,
model_type='GP',
acquisition_type=self.acquisition_type,
kernel=kernel,
X=np.array(X_init),
Y=np.array(Y_init),
exact_feval=True,
verbosity=True,
)
bo.num_acquisitions = self.run_info.result.n_eval
self.run_info.bo_obj = bo
def f_in_gpyopt(X):
Y = []
x_best = self.transform(self.run_info.result.x)
y_best = self.sign * self.run_info.result.y
for x in X:
y = f(x)
if y_best is None or y < y_best:
x_best = x
y_best = y
Y.append(y)
iter_callback(x=x_best, y=y_best, X_iter=X, Y_iter=Y)
return np.array(Y).reshape(len(Y), -1)
bo.f = bo._sign(f_in_gpyopt)
bo.objective = SingleObjective(
bo.f, bo.batch_size, bo.objective_name)
bo.run_optimization(max_iter=maxeval-len(X_init), eps=0,
verbosity=False)
result = OptimizerResult.from_GPyOpt_OptimizerResult(bo)
self.run_info.result.success = True
self.run_info.status = result.status
self.run_info.message = result.message
return self.run_info.result
def optimize():
"""
the parameters are:
pontrol_gain
min_periods_for_inference
min_move
SE0KLengthScale
SE0KSignalVariance
PKLengthScale
PKPeriodLength
PKSignalVariance
SE1KLengthScale
SE1KSignalVariance
min_periods_for_period_estimation
points_for_approximation
prediction_gain
"""
# the bounds are used to select which parameters to optimize,
# and in which range to optimize. If the range is zero (e.g., (1.0,1.0))
# the parameter isn't optimized at all.
bounds = [(0.7, 0.7), (2.0, 2.0), (0.2, 0.2), (700, 700), (18, 18),
(10, 10), (200, 200), (20, 20), (25, 25), (10, 10), (2, 2),
(100, 100), (0.5, 0.5)]
bounds = np.log(bounds)
# evaluate the current parameter set
opt_params = [0.7, 2, 0.2, 700, 18, 10, 200, 20, 25, 10, 2, 100, 0.5]
print eval_func([np.log(opt_params)])
# the initial_design_numdata defines how many random values are tested.
# apparently this is the most important value and should be as high as affordable.
# num_cores and batch_size define how many evaluations run simultaneously.
myBopt = BayesianOptimization(f=eval_func,
bounds=bounds,
num_cores=6,
batch_size=6,
maximize=True,
initial_design_numdata=24)
myBopt.run_optimization(verbosity=True, maximize=True)
print -myBopt.fx_opt
opt_params = np.exp(myBopt.x_opt)
print "{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}".format(
opt_params[0], opt_params[1], opt_params[2], opt_params[3],
opt_params[4], opt_params[5], opt_params[6], opt_params[7],
opt_params[8], opt_params[9], opt_params[10], opt_params[11],
opt_params[12])
def maximize_varmax_given_xi(xi, GP_model, PPBO_settings):
bounds = [{
'name': 'var_' + str(d),
'type': 'continuous',
'domain': (0, 1)
} for d in range(1, GP_model.D + 1)]
BO = BayesianOptimization(
lambda x: -varmax(xi, x, GP_model, PPBO_settings.mc_samples),
domain=bounds,
optimize_restarts=0,
normalize_Y=True)
BO.run_optimization(max_iter=PPBO_settings.BO_maxiter)
x_next = BO.x_opt
zero_coords = list(np.where(xi != 0)[0])
x_next[zero_coords] = 0
return x_next
def estimate_opt(x, y, my_pwlf):
# define the lower and upper bound for the number of line segments
bounds = [{
'name': 'var_1',
'type': 'discrete',
'domain': np.arange(2, 40)
}]
def my_obj(x):
# define some penalty parameter l
# you'll have to arbitrarily pick this
# it depends upon the noise in your data,
# and the value of your sum of square of residuals
l = y.mean() * 0.001
f = np.zeros(x.shape[0])
for i, j in enumerate(x):
my_pwlf.fit(j[0])
f[i] = my_pwlf.ssr + (l * j[0])
return f
np.random.seed(12121)
myBopt = BayesianOptimization(my_obj,
domain=bounds,
model_type='GP',
initial_design_numdata=10,
initial_design_type='latin',
exact_feval=True,
verbosity=True,
verbosity_model=False)
max_iter = 30
# perform the bayesian optimization to find the optimum number
# of line segments
myBopt.run_optimization(max_iter=max_iter, verbosity=True)
print('\n \n Opt found \n')
print('Optimum number of line segments:', myBopt.x_opt)
print('Function value:', myBopt.fx_opt)
myBopt.plot_acquisition()
myBopt.plot_convergence()
# perform the bayesian optimization to find the optimum number
# of line segments
myBopt.run_optimization(max_iter=max_iter, verbosity=True)
return myBopt.x_opt
def optimisation(self, init_num=5, max_iter=10):
# Define GPyOpt Bayesian Optimization object
myBopt = BayesianOptimization(f=self._objective,
domain=self.domain,
constraints=self.constraints,
acquisition_type='EI',
normalize_Y=True,
initial_design_numdata=init_num,
evaluator_type='local_penalization',
batch_size=int(self.num_cores),
num_cores=int(self.num_cores),
acquisition_jitter=0.01)
# run the bayesian optimisation
myBopt.run_optimization(max_iter=max_iter)
self.bo_obj = myBopt
# Select method to get optimum
#optmethod='point' # take optimum from BO evaluations
optmethod = 'interpol' # use posterior predictive to find optimum
if optmethod == 'point':
d_opt = self.bo_obj.x_opt
elif optmethod == 'interpol':
d_opt = methods.get_GP_optimum(self.bo_obj)
else:
raise NotImplementedError()
# Take some real-world data at optimum
y_obs = self.simobj.observe(d_opt)[0]
if self.utiltype == 'MI':
# Compute ratios r_obs and coefficients b_obs for final observation
r_obs, b_obs = self.utilobj.compute_final(d_opt,
y_obs,
num_cores=self.num_cores)
self.savedata = {
'd_opt': d_opt,
'y_obs': y_obs,
'r_obs': r_obs,
'b_obs': b_obs
}
else:
raise NotImplementedError()
def train_bayesian_opt(eval=True):
X_train, y_train, X_val, y_val = loadData()
knn_params = [{
'name': 'n_neighbors',
'type': 'discrete',
'domain': (3, 5, 7, 9, 11, 13)
}, {
'name': 'p',
'type': 'discrete',
'domain': (1, 2, 3)
}]
# Optimization objective
def cv_score(parameters):
parameters = parameters[0]
score = cross_val_score(KNeighborsClassifier(n_neighbors=int(
parameters[0]),
weights='uniform',
algorithm='brute',
p=int(parameters[1])),
X_train,
y_train,
scoring='accuracy').mean()
score = np.array(score)
return score
optimizer = BayesianOptimization(f=cv_score,
domain=knn_params,
model_type='GP',
acquisition_type='EI',
acquisition_jitter=0.001,
maximize=True)
optimizer.run_optimization(max_iter=20)
logger.info(str(optimizer.Y))
logger.info(str(optimizer.X))
logger.info(str(optimizer.X[-1]))
if eval == True:
train(params={
'n_neighbors': int(optimizer.X[-1][0]),
'p': int(optimizer.X[-1][1])
})
def optimise_sampler(num_samples, max_tuning_runs, hyper):
# tune hyper
print("TUNING HYPER-PARAMETERS for hyper=", hyper)
if (hyper.n_parameters() > 0):
# myBopt = BayesianOptimization(f=hyper, domain=hyper.bounds(), num_cores=os.environ['OMP_NUM_THREADS'])
myBopt = BayesianOptimization(f=hyper, domain=hyper.bounds())
myBopt.run_optimization(max_iter=max_tuning_runs)
x_opt = myBopt.x_opt
else:
x_opt = []
# take samples
print("TAKING SAMPLES")
p = multiprocessing.Pool(int(os.environ['OMP_NUM_THREADS']))
args = zip(range(num_samples), repeat(hyper), repeat(x_opt))
results = p.starmap(optimise, args)
return np.array(results)
def setUp(self):
np.random.seed(123)
domain = [{'name': 'var1', 'type': 'continuous', 'domain': (-5, 5), 'dimensionality': 5}]
space = Design_space(domain)
func = alpine1(input_dim=5, bounds=space.get_bounds())
bo = BayesianOptimization(f=func.f, domain=domain)
context = {'var1_1': 0.3, 'var1_2': 0.4}
context_manager = ContextManager(space, context)
x0 = np.array([[0, 0, 0, 0, 0]])
# initialize the model in a least intrusive way possible
bo.suggest_next_locations()
f = bo.acquisition.acquisition_function
f_df = bo.acquisition.acquisition_function_withGradients
self.problem_with_context = OptimizationWithContext(x0=x0, f=f, df=None, f_df=f_df, context_manager=context_manager)
self.x = np.array([[3, -3, 3]])
def bayesian_optimization(objective):
bds = [
{
'name': 'alpha',
'type': 'discrete',
'domain': np.arange(0.05, 0.4, 0.05)
}, {
'name': 'epsilon',
'type': 'discrete',
'domain': np.arange(0.05, 0.4, 0.05)
}
# {'name': 'gamma', 'type': 'discrete', 'domain': np.arange(0.5, 1, 0.05)}
]
# define el optimizador
optimizer = BayesianOptimization(f=objective,
domain=bds,
model_type='GP',
acquisition_type='EI',
acquisition_jitter=0.05,
verbosity=True,
maximize=True)
# realiza las 20 iteraciones de la optimizacion
optimizer.run_optimization(max_iter=30)
print(optimizer.X)
print(optimizer.Y)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
xx = optimizer.X[:, 0].reshape(len(optimizer.X[:, 0]), 1).reshape(-1)
yy = optimizer.X[:, 1].reshape(len(optimizer.X[:, 1]), 1).reshape(-1)
zz = -optimizer.Y.reshape(-1)
surf = ax.plot_trisurf(xx, yy, zz, cmap='viridis')
fig.colorbar(surf)
plt.xlabel('Alpha')
plt.ylabel('Epsilon')
plt.title('MountainCarContinuos V0 Optimization')
plt.show()
def test_normalization(self):
"""Make sure normalization works with wrong model."""
np.random.seed(1)
bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (1, 2)}]
# We get a first measurement at 1, but the model is so harshly
# violated that we will just continue to get measurements at 1.
# The minimum is at 2.
x0 = np.array([[1]])
f = lambda x: -1000 * x
# Remove hyperparameter optimization, not needed here
model = GPyOpt.models.GPModel(max_iters=0)
opt = BayesianOptimization(f, bounds, X=x0, model=model, normalize_Y=True)
opt.run_optimization(max_iter=1)
# Make sure that we did not sample the same point again
assert np.linalg.norm(opt.X[0] - opt.X[1]) > 1e-2
def optimizer_func(X, Y, BatchSize):
'''
Bayesian Optimizer Function
BatchSize is the number of suggestions for the next rounds
X should be the input variables
Y should be the metric to be optimized
'''
bds = [
{
'name': 'x1',
'type': 'continuous',
'domain': (0, 1)
},
{
'name': 'x2',
'type': 'continuous',
'domain': (0, 1)
},
{
'name': 'x3',
'type': 'continuous',
'domain': (0, 1)
},
]
constraints = [
{
'name': 'constr_1',
'constraint': 'x[:,0] + x[:,1] + x[:,2] -(1 + 0.005)'
}, ###<= 0
{
'name': 'constr_2',
'constraint': '(1- 0.005) - (x[:,0] + x[:,1] + x[:,2]) '
}
] ###<= 0
kernel = GPy.kern.Matern52(input_dim=len(bds), ARD=True)
optimizer = BayesianOptimization(
f=None,
domain=bds,
constraints=constraints,
model_type='GP',
acquisition_type='EI',
acquisition_jitter=0.1,
X=X,
Y=Y,
evaluator_type='local_penalization',
batch_size=BatchSize,
normalize_Y=True,
#noise_var = 0.02**2,
kernel=kernel)
return optimizer
def optimize_theta(self):
''' Optimizes all hyper-parameters (including noise) by maximizing the evidence '''
if self.verbose: print("Hyperparameter optimization begins...")
start = time.time()
#BayesianOptimization
#Higher lengthscale generates more accurate GP mean (and location of maximizer of mean)
#However, higher lengthscale also generates less accurate argmax distribution (argmax samples are widepread)
''' Bounds for hyperparameters: When COVARIANCE_SHRINKAGE = 1e-6'''
bounds = [{'name': 'sigma', 'type': 'continuous', 'domain': (0.0001,0.1)}, #Too low noise gives non-PSD covariance matrix
{'name': 'leghtscale', 'type': 'continuous', 'domain': (0.05,0.9)}, #Theoretical l max: 4*np.sqrt(self.D)
{'name': 'sigma_l', 'type': 'continuous', 'domain': (0.01,0.5)}] # since f is a utility function this parameter make no much sense.
BO = BayesianOptimization(lambda theta: -self.evidence(theta[0],self.fMAP), #theta[0] since need to unnest list one level
domain=bounds,
optimize_restarts=3,
normalize_Y=True,
initial_design_numdata=20)
BO.run_optimization(max_iter = 50)
if self.verbose: print('Optimization of hyperparameters took ' + str(time.time()-start) + ' seconds.')
self.theta = BO.x_opt
if self.verbose: print("The optimized theta is "+ str(self.theta))
def test_save_gp_no_filename(self):
myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d)
myBopt.run_optimization(max_iter=1, verbosity=False)
# Need to at least pass in filename or buffer
self.assertRaises(TypeError, lambda: myBopt.save_models())
def test_save_gp_default(self):
myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d)
myBopt.run_optimization(max_iter=1, verbosity=False)
myBopt.save_models(self.outfile_path)
self.check_output_model_file(['Iteration'])
run_id = sys.argv[2]
else:
raise Exception('Must specify a second run_id argument.')
print('run_id: %s' % run_id)
print('constraints:')
print(mixed_domain)
G, labels_matrix = load_blogcat()
input_dim = len(mixed_domain)
func = DwOpt(input_dim, bounds=mixed_domain, default_params=default_params, run_id=run_id)
bo = BayesianOptimization(f=func.f, # function to optimize
domain=mixed_domain, # box-constrains of the problem
initial_design_numdata=20, # number data initial design
acquisition_type='EI', # Expected Improvement
exact_feval=True, # True evaluations00
verbosity=True)
max_iter = 40
max_time = None
bo.run_optimization(max_iter, max_time)
save_opt(bo, run_id)
func.shutdown()
print('\nNumber of fails: %i' % func.fails)
if action == 'load':
if len(sys.argv) >= 3:
run_id = sys.argv[2]
else:
def test_save_gp_default_no_iters(self):
myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d)
# Exception should be raised as no iterations have been carried out yet
self.assertRaises(ValueError, lambda: myBopt.save_models(self.outfile_path))
def evalBOLikelihood(
self,
w,
testdata,
c0,
c1,
c_eval=0,
c_min=0.01,
c_max=0.2,
use_log=False,
true_dist=False,
vars_g=None,
npoints=50,
samples_ids=None,
weights_func=None):
'''
Find minimum of likelihood on testdata using decomposed
ratios and the weighted orthogonal morphing method to find the bases
'''
if true_dist:
vars = ROOT.TList()
for var in vars_g:
vars.Add(w.var(var))
x = ROOT.RooArgSet(vars)
else:
x = None
score = ROOT.RooArgSet(w.var('score'))
if use_log:
evaluateRatio = self.evaluateLogDecomposedRatio
post = 'log'
else:
evaluateRatio = self.evaluateDecomposedRatio
post = ''
# Compute bases if they don't exist for this range
if not os.path.isfile(
'3doubleindexes_{0:.2f}_{1:.2f}_{2:.2f}_{3:.2f}_{4}.dat'.format(
c_min[0],
c_min[1],
c_max[0],
c_max[1],
npoints)):
self.pre2DDoubleBasis(c_min=c_min, c_max=c_max, npoints=npoints)
all_indexes = np.loadtxt(
'3doubleindexes_{0:.2f}_{1:.2f}_{2:.2f}_{3:.2f}_{4}.dat'.format(
c_min[0], c_min[1], c_max[0], c_max[1], npoints))
all_indexes = np.array([[int(x) for x in rows]
for rows in all_indexes])
# Bkg used in the fit
# TODO: Harcoded this have to be changed
basis_value = 1
# Pre evaluate the values for each distribution
pre_pdf = [[range(self.nsamples) for _ in range(self.nsamples)], [
range(self.nsamples) for _ in range(self.nsamples)]]
pre_dist = [[range(self.nsamples) for _ in range(self.nsamples)], [
range(self.nsamples) for _ in range(self.nsamples)]]
# Only precompute distributions that will be used
unique_indexes = set()
for indexes in all_indexes:
unique_indexes |= set(indexes)
# change this enumerates
unique_indexes = list(unique_indexes)
for k in range(len(unique_indexes)):
for j in range(len(unique_indexes)):
index_k, index_j = (unique_indexes[k], unique_indexes[j])
# This save some time by only evaluating the needed samples
if index_k != basis_value:
continue
print 'Pre computing {0} {1}'.format(index_k, index_j)
if k != j:
f0pdf = w.function(
'bkghistpdf_{0}_{1}'.format(
index_k, index_j))
f1pdf = w.function(
'sighistpdf_{0}_{1}'.format(
index_k, index_j))
data = testdata
if self.preprocessing:
data = preProcessing(testdata, self.dataset_names[min(
k, j)], self.dataset_names[max(k, j)], self.scaler)
# outputs =
# predict('{0}/model/{1}/{2}/{3}_{4}_{5}.pkl'.format(self.dir,self.model_g,
outputs = predict(
'/afs/cern.ch/work/j/jpavezse/private/{0}_{1}_{2}.pkl'.format(
self.model_file, index_k, index_j), data, model_g=self.model_g)
f0pdfdist = np.array(
[self.evalDist(score, f0pdf, [xs]) for xs in outputs])
f1pdfdist = np.array(
[self.evalDist(score, f1pdf, [xs]) for xs in outputs])
pre_pdf[0][index_k][index_j] = f0pdfdist
pre_pdf[1][index_k][index_j] = f1pdfdist
else:
pre_pdf[0][index_k][index_j] = None
pre_pdf[1][index_k][index_j] = None
if true_dist:
f0 = w.pdf('f{0}'.format(index_k))
f1 = w.pdf('f{0}'.format(index_j))
if len(testdata.shape) > 1:
f0dist = np.array([self.evalDist(x, f0, xs)
for xs in testdata])
f1dist = np.array([self.evalDist(x, f1, xs)
for xs in testdata])
else:
f0dist = np.array([self.evalDist(x, f0, [xs])
for xs in testdata])
f1dist = np.array([self.evalDist(x, f1, [xs])
for xs in testdata])
pre_dist[0][index_k][index_j] = f0dist
pre_dist[1][index_k][index_j] = f1dist
indices = np.ones(testdata.shape[0], dtype=bool)
samples = []
# Usefull values to inspect after the training
target = self.F1_couplings[:]
def compute_one_alpha_part(weights, xs):
c1s_1 = np.multiply(weights,xs)
c1s_1 = np.multiply(weights,c1s_1)
alpha1 = c1s_1.sum()
return alpha1
exp_basis_weights = True
def vectorize(func):
def wrapper(X):
v = np.zeros(len(X))
for i, x_i in enumerate(X):
v[i] = func(x_i)
return v.reshape(-1, 1)
return wrapper
morph = MorphingWrapper()
morph.setSampleData(
nsamples=15,
ncouplings=3,
types=[
'S',
'S',
'S'],
ncomb=19,
morphed=self.F1_couplings,
samples=self.all_couplings)
def objective(theta):
n_effs = np.zeros(2)
alpha = np.zeros(2)
cs = []
cross_section_list = []
couplings_list = []
for ix,ind in enumerate(all_indexes):
target[1] = theta[0]
target[2] = theta[1]
print target
morph.resetTarget(target)
ind = np.array(ind)
morph.resetBasis([self.all_couplings[int(k)] for k in ind])
couplings = np.array(morph.getWeights())
cross_section = np.array(morph.getCrossSections())
cross_section_list.append(cross_section)
couplings_list.append(couplings)
# Compute F1 couplings and cross sections
c1_ = np.multiply(couplings, cross_section)
n_eff = c1_.sum()
n_tot = np.abs(c1_).sum()
n_effs[ix] = n_eff / n_tot
cs.append(c1_ / c1_.sum())
if exp_basis_weights == False:
alpha[ind] = compute_one_alpha_part(couplings,
cross_section)
if exp_basis_weights == True:
neff2 = 1./n_effs[1]
neff1 = 1./n_effs[0]
alpha1 = np.exp(-neff1**(1./3.))
alpha2 = np.exp(-neff2**(1./3.))
alpha[0] = alpha1/(alpha1 + alpha2)
alpha[1] = alpha2/(alpha1 + alpha2)
else:
alpha[0] = (1/2.)*(alpha2/(alpha1+alpha2))
alpha[1] = (1/2.)*(alpha1/(alpha1+alpha2))
# Compute Bkg weights
c0_arr_1 = np.zeros(15)
c0_arr_2 = np.zeros(15)
c0_arr_1[np.where(all_indexes[0] == basis_value)[0][0]] = 1.
c0_arr_2[np.where(all_indexes[1] == basis_value)[0][0]] = 1.
c0_arr_1 = c0_arr_1 / c0_arr_1.sum()
c0_arr_2 = c0_arr_2 / c0_arr_2.sum()
c1s = np.append(alpha[0] * cs[0], alpha[1] * cs[1])
c0_arr = np.append(0.5 * c0_arr_1, 0.5 * c0_arr_2)
n_eff_ratio = (alpha[0] * n_effs[0] +
alpha[1] * n_effs[1])
cross_section = np.append(cross_section_list[0], cross_section_list[1])
indexes = np.append(all_indexes[0], all_indexes[1])
completeRatios, trueRatios = evaluateRatio(w, testdata, x=x,
plotting=False, roc=False, c0arr=c0_arr, c1arr=c1s, true_dist=true_dist,
pre_dist=pre_dist, pre_evaluation=pre_pdf, cross_section=cross_section,
indexes=indexes)
completeRatios = 1. / completeRatios
if not use_log:
if n_eff_ratio < 0.3:
# TODO: Harcoded number
decomposedLikelihood = 20000
else:
decomposedLikelihood = -2.*np.mean(np.log(completeRatios))
else:
decomposedLikelihood = -np.mean(completeRatios.mean())
print completeRatios[completeRatios < 0.].shape
return decomposedLikelihood
bounds = [(c_min[0],c_max[0]),(c_min[1],c_max[1])]
solver = BayesianOptimization(vectorize(objective), bounds)
solver.run_optimization(max_iter=150, true_gradients=False)
approx_MLE = solver.x_opt
print("Approx. MLE =", approx_MLE)
return ((0.,0.),approx_MLE)
