def bayesian_optimization(num_iterations=50, num_runs=10, output_filename=None):
"""
Run a Bayesian optimization procedure with the given number of iterations
to find a good hyperparameter configuration on the network surrogate.
Run the whole search a given number of times and write the average
optimization trajectory (incumbent errors per epoch) as well as the
cumulative runtime of all configuration evaluations per epoch into
the file specified by the given filename.
"""
lower_bounds = np.array([b[0] for b in hyperparam_ranges])
upper_bounds = np.array([b[1] for b in hyperparam_ranges])
incumbent_errors = np.zeros(num_iterations)
cumulative_runtimes = np.zeros(num_iterations)
for run in range(num_runs):
results = fmin.bayesian_optimization(
objective_function, lower_bounds, upper_bounds,
num_iterations=num_iterations)
cumulative_runtimes += \
np.cumsum([runtime(config) for config in results["X"]])
incumbent_errors += np.array(results["incumbent_values"])
print("Finished run %d of %d." % (run, num_runs))
incumbent_errors /= num_runs
cumulative_runtimes /= num_runs
# Write average incumbent error per epoch into file with given filename
if output_filename is not None:
with open(output_filename, "w") as output_file:
for i in range(len(incumbent_errors)):
output_file.write("%d\t%f\t%f\n" %
(i, incumbent_errors[i], cumulative_runtimes[i]))
def optimization(self):
if (self.method == 'bayesian_optimization'):
results = bayesian_optimization(self.objective_function,
lower=self.lower_bounds,
upper=self.upper_bounds,
num_iterations=self.num_iterations,
acquisition_func='ei')
print(results['x_opt'], results['f_opt'])
self.results = {
'x_opt': results['x_opt'],
'f_opt': results['f_opt'],
'mse': results['incumbent_values']
}
elif (self.method == 'random_search'):
results = random_search(self.objective_function,
lower=self.lower_bounds,
upper=self.upper_bounds,
num_iterations=self.num_iterations)
print(results['x_opt'], results['f_opt'])
self.results = {
'x_opt': results['x_opt'],
'f_opt': results['f_opt'],
'mse': results['incumbent_values']
}
else:
pass
def bayesian_optimization(self):
""" Implementation of the Bayesian Optimization. """
# Set the parameters for the bayesian optimization
function = ho.objective_function
lower = np.array([
self._learning_rate[0], self._batch_size[0],
self._number_filters[0], self._number_filters[0],
self._number_filters[0]
])
upper = np.array([
self._learning_rate[1], self._batch_size[1],
self._number_filters[1], self._number_filters[1],
self._number_filters[1]
])
for i in range(RUNS):
results = bayesian_optimization(function,
lower,
upper,
num_iterations=ITERATIONS)
self.bo_performance.append(results['incumbent_values'])
self.bo_runtime.append(results['runtime'])
# Mean over all the runs
self.bo_performance = np.mean(self.bo_performance, axis=0)
# Cumulative sum of the mean for all the runs
self.bo_runtime = np.cumsum(np.mean(self.bo_runtime, axis=0), axis=0)
def test_bayesian_optimization_gp_mcmc(self):
res = bayesian_optimization(objective_function=objective,
lower=self.lower,
upper=self.upper,
n_init=2,
num_iterations=3)
assert len(res["x_opt"]) == 1
assert np.array(res["x_opt"]) >= 0
assert np.array(res["x_opt"]) <= 1
def test_bayesian_optimization(self):
res = bayesian_optimization(objective_function=objective,
lower=self.lower,
upper=self.upper,
n_init=2,
num_iterations=3)
assert len(res["x_opt"]) == 1
assert np.array(res["x_opt"]) >= 0
assert np.array(res["x_opt"]) <= 1
def test_rf(self):
res = bayesian_optimization(objective_function=objective,
lower=self.lower,
upper=self.upper,
n_init=2,
num_iterations=3,
model_type="rf")
assert len(res["x_opt"]) == 1
assert np.array(res["x_opt"]) >= 0
assert np.array(res["x_opt"]) <= 1
def robodrive(objective, hyperparameters, results_path, n_iterations=50):
"""
Distributed Bayesian optimization with Robo.
Parameters:
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `n_iterations` [int, default=50]
Number of optimization iterations
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Setup savefile
if rank < 10:
# Ensure results are sorted by rank
filename = 'hyperspace' + str(0) + str(rank)
else:
filename = 'hyperspace' + str(rank)
if not os.path.exists(results_path):
os.makedirs(results_path, exist_ok=True)
savefile = os.path.join(results_path, filename)
if rank == 0:
robospace = create_robospace(hyperparameters)
hyperspace = convert_robospace(robospace)
else:
hyperspace = None
space = comm.scatter(hyperspace, root=0)
lower = space[0]
upper = space[1]
results = bayesian_optimization(objective,
lower,
upper,
num_iterations=n_iterations)
with open(savefile, 'wb') as handle:
pickle.dump(results, handle, protocol=pickle.HIGHEST_PROTOCOL)
def get_weights(self, feature, X, Y, val_X, val_Y):
def objective_function(weights):
selected_idx = self._select(feature, weights, Y)
select_X, select_Y = X[selected_idx], Y[selected_idx]
self.model.train(select_X, select_Y)
error = self.model.loss(val_X, val_Y)
return error
return bayesian_optimization(
objective_function=objective_function,
lower=self.lower,
upper=self.upper,
num_iterations=self.num_iterations)['x_opt']
def optimizate_bayesian():
lower = np.array([-6, 32, 4, 4, 4])
upper = np.array([0, 512, 10, 10, 10])
list_y_performance = np.zeros((10, 50))
list_y_runtime = np.zeros((10, 50))
for i in range(10):
result = bayesian_optimization(objective_function,
lower,
upper,
num_iterations=50)
list_y_performance[i] = result.get('incumbent_values')
list_y_runtime[i] = result.get('runtime')
mean_performance = np.mean(list_y_performance, axis=0)
mean_runtime = np.cumsum(np.mean(list_y_runtime, axis=0))
return mean_performance, mean_runtime
def run_bayesian_optimization(args, kernel, objective_function, embedding_lst, eval_lst, BEST, WORST):
origin_rdl = embedding_lst[:]
origin_el = eval_lst[:]
lower = np.array([np.amin(embedding_lst)]*len(embedding_lst[0]))
upper = np.array([np.amax(embedding_lst)]*len(embedding_lst[0]))
results = []
best_ind = []
fix_budget = []
close_best = []
for i in range(args.num_run):
embedding_lst = origin_rdl[:]
eval_lst = origin_el[:]
logging.info("# %d run of bayesian optimization.", i)
# get the initialization
if i < len(embedding_lst)-2:
start = i
end = i+3
else:
start = i%(len(embedding_lst)-2)
end = i%(len(embedding_lst)-2)+3
x_init = np.array(embedding_lst[start:end])
y_init = np.array(eval_lst[start:end])
result = bayesian_optimization(objective_function, lower,upper, acquisition_func=args.acquisition_func,
model_type=args.model_type, num_iterations=len(origin_rdl),
X_init=x_init, Y_init=y_init, kernel=kernel,
sampling_method=args.sampling_method, replacement=args.replacement,
pool=np.array(embedding_lst), best=BEST)
results.append(result)
invs = result["incumbent_values"]
if BEST in invs:
best_ind.append(invs.index(BEST)+1)
else:
best_ind.append(WORST)
fix_budget.append(abs(min(invs[:args.budget])-BEST))
for i in range(len(invs)):
if invs[i]-BEST <= args.dif:
close_best.append(i+1)
break
return results, best_ind, fix_budget, close_best
def bayesian_opt():
bayesian_incumbent_list = []
bayesian_incumbent_mean_list = []
bayesian_runtime = []
for i in range(0, 10):
output_dict = bayesian_optimization(objective_function,
lower,
upper,
num_iterations=50)
bayesian_incumbent_mean_list.append(
np.mean(output_dict['incumbent_values']))
bayesian_incumbent_list.append(output_dict['incumbent_values'])
bayesian_runtime.append(output_dict['runtime'])
return np.mean(bayesian_incumbent_list,
axis=0), np.cumsum(np.mean(bayesian_runtime, axis=0),
axis=0), bayesian_incumbent_mean_list
def get_res(self):
lower = self.space.bound[0]
upper = self.space.bound[1]
res = bayesian_optimization(
self.fn,
lower,
upper,
maximizer=self.params.get('maximizer', 'scipy'),
acquisition_func=self.params.get('acquisition_func', 'log_ei'),
model_type=self.params.get('model_type', 'bohamiann'),
num_iterations=self.params['niter'],
output_path=str(self.output_path),
rng=self.rng,
)
return res
def bayesian_opt():
lower = np.array([-6, 32, 4, 4, 4])
upper = np.array([0, 512, 10, 10, 10])
f = objective_function
incumbents = np.zeros((10, 50))
incumbents_runtime = np.zeros((10, 50))
for i in range(10):
result = bayesian_optimization(f, lower, upper, num_iterations=50)
incumbents_runtime[i] = result["runtime"]
incumbents[i] = result["incumbent_values"]
inc = np.mean(incumbents, axis=0)
runtime = np.mean(incumbents_runtime, axis=0)
return inc, runtime
def optimization(self):
for optimization_method in self.optimization_methods:
t_start = time.time()
if (optimization_method == 'bayesian_optimization'):
results = bayesian_optimization(
self.objective_function,
lower=self.lower_bounds,
upper=self.upper_bounds,
num_iterations=self.num_iterations,
acquisition_func='ei')
t_end = time.time()
t = round(t_end - t_start)
x_opt = {}
for i in range(len(self.tune_params)):
x_opt[self.tune_params[i]] = results['x_opt'][i]
print(x_opt, results['f_opt'])
self.results['bayesian_optimization'] = {
'method': 'bayesian_optimization',
'x_opt': x_opt,
'f_opt': results['f_opt'],
'mse': results['incumbent_values'],
'time_consume': t
}
elif (optimization_method == 'random_search'):
results = random_search(self.objective_function,
lower=self.lower_bounds,
upper=self.upper_bounds,
num_iterations=self.num_iterations)
t_end = time.time()
t = round(t_end - t_start)
x_opt = {}
print(results['x_opt'])
for i in range(len(self.tune_params)):
x_opt[self.tune_params[i]] = results['x_opt'][i]
print(x_opt, results['f_opt'])
self.results['random_search'] = {
'method': 'random_search',
'x_opt': x_opt,
'f_opt': results['f_opt'],
'mse': results['incumbent_values'],
'time_consume': t
}
else:
pass
def gp_plain(self, seed, num_iterations=50):
"""Bayesian optimization with Gaussian Process(mcmc) as model and no contextual information
Returns np.array(1,50) mean of incumbents
"""
incumbent_value = np.ones((self.T, num_iterations))
for i in range(self.T):
rng1 = np.random.RandomState(seed)
if self.add_coef: self.f = Quadratic_function(self.coef[i])
result = bayesian_optimization(self.f.call,
self.lower_x,
self.upper_x,
num_iterations=num_iterations,
model_type="gp",
acquisition_func="ei",
n_init=1,
rng=rng1)
#Branin needs init=1
incumbent_value[i] = result["incumbent_values"]
return np.mean(incumbent_value, axis=0)
def train_weights(self,
feature,
X,
Y,
val_X,
val_Y,
select_num=1600,
isbalance=True):
def objective_function(weights):
selected_idx = self.select_data(feature, weights, select_num, Y,
isbalance)
select_X, select_Y = X[selected_idx], Y[selected_idx]
self.model.train(select_X, select_Y)
error = self.model.loss(val_X, val_Y)
return error
return bayesian_optimization(
objective_function=objective_function,
lower=self.lower,
upper=self.upper,
num_iterations=self.num_iterations)['x_opt']
def bayes_opt():
"""
Bayesian Optimization with RoBO.
Returns dictionary with results:
"x_opt" : the best found data point
"f_opt" : the corresponding function value
"incumbents": the incumbent (best found value) after each iteration
"incumbent_value": the function values of the incumbents
"runtime": the runtime in seconds after each iteration
"overhead": the optimization overhead
(i.e. time data we do not spend to evaluate the function)
of each iteration
"X": all data points that have been evaluated
"y": the corresponding function evaluations
"""
print('\n============= START Bayesian OPTIMIZATION =============\n')
print("""Optimization parameters:
- lower = {}
- upper = {}
- num_iter = {}
- maximizer = {}
- acq_func = {}
- model_type = {}
- n_init = {} """.format(lower, upper, args.num_iterations,
args.maximizer,
args.acquisition_func,
args.model_type, args.n_init))
results = bayesian_optimization(objective_function,
lower,
upper,
num_iterations=args.num_iterations,
maximizer=args.maximizer,
acquisition_func=args.acquisition_func,
model_type=args.model_type,
n_init=args.n_init)
print(results["x_opt"])
print(results["f_opt"])
print('\n============= END OPTIMIZATION =============\n')
output_path = "./experiments/fabolas/results/cnn_%s/ei_%d" % (dataset,
run_id)
elif dataset == "res_net":
f = ResidualNeuralNetworkOnCIFAR10(rng=rng)
num_iterations = 10
output_path = "./experiments/fabolas/results/res_%s/ei_%d" % (dataset,
run_id)
os.makedirs(output_path, exist_ok=True)
info = f.get_meta_information()
bounds = np.array(info['bounds'])
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=num_iterations,
n_init=2,
rng=rng,
output_path=output_path)
results["run_id"] = run_id
results['X'] = results['X'].tolist()
results['y'] = results['y'].tolist()
test_error = []
current_inc = None
current_inc_val = None
key = "incumbents"
for inc in results["incumbents"]:
if __name__ == '__main__':
r_all_errors = []
r_all_time = []
b_all_errors = []
b_all_time = []
for i in range(10):
print(i)
sys.stdout.flush()
plot_error, plot_time = random_search(objective_function,
runtime,
iterations=50)
r_all_errors.append(plot_error)
r_all_time.append(plot_time)
b_results = bayesian_optimization(objective_function,
lower,
upper,
num_iterations=50)
b_all_errors.append(b_results['incumbent_values'])
cur_time = 0.0
b_plot_time = []
for x in b_results['X']:
cur_time += runtime(x)
b_plot_time.append(cur_time)
b_all_time.append(b_plot_time)
for e in np.mean(r_all_errors, axis=0):
print('{:.6f}'.format(e))
print('=' * 80)
for t in np.mean(r_all_time, axis=0):
print('{:.2f}'.format(t))
print('=' * 80)
for e in np.mean(b_all_errors, axis=0):
from robo.fmin import bayesian_optimization
import logging
logging.basicConfig(level=logging.INFO)
def fn(x):
y = np.sin(3 * x[0]) * 4 * (x[0] - 1) * (x[0] + 2)
return y
lower = np.array([-2])
upper = np.array([3])
bo = bayesian_optimization(fn, lower, upper, num_iterations=10)
for it in bo:
print(it)
bo.incumbents
bo.incumbents_values
plt.plot(range(len(bo.incumbents_values)), bo.incumbents_values)
fn([150])
######################
# Improved interface #
f = SurrogateFCNet(path="/ihome/kleinaa/devel/git/HPOlib/surrogates/")
elif benchmark == "paramnet":
dataset = sys.argv[4]
f = SurrogateParamNet(dataset, "/ihome/kleinaa/devel/git/HPOlib/surrogates/")
benchmark += "_" + dataset
info = f.get_meta_information()
bounds = np.array(info['bounds'])
if method == "entropy_search":
results = entropy_search(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters, n_init=n_init)
elif method == "gp_mcmc":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
n_init=n_init, model_type="gp_mcmc")
elif method == "gp":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
n_init=n_init, model_type="gp")
elif method == "rf":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
n_init=n_init, model_type="rf")
elif method == "random_search":
results = random_search(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters)
elif method == "bohamiann":
results = bohamiann(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
import numpy as np from hpolib.benchmarks.synthetic_functions import Branin from robo.fmin import bayesian_optimization f = Branin() info = f.get_meta_information() bounds = np.array(info['bounds']) # Start Bayesian optimization to optimize the objective function results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1], num_iterations=50) print(results["x_opt"]) print(results["f_opt"])
for i in range(len(x)):
params[hps[i]] = int(x[i])
# n_train = len(X_train)
n_budget = s
idx_train = np.random.randint(low=0, high=n_budget, size=n_budget)
x_train_fab = X_train.iloc[idx_train]
y_train_fab = y_train.iloc[idx_train]
start_time = time.time()
val_loss = train_evaluate_rf(x_train_fab, y_train_fab, X_val, y_val,
params)
cost = time.time() - start_time
print('loss: % f, Cost: % f ' % (val_loss, cost))
return val_loss, cost
# results = bayesian_optimization(objective_func_bo, lower=lower, upper=upper, num_iterations=5)
# results = fabolas(objective_func_fabolas, lower=lower, upper=upper, s_min=100, s_max=50000, num_iterations=120)
results = bayesian_optimization(objective_func_bo,
lower,
upper,
model_type='bohamiann',
num_iterations=20)
bla = 0
def optimize(self) -> TuningResult:
"""
Method performs a hyperparameter optimization run according to the selected HPO-method.
:return: result: TuningResult
TuningResult-object that contains the results of this optimization run.
:return:
"""
# Convert the skopt hyperparameter space into a continuous space for RoBO
hp_space_lower = np.zeros(shape=(len(self.hp_space), ))
hp_space_upper = np.zeros(shape=(len(self.hp_space), ))
for i in range(len(self.hp_space)):
if type(self.hp_space[i]) == skopt.space.space.Integer:
hp_space_lower[i, ] = self.hp_space[i].low
hp_space_upper[i, ] = self.hp_space[i].high
elif type(self.hp_space[i]) == skopt.space.space.Categorical:
n_choices = len(list(self.hp_space[i].categories))
hp_space_lower[i, ] = 0
hp_space_upper[i, ] = n_choices - 1
elif type(self.hp_space[i]) == skopt.space.space.Real:
hp_space_lower[i, ] = self.hp_space[i].low
hp_space_upper[i, ] = self.hp_space[i].high
else:
raise Exception(
'The skopt HP-space could not be converted correctly!')
# Set the random seed of the random number generator
rand_num_generator = np.random.RandomState(seed=self.random_seed)
# Optimize on the predefined n_func_evals and measure the wall clock times
start_time = time.time()
self.times = [] # Initialize a list for saving the wall clock times
# Use a warmstart configuration (only possible for BOHAMIANN, not FABOLAS)
if self.do_warmstart == 'Yes':
# Initialize numpy arrays for saving the warmstart configuration and the warmstart loss
warmstart_config = np.zeros(shape=(1, len(self.hp_space)))
warmstart_loss = np.zeros(shape=(1, 1))
# Retrieve the default hyperparameters and the default loss for the ML-algorithm
default_params = self.get_warmstart_configuration()
try:
# Dictionary for saving the warmstart HP-configuration (only contains the HPs, which are part of the
# 'tuned' HP-space
warmstart_dict = {}
# Iterate over all HPs of this ML-algorithm's tuned HP-space and append the default values to
# the numpy array
for i in range(len(self.hp_space)):
this_param = self.hp_space[i].name
# Categorical HPs need to be encoded as integer values for RoBO
if type(self.hp_space[i]) == skopt.space.space.Categorical:
choices = self.hp_space[i].categories
this_warmstart_value_cat = default_params[this_param]
dict_value = this_warmstart_value_cat
# Find the index of the default / warmstart HP in the list of possible choices
for j in range(len(choices)):
if this_warmstart_value_cat == choices[j]:
this_warmstart_value = j
# For all non-categorical HPs
else:
this_warmstart_value = default_params[this_param]
dict_value = this_warmstart_value
# For some HPs (e.g. max_depth of RF) the default value is None, although their typical dtype is
# different (e.g. int)
if this_warmstart_value is None:
# Try to impute these values by the mean value
this_warmstart_value = int(
0.5 *
(self.hp_space[i].low + self.hp_space[i].high))
dict_value = this_warmstart_value
# Pass the warmstart value to the according numpy array
warmstart_config[0, i] = this_warmstart_value
warmstart_dict[this_param] = dict_value
# Pass the default loss to the according numpy array
warmstart_loss[0, 0] = self.get_warmstart_loss(
warmstart_dict=warmstart_dict)
# Pass the warmstart configuration as a kwargs dict
kwargs = {'X_init': warmstart_config, 'Y_init': warmstart_loss}
# Set flag to indicate that a warmstart took place
did_warmstart = True
except:
print('Warmstarting RoBO failed!')
kwargs = {}
# Set flag to indicate that NO warmstart took place
did_warmstart = False
# No warmstart requested
else:
kwargs = {}
# Set flag to indicate that NO warmstart took place
did_warmstart = False
# Select the specified HPO-tuning method
try:
if self.hpo_method == 'Fabolas':
# Budget correct? // Set further parameters?
s_max = len(
self.x_train
) # Maximum number of data points for the training data set
s_min = int(
0.05 * s_max
) # Maximum number of data points for the training data set
n_init = int(self.n_func_evals /
3) # Requirement of the fabolas implementation
result_dict = fabolas(
objective_function=self.objective_fabolas,
s_min=s_min,
s_max=s_max,
lower=hp_space_lower,
upper=hp_space_upper,
num_iterations=self.n_func_evals,
rng=rand_num_generator,
n_init=n_init)
run_successful = True
elif self.hpo_method == 'Bohamiann':
if did_warmstart:
# A single initial design point (warm start hyperparameter configuration)
kwargs['n_init'] = 1
# Budget correct? // Set further parameters?
result_dict = bayesian_optimization(
objective_function=self.objective_bohamiann,
lower=hp_space_lower,
upper=hp_space_upper,
model_type='bohamiann',
num_iterations=self.n_func_evals,
rng=rand_num_generator,
**kwargs)
run_successful = True
else:
raise Exception('Unknown HPO-method!')
# Algorithm crashed
except:
# Add a warning here
run_successful = False
# If the optimization run was successful, determine the optimization results
if run_successful:
for i in range(len(self.times)):
# Subtract the start time to receive the wall clock time of each function evaluation
self.times[i] = self.times[i] - start_time
wall_clock_time = max(self.times)
# Insert timestamp of 0.0 for the warm start hyperparameter configuration
if did_warmstart:
self.times.insert(0, 0.0)
# Timestamps
timestamps = self.times
# Losses (not incumbent losses)
losses = result_dict['y']
evaluation_ids = list(range(1, len(losses) + 1))
best_loss = min(losses)
configurations = ()
for config in result_dict['X']:
# Cut off the unused Fabolas budget value at the end
config = config[:len(self.hp_space)]
config_dict = {}
for i in range(len(config)):
if type(self.hp_space[i]) == skopt.space.space.Integer:
config_dict[self.hp_space[i].name] = int(
round(config[i]))
elif type(
self.hp_space[i]) == skopt.space.space.Categorical:
config_dict[self.hp_space[i].name] = list(
self.hp_space[i].categories)[int(round(config[i]))]
elif type(self.hp_space[i]) == skopt.space.space.Real:
config_dict[self.hp_space[i].name] = config[i]
else:
raise Exception(
'The continuous HP-space could not be converted correctly!'
)
configurations = configurations + (config_dict, )
# Find the best hyperparameter configuration (incumbent)
best_configuration = {}
x_opt = result_dict['x_opt']
for i in range(len(x_opt)):
if type(self.hp_space[i]) == skopt.space.space.Integer:
best_configuration[self.hp_space[i].name] = int(
round(x_opt[i]))
elif type(self.hp_space[i]) == skopt.space.space.Categorical:
best_configuration[self.hp_space[i].name] = list(
self.hp_space[i].categories)[int(round(x_opt[i]))]
elif type(self.hp_space[i]) == skopt.space.space.Real:
best_configuration[self.hp_space[i].name] = x_opt[i]
else:
raise Exception(
'The continuous HP-space could not be converted correctly!'
)
# Run not successful (algorithm crashed)
else:
evaluation_ids, timestamps, losses, configurations, best_loss, best_configuration, wall_clock_time = \
self.impute_results_for_crash()
# Pass the results to a TuningResult-Object
result = TuningResult(evaluation_ids=evaluation_ids,
timestamps=timestamps,
losses=losses,
configurations=configurations,
best_loss=best_loss,
best_configuration=best_configuration,
wall_clock_time=wall_clock_time,
successful=run_successful,
did_warmstart=did_warmstart)
return result
def get_robo_results(count):
lower = numpy.array([-5, 0])
upper = numpy.array([10, 15])
results = bayesian_optimization(branin_for_original_robo, lower, upper, num_iterations=count)
return results
benchmark += "_" + dataset
info = f.get_meta_information()
bounds = np.array(info['bounds'])
if method == "entropy_search":
results = entropy_search(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
n_init=n_init)
elif method == "gp_mcmc":
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
n_init=n_init,
model_type="gp_mcmc")
elif method == "gp":
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
n_init=n_init,
model_type="gp")
elif method == "rf":
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
n_init=n_init,
import numpy as np
from robo.fmin import bayesian_optimization
import logging
logging.basicConfig(level=logging.INFO)
# The optimization function that we want to optimize.
# It gets a numpy array with shape (1,D) where D is the number of input dimensions
def objective_function(x):
y = np.sin(3 * x[0]) * 4 * (x[0] - 1) * (x[0] + 2)
return y
# Defining the bounds and dimensions of the input space
lower = np.array([0])
upper = np.array([6])
# Start Bayesian optimization to optimize the objective function
results = bayesian_optimization(objective_function, lower, upper, num_iterations=50)
print(results["x_opt"])
print(results["f_opt"])
info = f.get_meta_information()
bounds = np.array(info['bounds'])
if method == "entropy_search":
results = entropy_search(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
n_init=n_init)
elif method == "gp_mcmc":
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
acquisition_func=acquisition,
maximizer=maximizer,
n_init=n_init,
model_type="gp_mcmc",
rng=rng)
elif method == "gp":
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=n_iters,
acquisition_func=acquisition,
maximizer=maximizer,
n_init=n_init,
model_type="gp")
elif method == "rf":
results = bayesian_optimization(f,
f = hpobench.SinTwo()
elif benchmark == "bohachevsky":
f = hpobench.Bohachevsky()
elif benchmark == "levy":
f = hpobench.Levy()
info = f.get_meta_information()
bounds = np.array(info['bounds'])
if method == "entropy_search":
results = entropy_search(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters, n_init=n_init)
elif method == "gp_mcmc":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
acquisition_func=acquisition,
maximizer=maximizer,
n_init=n_init, model_type="gp_mcmc", rng=rng)
elif method == "gp":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
acquisition_func=acquisition,
maximizer=maximizer,
n_init=n_init, model_type="gp")
elif method == "rf":
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters,
n_init=n_init, model_type="rf")
elif method == "random_search":
results = random_search(f, bounds[:, 0], bounds[:, 1],
num_iterations=n_iters)
import numpy as np
from robo.fmin import bayesian_optimization
import logging
logging.basicConfig(level=logging.INFO)
def fn(x):
y = np.sin(3 * x[0]) * 4 * (x[0] - 1) * (x[0] + 2)
return y
lower = np.array([-2])
upper = np.array([3])
bo = bayesian_optimization(fn, lower, upper, num_iterations=10)
for it in bo:
print(it)
bo.incumbents
bo.incumbents_values
plt.plot(range(len(bo.incumbents_values)), bo.incumbents_values)
fn([150])
######################
# Improved interface #
######################
:param x_sample: domain sampled from bayesian optimization
:return: the corresponding evaluation result
'''
for i in range(x.shape[0]):
if (sample == x[i]).all():
return y[i]
result = np.zeros((len(y)-3, len(y)))
for i in range(len(y)-3):
print("step {0}/{1}".format(i+1, len(y)-3))
label_ids = np.array([i, i+1, i+2])
pool = np.array([x[i] for i in range(x.shape[0]) if i not in label_ids])
bo_res = bayesian_optimization(objective_function, lower, upper, acquisition_func=args.acquisition_func,
model_type=args.model_type, num_iterations=len(y),
X_init=x[label_ids], Y_init=y[label_ids], kernel=kernel,
sampling_method=args.sampling_method, replacement=False,
pool=pool, best=100000)
x_samples = np.array(bo_res['X'])
print(x_samples.shape[0])
sampling_order = []
for j in range(x_samples.shape[0]):
for k in range(x.shape[0]):
if (x_samples[j] == x[k]).all():
sampling_order.append(k)
print(sampling_order)
result[i] = np.array(sampling_order)
model_name = "bo_" + args.acquisition_func + "_" + args.kernel + "_" + args.model_type
output_file = args.output + "/" + args.architecture + "/" + args.dataset + "/" + \
y_test,
parser_output_path=parser_output_path,
perl_script_path=perl_script_path)
run_dict[baseline].append((val_accuracy, test_accuracy))
# define the lower and upper bounds of the input space [-1, 1]
lower = np.array(len(feature_names) * [-1])
upper = np.array(len(feature_names) * [1])
print('Lower limits shape:', lower.shape)
print('Upper limits shape:', upper.shape)
print('Running Bayesian Optimization...')
print("Bayesian Opt number of iteration : {}".format(
args.num_iterations))
res = bayesian_optimization(objective_function,
lower=lower,
upper=upper,
num_iterations=args.num_iterations)
best_feature_weights = res['x_opt']
print('Best feature weights', best_feature_weights)
print("Selecting data based on best feature weights :")
train_subset, labels_subset = task_utils.get_data_subsets(
feature_values, best_feature_weights, X_train, y_train,
args.task, num_train_examples)
val_accuracy, test_accuracy = train_and_evaluate(train_subset,
labels_subset,
X_val,
y_val,
X_test,
y_test,
"""
This example shows how RoBO can be combined with HPOlib.
Before you run it, make sure that you installed it.
For further information have a look here https://github.com/automl/HPOlib2.git
"""
import numpy as np
from hpolib.benchmarks.synthetic_functions import Branin
from robo.fmin import bayesian_optimization
f = Branin()
info = f.get_meta_information()
bounds = np.array(info['bounds'])
# Start Bayesian optimization to optimize the objective function
results = bayesian_optimization(f,
bounds[:, 0],
bounds[:, 1],
num_iterations=50)
print(results["x_opt"])
print(results["f_opt"])
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
from robo.fmin import bayesian_optimization
# Global Configuration
try:
CFG = sys.argv[1]
CFG = CFG.replace(".py", "")
SEED = int(sys.argv[2])
except:
logger.info("USAGE: python run.py CONFIGURATION_FILE_PATH SEED_VALUE")
sys.exit(-1)
PREFIX = CFG+("_%i" % SEED)
CFG = CFG+".py"
OUTFILE = "%s_bo_posterior" % PREFIX
logger.info("OPENING CODE FROM CFG=%s (SEED=%i) => OUTPUT:%s" % (CFG, SEED, OUTFILE))
exec(open(CFG).read())
# BlackBox BO over posterior
rng = np.random.RandomState(SEED)
res = bayesian_optimization(objective_function, lower, upper, num_iterations=num_iterations, X_init=X_init, Y_init=Y_init, n_init=n_init, rng=rng,)
output_path = "./experiments/fabolas/results/cnn_%s/ei_%d" % (dataset, run_id)
elif dataset == "svhn":
f = ConvolutionalNeuralNetworkOnSVHN(rng=rng)
num_iterations = 15
output_path = "./experiments/fabolas/results/cnn_%s/ei_%d" % (dataset, run_id)
elif dataset == "res_net":
f = ResidualNeuralNetworkOnCIFAR10(rng=rng)
num_iterations = 10
output_path = "./experiments/fabolas/results/%s/ei_%d" % (dataset, run_id)
os.makedirs(output_path, exist_ok=True)
info = f.get_meta_information()
bounds = np.array(info['bounds'])
results = bayesian_optimization(f, bounds[:, 0], bounds[:, 1],
num_iterations=num_iterations, n_init=2,
rng=rng, output_path=output_path)
results["run_id"] = run_id
results['X'] = results['X'].tolist()
results['y'] = results['y'].tolist()
test_error = []
current_inc = None
current_inc_val = None
key = "incumbents"
for inc in results["incumbents"]:
print(inc)
if current_inc == inc:
