def main(penalty, aleatoric_weight, random_trials, bayes_opt_iters, grid_size,
kernel_type, opt_func):
"""
Optimise the heteroscedastic Branin-Hoo function.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of $\beta of ANPEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
param: grid_size: int specifying the side length of the 2D grid to initialise on.
param: kernel_type: str specifying the type of kernel. One of ['matern_12', 'matern_32', 'matern_52', 'rbf']
param: opt_func: str specifying the optimisation function. One of ['hosaki', 'branin', 'goldstein']
"""
heteroscedastic = True
noise_level = 0
n_restarts = 20
# We perform random trials of Bayesian Optimisation
rand_running_sum = np.zeros(bayes_opt_iters)
rand_squares = np.zeros(bayes_opt_iters)
homo_running_sum = np.zeros(bayes_opt_iters)
homo_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_running_sum = np.zeros(bayes_opt_iters)
aug_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
rand_noise_running_sum = np.zeros(bayes_opt_iters)
rand_noise_squares = np.zeros(bayes_opt_iters)
homo_noise_running_sum = np.zeros(bayes_opt_iters)
homo_noise_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_noise_running_sum = np.zeros(bayes_opt_iters)
aug_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
for i in range(random_trials):
numpy_seed = i
np.random.seed(numpy_seed)
if opt_func == 'hosaki':
bounds = np.array([[0.0, 5.0], [
0.0, 5.0
]]) # bounds of the Bayesian Optimisation problem for Hosaki
else:
bounds = np.array([[0.0, 1.0],
[0.0, 1.0]]) # bounds for other test functions
# Initial noisy data points sampled uniformly at random from the input space.
if opt_func == 'hosaki':
X_init = np.random.uniform(0.0, 5.0, size=(grid_size**2, 2))
Y_init = hosaki_function(X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
x1_star = np.arange(0.0, 5.0, 0.5)
x2_star = np.arange(0.0, 5.0, 0.5)
else:
X_init = np.random.uniform(0.0, 1.0, size=(grid_size**2, 2))
x1_star = np.arange(0.0, 1.0, 0.1)
x2_star = np.arange(0.0, 1.0, 0.1)
if opt_func == 'branin':
Y_init = branin_function(X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
else:
Y_init = goldstein_price_function(
X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
plot_sample = np.array(np.meshgrid(x1_star, x2_star)).T.reshape(
-1, 2) # Where 2 gives the dimensionality
# Initialize samples
if heteroscedastic: # if heteroscedastic extract only the noisy evaluation of the objective function
Y_init = Y_init[0]
homo_X_sample = X_init
homo_Y_sample = Y_init
het_X_sample = X_init
het_Y_sample = Y_init
aug_X_sample = X_init
aug_Y_sample = Y_init
aug_het_X_sample = X_init
aug_het_Y_sample = Y_init
# initial GP hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0 # optimise noise for homoscedastic GP
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
rand_best_so_far = -300
homo_best_so_far = -300 # value to beat
het_best_so_far = -300
aug_best_so_far = -300
aug_het_best_so_far = -300
rand_noise_best_so_far = 300 # value to beat
homo_noise_best_so_far = 300
het_noise_best_so_far = 300
aug_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
rand_obj_val_list = []
homo_obj_val_list = []
het_obj_val_list = []
aug_obj_val_list = []
aug_het_obj_val_list = []
rand_noise_val_list = []
homo_noise_val_list = []
het_noise_val_list = []
aug_noise_val_list = []
aug_het_noise_val_list = []
rand_collected_x1 = []
rand_collected_x2 = []
homo_collected_x1 = []
homo_collected_x2 = []
het_collected_x1 = []
het_collected_x2 = []
aug_collected_x1 = []
aug_collected_x2 = []
aug_het_collected_x1 = []
aug_het_collected_x2 = []
for j in range(bayes_opt_iters):
print(j)
# random sampling baseline
seed = bayes_opt_iters * i + j # This approach
print(f'Seed is: {seed}')
np.random.seed(seed)
if opt_func == 'hosaki':
random_x1_next = np.random.uniform(0.0, 5.0, size=(1, ))
random_x2_next = np.random.uniform(0.0, 5.0, size=(1, ))
else:
random_x1_next = np.random.uniform(0.0, 1.0, size=(1, ))
random_x2_next = np.random.uniform(0.0, 1.0, size=(1, ))
random_X_next = np.array(
np.meshgrid(random_x1_next, random_x2_next)).T.reshape(-1, 2)
rand_collected_x1.append(random_x1_next)
rand_collected_x2.append(random_x2_next)
f_plot = True
if j > 0:
f_plot = False
if opt_func == 'hosaki':
random_Y_next = hosaki_function(
random_x1_next,
random_x2_next,
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
random_Y_next = random_Y_next[0]
_, rand_noise_val, random_composite_obj_val = hosaki_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
random_composite_obj_val -= penalty * rand_noise_val
else:
random_composite_obj_val = hosaki_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
random_Y_next = branin_function(
random_x1_next,
random_x2_next,
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
random_Y_next = random_Y_next[0]
_, rand_noise_val, random_composite_obj_val = branin_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
random_composite_obj_val -= penalty * rand_noise_val
else:
random_composite_obj_val = branin_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
else:
random_Y_next = goldstein_price_function(
random_x1_next,
random_x2_next,
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
random_Y_next = random_Y_next[0]
_, rand_noise_val, random_composite_obj_val = goldstein_price_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
random_composite_obj_val -= penalty * rand_noise_val
else:
random_composite_obj_val = goldstein_price_function(
random_x1_next,
random_x2_next,
noise=0.0,
heteroscedastic=heteroscedastic)
f_plot = False
if random_composite_obj_val > rand_best_so_far:
rand_best_so_far = random_composite_obj_val
rand_obj_val_list.append(random_composite_obj_val)
else:
rand_obj_val_list.append(rand_best_so_far)
if heteroscedastic:
if rand_noise_val < rand_noise_best_so_far:
rand_noise_best_so_far = rand_noise_val
rand_noise_val_list.append(rand_noise_val)
else:
rand_noise_val_list.append(rand_noise_best_so_far)
# Obtain next sampling point from the acquisition function (expected_improvement)
homo_X_next = my_propose_location(my_expected_improvement,
homo_X_sample,
homo_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300)
homo_collected_x1.append(homo_X_next[:, 0])
homo_collected_x2.append(homo_X_next[:, 1])
print(homo_X_next)
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
homo_Y_next = hosaki_function(homo_X_next[:, 0],
homo_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
homo_Y_next = homo_Y_next[0]
_, homo_noise_val, homo_composite_obj_val = hosaki_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
homo_composite_obj_val -= penalty * homo_noise_val
else:
homo_composite_obj_val = hosaki_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
homo_Y_next = branin_function(homo_X_next[:, 0],
homo_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
homo_Y_next = homo_Y_next[0]
_, homo_noise_val, homo_composite_obj_val = branin_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
homo_composite_obj_val -= penalty * homo_noise_val
else:
homo_composite_obj_val = branin_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
homo_Y_next = goldstein_price_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
homo_Y_next = homo_Y_next[0]
_, homo_noise_val, homo_composite_obj_val = goldstein_price_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
homo_composite_obj_val -= penalty * homo_noise_val
else:
homo_composite_obj_val = goldstein_price_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if homo_composite_obj_val > homo_best_so_far:
homo_best_so_far = homo_composite_obj_val
homo_obj_val_list.append(homo_composite_obj_val)
else:
homo_obj_val_list.append(homo_best_so_far)
if heteroscedastic:
if homo_noise_val < homo_noise_best_so_far:
homo_noise_best_so_far = homo_noise_val
homo_noise_val_list.append(homo_noise_val)
else:
homo_noise_val_list.append(homo_noise_best_so_far)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(
heteroscedastic_expected_improvement,
het_X_sample,
het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300,
aleatoric_weight=aleatoric_weight)
het_collected_x1.append(het_X_next[:, 0])
het_collected_x2.append(het_X_next[:, 1])
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
het_Y_next = hosaki_function(het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = hosaki_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = hosaki_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
het_Y_next = branin_function(het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = branin_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = branin_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
het_Y_next = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if het_composite_obj_val > het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if heteroscedastic:
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the augmented expected improvement (AEI)
aug_X_next = my_propose_location(augmented_expected_improvement,
aug_X_sample,
aug_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300,
aleatoric_weight=aleatoric_weight,
aei=True)
aug_collected_x1.append(aug_X_next[:, 0])
aug_collected_x2.append(aug_X_next[:, 1])
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
aug_Y_next = hosaki_function(aug_X_next[:, 0],
aug_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_Y_next = aug_Y_next[0]
_, aug_noise_val, aug_composite_obj_val = hosaki_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_composite_obj_val -= penalty * aug_noise_val
else:
aug_composite_obj_val = hosaki_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
aug_Y_next = branin_function(aug_X_next[:, 0],
aug_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_Y_next = aug_Y_next[0]
_, aug_noise_val, aug_composite_obj_val = branin_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_composite_obj_val -= penalty * aug_noise_val
else:
aug_composite_obj_val = branin_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
aug_Y_next = goldstein_price_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_Y_next = aug_Y_next[0]
_, aug_noise_val, aug_composite_obj_val = goldstein_price_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_composite_obj_val -= penalty * aug_noise_val
else:
aug_composite_obj_val = goldstein_price_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if aug_composite_obj_val > aug_best_so_far:
aug_best_so_far = aug_composite_obj_val
aug_obj_val_list.append(aug_composite_obj_val)
else:
aug_obj_val_list.append(aug_best_so_far)
if heteroscedastic:
if aug_noise_val < aug_noise_best_so_far:
aug_noise_best_so_far = aug_noise_val
aug_noise_val_list.append(aug_noise_val)
else:
aug_noise_val_list.append(aug_noise_best_so_far)
# Add sample to previous sample
aug_X_sample = np.vstack((aug_X_sample, aug_X_next))
aug_Y_sample = np.vstack((aug_Y_sample, aug_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(
heteroscedastic_augmented_expected_improvement,
aug_het_X_sample,
aug_het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300,
aleatoric_weight=aleatoric_weight)
aug_het_collected_x1.append(aug_het_X_next[:, 0])
aug_het_collected_x2.append(aug_het_X_next[:, 1])
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
aug_het_Y_next = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
aug_het_Y_next = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
aug_het_Y_next = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if aug_het_composite_obj_val > aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if heteroscedastic:
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
rand_running_sum += np.array(rand_obj_val_list,
dtype=np.float64).flatten()
rand_squares += np.array(rand_obj_val_list,
dtype=np.float64).flatten()**2
homo_running_sum += np.array(homo_obj_val_list,
dtype=np.float64).flatten()
homo_squares += np.array(homo_obj_val_list,
dtype=np.float64).flatten()**2
hetero_running_sum += np.array(het_obj_val_list,
dtype=np.float64).flatten()
hetero_squares += np.array(het_obj_val_list,
dtype=np.float64).flatten()**2
aug_running_sum += np.array(aug_obj_val_list,
dtype=np.float64).flatten()
aug_squares += np.array(aug_obj_val_list,
dtype=np.float64).flatten()**2
aug_het_running_sum += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()
aug_het_squares += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()**2
if heteroscedastic:
rand_noise_running_sum += np.array(
rand_noise_val_list, dtype=np.float64).flatten(
) # just the way to average out across all random trials
rand_noise_squares += np.array(
rand_noise_val_list,
dtype=np.float64).flatten()**2 # likewise for errors
homo_noise_running_sum += np.array(homo_noise_val_list,
dtype=np.float64).flatten()
homo_noise_squares += np.array(homo_noise_val_list,
dtype=np.float64).flatten()**2
hetero_noise_running_sum += np.array(het_noise_val_list,
dtype=np.float64).flatten()
hetero_noise_squares += np.array(het_noise_val_list,
dtype=np.float64).flatten()**2
aug_noise_running_sum += np.array(aug_noise_val_list,
dtype=np.float64).flatten()
aug_noise_squares += np.array(aug_noise_val_list,
dtype=np.float64).flatten()**2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()
aug_het_noise_squares += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()**2
# results are negated to turn problem into minimisation for consistency.
rand_means = -rand_running_sum / random_trials
rand_errs = (np.sqrt(rand_squares / random_trials - rand_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
homo_means = -homo_running_sum / random_trials
homo_errs = (np.sqrt(homo_squares / random_trials - homo_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
hetero_means = -hetero_running_sum / random_trials
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_means = -aug_running_sum / random_trials
aug_errs = (np.sqrt(aug_squares / random_trials - aug_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_het_means = -aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
if heteroscedastic:
rand_noise_means = rand_noise_running_sum / random_trials
homo_noise_means = homo_noise_running_sum / random_trials
hetero_noise_means = hetero_noise_running_sum / random_trials
rand_noise_errs = (
np.sqrt(rand_noise_squares / random_trials -
rand_noise_means**2)) / np.sqrt(random_trials)
homo_noise_errs = (
np.sqrt(homo_noise_squares / random_trials -
homo_noise_means**2)) / np.sqrt(random_trials)
hetero_noise_errs = (
np.sqrt(hetero_noise_squares / random_trials -
hetero_noise_means**2)) / np.sqrt(random_trials)
aug_noise_means = aug_noise_running_sum / random_trials
aug_noise_errs = (np.sqrt(aug_noise_squares / random_trials -
aug_noise_means**2)) / np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (
np.sqrt(aug_het_noise_squares / random_trials -
aug_het_noise_means**2)) / np.sqrt(random_trials)
print('List of average random values is: ' + str(rand_means))
print('List of random errors is: ' + str(rand_errs))
print('List of average homoscedastic values is: ' + str(homo_means))
print('List of homoscedastic errors is: ' + str(homo_errs))
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_errs))
print('List of average AEI values is: ' + str(aug_means))
print('List of AEI errors is: ' + str(aug_errs))
print('List of average het-AEI values is: ' + str(aug_het_means))
print('List of het-AEI errors is: ' + str(aug_het_errs))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_rand = np.array(rand_means) - np.array(rand_errs)
upper_rand = np.array(rand_means) + np.array(rand_errs)
lower_homo = np.array(homo_means) - np.array(homo_errs)
upper_homo = np.array(homo_means) + np.array(homo_errs)
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_aei = np.array(aug_means) - np.array(aug_errs)
upper_aei = np.array(aug_means) + np.array(aug_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, rand_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x,
lower_rand,
upper_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_homo,
upper_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_hetero,
upper_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x, lower_aei, upper_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x,
lower_het_aei,
upper_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
if heteroscedastic:
if penalty != 1:
plt.ylabel('f(x) + ' + str(penalty) + 'g(x)', fontsize=14)
else:
plt.ylabel('f(x) + g(x)', fontsize=14)
else:
plt.ylabel('f(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
if noise_level > 0:
tag = 'with_noise_' + str(noise_level)
else:
if heteroscedastic:
tag = 'heteroscedastic'
else:
tag = ''
plt.savefig(
'kernel_figures/{}/{}_kernel_type_{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}'
'_hundred_times_penalty_is_{}_aleatoric_weight_is_{}_{}'.format(
opt_func, kernel_type, bayes_opt_iters, random_trials, grid_size,
numpy_seed, int(100 * penalty), aleatoric_weight, tag),
bbox_inches='tight')
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
if heteroscedastic:
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_rand = np.array(rand_noise_means) - np.array(
rand_noise_errs)
upper_noise_rand = np.array(rand_noise_means) + np.array(
rand_noise_errs)
lower_noise_homo = np.array(homo_noise_means) - np.array(
homo_noise_errs)
upper_noise_homo = np.array(homo_noise_means) + np.array(
homo_noise_errs)
lower_noise_hetero = np.array(hetero_noise_means) - np.array(
hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(
hetero_noise_errs)
lower_noise_aei = np.array(aug_noise_means) - np.array(aug_noise_errs)
upper_noise_aei = np.array(aug_noise_means) + np.array(aug_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(
aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(
aug_het_noise_errs)
#plt.plot(iter_x, best_noise_plot, '--', color='k', label='Optimal')
plt.plot(iter_x, rand_noise_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_noise_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_noise_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_noise_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_noise_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x,
lower_noise_rand,
upper_noise_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_homo,
upper_noise_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_hetero,
upper_noise_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_aei,
upper_noise_aei,
color='tab:red',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_het_aei,
upper_noise_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
plt.savefig(
'kernel_figures/{}/{}_heteroscedastic_{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}_'
'noise_only_hundred_times_penalty_is_{}_aleatoric_weight_is_{}_new_aei'
.format(opt_func, kernel_type,
bayes_opt_iters, random_trials, grid_size, numpy_seed,
int(100 * penalty), aleatoric_weight),
bbox_inches='tight')
# Save data for cosmetic plotting
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/rand_means.txt',
rand_means)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/homo_means.txt',
homo_means)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/hetero_means.txt',
hetero_means)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/aug_means.txt',
aug_means)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/aug_het_means.txt',
aug_het_means)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/lower_rand.txt',
lower_rand)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/upper_rand.txt',
upper_rand)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/lower_homo.txt',
lower_homo)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/upper_homo.txt',
upper_homo)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/lower_hetero.txt',
lower_hetero)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/upper_hetero.txt',
upper_hetero)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/lower_aei.txt',
lower_aei)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/upper_aei.txt',
upper_aei)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/lower_het_aei.txt',
lower_het_aei)
np.savetxt(f'synth_saved_data/{kernel_type}/{opt_func}/upper_het_aei.txt',
upper_het_aei)
if heteroscedastic:
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/rand_noise_means.txt',
rand_noise_means)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/homo_noise_means.txt',
homo_noise_means)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/hetero_noise_means.txt',
hetero_noise_means)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/aug_noise_means.txt',
aug_noise_means)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/aug_het_noise_means.txt',
aug_het_noise_means)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/lower_noise_rand.txt',
lower_noise_rand)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/upper_noise_rand.txt',
upper_noise_rand)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/lower_noise_homo.txt',
lower_noise_homo)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/upper_noise_homo.txt',
upper_noise_homo)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/lower_noise_hetero.txt',
lower_noise_hetero)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/upper_noise_hetero.txt',
upper_noise_hetero)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/lower_noise_aei.txt',
lower_noise_aei)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/upper_noise_aei.txt',
upper_noise_aei)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/lower_noise_het_aei.txt',
lower_noise_het_aei)
np.savetxt(
f'synth_saved_data/{kernel_type}/{opt_func}/upper_noise_het_aei.txt',
upper_noise_het_aei)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(
heteroscedastic_one_off_expected_improvement,
het_X_sample,
het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_penalty)
het_collected_x.append(het_X_next)
# Obtain next noisy sample from the objective function
het_Y_next = linear_sin_noise(het_X_next,
noise_coeff,
plot_sample,
def main(penalty, aleatoric_weight, random_trials, bayes_opt_iters,
init_set_size):
"""
Script for running the soil phosphorus fraction optimisation experiment.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of $\beta of ANPEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
param: init_set_size: int specifying the side length of the 2D grid to initialise on.
"""
# We perform random trials of Bayesian Optimisation
rand_running_sum = np.zeros(bayes_opt_iters)
rand_squares = np.zeros(bayes_opt_iters)
homo_running_sum = np.zeros(bayes_opt_iters)
homo_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_running_sum = np.zeros(bayes_opt_iters)
aug_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
rand_noise_running_sum = np.zeros(bayes_opt_iters)
rand_noise_squares = np.zeros(bayes_opt_iters)
homo_noise_running_sum = np.zeros(bayes_opt_iters)
homo_noise_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_noise_running_sum = np.zeros(bayes_opt_iters)
aug_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
xs, ys, std = soil_bo(fplot_data=False)
# ys = np.log(ys)
# Y_scaler = StandardScaler().fit(ys)
# ys = Y_scaler.transform(ys)
for i in range(random_trials):
numpy_seed = i
np.random.seed(numpy_seed)
xs_train, xs_test, ys_train, ys_test = train_test_split(
xs,
ys,
test_size=init_set_size,
shuffle=True,
random_state=numpy_seed) # use (xs_test, ys_test) to initialise
init_num_samples = len(ys_test)
bounds = np.array([0, 2]).reshape(
-1, 1) # bounds of the Bayesian Optimisation problem.
plot_sample = np.linspace(0, 2, 50).reshape(
-1, 1) # samples for plotting purposes
X_init = xs_test
Y_init = ys_test
# Initialize samples
homo_X_sample = X_init.reshape(-1, 1)
homo_Y_sample = Y_init.reshape(-1, 1)
het_X_sample = X_init.reshape(-1, 1)
het_Y_sample = Y_init.reshape(-1, 1)
aug_X_sample = X_init.reshape(-1, 1)
aug_Y_sample = Y_init.reshape(-1, 1)
aug_het_X_sample = X_init.reshape(-1, 1)
aug_het_Y_sample = Y_init.reshape(-1, 1)
# initial GP hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
rand_best_so_far = 300
homo_best_so_far = 300 # value to beat
het_best_so_far = 300
aug_best_so_far = 300
aug_het_best_so_far = 300
rand_noise_best_so_far = 300 # value to beat
homo_noise_best_so_far = 300
het_noise_best_so_far = 300
aug_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
rand_obj_val_list = []
homo_obj_val_list = []
het_obj_val_list = []
aug_obj_val_list = []
aug_het_obj_val_list = []
rand_noise_val_list = []
homo_noise_val_list = []
het_noise_val_list = []
aug_noise_val_list = []
aug_het_noise_val_list = []
rand_collected_x = []
homo_collected_x = []
het_collected_x = []
aug_collected_x = []
aug_het_collected_x = []
for i in range(bayes_opt_iters):
print(i)
# take random point from uniform distribution
rand_X_next = np.random.uniform(0, 2)
# Obtain next noisy sample from the objective function
rand_X_next = min(xs_train, key=lambda x: abs(x - rand_X_next)
) # Closest point in the heldout set.
rand_index = list(xs).index(rand_X_next)
rand_Y_next = ys[rand_index]
rand_composite_obj_val = rand_Y_next + penalty * std[rand_index]
rand_noise_val = std[rand_index]
rand_collected_x.append(rand_X_next)
# check if random point's Y value is better than best so far
if rand_composite_obj_val < rand_best_so_far:
rand_best_so_far = rand_composite_obj_val
rand_obj_val_list.append(rand_composite_obj_val)
else:
rand_obj_val_list.append(rand_best_so_far)
# if yes, save it, if no, save best so far into list of best y-value per iteration in rand_composite_obj_val
if rand_noise_val < rand_noise_best_so_far:
rand_noise_best_so_far = rand_noise_val
rand_noise_val_list.append(rand_noise_val)
else:
rand_noise_val_list.append(rand_noise_best_so_far)
# Obtain next sampling point from the acquisition function (expected_improvement)
homo_X_next = my_propose_location(my_expected_improvement,
homo_X_sample,
homo_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=3,
min_val=300)
homo_collected_x.append(homo_X_next)
# Obtain next noisy sample from the objective function
homo_X_next = min(xs_train, key=lambda x: abs(x - homo_X_next))
homo_index = list(xs).index(homo_X_next)
homo_Y_next = ys[homo_index]
homo_composite_obj_val = homo_Y_next + penalty * std[homo_index]
homo_noise_val = std[homo_index]
if homo_composite_obj_val < homo_best_so_far:
homo_best_so_far = homo_composite_obj_val
homo_obj_val_list.append(homo_composite_obj_val)
else:
homo_obj_val_list.append(homo_best_so_far)
if homo_noise_val < homo_noise_best_so_far:
homo_noise_best_so_far = homo_noise_val
homo_noise_val_list.append(homo_noise_val)
else:
homo_noise_val_list.append(homo_noise_best_so_far)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(
heteroscedastic_expected_improvement,
het_X_sample,
het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_weight)
het_collected_x.append(het_X_next)
# Obtain next noisy sample from the objective function
het_X_next = min(xs_train, key=lambda x: abs(x - het_X_next))
het_index = list(xs).index(het_X_next)
het_Y_next = ys[het_index]
het_composite_obj_val = het_Y_next + penalty * std[het_index]
het_noise_val = std[het_index]
if het_composite_obj_val < het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the augmented expected improvement (AEI)
aug_X_next = my_propose_location(augmented_expected_improvement,
aug_X_sample,
aug_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_weight,
aei=True)
aug_collected_x.append(aug_X_next)
# Obtain next noisy sample from the objective function
aug_X_next = min(xs_train, key=lambda x: abs(x - aug_X_next))
aug_index = list(xs).index(aug_X_next)
aug_Y_next = ys[aug_index]
aug_composite_obj_val = aug_Y_next + penalty * std[aug_index]
aug_noise_val = std[aug_index]
if aug_composite_obj_val < aug_best_so_far:
aug_best_so_far = aug_composite_obj_val
aug_obj_val_list.append(aug_composite_obj_val)
else:
aug_obj_val_list.append(aug_best_so_far)
if aug_noise_val < aug_noise_best_so_far:
aug_noise_best_so_far = aug_noise_val
aug_noise_val_list.append(aug_noise_val)
else:
aug_noise_val_list.append(aug_noise_best_so_far)
# Add sample to previous sample
aug_X_sample = np.vstack((aug_X_sample, aug_X_next))
aug_Y_sample = np.vstack((aug_Y_sample, aug_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(
heteroscedastic_augmented_expected_improvement,
aug_het_X_sample,
aug_het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_weight)
aug_het_collected_x.append(aug_het_X_next)
# Obtain next noisy sample from the objective function
aug_het_X_next = min(xs_train,
key=lambda x: abs(x - aug_het_X_next))
aug_het_index = list(xs).index(aug_het_X_next)
aug_het_Y_next = ys[aug_het_index]
aug_het_composite_obj_val = aug_het_Y_next + penalty * std[
aug_het_index]
aug_het_noise_val = std[aug_het_index]
if aug_het_composite_obj_val < aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
rand_running_sum += np.array(
rand_obj_val_list, dtype=np.float64).flatten(
) # just the way to average out across all random trials
rand_squares += np.array(
rand_obj_val_list,
dtype=np.float64).flatten()**2 # likewise for errors
homo_running_sum += np.array(homo_obj_val_list,
dtype=np.float64).flatten()
homo_squares += np.array(homo_obj_val_list,
dtype=np.float64).flatten()**2
hetero_running_sum += np.array(het_obj_val_list,
dtype=np.float64).flatten()
hetero_squares += np.array(het_obj_val_list,
dtype=np.float64).flatten()**2
aug_running_sum += np.array(aug_obj_val_list,
dtype=np.float64).flatten()
aug_squares += np.array(aug_obj_val_list,
dtype=np.float64).flatten()**2
aug_het_running_sum += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()
aug_het_squares += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()**2
rand_noise_running_sum += np.array(
rand_noise_val_list, dtype=np.float64).flatten(
) # just the way to average out across all random trials
rand_noise_squares += np.array(
rand_noise_val_list,
dtype=np.float64).flatten()**2 # likewise for errors
homo_noise_running_sum += np.array(homo_noise_val_list,
dtype=np.float64).flatten()
homo_noise_squares += np.array(homo_noise_val_list,
dtype=np.float64).flatten()**2
hetero_noise_running_sum += np.array(het_noise_val_list,
dtype=np.float64).flatten()
hetero_noise_squares += np.array(het_noise_val_list,
dtype=np.float64).flatten()**2
aug_noise_running_sum += np.array(aug_noise_val_list,
dtype=np.float64).flatten()
aug_noise_squares += np.array(aug_noise_val_list,
dtype=np.float64).flatten()**2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()
aug_het_noise_squares += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()**2
rand_means = rand_running_sum / random_trials
rand_errs = (np.sqrt(rand_squares / random_trials -
rand_means**2)) / np.sqrt(random_trials)
homo_means = homo_running_sum / random_trials
hetero_means = hetero_running_sum / random_trials
homo_errs = (np.sqrt(homo_squares / random_trials - homo_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_means = aug_running_sum / random_trials
aug_errs = (np.sqrt(aug_squares / random_trials - aug_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_het_means = aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
rand_noise_means = rand_noise_running_sum / random_trials
homo_noise_means = homo_noise_running_sum / random_trials
hetero_noise_means = hetero_noise_running_sum / random_trials
rand_noise_errs = (np.sqrt(rand_noise_squares / random_trials -
rand_noise_means**2)) / np.sqrt(random_trials)
homo_noise_errs = (np.sqrt(homo_noise_squares / random_trials -
homo_noise_means**2)) / np.sqrt(random_trials)
hetero_noise_errs = (
np.sqrt(hetero_noise_squares / random_trials -
hetero_noise_means**2)) / np.sqrt(random_trials)
aug_noise_means = aug_noise_running_sum / random_trials
aug_noise_errs = (np.sqrt(aug_noise_squares / random_trials -
aug_noise_means**2)) / np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (
np.sqrt(aug_het_noise_squares / random_trials -
aug_het_noise_means**2)) / np.sqrt(random_trials)
print('List of average homoscedastic values is: ' + str(homo_means))
print('List of homoscedastic errors is: ' + str(homo_errs))
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_errs))
print('List of average AEI values is: ' + str(aug_means))
print('List of AEI errors is: ' + str(aug_errs))
print('List of average het-AEI values is: ' + str(aug_het_errs))
print('List of het-AEI errors is: ' + str(aug_het_errs))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_rand = np.array(rand_means) - np.array(rand_errs)
upper_rand = np.array(rand_means) + np.array(rand_errs)
lower_homo = np.array(homo_means) - np.array(homo_errs)
upper_homo = np.array(homo_means) + np.array(homo_errs)
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_aei = np.array(aug_means) - np.array(aug_errs)
upper_aei = np.array(aug_means) + np.array(aug_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, rand_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x,
lower_rand,
upper_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_homo,
upper_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_hetero,
upper_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x, lower_aei, upper_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x,
lower_het_aei,
upper_het_aei,
color='tab:purple',
alpha=0.1)
plt.yticks([75, 150, 225])
#plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
if penalty != 1:
plt.ylabel(f'Phosphorous Fraction + {penalty}*Noise', fontsize=14)
else:
plt.ylabel(f'Phosphorous Fraction + Noise', fontsize=14)
plt.tick_params(labelsize=14)
plt.ylim([0, 150])
#plt.legend(loc=1, fontsize=12)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
plt.savefig(
'soil_figures/bayesopt_plot{}_iters_{}_random_trials_and_init_num_samples_of_{}'
'_and_seed_{}_new_penalty_is_{}_aleatoric_weight_is_{}_new_aei'.format(
bayes_opt_iters, random_trials, init_num_samples, numpy_seed,
penalty, aleatoric_weight),
bbox_inches='tight')
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_rand = np.array(rand_noise_means) - np.array(rand_noise_errs)
upper_noise_rand = np.array(rand_noise_means) + np.array(rand_noise_errs)
lower_noise_homo = np.array(homo_noise_means) - np.array(homo_noise_errs)
upper_noise_homo = np.array(homo_noise_means) + np.array(homo_noise_errs)
lower_noise_hetero = np.array(hetero_noise_means) - np.array(
hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(
hetero_noise_errs)
lower_noise_aei = np.array(aug_noise_means) - np.array(aug_noise_errs)
upper_noise_aei = np.array(aug_noise_means) + np.array(aug_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(
aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(
aug_het_noise_errs)
plt.plot(iter_x, rand_noise_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_noise_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_noise_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_noise_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_noise_means, color='tab:purple', label='HAEI')
plt.yticks([10, 20, 30, 40])
plt.fill_between(iter_x,
lower_noise_rand,
upper_noise_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_homo,
upper_noise_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_hetero,
upper_noise_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_aei,
upper_noise_aei,
color='tab:red',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_het_aei,
upper_noise_het_aei,
color='tab:purple',
alpha=0.1)
#plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('Aleatoric Noise', fontsize=14)
plt.tick_params(labelsize=14)
#plt.legend(loc=1, fontsize=12)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
plt.savefig(
'soil_figures/bayesopt_plot{}_iters_{}_random_trials_and_init_num_samples_of_{}_and_seed_{}_'
'noise_only_penalty_is_{}_aleatoric_weight_is_{}_new_aei'.format(
bayes_opt_iters, random_trials, init_num_samples, numpy_seed,
penalty, aleatoric_weight),
bbox_inches='tight')
def main(penalty, aleatoric_weight, random_trials, bayes_opt_iters, grid_size):
"""
Optimise the heteroscedastic Branin-Hoo function.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of $\beta of ANPEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
param: grid_size: int specifying the side length of the 2D grid to initialise on.
"""
standardised = True # Whether or not to use the standardised Branin function from Picheny and Ginsbourger 2012
plot_collected = True # Whether to plot collected data points
# We perform random trials of Bayesian Optimisation
rand_running_sum = np.zeros(bayes_opt_iters)
rand_squares = np.zeros(bayes_opt_iters)
homo_running_sum = np.zeros(bayes_opt_iters)
homo_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_running_sum = np.zeros(bayes_opt_iters)
aug_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
rand_noise_running_sum = np.zeros(bayes_opt_iters)
rand_noise_squares = np.zeros(bayes_opt_iters)
homo_noise_running_sum = np.zeros(bayes_opt_iters)
homo_noise_squares = np.zeros(
bayes_opt_iters
) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_noise_running_sum = np.zeros(bayes_opt_iters)
aug_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
for i in range(random_trials):
numpy_seed = i
np.random.seed(numpy_seed)
if standardised:
bounds = np.array([[0, 1.0], [0, 1.0]])
else:
bounds = np.array(
[[-5.0, 10.0],
[0.0, 15.0]]) # bounds of the Bayesian Optimisation problem.
# Initial noisy data points sampled uniformly at random from the input space.
if standardised:
x1 = np.random.uniform(0, 1, size=(grid_size, ))
x2 = np.random.uniform(0, 1, size=(grid_size, ))
else:
x1 = np.random.uniform(-5.0, 10.0, size=(grid_size, ))
x2 = np.random.uniform(0.0, 15.0, size=(grid_size, ))
X_init = np.array(np.meshgrid(x1, x2)).T.reshape(-1, 2)
Y_init = heteroscedastic_branin(X_init[:, 0], X_init[:, 1])
if standardised:
x1_star = np.arange(0, 1, 0.05)
x2_star = np.arange(0, 1, 0.05)
else:
x1_star = np.arange(-5.0, 10.0, 0.5)
x2_star = np.arange(0.0, 15.0, 0.5)
plot_sample = np.array(np.meshgrid(x1_star, x2_star)).T.reshape(
-1, 2) # Where 2 gives the dimensionality
# Initialize samples
homo_X_sample = X_init
homo_Y_sample = Y_init
het_X_sample = X_init
het_Y_sample = Y_init
aug_X_sample = X_init
aug_Y_sample = Y_init
aug_het_X_sample = X_init
aug_het_Y_sample = Y_init
# initial GP hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0 # optimise noise for homoscedastic GP
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
rand_best_so_far = 300
homo_best_so_far = 300 # value to beat
het_best_so_far = 300
aug_best_so_far = 300
aug_het_best_so_far = 300
rand_noise_best_so_far = 300 # value to beat
homo_noise_best_so_far = 300
het_noise_best_so_far = 300
aug_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
rand_obj_val_list = []
homo_obj_val_list = []
het_obj_val_list = []
aug_obj_val_list = []
aug_het_obj_val_list = []
rand_noise_val_list = []
homo_noise_val_list = []
het_noise_val_list = []
aug_noise_val_list = []
aug_het_noise_val_list = []
rand_collected_x1 = []
rand_collected_x2 = []
homo_collected_x1 = []
homo_collected_x2 = []
het_collected_x1 = []
het_collected_x2 = []
aug_collected_x1 = []
aug_collected_x2 = []
aug_het_collected_x1 = []
aug_het_collected_x2 = []
for i in range(bayes_opt_iters):
print(i)
# random sampling baseline
if standardised:
random_x1_next = np.random.uniform(0, 1, size=(1, ))
random_x2_next = np.random.uniform(0, 1, size=(1, ))
else:
random_x1_next = np.random.uniform(-5.0, 10.0, size=(1, ))
random_x2_next = np.random.uniform(0.0, 15.0, size=(1, ))
random_X_next = np.array(
np.meshgrid(random_x1_next, random_x2_next)).T.reshape(-1, 2)
rand_collected_x1.append(random_x1_next)
rand_collected_x2.append(random_x2_next)
f_plot = True
if i > 0:
f_plot = False
random_Y_next = heteroscedastic_branin(random_x1_next,
random_x2_next,
standardised=standardised,
f_plot=f_plot,
penalty=penalty)
random_composite_obj_val, rand_noise_val = min_branin_noise_function(
random_x1_next,
random_x2_next,
standardised=standardised,
penalty=penalty)
f_plot = False
if random_composite_obj_val < rand_best_so_far:
rand_best_so_far = random_composite_obj_val
rand_obj_val_list.append(random_composite_obj_val)
else:
rand_obj_val_list.append(rand_best_so_far)
if rand_noise_val < rand_noise_best_so_far:
rand_noise_best_so_far = rand_noise_val
rand_noise_val_list.append(rand_noise_val)
else:
rand_noise_val_list.append(rand_noise_best_so_far)
# Obtain next sampling point from the acquisition function (expected_improvement)
homo_X_next = my_propose_location(my_expected_improvement,
homo_X_sample,
homo_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=3,
min_val=300)
homo_collected_x1.append(homo_X_next[:, 0])
homo_collected_x2.append(homo_X_next[:, 1])
# Obtain next noisy sample from the objective function
homo_Y_next = heteroscedastic_branin(homo_X_next[:, 0],
homo_X_next[:, 1],
standardised=standardised,
f_plot=f_plot,
penalty=penalty)
homo_composite_obj_val, homo_noise_val = min_branin_noise_function(
homo_X_next[:, 0],
homo_X_next[:, 1],
standardised=standardised,
penalty=penalty)
if homo_composite_obj_val < homo_best_so_far:
homo_best_so_far = homo_composite_obj_val
homo_obj_val_list.append(homo_composite_obj_val)
else:
homo_obj_val_list.append(homo_best_so_far)
if homo_noise_val < homo_noise_best_so_far:
homo_noise_best_so_far = homo_noise_val
homo_noise_val_list.append(homo_noise_val)
else:
homo_noise_val_list.append(homo_noise_best_so_far)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(
heteroscedastic_expected_improvement,
het_X_sample,
het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_weight)
het_collected_x1.append(het_X_next[:, 0])
het_collected_x2.append(het_X_next[:, 1])
# Obtain next noisy sample from the objective function
het_Y_next = heteroscedastic_branin(het_X_next[:, 0],
het_X_next[:, 1],
standardised=standardised,
f_plot=f_plot,
penalty=penalty)
het_composite_obj_val, het_noise_val = min_branin_noise_function(
het_X_next[:, 0],
het_X_next[:, 1],
standardised=standardised,
penalty=penalty)
if het_composite_obj_val < het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the augmented expected improvement (AEI)
aug_X_next = my_propose_location(augmented_expected_improvement,
aug_X_sample,
aug_Y_sample,
noise,
l_init,
sigma_f_init,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=aleatoric_weight,
aei=True)
aug_collected_x1.append(aug_X_next[:, 0])
aug_collected_x2.append(aug_X_next[:, 1])
# Obtain next noisy sample from the objective function
aug_Y_next = heteroscedastic_branin(aug_X_next[:, 0],
aug_X_next[:, 1],
standardised=standardised,
f_plot=f_plot,
penalty=penalty)
aug_composite_obj_val, aug_noise_val = min_branin_noise_function(
aug_X_next[:, 0],
aug_X_next[:, 1],
standardised=standardised,
penalty=penalty)
if aug_composite_obj_val < aug_best_so_far:
aug_best_so_far = aug_composite_obj_val
aug_obj_val_list.append(aug_composite_obj_val)
else:
aug_obj_val_list.append(aug_best_so_far)
if aug_noise_val < aug_noise_best_so_far:
aug_noise_best_so_far = aug_noise_val
aug_noise_val_list.append(aug_noise_val)
else:
aug_noise_val_list.append(aug_noise_best_so_far)
# Add sample to previous sample
aug_X_sample = np.vstack((aug_X_sample, aug_X_next))
aug_Y_sample = np.vstack((aug_Y_sample, aug_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(
heteroscedastic_augmented_expected_improvement,
aug_het_X_sample,
aug_het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=3,
min_val=300,
aleatoric_weight=500)
aug_het_collected_x1.append(aug_het_X_next[:, 0])
aug_het_collected_x2.append(aug_het_X_next[:, 1])
# Obtain next noisy sample from the objective function
aug_het_Y_next = heteroscedastic_branin(aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
standardised=standardised,
f_plot=f_plot,
penalty=penalty)
aug_het_composite_obj_val, aug_het_noise_val = min_branin_noise_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
standardised=standardised,
penalty=penalty)
if aug_het_composite_obj_val < aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
rand_running_sum += np.array(rand_obj_val_list,
dtype=np.float64).flatten()
rand_squares += np.array(rand_obj_val_list,
dtype=np.float64).flatten()**2
homo_running_sum += np.array(homo_obj_val_list,
dtype=np.float64).flatten()
homo_squares += np.array(homo_obj_val_list,
dtype=np.float64).flatten()**2
hetero_running_sum += np.array(het_obj_val_list,
dtype=np.float64).flatten()
hetero_squares += np.array(het_obj_val_list,
dtype=np.float64).flatten()**2
aug_running_sum += np.array(aug_obj_val_list,
dtype=np.float64).flatten()
aug_squares += np.array(aug_obj_val_list,
dtype=np.float64).flatten()**2
aug_het_running_sum += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()
aug_het_squares += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()**2
rand_noise_running_sum += np.array(
rand_noise_val_list, dtype=np.float64).flatten(
) # just the way to average out across all random trials
rand_noise_squares += np.array(
rand_noise_val_list,
dtype=np.float64).flatten()**2 # likewise for errors
homo_noise_running_sum += np.array(homo_noise_val_list,
dtype=np.float64).flatten()
homo_noise_squares += np.array(homo_noise_val_list,
dtype=np.float64).flatten()**2
hetero_noise_running_sum += np.array(het_noise_val_list,
dtype=np.float64).flatten()
hetero_noise_squares += np.array(het_noise_val_list,
dtype=np.float64).flatten()**2
aug_noise_running_sum += np.array(aug_noise_val_list,
dtype=np.float64).flatten()
aug_noise_squares += np.array(aug_noise_val_list,
dtype=np.float64).flatten()**2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()
aug_het_noise_squares += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()**2
rand_means = rand_running_sum / random_trials
rand_errs = (np.sqrt(rand_squares / random_trials - rand_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
homo_means = homo_running_sum / random_trials
hetero_means = hetero_running_sum / random_trials
homo_errs = (np.sqrt(homo_squares / random_trials - homo_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_means = aug_running_sum / random_trials
aug_errs = (np.sqrt(aug_squares / random_trials - aug_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_het_means = aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
rand_noise_means = rand_noise_running_sum / random_trials
homo_noise_means = homo_noise_running_sum / random_trials
hetero_noise_means = hetero_noise_running_sum / random_trials
rand_noise_errs = (np.sqrt(rand_noise_squares / random_trials -
rand_noise_means**2)) / np.sqrt(random_trials)
homo_noise_errs = (np.sqrt(homo_noise_squares / random_trials -
homo_noise_means**2)) / np.sqrt(random_trials)
hetero_noise_errs = (
np.sqrt(hetero_noise_squares / random_trials -
hetero_noise_means**2)) / np.sqrt(random_trials)
aug_noise_means = aug_noise_running_sum / random_trials
aug_noise_errs = (np.sqrt(aug_noise_squares / random_trials -
aug_noise_means**2)) / np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (
np.sqrt(aug_het_noise_squares / random_trials -
aug_het_noise_means**2)) / np.sqrt(random_trials)
print('List of average random values is: ' + str(rand_means))
print('List of random errors is: ' + str(rand_errs))
print('List of average homoscedastic values is: ' + str(homo_means))
print('List of homoscedastic errors is: ' + str(homo_errs))
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_errs))
print('List of average AEI values is: ' + str(aug_means))
print('List of AEI errors is: ' + str(aug_errs))
print('List of average het-AEI values is: ' + str(aug_het_means))
print('List of het-AEI errors is: ' + str(aug_het_errs))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_rand = np.array(rand_means) - np.array(rand_errs)
upper_rand = np.array(rand_means) + np.array(rand_errs)
lower_homo = np.array(homo_means) - np.array(homo_errs)
upper_homo = np.array(homo_means) + np.array(homo_errs)
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_aei = np.array(aug_means) - np.array(aug_errs)
upper_aei = np.array(aug_means) + np.array(aug_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, rand_means, color='tab:orange', label='Random Sampling')
plt.plot(iter_x, homo_means, color='tab:blue', label='Homoscedastic')
plt.plot(iter_x,
hetero_means,
color='tab:green',
label='Heteroscedastic ANPEI')
plt.plot(iter_x, aug_means, color='tab:red', label='Homoscedastic AEI')
plt.plot(iter_x,
aug_het_means,
color='tab:purple',
label='Heteroscedastic AEI')
plt.fill_between(iter_x,
lower_rand,
upper_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_homo,
upper_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_hetero,
upper_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x, lower_aei, upper_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x,
lower_het_aei,
upper_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('f(x) + g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc=1, fontsize=12)
plt.savefig(
'toy_branin_figures/bayesopt_plot{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}'
'_new_standardised_is_{}_penalty_is_{}_aleatoric_weight_is_{}_new_aei'.
format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised, penalty, aleatoric_weight))
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_rand = np.array(rand_noise_means) - np.array(rand_noise_errs)
upper_noise_rand = np.array(rand_noise_means) + np.array(rand_noise_errs)
lower_noise_homo = np.array(homo_noise_means) - np.array(homo_noise_errs)
upper_noise_homo = np.array(homo_noise_means) + np.array(homo_noise_errs)
lower_noise_hetero = np.array(hetero_noise_means) - np.array(
hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(
hetero_noise_errs)
lower_noise_aei = np.array(aug_noise_means) - np.array(aug_noise_errs)
upper_noise_aei = np.array(aug_noise_means) + np.array(aug_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(
aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(
aug_het_noise_errs)
#best_noise_plot = np.zeros(len(iter_x))
#plt.plot(iter_x, best_noise_plot, '--', color='k', label='Optimal')
plt.plot(iter_x,
rand_noise_means,
color='tab:orange',
label='Random Sampling')
plt.plot(iter_x, homo_noise_means, color='tab:blue', label='Homoscedastic')
plt.plot(iter_x,
hetero_noise_means,
color='tab:green',
label='Heteroscedastic ANPEI')
plt.plot(iter_x,
aug_noise_means,
color='tab:red',
label='Homoscedastic AEI')
plt.plot(iter_x,
aug_het_noise_means,
color='tab:purple',
label='Heteroscedastic AEI')
plt.fill_between(iter_x,
lower_noise_rand,
upper_noise_rand,
color='tab:orange',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_homo,
upper_noise_homo,
color='tab:blue',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_hetero,
upper_noise_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_aei,
upper_noise_aei,
color='tab:red',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_het_aei,
upper_noise_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc=1, fontsize=12)
plt.savefig(
'toy_branin_figures/bayesopt_plot{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}_'
'noise_only_standardised_is_{}_penalty_is_{}_aleatoric_weight_is_{}_new_aei'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised, penalty, aleatoric_weight))
if plot_collected:
plt.cla()
plt.plot(np.array(rand_collected_x1),
np.array(rand_collected_x2),
'+',
color='tab:orange',
markersize='12',
linewidth='8')
plt.xlabel('x1')
plt.ylabel('x2')
plt.title('Collected Data Points')
plt.savefig(
'toy_branin_figures/collected_points/bayesopt_plot{}_iters_{}_random_trials_and'
'_grid_size_of_{}_and_seed_{}_with_het_aei_full_unc_new_rand_standardised_is_{}'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised))
plt.close()
plt.cla()
plt.plot(np.array(homo_collected_x1),
np.array(homo_collected_x2),
'+',
color='tab:blue',
markersize='12',
linewidth='8')
plt.xlabel('x1')
plt.ylabel('x2')
plt.title('Collected Data Points')
plt.savefig(
'toy_branin_figures/collected_points/bayesopt_plot{}_iters_{}_random_trials_and'
'_grid_size_of_{}_and_seed_{}_with_het_aei_full_unc_new_rand_homo_standardised_is_{}'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised))
plt.close()
plt.cla()
plt.plot(np.array(het_collected_x1),
np.array(het_collected_x2),
'+',
color='tab:green',
markersize='12',
linewidth='8')
plt.xlabel('x1')
plt.ylabel('x2')
plt.title('Collected Data Points')
plt.savefig(
'toy_branin_figures/collected_points/bayesopt_plot{}_iters_{}_random_trials_and'
'_grid_size_of_{}_and_seed_{}_with_het_aei_full_unc_new_het_anpei_standardised_is_{}'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised))
plt.close()
plt.cla()
plt.plot(np.array(aug_collected_x1),
np.array(aug_collected_x2),
'+',
color='tab:red',
markersize='12',
linewidth='8')
plt.xlabel('x1')
plt.ylabel('x2')
plt.title('Collected Data Points')
plt.savefig(
'toy_branin_figures/collected_points/bayesopt_plot{}_iters_{}_random_trials_and'
'_grid_size_of_{}_and_seed_{}_with_het_aei_full_unc_new_rand_aug_standardised_is_{}'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised))
plt.close()
plt.cla()
plt.plot(np.array(aug_het_collected_x1),
np.array(aug_het_collected_x2),
'+',
color='tab:purple',
markersize='12',
linewidth='8')
plt.xlabel('x1')
plt.ylabel('x2')
plt.title('Collected Data Points')
plt.savefig(
'toy_branin_figures/collected_points/bayesopt_plot{}_iters_{}_random_trials_and'
'_grid_size_of_{}_and_seed_{}_with_het_aei_full_unc_new_rand_aug_het_standardised_is_{}'
.format(bayes_opt_iters, random_trials, grid_size, numpy_seed,
standardised))
plt.close()
def main(penalty, aleatoric_weight, random_trials, bayes_opt_iters, init_set_size, n_components, path):
"""
Script for running the soil phosphorus fraction optimisation experiment.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of $\beta of ANPEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
param: init_set_size: int specifying the side length of the 2D grid to initialise on.
param: n_components: int specifying the number of PCA principle components to keep.
param: path: str specifying the path to the Freesolv.txt file.
"""
task = 'FreeSolv'
use_frag = True
use_exp = True # use experimental values.
xs, ys, std = parse_dataset(task, path, use_frag, use_exp)
warnings.filterwarnings('ignore')
# Number of iterations
bayes_opt_iters = 10
random_trials = 50
# We perform random trials of Bayesian Optimisation
rand_running_sum = np.zeros(bayes_opt_iters)
rand_squares = np.zeros(bayes_opt_iters)
homo_running_sum = np.zeros(bayes_opt_iters)
homo_squares = np.zeros(bayes_opt_iters) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_running_sum = np.zeros(bayes_opt_iters)
aug_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
rand_noise_running_sum = np.zeros(bayes_opt_iters)
rand_noise_squares = np.zeros(bayes_opt_iters)
homo_noise_running_sum = np.zeros(bayes_opt_iters)
homo_noise_squares = np.zeros(bayes_opt_iters) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_noise_running_sum = np.zeros(bayes_opt_iters)
aug_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
for i in range(random_trials):
start_seed = 47
numpy_seed = i + start_seed # set to avoid segfault issue
# ('Process finished with exit code 139 (interrupted by signal 11: SIGSEGV)') when i = 0
# test in this instance is the initialisation set for Bayesian Optimisation and train is the heldout set.
xs_train, xs_test, ys_train, ys_test = train_test_split(xs, ys, test_size=init_set_size, random_state=numpy_seed, shuffle=True)
pca = PCA(n_components)
xs_test = pca.fit_transform(xs_test)
print('Fraction of variance retained is: ' + str(sum(pca.explained_variance_ratio_)))
xs_train = pca.transform(xs_train)
_, _, std_train, std_test = train_test_split(xs, std, test_size=init_set_size, random_state=numpy_seed, shuffle=True)
ys_train = ys_train.reshape(-1, 1)
ys_test = ys_test.reshape(-1, 1)
init_num_samples = len(ys_test)
bounds = np.array([np.array([np.min(xs_train[:, i]), np.max(xs_train[:, i])]) for i in range(xs_train.shape[1])])
# Can only plot in 2D
if n_components == 2:
x1_star = np.arange(np.min(xs_train[:, 0]), np.max(xs_train[:, 0]), 0.2)
x2_star = np.arange(np.min(xs_train[:, 1]), np.max(xs_train[:, 1]), 0.2)
plot_sample = np.array(np.meshgrid(x1_star, x2_star)).T.reshape(-1, 2) # Where 2 gives the dimensionality
else:
plot_sample = None
X_init = xs_test
Y_init = ys_test
# Initialize samples
homo_X_sample = X_init
homo_Y_sample = Y_init
het_X_sample = X_init
het_Y_sample = Y_init
aug_X_sample = X_init
aug_Y_sample = Y_init
aug_het_X_sample = X_init
aug_het_Y_sample = Y_init
# initial GP hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
rand_best_so_far = 300
homo_best_so_far = 300 # value to beat
het_best_so_far = 300
aug_best_so_far = 300
aug_het_best_so_far = 300
rand_noise_best_so_far = 300
homo_noise_best_so_far = 300 # value to beat
het_noise_best_so_far = 300
aug_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
rand_obj_val_list = []
homo_obj_val_list = []
het_obj_val_list = []
aug_obj_val_list = []
aug_het_obj_val_list = []
rand_noise_val_list = []
homo_noise_val_list = []
het_noise_val_list = []
aug_noise_val_list = []
aug_het_noise_val_list = []
rand_collected_x = []
homo_collected_x = []
het_collected_x = []
aug_collected_x = []
aug_het_collected_x = []
for j in range(bayes_opt_iters):
print(j)
# take random point from uniform distribution
rand_X_next = np.random.uniform(np.min(xs_train, axis=0), np.max(xs_train, axis=0)) # this just takes X not the sin function itself
# Obtain next noisy sample from the objective function
rand_X_next = min(xs_train, key=lambda x: np.linalg.norm(x - rand_X_next)) # Closest point in the heldout set.
rand_index = list(xs_train[:, 0]).index(rand_X_next[0]) # index by first dimension
rand_Y_next = ys_train[rand_index]
rand_composite_obj_val = rand_Y_next + penalty*std_train[rand_index]
rand_noise_val = std_train[rand_index]
rand_collected_x.append(rand_X_next)
# check if random point's Y value is better than best so far
if rand_composite_obj_val < rand_best_so_far:
rand_best_so_far = rand_composite_obj_val
rand_obj_val_list.append(rand_composite_obj_val)
else:
rand_obj_val_list.append(rand_best_so_far)
# if yes, save it, if no, save best so far into list of best y-value per iteration in rand_composite_obj_val
if rand_noise_val < rand_noise_best_so_far:
rand_noise_best_so_far = rand_noise_val
rand_noise_val_list.append(rand_noise_val)
else:
rand_noise_val_list.append(rand_noise_best_so_far)
# Obtain next sampling point from the acquisition function (expected_improvement)
homo_X_next = my_propose_location(my_expected_improvement, homo_X_sample, homo_Y_sample, noise, l_init, sigma_f_init,
bounds, plot_sample, n_restarts=3, min_val=300)
homo_collected_x.append(homo_X_next)
# Obtain next noisy sample from the objective function
homo_X_next = min(xs_train, key=lambda x: np.linalg.norm(x - homo_X_next)) # Closest point in the heldout set.
homo_index = list(xs_train[:, 0]).index(homo_X_next[0]) # index by first dimension
homo_Y_next = ys_train[homo_index]
homo_composite_obj_val = homo_Y_next + penalty*std_train[homo_index]
homo_noise_val = std_train[homo_index]
homo_collected_x.append(homo_X_next)
if homo_composite_obj_val < homo_best_so_far:
homo_best_so_far = homo_composite_obj_val
homo_obj_val_list.append(homo_composite_obj_val)
else:
homo_obj_val_list.append(homo_best_so_far)
if homo_noise_val < homo_noise_best_so_far:
homo_noise_best_so_far = homo_noise_val
homo_noise_val_list.append(homo_noise_val)
else:
homo_noise_val_list.append(homo_noise_best_so_far)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(heteroscedastic_expected_improvement, het_X_sample,
het_Y_sample, noise, l_init, sigma_f_init, l_noise_init,
sigma_f_noise_init, gp2_noise, num_iters, sample_size, bounds,
plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight)
het_collected_x.append(het_X_next)
# Obtain next noisy sample from the objective function
het_X_next = min(xs_train, key=lambda x: np.linalg.norm(x - het_X_next))
het_index = list(xs_train[:, 0]).index(het_X_next[0])
het_Y_next = ys_train[het_index]
het_composite_obj_val = het_Y_next + penalty*std_train[het_index]
het_noise_val = std_train[het_index]
het_collected_x.append(het_X_next)
if het_composite_obj_val < het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the augmented expected improvement (AEI)
aug_X_next = my_propose_location(augmented_expected_improvement, aug_X_sample, aug_Y_sample, noise, l_init, sigma_f_init,
bounds, plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight, aei=True)
aug_collected_x.append(aug_X_next)
# Obtain next noisy sample from the objective function
aug_X_next = min(xs_train, key=lambda x: np.linalg.norm(x - aug_X_next))
aug_index = list(xs_train[:, 0]).index(aug_X_next[0])
aug_Y_next = ys_train[aug_index]
aug_composite_obj_val = aug_Y_next + penalty*std_train[aug_index]
aug_noise_val = std_train[aug_index]
aug_collected_x.append(het_X_next)
if aug_composite_obj_val < aug_best_so_far:
aug_best_so_far = aug_composite_obj_val
aug_obj_val_list.append(aug_composite_obj_val)
else:
aug_obj_val_list.append(aug_best_so_far)
if aug_noise_val < aug_noise_best_so_far:
aug_noise_best_so_far = aug_noise_val
aug_noise_val_list.append(aug_noise_val)
else:
aug_noise_val_list.append(aug_noise_best_so_far)
# Add sample to previous sample
aug_X_sample = np.vstack((aug_X_sample, aug_X_next))
aug_Y_sample = np.vstack((aug_Y_sample, aug_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(heteroscedastic_augmented_expected_improvement, aug_het_X_sample,
aug_het_Y_sample, noise, l_init, sigma_f_init, l_noise_init,
sigma_f_noise_init, gp2_noise, num_iters, sample_size, bounds,
plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight)
aug_het_collected_x.append(aug_het_X_next)
# Obtain next noisy sample from the objective function
aug_het_X_next = min(xs_train, key=lambda x: np.linalg.norm(x - aug_het_X_next))
aug_het_index = list(xs_train[:, 0]).index(aug_het_X_next[0])
aug_het_Y_next = ys_train[aug_het_index]
aug_het_composite_obj_val = aug_het_Y_next + penalty*std_train[aug_het_index]
aug_het_noise_val = std_train[aug_het_index]
aug_het_collected_x.append(aug_het_X_next)
if aug_het_composite_obj_val < aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
rand_running_sum += np.array(rand_obj_val_list, dtype=np.float64).flatten()
rand_squares += np.array(rand_obj_val_list, dtype=np.float64).flatten() ** 2
homo_running_sum += np.array(homo_obj_val_list, dtype=np.float64).flatten()
homo_squares += np.array(homo_obj_val_list, dtype=np.float64).flatten() ** 2
hetero_running_sum += np.array(het_obj_val_list, dtype=np.float64).flatten()
hetero_squares += np.array(het_obj_val_list, dtype=np.float64).flatten() ** 2
aug_running_sum += np.array(aug_obj_val_list, dtype=np.float64).flatten()
aug_squares += np.array(aug_obj_val_list, dtype=np.float64).flatten() ** 2
aug_het_running_sum += np.array(aug_het_obj_val_list, dtype=np.float64).flatten()
aug_het_squares += np.array(aug_het_obj_val_list, dtype=np.float64).flatten() ** 2
rand_noise_running_sum += np.array(rand_noise_val_list, dtype=np.float64).flatten() # just the way to average out across all random trials
rand_noise_squares += np.array(rand_noise_val_list, dtype=np.float64).flatten() ** 2 # likewise for errors
homo_noise_running_sum += np.array(homo_noise_val_list, dtype=np.float64).flatten()
homo_noise_squares += np.array(homo_noise_val_list, dtype=np.float64).flatten() ** 2
hetero_noise_running_sum += np.array(het_noise_val_list, dtype=np.float64).flatten()
hetero_noise_squares += np.array(het_noise_val_list, dtype=np.float64).flatten() ** 2
aug_noise_running_sum += np.array(aug_noise_val_list, dtype=np.float64).flatten()
aug_noise_squares += np.array(aug_noise_val_list, dtype=np.float64).flatten() ** 2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list, dtype=np.float64).flatten()
aug_het_noise_squares += np.array(aug_het_noise_val_list, dtype=np.float64).flatten() ** 2
print(f'trial {i} complete')
if init_set_size == 0.2:
seed_index = i + start_seed + 1
np.savetxt(f'freesolv_data/02_pen_1/rand_means/rand_means_{start_seed}_{seed_index}.txt', rand_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/rand_means/rand_squares_{start_seed}_{seed_index}.txt', rand_squares)
np.savetxt(f'freesolv_data/02_pen_1/homo_means/homo_means_{start_seed}_{seed_index}.txt', homo_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/homo_means/homo_squares_{start_seed}_{seed_index}.txt', homo_squares)
np.savetxt(f'freesolv_data/02_pen_1/het_means/hetero_means_{start_seed}_{seed_index}.txt', hetero_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/het_means/hetero_squares_{start_seed}_{seed_index}.txt', hetero_squares)
np.savetxt(f'freesolv_data/02_pen_1/aug_means/aug_means_{start_seed}_{seed_index}.txt', aug_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/aug_means/aug_squares_{start_seed}_{seed_index}.txt', aug_squares)
np.savetxt(f'freesolv_data/02_pen_1/aug_het_means/aug_het_means_{start_seed}_{seed_index}.txt', aug_het_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/aug_het_means/aug_het_squares_{start_seed}_{seed_index}.txt', aug_het_squares)
np.savetxt(f'freesolv_data/02_pen_1/rand_noise/rand_means_{start_seed}_{seed_index}.txt', rand_noise_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/rand_noise/rand_squares_{start_seed}_{seed_index}.txt', rand_noise_squares)
np.savetxt(f'freesolv_data/02_pen_1/homo_noise/homo_means_{start_seed}_{seed_index}.txt', homo_noise_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/homo_noise/homo_squares_{start_seed}_{seed_index}.txt', homo_noise_squares)
np.savetxt(f'freesolv_data/02_pen_1/het_noise/hetero_means_{start_seed}_{seed_index}.txt', hetero_noise_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/het_noise/hetero_squares_{start_seed}_{seed_index}.txt', hetero_noise_squares)
np.savetxt(f'freesolv_data/02_pen_1/aug_noise/aug_means_{start_seed}_{seed_index}.txt', aug_noise_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/aug_noise/aug_squares_{start_seed}_{seed_index}.txt', aug_noise_squares)
np.savetxt(f'freesolv_data/02_pen_1/aug_het_noise/aug_het_means_{start_seed}_{seed_index}.txt', aug_het_noise_running_sum)
np.savetxt(f'freesolv_data/02_pen_1/aug_het_noise/aug_het_squares_{start_seed}_{seed_index}.txt', aug_het_noise_squares)
rand_means = rand_running_sum / random_trials
rand_errs = (np.sqrt(rand_squares / random_trials - rand_means **2))/np.sqrt(random_trials)
homo_means = homo_running_sum / random_trials
hetero_means = hetero_running_sum / random_trials
homo_errs = (np.sqrt(homo_squares / random_trials - homo_means ** 2, dtype=np.float64))/np.sqrt(random_trials)
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means ** 2, dtype=np.float64))/np.sqrt(random_trials)
aug_means = aug_running_sum / random_trials
aug_errs = (np.sqrt(aug_squares / random_trials - aug_means ** 2, dtype=np.float64))/np.sqrt(random_trials)
aug_het_means = aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means **2, dtype=np.float64))/np.sqrt(random_trials)
rand_noise_means = rand_noise_running_sum / random_trials
homo_noise_means = homo_noise_running_sum / random_trials
hetero_noise_means = hetero_noise_running_sum / random_trials
rand_noise_errs = (np.sqrt(rand_noise_squares / random_trials - rand_noise_means ** 2))/np.sqrt(random_trials)
homo_noise_errs = (np.sqrt(homo_noise_squares / random_trials - homo_noise_means ** 2))/np.sqrt(random_trials)
hetero_noise_errs = (np.sqrt(hetero_noise_squares / random_trials - hetero_noise_means ** 2))/np.sqrt(random_trials)
aug_noise_means = aug_noise_running_sum / random_trials
aug_noise_errs = (np.sqrt(aug_noise_squares / random_trials - aug_noise_means ** 2))/np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (np.sqrt(aug_het_noise_squares / random_trials - aug_het_noise_means ** 2))/np.sqrt(random_trials)
print('List of average random values is: ' + str(rand_means))
print('List of random errors is: ' + str(rand_noise_means))
print('List of average homoscedastic values is: ' + str(homo_means))
print('List of homoscedastic errors is: ' + str(homo_noise_means))
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_noise_means))
print('List of average AEI values is: ' + str(aug_means))
print('List of AEI errors is: ' + str(aug_noise_means))
print('List of average het-AEI values is: ' + str(aug_het_means))
print('List of het-AEI errors is: ' + str(aug_het_noise_means))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_rand = np.array(rand_means) - np.array(rand_errs)
upper_rand = np.array(rand_means) + np.array(rand_errs)
lower_homo = np.array(homo_means) - np.array(homo_errs)
upper_homo = np.array(homo_means) + np.array(homo_errs)
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_aei = np.array(aug_means) - np.array(aug_errs)
upper_aei = np.array(aug_means) + np.array(aug_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, rand_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x, lower_rand, upper_rand, color='tab:orange', alpha=0.1)
plt.fill_between(iter_x, lower_homo, upper_homo, color='tab:blue', alpha=0.1)
plt.fill_between(iter_x, lower_hetero, upper_hetero, color='tab:green', alpha=0.1)
plt.fill_between(iter_x, lower_aei, upper_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x, lower_het_aei, upper_het_aei, color='tab:purple', alpha=0.1)
#plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
if penalty != 1:
plt.ylabel(f'Hydration Free Energy (kcal/mol) + {penalty}*Noise', fontsize=14)
else:
plt.ylabel(f'Hydration Free Energy (kcal/mol) + Noise', fontsize=14)
plt.tick_params(labelsize=14)
#plt.legend(loc=1)
plt.legend(loc='lower left', bbox_to_anchor=(0.0, -0.425), ncol=3, borderaxespad=0, fontsize=14, frameon=False)
plt.savefig('new_freesolv_figures/bayesopt_plot{}_iters_{}_random_trials_and_init_num_samples_of_{}_and_seed_{}_'
'new_acq_penalty_is_{}_aleatoric_weight_is_{}_n_components_is_{}_new_aei_comp_seed_check'.
format(bayes_opt_iters, random_trials, init_num_samples, numpy_seed, penalty, aleatoric_weight, n_components), bbox_inches='tight')
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_rand = np.array(rand_noise_means) - np.array(rand_noise_errs)
upper_noise_rand = np.array(rand_noise_means) + np.array(rand_noise_errs)
lower_noise_homo = np.array(homo_noise_means) - np.array(homo_noise_errs)
upper_noise_homo = np.array(homo_noise_means) + np.array(homo_noise_errs)
lower_noise_hetero = np.array(hetero_noise_means) - np.array(hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(hetero_noise_errs)
lower_noise_aei = np.array(aug_noise_means) - np.array(aug_noise_errs)
upper_noise_aei = np.array(aug_noise_means) + np.array(aug_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(aug_het_noise_errs)
plt.plot(iter_x, rand_noise_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_noise_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_noise_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_noise_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_noise_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x, lower_noise_rand, upper_noise_rand, color='tab:orange', alpha=0.1)
plt.fill_between(iter_x, lower_noise_homo, upper_noise_homo, color='tab:blue', alpha=0.1)
plt.fill_between(iter_x, lower_noise_hetero, upper_noise_hetero, color='tab:green', alpha=0.1)
plt.fill_between(iter_x, lower_noise_aei, upper_noise_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x, lower_noise_het_aei, upper_noise_het_aei, color='tab:purple', alpha=0.1)
#plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('Aleatoric Noise', fontsize=14)
plt.tick_params(labelsize=14)
#plt.legend(loc=1)
plt.legend(loc='lower left', bbox_to_anchor=(0.0, -0.425), ncol=3, borderaxespad=0, fontsize=14, frameon=False)
plt.savefig('new_freesolv_figures/bayesopt_plot{}_iters_{}_random_trials_and_init_num_samples_of_{}_and_seed_{}_'
'noise_only_new_acq_penalty_is_{}_aleatoric_weight_is_{}_n_components_is_{}_new_aei_comp_seed_check'.
format(bayes_opt_iters, random_trials, init_num_samples, numpy_seed, penalty, aleatoric_weight, n_components), bbox_inches='tight')
def main(penalty, aleatoric_weight, aleatoric_weight_aug, random_trials,
bayes_opt_iters, grid_size, exp_type, opt_func, noise_level):
"""
Optimise the heteroscedastic Branin-Hoo function.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of $\beta of ANPEI
param: aleatoric_weight_aug: float specifying the value of $\gamma of HAEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
param: grid_size: int specifying the side length of the 2D grid to initialise on.
param: exp_type: str specifying the type of experiment. One of ['hetero', 'homoscedastic', 'noiseless']
param: opt_func: str specifying the optimisation function. One of ['hosaki', 'branin', 'goldstein']
param: noise_level: int specifying the noise level for homoscedastic noise. Should be 0 when heteroscedastic.
"""
if noise_level != 0:
assert exp_type == 'homoscedastic'
if exp_type == 'hetero':
assert noise_level == 0
heteroscedastic = True
if heteroscedastic is not True and noise_level == 0:
assert exp_type == 'noiseless'
n_restarts = 20
# We perform random trials of Bayesian Optimisation
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
for i in range(random_trials):
numpy_seed = i
np.random.seed(numpy_seed)
if opt_func == 'hosaki':
bounds = np.array([[0.0, 5.0], [
0.0, 5.0
]]) # bounds of the Bayesian Optimisation problem for Hosaki
else:
bounds = np.array([[0.0, 1.0],
[0.0, 1.0]]) # bounds for other test functions
# Initial noisy data points sampled uniformly at random from the input space.
if opt_func == 'hosaki':
X_init = np.random.uniform(0.0, 5.0, size=(grid_size**2, 2))
Y_init = hosaki_function(X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
x1_star = np.arange(0.0, 5.0, 0.5)
x2_star = np.arange(0.0, 5.0, 0.5)
else:
X_init = np.random.uniform(0.0, 1.0, size=(grid_size**2, 2))
x1_star = np.arange(0.0, 1.0, 0.1)
x2_star = np.arange(0.0, 1.0, 0.1)
if opt_func == 'branin':
Y_init = branin_function(X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
else:
Y_init = goldstein_price_function(
X_init[:, 0],
X_init[:, 1],
heteroscedastic=heteroscedastic)
plot_sample = np.array(np.meshgrid(x1_star, x2_star)).T.reshape(
-1, 2) # Where 2 gives the dimensionality
# Initialize samples
Y_init = Y_init[0]
homo_X_sample = X_init
homo_Y_sample = Y_init
het_X_sample = X_init
het_Y_sample = Y_init
aug_X_sample = X_init
aug_Y_sample = Y_init
aug_het_X_sample = X_init
aug_het_Y_sample = Y_init
# initial GP hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0 # optimise noise for homoscedastic GP
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
het_best_so_far = -300
aug_het_best_so_far = -300
het_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
het_obj_val_list = []
aug_het_obj_val_list = []
het_noise_val_list = []
aug_het_noise_val_list = []
het_collected_x1 = []
het_collected_x2 = []
aug_het_collected_x1 = []
aug_het_collected_x2 = []
for j in range(bayes_opt_iters):
print(j)
# random sampling baseline
seed = bayes_opt_iters * i + j # This approach
print(f'Seed is: {seed}')
np.random.seed(seed)
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(
heteroscedastic_expected_improvement,
het_X_sample,
het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300,
aleatoric_weight=aleatoric_weight)
het_collected_x1.append(het_X_next[:, 0])
het_collected_x2.append(het_X_next[:, 1])
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
het_Y_next = hosaki_function(het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = hosaki_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = hosaki_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
het_Y_next = branin_function(het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = branin_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = branin_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
het_Y_next = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
het_Y_next = het_Y_next[0]
_, het_noise_val, het_composite_obj_val = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
het_composite_obj_val -= penalty * het_noise_val
else:
het_composite_obj_val = goldstein_price_function(
het_X_next[:, 0],
het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if het_composite_obj_val > het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if heteroscedastic:
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(
heteroscedastic_augmented_expected_improvement,
aug_het_X_sample,
aug_het_Y_sample,
noise,
l_init,
sigma_f_init,
l_noise_init,
sigma_f_noise_init,
gp2_noise,
num_iters,
sample_size,
bounds,
plot_sample,
n_restarts=n_restarts,
min_val=300,
aleatoric_weight=aleatoric_weight_aug)
aug_het_collected_x1.append(aug_het_X_next[:, 0])
aug_het_collected_x2.append(aug_het_X_next[:, 1])
# Obtain next noisy sample from the objective function
if opt_func == 'hosaki':
aug_het_Y_next = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = hosaki_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
elif opt_func == 'branin':
aug_het_Y_next = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = branin_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
else:
aug_het_Y_next = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=noise_level,
heteroscedastic=heteroscedastic)
if heteroscedastic:
aug_het_Y_next = aug_het_Y_next[0]
_, aug_het_noise_val, aug_het_composite_obj_val = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
aug_het_composite_obj_val -= penalty * aug_het_noise_val
else:
aug_het_composite_obj_val = goldstein_price_function(
aug_het_X_next[:, 0],
aug_het_X_next[:, 1],
noise=0.0,
heteroscedastic=heteroscedastic)
if aug_het_composite_obj_val > aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
hetero_running_sum += np.array(het_obj_val_list,
dtype=np.float64).flatten()
hetero_squares += np.array(het_obj_val_list,
dtype=np.float64).flatten()**2
aug_het_running_sum += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()
aug_het_squares += np.array(aug_het_obj_val_list,
dtype=np.float64).flatten()**2
hetero_noise_running_sum += np.array(het_noise_val_list,
dtype=np.float64).flatten()
hetero_noise_squares += np.array(het_noise_val_list,
dtype=np.float64).flatten()**2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()
aug_het_noise_squares += np.array(aug_het_noise_val_list,
dtype=np.float64).flatten()**2
# results are negated to turn problem into minimisation for consistency.
hetero_means = -hetero_running_sum / random_trials
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
aug_het_means = -aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means**2,
dtype=np.float64)) / np.sqrt(random_trials)
hetero_noise_means = hetero_noise_running_sum / random_trials
hetero_noise_errs = (
np.sqrt(hetero_noise_squares / random_trials -
hetero_noise_means**2)) / np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (
np.sqrt(aug_het_noise_squares / random_trials -
aug_het_noise_means**2)) / np.sqrt(random_trials)
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_errs))
print('List of average het-AEI values is: ' + str(aug_het_means))
print('List of het-AEI errors is: ' + str(aug_het_errs))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, hetero_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_het_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x,
lower_hetero,
upper_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x,
lower_het_aei,
upper_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
if penalty != 1:
plt.ylabel('f(x) +' + str(penalty) + 'g(x)', fontsize=14)
else:
plt.ylabel('f(x) + g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
tag = 'heteroscedastic'
plt.savefig(
'gamma_figures/{}_{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}'
'_hundred_times_penalty_is_{}_aleatoric_weight_aug_is_{}_{}_aug'.
format(opt_func, bayes_opt_iters, random_trials, grid_size, numpy_seed,
int(100 * penalty), aleatoric_weight_aug, tag),
bbox_inches='tight')
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_hetero = np.array(hetero_noise_means) - np.array(
hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(
hetero_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(
aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(
aug_het_noise_errs)
plt.plot(iter_x, hetero_noise_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_het_noise_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x,
lower_noise_hetero,
upper_noise_hetero,
color='tab:green',
alpha=0.1)
plt.fill_between(iter_x,
lower_noise_het_aei,
upper_noise_het_aei,
color='tab:purple',
alpha=0.1)
plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.legend(loc='lower left',
bbox_to_anchor=(0.0, -0.425),
ncol=3,
borderaxespad=0,
fontsize=14,
frameon=False)
plt.savefig(
'gamma_figures/{}_{}_iters_{}_random_trials_and_grid_size_of_{}_and_seed_{}_'
'noise_only_hundred_times_penalty_is_{}_aleatoric_weight_aug_is_{}_aug'
.format(opt_func, bayes_opt_iters, random_trials, grid_size,
numpy_seed, int(100 * penalty), aleatoric_weight_aug),
bbox_inches='tight')
# Save data for cosmetic plotting
np.savetxt(
f'synth_saved_data/gamma/hetero_means_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
hetero_means)
np.savetxt(
f'synth_saved_data/gamma/aug_het_means_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
aug_het_means)
np.savetxt(
f'synth_saved_data/gamma/lower_hetero_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
lower_hetero)
np.savetxt(
f'synth_saved_data/gamma/upper_hetero_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
upper_hetero)
np.savetxt(
f'synth_saved_data/gamma/lower_het_aei_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
lower_het_aei)
np.savetxt(
f'synth_saved_data/gamma/upper_het_aei_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
upper_het_aei)
np.savetxt(
f'synth_saved_data/gamma/hetero_noise_means_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
hetero_noise_means)
np.savetxt(
f'synth_saved_data/gamma/aug_het_noise_means_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
aug_het_noise_means)
np.savetxt(
f'synth_saved_data/gamma/lower_noise_hetero_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
lower_noise_hetero)
np.savetxt(
f'synth_saved_data/gamma/upper_noise_hetero_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
upper_noise_hetero)
np.savetxt(
f'synth_saved_data/gamma/lower_noise_het_aei_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
lower_noise_het_aei)
np.savetxt(
f'synth_saved_data/gamma/upper_noise_het_aei_aleatoric_weight_is_{aleatoric_weight_aug}_aug.txt',
upper_noise_het_aei)
def main(penalty, aleatoric_weight, random_trials, bayes_opt_iters):
"""
Run experiments on the sin wave function.
param: penalty: $\alpha$ parameter specifying weight of noise component to objective
param: aleatoric_weight: float specifying the value of both $\beta and $\gamma of ANPEI and HAEI
param: random_trials: int specifying the number of random initialisations
param: bayes_opt_iters: int specifying the number of iterations of BayesOpt
"""
coefficient = 0.2 # tunes the relative size of the maxima in the function (used when modification = True)
noise_coeff = 0.5 # noise coefficient will be noise(X) will be linear e.g. 0.2 * X
# We perform random trials of Bayesian Optimisation
rand_running_sum = np.zeros(bayes_opt_iters)
rand_squares = np.zeros(bayes_opt_iters)
homo_running_sum = np.zeros(bayes_opt_iters)
homo_squares = np.zeros(bayes_opt_iters) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_running_sum = np.zeros(bayes_opt_iters)
hetero_squares = np.zeros(bayes_opt_iters)
aug_running_sum = np.zeros(bayes_opt_iters)
aug_squares = np.zeros(bayes_opt_iters)
aug_het_running_sum = np.zeros(bayes_opt_iters)
aug_het_squares = np.zeros(bayes_opt_iters)
# We compute the objective corresponding to aleatoric noise only
rand_noise_running_sum = np.zeros(bayes_opt_iters)
rand_noise_squares = np.zeros(bayes_opt_iters)
homo_noise_running_sum = np.zeros(bayes_opt_iters)
homo_noise_squares = np.zeros(bayes_opt_iters) # Following the single-pass estimator given on pg. 192 of mathematics for machine learning
hetero_noise_running_sum = np.zeros(bayes_opt_iters)
hetero_noise_squares = np.zeros(bayes_opt_iters)
aug_noise_running_sum = np.zeros(bayes_opt_iters)
aug_noise_squares = np.zeros(bayes_opt_iters)
aug_het_noise_running_sum = np.zeros(bayes_opt_iters)
aug_het_noise_squares = np.zeros(bayes_opt_iters)
fplot = True # plot data
for i in range(random_trials): # random trials to get average across x number of trials
numpy_seed = i
np.random.seed(numpy_seed)
bounds = np.array([0, 10]).reshape(-1, 1) # bounds of the Bayesian Optimisation problem.
# Initial noisy data points sampled uniformly at random from the input space.
init_num_samples = 25
X_init = np.random.uniform(0, 10, init_num_samples).reshape(-1, 1) # sample 10 points at random from the bounds to initialise with
plot_sample = np.linspace(0, 10, 50).reshape(-1, 1) # samples for plotting purposes
if i > 0:
fplot = False
Y_init = linear_sin_noise(X_init, noise_coeff, plot_sample, coefficient, fplot=fplot)
# Initialize samples
homo_X_sample = X_init.reshape(-1, 1)
homo_Y_sample = Y_init.reshape(-1, 1)
het_X_sample = X_init.reshape(-1, 1)
het_Y_sample = Y_init.reshape(-1, 1)
aug_X_sample = X_init.reshape(-1, 1)
aug_Y_sample = Y_init.reshape(-1, 1)
aug_het_X_sample = X_init.reshape(-1, 1)
aug_het_Y_sample = Y_init.reshape(-1, 1)
# initial BayesOpt hypers
l_init = 1.0
sigma_f_init = 1.0
noise = 1.0
l_noise_init = 1.0
sigma_f_noise_init = 1.0
gp2_noise = 1.0
num_iters = 10
sample_size = 100
rand_best_so_far = -300 # value to beat
homo_best_so_far = -300
het_best_so_far = -300
aug_best_so_far = -300
aug_het_best_so_far = -300
rand_noise_best_so_far = 300 # value to beat
homo_noise_best_so_far = 300
het_noise_best_so_far = 300
aug_noise_best_so_far = 300
aug_het_noise_best_so_far = 300
rand_obj_val_list = []
homo_obj_val_list = []
het_obj_val_list = []
aug_obj_val_list = []
aug_het_obj_val_list = []
rand_noise_val_list = []
homo_noise_val_list = []
het_noise_val_list = []
aug_noise_val_list = []
aug_het_noise_val_list = []
for i in range(bayes_opt_iters):
# number of BO iterations i.e. number of times sampled from black-box function using the acquisition function.
# random sampling first
# take random point from uniform distribution
rand_X_next = np.random.uniform(0, 10) # this just takes X not the sin function itself
# check if random point's Y value is better than best so far
rand_composite_obj_val, rand_noise_val = max_sin_noise_objective(rand_X_next, noise_coeff, coefficient,
fplot=False, penalty=penalty)
if rand_composite_obj_val > rand_best_so_far:
rand_best_so_far = rand_composite_obj_val
rand_obj_val_list.append(rand_composite_obj_val)
else:
rand_obj_val_list.append(rand_best_so_far)
# if yes, save it, if no, save best so far into list of best y-value per iteration in rand_composite_obj_val
if rand_noise_val < rand_noise_best_so_far:
rand_noise_best_so_far = rand_noise_val
rand_noise_val_list.append(rand_noise_val)
else:
rand_noise_val_list.append(rand_noise_best_so_far)
print(i)
# Obtain next sampling point from the acquisition function (expected_improvement)
homo_X_next = my_propose_location(my_expected_improvement, homo_X_sample, homo_Y_sample, noise, l_init,
sigma_f_init, bounds, plot_sample, n_restarts=3, min_val=300)
# Obtain next noisy sample from the objective function
homo_Y_next = linear_sin_noise(homo_X_next, noise_coeff, plot_sample, coefficient, fplot=False)
homo_composite_obj_val, homo_noise_val = max_sin_noise_objective(homo_X_next, noise_coeff, coefficient, fplot=False, penalty=penalty)
# if the new Y-value is better than our best so far, save it as best so far and append it to best Y-values list in *_composite_obj_val
if homo_composite_obj_val > homo_best_so_far:
homo_best_so_far = homo_composite_obj_val
homo_obj_val_list.append(homo_composite_obj_val)
else:
homo_obj_val_list.append(homo_best_so_far)
if homo_noise_val < homo_noise_best_so_far:
homo_noise_best_so_far = homo_noise_val
homo_noise_val_list.append(homo_noise_val)
else:
homo_noise_val_list.append(homo_noise_best_so_far)
# Add sample to previous samples
homo_X_sample = np.vstack((homo_X_sample, homo_X_next))
homo_Y_sample = np.vstack((homo_Y_sample, homo_Y_next))
# Obtain next sampling point from the het acquisition function (ANPEI)
het_X_next = heteroscedastic_propose_location(heteroscedastic_expected_improvement, het_X_sample,
het_Y_sample, noise, l_init, sigma_f_init, l_noise_init,
sigma_f_noise_init, gp2_noise, num_iters, sample_size, bounds,
plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight)
# Obtain next noisy sample from the objective function
het_Y_next = linear_sin_noise(het_X_next, noise_coeff, plot_sample, coefficient, fplot=False)
het_composite_obj_val, het_noise_val = max_sin_noise_objective(het_X_next, noise_coeff, coefficient,
fplot=False, penalty=penalty)
if het_composite_obj_val > het_best_so_far:
het_best_so_far = het_composite_obj_val
het_obj_val_list.append(het_composite_obj_val)
else:
het_obj_val_list.append(het_best_so_far)
if het_noise_val < het_noise_best_so_far:
het_noise_best_so_far = het_noise_val
het_noise_val_list.append(het_noise_val)
else:
het_noise_val_list.append(het_noise_best_so_far)
# Add sample to previous samples
het_X_sample = np.vstack((het_X_sample, het_X_next))
het_Y_sample = np.vstack((het_Y_sample, het_Y_next))
# Obtain next sampling point from the augmented expected improvement (AEI)
aug_X_next = my_propose_location(augmented_expected_improvement, aug_X_sample, aug_Y_sample, noise,l_init,
sigma_f_init, bounds, plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight, aei=True)
# Obtain next noisy sample from the objective function
aug_Y_next = linear_sin_noise(aug_X_next, noise_coeff, plot_sample, coefficient, fplot=False)
aug_composite_obj_val, aug_noise_val = max_sin_noise_objective(aug_X_next, noise_coeff, coefficient, fplot=False, penalty=penalty)
if aug_composite_obj_val > aug_best_so_far:
aug_best_so_far = aug_composite_obj_val
aug_obj_val_list.append(aug_composite_obj_val)
else:
aug_obj_val_list.append(aug_best_so_far)
if aug_noise_val < aug_noise_best_so_far:
aug_noise_best_so_far = aug_noise_val
aug_noise_val_list.append(aug_noise_val)
else:
aug_noise_val_list.append(aug_noise_best_so_far)
# Add sample to previous sample
aug_X_sample = np.vstack((aug_X_sample, aug_X_next))
aug_Y_sample = np.vstack((aug_Y_sample, aug_Y_next))
# Obtain next sampling point from the heteroscedastic augmented expected improvement (het-AEI)
aug_het_X_next = heteroscedastic_propose_location(heteroscedastic_augmented_expected_improvement,
aug_het_X_sample,aug_het_Y_sample, noise, l_init, sigma_f_init,
l_noise_init, sigma_f_noise_init, gp2_noise, num_iters, sample_size,
bounds, plot_sample, n_restarts=3, min_val=300, aleatoric_weight=aleatoric_weight)
# Obtain next noisy sample from the objective function
aug_het_Y_next = linear_sin_noise(aug_het_X_next, noise_coeff, plot_sample, coefficient, fplot=False)
aug_het_composite_obj_val, aug_het_noise_val = max_sin_noise_objective(aug_het_X_next, noise_coeff, coefficient, fplot=False, penalty=penalty)
if aug_het_composite_obj_val > aug_het_best_so_far:
aug_het_best_so_far = aug_het_composite_obj_val
aug_het_obj_val_list.append(aug_het_composite_obj_val)
else:
aug_het_obj_val_list.append(aug_het_best_so_far)
if aug_het_noise_val < aug_het_noise_best_so_far:
aug_het_noise_best_so_far = aug_het_noise_val
aug_het_noise_val_list.append(aug_het_noise_val)
else:
aug_het_noise_val_list.append(aug_het_noise_best_so_far)
# Add sample to previous sample
aug_het_X_sample = np.vstack((aug_het_X_sample, aug_het_X_next))
aug_het_Y_sample = np.vstack((aug_het_Y_sample, aug_het_Y_next))
# adding the best values in order of iteration on top of each other element wise
rand_running_sum += np.array(rand_obj_val_list) # just the way to average out across all random trials
rand_squares += np.array(rand_obj_val_list) ** 2 # likewise for errors
homo_running_sum += np.array(homo_obj_val_list)
homo_squares += np.array(homo_obj_val_list) ** 2
hetero_running_sum += np.array(het_obj_val_list)
hetero_squares += np.array(het_obj_val_list) ** 2
aug_running_sum += np.array(aug_obj_val_list)
aug_squares += np.array(aug_obj_val_list) ** 2
aug_het_running_sum += np.array(aug_het_obj_val_list)
aug_het_squares += np.array(aug_het_obj_val_list) ** 2
rand_noise_running_sum += np.array(rand_noise_val_list) # just the way to average out across all random trials
rand_noise_squares += np.array(rand_noise_val_list) ** 2 # likewise for errors
homo_noise_running_sum += np.array(homo_noise_val_list)
homo_noise_squares += np.array(homo_noise_val_list) ** 2
hetero_noise_running_sum += np.array(het_noise_val_list)
hetero_noise_squares += np.array(het_noise_val_list) ** 2
aug_noise_running_sum += np.array(aug_noise_val_list)
aug_noise_squares += np.array(aug_noise_val_list) ** 2
aug_het_noise_running_sum += np.array(aug_het_noise_val_list)
aug_het_noise_squares += np.array(aug_het_noise_val_list) ** 2
rand_means = rand_running_sum / random_trials
homo_means = homo_running_sum / random_trials
hetero_means = hetero_running_sum / random_trials
rand_errs = (np.sqrt(rand_squares / random_trials - rand_means ** 2))/np.sqrt(random_trials)
homo_errs = (np.sqrt(homo_squares / random_trials - homo_means ** 2))/np.sqrt(random_trials)
hetero_errs = (np.sqrt(hetero_squares / random_trials - hetero_means ** 2))/np.sqrt(random_trials)
aug_means = aug_running_sum / random_trials
aug_errs = (np.sqrt(aug_squares / random_trials - aug_means ** 2))/np.sqrt(random_trials)
aug_het_means = aug_het_running_sum / random_trials
aug_het_errs = (np.sqrt(aug_het_squares / random_trials - aug_het_means ** 2))/np.sqrt(random_trials)
rand_noise_means = rand_noise_running_sum / random_trials
homo_noise_means = homo_noise_running_sum / random_trials
hetero_noise_means = hetero_noise_running_sum / random_trials
rand_noise_errs = (np.sqrt(rand_noise_squares / random_trials - rand_noise_means ** 2))/np.sqrt(random_trials)
homo_noise_errs = (np.sqrt(homo_noise_squares / random_trials - homo_noise_means ** 2))/np.sqrt(random_trials)
hetero_noise_errs = (np.sqrt(hetero_noise_squares / random_trials - hetero_noise_means ** 2))/np.sqrt(random_trials)
aug_noise_means = aug_noise_running_sum / random_trials
aug_noise_errs = (np.sqrt(aug_noise_squares / random_trials - aug_noise_means ** 2))/np.sqrt(random_trials)
aug_het_noise_means = aug_het_noise_running_sum / random_trials
aug_het_noise_errs = (np.sqrt(aug_het_noise_squares / random_trials - aug_het_noise_means ** 2))/np.sqrt(random_trials)
print('List of average random sampling values is: ' + str(rand_means))
print('List of random sampling errors is:' + str(rand_errs))
print('List of average homoscedastic values is: ' + str(homo_means))
print('List of homoscedastic errors is: ' + str(homo_errs))
print('List of average heteroscedastic values is ' + str(hetero_means))
print('List of heteroscedastic errors is: ' + str(hetero_errs))
print('List of average AEI values is: ' + str(aug_means))
print('List of AEI errors is: ' + str(aug_errs))
print('List of average het-AEI values is: ' + str(aug_het_means))
print('List of het-AEI errors is: ' + str(aug_het_errs))
iter_x = np.arange(1, bayes_opt_iters + 1)
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_rand = np.array(rand_means) - np.array(rand_errs)
upper_rand = np.array(rand_means) + np.array(rand_errs)
lower_homo = np.array(homo_means) - np.array(homo_errs)
upper_homo = np.array(homo_means) + np.array(homo_errs)
lower_hetero = np.array(hetero_means) - np.array(hetero_errs)
upper_hetero = np.array(hetero_means) + np.array(hetero_errs)
lower_aei = np.array(aug_means) - np.array(aug_errs)
upper_aei = np.array(aug_means) + np.array(aug_errs)
lower_het_aei = np.array(aug_het_means) - np.array(aug_het_errs)
upper_het_aei = np.array(aug_het_means) + np.array(aug_het_errs)
plt.plot(iter_x, rand_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x, lower_rand, upper_rand, color='tab:orange', alpha=0.1)
plt.fill_between(iter_x, lower_homo, upper_homo, color='tab:blue', alpha=0.1)
plt.fill_between(iter_x, lower_hetero, upper_hetero, color='tab:green', alpha=0.1)
plt.fill_between(iter_x, lower_aei, upper_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x, lower_het_aei, upper_het_aei, color='tab:purple', alpha=0.1)
#plt.title('Best Objective Function Value Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
if penalty > 1:
plt.ylabel(f'f(x) - {penalty}*g(x)', fontsize=14)
else:
plt.ylabel('f(x) - g(x)', fontsize=14)
plt.yticks([3, 4, 5, 6, 7])
plt.tick_params(labelsize=14)
#plt.legend(loc=4, fontsize=14)
plt.legend(loc='lower left', bbox_to_anchor=(0.0, -0.425), ncol=3, borderaxespad=0, fontsize=14, frameon=False)
#plt.tight_layout()
plt.savefig('toy_figures/bayesopt_plot{}_iters_{}_random_trials_and_{}_coefficient_times_100_and_noise_coeff_times_'
'100_of_{}_init_num_samples_of_{}_and_seed_{}_new_penalty_is_{}_aleatoric_weight_is_{}_new_aei2'.format(bayes_opt_iters, random_trials,
int(coefficient * 100), int(noise_coeff * 100),
init_num_samples, numpy_seed, penalty, aleatoric_weight), bbox_inches='tight')
plt.close()
# clear figure from previous fplot returns if fiddling with form of function
plt.cla()
ax = plt.gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
lower_noise_rand = np.array(rand_noise_means) - np.array(rand_noise_errs)
upper_noise_rand = np.array(rand_noise_means) + np.array(rand_noise_errs)
lower_noise_homo = np.array(homo_noise_means) - np.array(homo_noise_errs)
upper_noise_homo = np.array(homo_noise_means) + np.array(homo_noise_errs)
lower_noise_hetero = np.array(hetero_noise_means) - np.array(hetero_noise_errs)
upper_noise_hetero = np.array(hetero_noise_means) + np.array(hetero_noise_errs)
lower_noise_aei = np.array(aug_noise_means) - np.array(aug_noise_errs)
upper_noise_aei = np.array(aug_noise_means) + np.array(aug_noise_errs)
lower_noise_het_aei = np.array(aug_het_noise_means) - np.array(aug_het_noise_errs)
upper_noise_het_aei = np.array(aug_het_noise_means) + np.array(aug_het_noise_errs)
plt.plot(iter_x, rand_noise_means, color='tab:orange', label='RS')
plt.plot(iter_x, homo_noise_means, color='tab:blue', label='EI')
plt.plot(iter_x, hetero_noise_means, color='tab:green', label='ANPEI')
plt.plot(iter_x, aug_noise_means, color='tab:red', label='AEI')
plt.plot(iter_x, aug_het_noise_means, color='tab:purple', label='HAEI')
plt.fill_between(iter_x, lower_noise_rand, upper_noise_rand, color='tab:orange', alpha=0.1)
plt.fill_between(iter_x, lower_noise_homo, upper_noise_homo, color='tab:blue', alpha=0.1)
plt.fill_between(iter_x, lower_noise_hetero, upper_noise_hetero, color='tab:green', alpha=0.1)
plt.fill_between(iter_x, lower_noise_aei, upper_noise_aei, color='tab:red', alpha=0.1)
plt.fill_between(iter_x, lower_noise_het_aei, upper_noise_het_aei, color='tab:purple', alpha=0.1)
#plt.title('Lowest Aleatoric Noise Found so Far', fontsize=16)
plt.xlabel('Function Evaluations', fontsize=14)
plt.ylabel('g(x)', fontsize=14)
plt.tick_params(labelsize=14)
plt.yticks([1, 2, 3])
#plt.legend(loc=1, fontsize=14)
plt.legend(loc='lower left', bbox_to_anchor=(0.0, -0.425), ncol=3, borderaxespad=0, fontsize=14, frameon=False)
#plt.tight_layout()
plt.savefig('toy_figures/bayesopt_plot{}_iters_{}_random_trials_and_{}_coefficient_times_100_and_noise_coeff_times_'
'100_of_{}_init_num_samples_of_{}_and_seed_{}_noise_only_penalty_is_{}_aleatoric_weight_is_{}_new_aei2'.format(bayes_opt_iters, random_trials,
int(coefficient * 100), int(noise_coeff * 100),
init_num_samples, numpy_seed, penalty, aleatoric_weight), bbox_inches='tight')