def plot_gp(optimizer, x=None, y=None, set_xlim=(-2, 10)):
"""
Plot the Gaussian posterior and the utility function after one or more
optimization steps.
Taken from
https://github.com/fmfn/BayesianOptimization/blob/master/examples/visualization.ipynb
"""
fig = plt.figure(figsize=(10, 5))
steps = len(optimizer.space)
fig.suptitle(
'Gaussian Process and Utility Function After {} Steps'.format(steps),
fontdict={'size': 30}
)
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
axis = plt.subplot(gs[0])
acq = plt.subplot(gs[1])
x_obs = np.array([[res["params"]["x"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
if x is not None:
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
else:
bounds = optimizer._space._bounds[0]
x0, x1 = bounds
x = np.linspace(x0, x1, 1000).reshape(-1, 1)
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
if (x is not None) and (y is not None):
axis.plot(x, y, linewidth=3, label='Target')
axis.plot(x_obs.flatten(), y_obs, 'D', markersize=8,
label=u'Observations', color='r')
axis.plot(x, mu, '--', color='k', label='Prediction')
axis.fill(np.concatenate([x, x[::-1]]),
np.concatenate([mu - 1.9600 * sigma,
(mu + 1.9600 * sigma)[::-1]]),
alpha=.6, fc='c', ec='None', label='95% confidence interval')
#axis.set_xlim(set_xlim)
axis.set_ylim((None, None))
axis.set_ylabel('f(x)', fontdict={'size': 20})
axis.set_xlabel('x', fontdict={'size': 20})
utility_function = UtilityFunction(kind="ucb", kappa=5, xi=0)
utility = utility_function.utility(x, optimizer._gp, 0)
acq.plot(x, utility, label='Utility Function', color='purple')
acq.plot(x[np.argmax(utility)], np.max(utility), '*', markersize=15,
label=u'Next Best Guess', markerfacecolor='gold',
markeredgecolor='k', markeredgewidth=1)
acq.set_xlim(set_xlim)
acq.set_ylim((0, np.max(utility) + 0.5))
acq.set_ylabel('Utility', fontdict={'size': 20})
acq.set_xlabel('x', fontdict={'size': 20})
axis.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
acq.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
def plot_gp(optimizer, x, y):
fig = plt.figure(figsize=(16, 10))
steps = len(optimizer.space)
fig.suptitle(
'Gaussian Process and Utility Function After {} Steps'.format(steps),
fontdict={'size': 30})
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
axis = plt.subplot(gs[0])
acq = plt.subplot(gs[1])
x_obs = np.array([[res["params"]["x"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
axis.plot(x, y, linewidth=3, label='Target')
axis.plot(x_obs.flatten(),
y_obs,
'D',
markersize=8,
label=u'Observations',
color='r')
axis.plot(x, mu, '--', color='k', label='Prediction')
axis.fill(np.concatenate([x, x[::-1]]),
np.concatenate(
[mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=.6,
fc='c',
ec='None',
label='95% confidence interval')
axis.set_xlim((-2, 10))
axis.set_ylim((None, None))
axis.set_ylabel('f(x)', fontdict={'size': 20})
axis.set_xlabel('x', fontdict={'size': 20})
utility_function = UtilityFunction(kind="ei", kappa=None, xi=0)
utility = utility_function.utility(x, optimizer._gp, 0)
acq.plot(x, utility, label='Utility Function', color='purple')
acq.plot(x[np.argmax(utility)],
np.max(utility),
'*',
markersize=15,
label=u'Next Best Guess',
markerfacecolor='gold',
markeredgecolor='k',
markeredgewidth=1)
acq.set_xlim((-2, 10))
acq.set_ylim((0, np.max(utility) + 0.5))
acq.set_ylabel('Utility', fontdict={'size': 20})
acq.set_xlabel('x', fontdict={'size': 20})
axis.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
acq.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
plt.show()
def plot_gp(optimizer, x):
fig = plt.figure()
steps = len(optimizer.space)
fig.suptitle('Gaussian Process after {} steps'.format(steps),
fontdict={'size': 30})
axis = fig.add_subplot(111)
#gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
#axis = plt.subplot(gs[0])
#acq = plt.subplot(gs[1])
x_obs = numpy.array([[res["params"]["learning_rate"]]
for res in optimizer.res])
y_obs = numpy.array([res["target"] for res in optimizer.res])
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
#axis.plot(x, y, linewidth=3, label='Target')
unc = 0.0033 # calculated for Leptonic ttH vs ttGG
axis.errorbar(x_obs.flatten(),
y_obs,
yerr=numpy.ones(len(y_obs)) * unc,
label='Observations',
color='r',
marker='o',
markersize=8,
ls="none")
axis.plot(x, mu, '--', color='k', label='Prediction')
axis.fill(numpy.concatenate([x, x[::-1]]),
numpy.concatenate(
[mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=.6,
fc='c',
ec='None',
label='95% confidence interval')
axis.set_xlim((-5, -1))
axis.set_ylim((0.7, 0.82))
axis.set_ylabel('AUC', fontdict={'size': 20})
axis.set_xlabel('Learning Rate', fontdict={'size': 20})
utility_function = UtilityFunction(kind="ucb", kappa=5, xi=0)
#utility_function = UtilityFunction(kind="ei", xi=float(args.xi))
utility = utility_function.utility(x, optimizer._gp, 0)
#acq.plot(x, utility, label='Utility Function', color='purple')
#acq.plot(x[numpy.argmax(utility)], numpy.max(utility), '*', markersize=15,
#label=u'Next Best Guess', markerfacecolor='gold', markeredgecolor='k', markeredgewidth=1)
#acq.set_xlim((-2, 10))
#acq.set_ylim((0, numpy.max(utility) + 0.5))
#acq.set_ylabel('Utility', fontdict={'size':20})
#acq.set_xlabel('x', fontdict={'size':20})
axis.legend(loc='upper left')
#acq.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
plt.savefig('optimization_step_%d.pdf' % (steps), bbox_inches='tight')
plt.clf()
def plot_gp(optimizer, x, y):
steps = len(optimizer.space)
x_obs = np.array([[res["params"]["x"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
fig, axes = plt.subplots(2, 1, figsize=(9, 7))
ax = axes[0]
ax.plot(x, y, linewidth=3, label='Target')
ax.plot(x_obs.flatten(), y_obs, 'd', label=u'Observations', color='r')
ax.plot(x, mu, '--', color='k', label='Prediction')
ax.fill(np.concatenate([x, x[::-1]]),
np.concatenate([mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=.6,
fc='c',
ec='None',
label='95% confidence interval')
ax.set_ylabel('f(x)')
ax.set_xlabel('x')
ax.set_title(f'Gaussian Process and Utility Function After {steps} Steps')
ax.grid()
ax.legend(bbox_to_anchor=(1, 0.5))
utility_function = UtilityFunction(kind="ucb", kappa=5, xi=0)
utility = utility_function.utility(x, optimizer._gp, 0)
ax = axes[1]
ax.plot(x, utility, label='Utility function', color='purple')
ax.plot(x[np.argmax(utility)],
np.max(utility),
'*',
markersize=15,
label=u'Next best guess',
markerfacecolor='gold',
markeredgecolor='k',
markeredgewidth=1)
ax.set_ylim((0, np.max(utility) + 0.5))
ax.set_ylabel('Utility')
ax.set_xlabel('x')
ax.grid()
ax.legend(bbox_to_anchor=(1, 0.5))
return fig
axis.set_xlabel('NIter', fontdict={'size': 20})
acq.set_xlabel('NIter', fontdict={'size': 20})
axis.legend(loc=1, borderaxespad=0.)
acq.legend(loc=1, borderaxespad=0.)
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
def refit():
x_obs = np.array([[res["params"]["x"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
optimizer._gp.fit(x_obs, y_obs)
optimizer.probe({'x': 5}, lazy=False)
prev_probe = None
next_probe = 5
while prev_probe != next_probe:
refit()
if prev_probe == 1:
plot_gp(optimizer, x, y=None)
plt.show()
refit()
utility_int = utility_function.utility(x_int, optimizer._gp, 0)
prev_probe = next_probe
next_probe = np.argmax(utility_int)
optimizer.probe({'x': next_probe}, lazy=False)
def plot_gp(optimizer, x, y, util_func="ucb", kappa=5, xi=0.01):
plt.figure(figsize=(14, 8))
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
axis = plt.subplot(gs[0])
acq = plt.subplot(gs[1])
x_obs = np.array([[res["params"]["p"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
steps = len(optimizer.space)
current_max_target_function = optimizer.max["target"]
current_best_param = optimizer.max["params"]["p"]
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
# axis.plot(x, y, linewidth=3, label="Target")
axis.plot(x_obs.flatten(), y_obs, "D", markersize=8, label="Observations", color="r")
axis.plot(x, mu, "--", color="k", label="Prediction")
axis.fill(
np.concatenate([x, x[::-1]]),
np.concatenate([mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=0.4,
fc="c",
ec="None",
label="95% confidence interval",
)
# axis.set_xlim((x_obs.min(), x_obs.max()))
# TODO 20190614 if the score is negative, the current ylim will trim out some range.
# a workaround is to add sign of the value, but not tested
axis.set_ylim((0.85 * y_obs.min() * np.sign(y_obs.min()),
1.15 * y_obs.max() * np.sign(y_obs.max())))
axis.set_ylabel("tsne_with_metric_and_constraint", fontdict={"size": 16})
utility_function = UtilityFunction(kind=util_func, kappa=kappa, xi=xi)
utility = utility_function.utility(x, optimizer._gp, y_max=current_max_target_function)
acq.plot(x, utility, label=f"Utility Function ({util_func})", color="purple")
acq.plot(
x[np.argmax(utility)],
np.max(utility),
"*",
markersize=15,
label="Next Best Guess",
markerfacecolor="gold",
markeredgecolor="k",
markeredgewidth=1,
)
# acq.set_xlim((x_obs.min(), x_obs.max()))
# acq.set_ylim((0, np.max(utility) + 0.5))
acq.set_ylabel(f"Utility ({util_func})", fontdict={"size": 16})
acq.set_xlabel("perplexity", fontdict={"size": 16})
axis.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.0)
acq.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.0)
# debug next best guess
next_best_guess_param = x[np.argmax(utility)]
acq.set_title(f"Next best guess param: {next_best_guess_param}", fontdict={"size": 16})
# draw indicator vline @ the next perplexity
acq.axvline(next_best_guess_param, color='g', linestyle='--', alpha=0.4)
axis.axvline(next_best_guess_param, color='g', linestyle='--', alpha=0.4)
# draw indicator hline @ the current max value of the target function
axis.axhline([current_max_target_function], color='r', linestyle='--', alpha=0.4)
debug_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
debug_method_name = {
"ucb": f"ucb_kappa{kappa}",
"ei": f"ei_xi{xi}",
"poi": f"poi_xi{xi}"
}[util_func]
axis.set_title(f"Figure created @ {debug_time}", size=12)
plt.suptitle(
f"GP ({debug_method_name} utility function) after {steps} steps"
f" with best predicted perlexity = {current_best_param:.2f}", size=20)
fig_name = (f"./plots/{score_name}/{debug_method_name}"
f"_constraint{constraint_proportion}"
f"_{dataset_name}_step{steps}.png")
plt.savefig(fig_name, bbox_inches="tight")
mlflow.log_artifact(fig_name)
def plot_gp(bayes_optimizer, xrange,
y=None,
title=None,
plot_target=True,
plot_observations=True,
plot_prediction=True,
plot_ci=True,
plot_utility=True,
plot_next_best=True):
x = np.linspace(xrange[0], xrange[1], 10000).reshape(-1, 1)
fig = plt.figure(figsize=(16, 10))
steps = len(bayes_optimizer.space)
fig.suptitle(
'{} After {} Iterations'.format(title or "Bayesian Optimization", steps),
fontsize=30,
)
if plot_utility:
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
else:
gs = gridspec.GridSpec(1, 1, height_ratios=[1])
axis = plt.subplot(gs[0])
x_obs = np.array([[res["params"]["x"]] for res in bayes_optimizer.res])
y_obs = np.array([res["target"] for res in bayes_optimizer.res])
mu, sigma = posterior(bayes_optimizer, x_obs, y_obs, x)
if y is not None and plot_target:
axis.plot(x, y, linewidth=3, label='Target')
if plot_observations:
axis.plot(x_obs.flatten(), y_obs, 'D', markersize=8, label=u'Observations', color='r')
if plot_prediction:
axis.plot(x, mu, '--', color='k', label='Prediction')
if plot_ci:
axis.fill(np.concatenate([x, x[::-1]]),
np.concatenate([mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=.6, fc='c', ec='None', label='95% confidence interval')
axis.set_xlim(xrange)
axis.set_ylim((None, None))
axis.set_ylabel('f(x)', fontdict={'size': 20})
axis.set_xlabel('x', fontdict={'size': 20})
axis.legend(loc=0, borderaxespad=0.6)
if plot_utility:
acq = plt.subplot(gs[1])
utility_function = UtilityFunction(kind="ucb", kappa=5, xi=0)
utility = utility_function.utility(x, bayes_optimizer._gp, 0)
acq.plot(x, utility, label='Utility Function', color='purple')
if plot_next_best:
acq.plot(x[np.argmax(utility)], np.max(utility), '*', markersize=15,
label=u'Next Best Guess', markerfacecolor='gold', markeredgecolor='k', markeredgewidth=1)
acq.set_xlim(xrange)
acq.set_ylim((0, np.max(utility) + 0.5))
acq.set_ylabel('Utility', fontsize=20)
acq.set_xlabel('x', fontsize=20)
acq.legend(loc=0, borderaxespad=0.6)
plt.show()
class PlotProgress_2D:
def __init__(self, optimizer, true_function=None, cost=None):
self.optimizer = optimizer
self.true_function = true_function
self.cost = cost
self.phi, self.theta = 30, 45 # Default 3D plot camera rotations
self.N = 100
self._first_plot = True
self.target_space = optimizer._space
self.bounds = self.target_space._bounds
self.axis_names = self.target_space._keys
if self.target_space.dim != 2:
raise NotImplementedError('3d or higher dimensional visualization '
'not supported.')
def plot(self):
if self._first_plot:
self._setup_first()
self._update_plots()
self._first_plot = False
plt.pause(0.0001)
def _setup_grid(self):
self.X_ = np.linspace(self.bounds[0][0], self.bounds[0][1], self.N)
self.Y_ = np.linspace(self.bounds[1][0], self.bounds[1][1], self.N)
self.X, self.Y = np.meshgrid(self.X_, self.Y_)
self.design_matrix = np.array(list(itertools.product(self.X_,
self.Y_)))
def _setup_first(self):
self.fig = plt.figure()
if (self.true_function is not None) and (self.cost is not None):
self.ax_true = self.fig.add_subplot(2, 2, 1, projection='3d')
self.ax_opt = self.fig.add_subplot(2, 2, 2, projection='3d')
self.ax_cost = self.fig.add_subplot(2, 2, 3)
self.ax_utility = self.fig.add_subplot(2, 2, 4, projection='3d')
elif self.true_function is not None:
self.ax_true = self.fig.add_subplot(2, 2, 1, projection='3d')
self.ax_opt = self.fig.add_subplot(2, 2, 2, projection='3d')
self.ax_utility = self.fig.add_subplot(2, 2, 4, projection='3d')
elif self.cost is not None:
self.ax_opt = self.fig.add_subplot(2, 2, 2, projection='3d')
self.ax_cost = self.fig.add_subplot(2, 2, 3)
self.ax_utility = self.fig.add_subplot(2, 2, 4, projection='3d')
else:
self.ax_opt = self.fig.add_subplot(2, 2, 2, projection='3d')
self.ax_utility = self.fig.add_subplot(2, 2, 4, projection='3d')
self._setup_grid()
if self.true_function is not None:
self.ax_true.plot_surface(self.X, self.Y,
self.true_function(self.X, self.Y))
self.ax_true.view_init(self.phi, self.theta)
def _update_plots(self):
self._update_ax_utility()
self._update_ax_opt()
self._update_ax_cost()
plt.pause(1e-15)
def _update_ax_opt(self):
self.ax_opt.clear()
opt = self.optimizer
ax_names = self.axis_names
x_obs = np.array([[res["params"][ax_names[0]]] for res in opt.res])
y_obs = np.array([[res["params"][ax_names[1]]] for res in opt.res])
target_obs = np.array([res["target"] for res in opt.res])
mu, sigma = posterior(self.optimizer, x_obs, y_obs, target_obs,
self.design_matrix)
mu = np.resize(mu, (self.N, self.N))
self.ax_opt.clear()
self.ax_opt.plot_surface(self.X, self.Y, mu)
self.ax_opt.view_init(self.phi, self.theta)
self.ax_opt.plot(
[self.next_point[0], self.next_point[0]],
[self.next_point[1], self.next_point[1]],
[self.ax_opt.get_zlim()[0],
self.ax_opt.get_zlim()[1]],
'r-',
linewidth=3)
def _update_ax_utility(self):
self.ax_utility.clear()
# 2.576 is the default kappa value in bayes_opt source (for whatever
# reason). The xi parameter is ignored when using UCB utility.
self.utility_function = UtilityFunction(kind='ucb', kappa=2.576, xi=0)
# Last argument (y_max) ignored when using UCB utility.
utility = self.utility_function.utility(self.design_matrix,
self.optimizer._gp, 0)
utility = np.resize(utility, (self.N, self.N))
self.ax_utility.plot_surface(self.X, self.Y, utility, alpha=0.8)
ind = utility.argmax()
i, j = np.unravel_index(ind, utility.shape)
self.ax_utility.scatter(self.X_[i],
self.Y_[j],
np.amax(utility),
s=50,
c='r')
self.ax_utility.view_init(self.phi, self.theta)
self.next_point = [self.X_[i], self.Y_[i], np.amax(utility)]
def _update_ax_cost(self):
t = np.empty(len(self.optimizer.res))
for i, res in enumerate(self.optimizer.res):
t[i] = res['target']
self.ax_cost.plot(t)
def plot_gp(optimizer, x):
fig = plt.figure(figsize=(16, 10))
steps = len(optimizer.space)
fig.suptitle(
'Gaussian Process and Utility Function After {} Steps'.format(steps),
fontdict={'size': 30})
gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
axis = plt.subplot(gs[0])
acq = plt.subplot(gs[1])
x_obs = np.array([[res["params"]["alpha"]] for res in optimizer.res])
y_obs = np.array([res["target"] for res in optimizer.res])
mu, sigma = posterior(optimizer, x_obs, y_obs, x)
# axis.plot(x, y, linewidth=3, label='Target')
axis.plot(x_obs.flatten(),
y_obs,
'D',
markersize=8,
label='Observations',
color='r')
axis.plot(x, mu, '--', color='k', label='Prediction')
axis.fill(np.concatenate([x, x[::-1]]),
np.concatenate(
[mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]),
alpha=.6,
fc='c',
ec='None',
label='95% confidence interval')
axis.set_xlim((0, 1))
axis.set_ylim((None, None))
axis.set_ylabel('f(x)', fontdict={'size': 20})
axis.set_xlabel('x', fontdict={'size': 20})
utility_function = UtilityFunction(kind="ucb", kappa=5, xi=0)
next_point_suggestion = optimizer.suggest(utility_function)
print("Next point suggestion : ", next_point_suggestion)
utility = utility_function.utility(x, optimizer._gp, 0)
acq.plot(x, utility, label='Utility Function', color='purple')
acq.plot(x[np.argmax(utility)],
np.max(utility),
'*',
markersize=15,
label=u'Next Best Guess',
markerfacecolor='gold',
markeredgecolor='k',
markeredgewidth=1)
acq.set_xlim((0, 1))
acq.set_ylim((np.min(utility) - 0.1, np.max(utility) + 0.1))
acq.set_ylabel('Utility', fontdict={'size': 20})
acq.set_xlabel('x', fontdict={'size': 20})
axis.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
acq.legend(loc=2, bbox_to_anchor=(1.01, 1), borderaxespad=0.)
PATH = './BO/' + time.strftime('%y%m%d_%H%M',
time.localtime(BO_start)) + '/'
os.makedirs(PATH, exist_ok=True)
c_time = time.time()
file_name = str(len(optimizer.space)) + 'steps_' + time.strftime(
'%y%m%d_%H%M', time.localtime(c_time)) + '.png'
plt.savefig(PATH + file_name)
plt.show()