def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment,
data=ax_client.experiment.fetch_data())
html_elements = [
plot_config_to_html(ax_client.get_optimization_trace())
]
model_params = get_range_parameters(model)
try:
if len(model_params) > 1:
html_elements.append(
plot_config_to_html(
interact_contour(
model=model,
metric_name=self.YR.args.eval_primary_metric)))
else:
html_elements.append(
plot_config_to_html(
interact_slice(
model=model,
param_name=model_params[0].name,
metric_name=self.YR.args.eval_primary_metric)))
except TypeError:
pass
with open(
os.path.join(self.bayes_opt_root_experiment_folder,
"optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name,
html_elements))
def testContours(self):
exp = get_branin_experiment(with_str_choice_param=True,
with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_contour_plotly(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour_plotly(model, list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour(model, list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = plot = plot_contour(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
# Make sure all parameters and metrics are displayed in tooltips
tooltips = list(exp.parameters.keys()) + list(exp.metrics.keys())
for d in plot.data["data"]:
# Only check scatter plots hoverovers
if d["type"] != "scatter":
continue
for text in d["text"]:
for tt in tooltips:
self.assertTrue(tt in text)
def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment, data=ax_client.experiment.fetch_data())
html_elements = []
html_elements.append(plot_config_to_html(ax_client.get_optimization_trace()))
try:
html_elements.append(plot_config_to_html(interact_contour(model=model, metric_name=self.YR.args.eval_primary_metric)))
except:
pass
with open(os.path.join(self.bayes_opt_root_experiment_folder, "optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name, html_elements))
def testContours(self):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_contour_plotly(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour_plotly(model, list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = plot = plot_contour(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = interact_contour(model, list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
# Save all experiment parameters
df = exp.eval().df
df.to_csv(os.path.join(output_dir, 'exp_eval.csv'), index=False)
# Save best parameter
best_arm_name = df.arm_name[df['mean'] == df['mean'].max()].values[0]
exp_arm = {k: v.parameters for k, v in exp.arms_by_name.items()}
exp_arm['best'] = best_arm_name
print('Best arm:\n', str(exp.arms_by_name[best_arm_name]))
with open(os.path.join(output_dir, 'exp_arm.json'), 'w') as f:
json.dump(exp_arm, f)
# Contour Plot
os.makedirs(os.path.join(output_dir, 'contour_plot'))
for metric in ['f1', 'precision', 'recall', 'accuracy']:
contour_plot = interact_contour(model=gpei, metric_name=metric)
plot(contour_plot.data,
filename=os.path.join(output_dir, 'contour_plot',
'{}.html'.format(metric)))
# Tradeoff Plot
tradeoff_plot = plot_objective_vs_constraints(gpei, 'f1', rel=False)
plot(tradeoff_plot.data,
filename=os.path.join(output_dir, 'tradeoff_plot.html'))
# Slice Plot
# show the metric outcome as a function of one parameter while fixing the others
os.makedirs(os.path.join(output_dir, 'slice_plot'))
for param in ["lr", "decay", "warmups", "eps"]:
slice_plot = plot_slice(gpei, param, "f1")
plot(slice_plot.data,