def testContours(self):
exp = get_branin_experiment(with_str_choice_param=True,
with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_contour_plotly(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour_plotly(model, list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour(model, list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = plot = plot_contour(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
# Make sure all parameters and metrics are displayed in tooltips
tooltips = list(exp.parameters.keys()) + list(exp.metrics.keys())
for d in plot.data["data"]:
# Only check scatter plots hoverovers
if d["type"] != "scatter":
continue
for text in d["text"]:
for tt in tooltips:
self.assertTrue(tt in text)
def testContours(self):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_contour_plotly(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour_plotly(model, list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = plot = plot_contour(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = interact_contour(model, list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
def plot_all(model, objectives, name="", rend=False):
for o in objectives:
plot = plot_contour(
model=model,
param_x="roll-p",
param_y="roll-d",
metric_name=o,
)
plot[0]['layout']['title'] = o
data = plot[0]['data']
lay = plot[0]['layout']
for i, d in enumerate(data):
if i > 1:
d['cliponaxis'] = False
fig = {
"data": data,
"layout": lay,
}
go.Figure(fig).write_image(name + o + ".png")
if rend: render(plot)
def get_contour_plot(
self,
param_x: Optional[str] = None,
param_y: Optional[str] = None,
metric_name: Optional[str] = None,
) -> AxPlotConfig:
"""Retrieves a plot configuration for a contour plot of the response
surface. For response surfaces with more than two parameters,
selected two parameters will appear on the axes, and remaining parameters
will be affixed to the middle of their range. If contour params arguments
are not provided, the first two parameters in the search space will be
used. If contour metrics are not provided, objective will be used.
Args:
param_x: name of parameters to use on x-axis for
the contour response surface plots.
param_y: name of parameters to use on y-axis for
the contour response surface plots.
metric_name: Name of the metric, for which to plot the response
surface.
"""
if not self.experiment.trials:
raise ValueError("Cannot generate plot as there are no trials.")
if len(self.experiment.parameters) < 2:
raise ValueError(
"Cannot create a contour plot as experiment has less than 2 "
"parameters, but a contour-related argument was provided."
)
if (param_x or param_y) and not (param_x and param_y):
raise ValueError(
"If `param_x` is provided, `param_y` is "
"required as well, and vice-versa."
)
objective_name = self.objective_name
if not metric_name:
metric_name = objective_name
if not param_x or not param_y:
parameter_names = list(self.experiment.parameters.keys())
param_x = parameter_names[0]
param_y = parameter_names[1]
if param_x not in self.experiment.parameters:
raise ValueError(
f'Parameter "{param_x}" not found in the optimization search space.'
)
if param_y not in self.experiment.parameters:
raise ValueError(
f'Parameter "{param_y}" not found in the optimization search space.'
)
if metric_name not in self.experiment.metrics:
raise ValueError(
f'Metric "{metric_name}" is not associated with this optimization.'
)
if self.generation_strategy.model is not None:
try:
logger.info(
f"Retrieving contour plot with parameter '{param_x}' on X-axis "
f"and '{param_y}' on Y-axis, for metric '{metric_name}'. "
"Ramaining parameters are affixed to the middle of their range."
)
return plot_contour(
model=not_none(self.generation_strategy.model),
param_x=param_x,
param_y=param_y,
metric_name=metric_name,
)
except NotImplementedError:
# Some models don't implement '_predict', which is needed
# for the contour plots.
logger.info(
f"Model {self.generation_strategy.model} does not implement "
"`predict`, so it cannot be used to generate a response "
"surface plot."
)
raise ValueError(
f'Could not obtain contour plot of "{metric_name}" for parameters '
f'"{param_x}" and "{param_y}", as a model with predictive ability, '
"such as a Gaussian Process, has not yet been trained in the course "
"of this optimization."
)
def hyperparam_plot(exp_model, param_x, param_y):
render(plot_contour(exp_model, param_x, param_y, metric_name='-reward'))
def mems_exp(cfg):
log.info("============= Configuration =============")
log.info(f"Config:\n{cfg.pretty()}")
log.info("=========================================")
search_space = gen_search_space(cfg.problem)
outcome_con = gen_outcome_constraints(cfg.problem)
exp = SimpleExperiment(
name=cfg.problem.name,
search_space=SearchSpace(search_space),
evaluation_function=jumper,
objective_name="Energy_(uJ)",
minimize=False,
outcome_constraints=outcome_con,
)
optimization_config = OptimizationConfig(objective=Objective(
metric=MEMsMetric(name="Energy_(uJ)"),
minimize=False,
), )
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
exp.runner = MyRunner()
exp.optimization_config = optimization_config
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
print(f"Running {cfg.bo.random} Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
# data = exp.fetch_data()
num_opt = cfg.bo.optimized
for i in range(num_opt):
if (i % 5) == 0 and cfg.plot_during:
plot = plot_contour(
model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)",
)
data = plot[0]['data']
lay = plot[0]['layout']
render(plot)
print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
plot_learn = plot_learning(exp, cfg)
# go.Figure(plot_learn).show()
save_fig([plot_learn], "optimize")
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
plot = plot_contour(model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
save_fig(plot, dir=f"N_L_Energy")
# render(plot)
plot = plot_contour(model=gpei,
param_x="N",
param_y="w",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
# render(plot)
save_fig(plot, dir=f"N_w_Energy")
plot = plot_contour(model=gpei,
param_x="w",
param_y="L",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
save_fig(plot, dir=f"w_L_Energy")
def mems_exp(cfg):
log.info("============= Configuration =============")
log.info(f"Config:\n{cfg.pretty()}")
log.info("=========================================")
raise NotImplementedError("TODO load experimental data and verify likelihood of differnet points")
search_space = gen_search_space(cfg.problem)
outcome_con = gen_outcome_constraints(cfg.problem)
exp = SimpleExperiment(
name=cfg.problem.name,
search_space=SearchSpace(search_space),
evaluation_function=jumper,
objective_name="Energy_(uJ)",
minimize=False,
outcome_constraints=outcome_con,
)
optimization_config = OptimizationConfig(
objective=Objective(
metric=MEMsMetric(name="Energy_(uJ)"),
minimize=False,
),
)
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
exp.runner = MyRunner()
exp.optimization_config = optimization_config
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
print(f"Running {cfg.bo.random} Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
# data = exp.fetch_data()
num_opt = cfg.bo.optimized
for i in range(num_opt):
if (i % 5) == 0 and cfg.plot_during:
plot = plot_contour(model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)", )
data = plot[0]['data']
lay = plot[0]['layout']
render(plot)
print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
from ax.models.torch.botorch_defaults import predict_from_model
import torch
X = torch.Tensor([[2, 7e-4, 1e-4], [1, 5e-4, 1e-4]]).double()
mean, cov = predict_from_model(gpei.model.model, X)
# X(Tensor) – n x d parameters
ll = log_likelihood(X, mean, cov)
plot_ll(ll)
def make_stat(input_data, experiment, generation_strategy):
def prepare_data_for_plots():
## preparing data
res = pd.DataFrame()
for trial in experiment.data_by_trial:
last_entry = list(experiment.data_by_trial[trial].keys())[-1]
res = res.append(experiment.data_by_trial[trial][last_entry].df)
res = res.sort_values(
by=['trial_index', 'arm_name', 'metric_name']).drop_duplicates(
subset=['arm_name', 'metric_name'], keep='last')
generation_strategy._set_or_update_model(data=Data(df=res))
def prepare_data_for_stat_table():
res = pd.DataFrame()
for trial in experiment.data_by_trial:
last_entry = list(experiment.data_by_trial[trial].keys())[-1]
res = res.append(experiment.data_by_trial[trial][last_entry].df)
arm_dict = pd.DataFrame(experiment.arms_by_name.items())
arm_dict[1] = arm_dict[1].apply(lambda x: x.parameters)
arm_dict.columns = ['arm_name', 'arm_params']
arm_table = pd.merge(res, arm_dict, on='arm_name', how='left')
arm_table['arm_params'] = arm_table['arm_params'].apply(json.dumps)
return arm_table
def get_mean_diff(x):
group_vals = np.array(x)
diff = np.subtract.outer(group_vals, group_vals)
diff_clean = diff[~np.eye(diff.shape[0], dtype=bool)].reshape(
len(group_vals),
len(group_vals) - 1)
return diff_clean.mean(axis=1)
def get_winner_confidence(arm_table, control_index, best_index, metric):
if 'ARPU' in metric:
mu1, sem1, n1 = arm_table.loc[best_index,
['mean', 'sem', 'n']].values.T
mu2, sem2, n2 = arm_table.loc[control_index,
['mean', 'sem', 'n']].values.T
std1 = sem1 * np.sqrt(n1)
std2 = sem2 * np.sqrt(n2)
satSE = np.sqrt(std1**2 / n1 + std2**2 / n2)
Z = (mu1 - mu2) / satSE
pval = stats.t.sf(np.abs(Z), n1 + n2 - 2) * 2
else:
p1, n1 = arm_table.loc[best_index][['mean', 'n']].values.T
p2, n2 = arm_table.loc[control_index][['mean', 'n']].values.T
SE = np.sqrt((p1 * (1 - p1) / n1) + (p2 * (1 - p2) / n2))
Z = (p1 - p2) / SE
pval = stats.norm.sf(Z) * 2
confidence = 1 - pval
return Z, pval, confidence
if input_data['query_type'] == 'experiment_info':
exp_json = object_to_json(experiment)
end_time = ''
if not [
trial for stage_type in [0, 1, 4]
for trial in experiment.trials_by_status[stage_type]
]:
try:
end_time = experiment.trials_by_status[3][
-1].time_completed.strftime('%Y-%m-%d %H:%M:%S')
except:
pass
experiment_info = {
'experiment_name': exp_json['name'],
'experiment_description': exp_json['description'],
'status': {
trial: data['status']
for trial, data in exp_json['trials'].items()
},
'tеst_started': exp_json['time_created']['value'],
'test_ended': end_time,
'ends_approx': '????'
}
return experiment_info
# TODO: show dates on x axis for rollout plot
elif input_data['query_type'] == 'rollout_plot':
plot_config = plot_bandit_rollout(experiment).data
plot_config['layout']['showlegend'] = True
rollout_plot = plot(plot_config, show_link=False, output_type='div')
return rollout_plot
elif input_data['query_type'] == 'marginal_effects_plots':
prepare_data_for_plots()
me_plots = {}
params_list = list(experiment.parameters.keys())
for metric in experiment.metrics.keys():
try:
if experiment.description['module'] == 'bayesian_optimization':
if len(params_list) == 1:
plot_config = plot_slice(
model=generation_strategy.model,
param_name=params_list[0],
metric_name=metric).data
slice_plot = plot(plot_config,
show_link=False,
output_type='div')
me_plots[metric] = slice_plot
elif len(params_list) >= 2:
for i, j in itertools.combinations(params_list, 2):
plot_config = plot_contour(
model=generation_strategy.model,
param_x=i,
param_y=j,
metric_name=metric).data
counter_plot = plot(plot_config,
show_link=False,
output_type='div')
me_plots[metric + '_' + i + '_' + j] = counter_plot
elif experiment.description['module'] == 'bandit':
plot_config = plot_marginal_effects(
model=generation_strategy.model, metric=metric).data
marginal_effects_plot = plot(plot_config,
show_link=False,
output_type='div')
me_plots[metric] = marginal_effects_plot
except (ValueError, KeyError) as e:
me_plots[metric] = str(e)
return me_plots
elif input_data['query_type'] == 'predicted_outcomes_plots':
prepare_data_for_plots()
me_plots = {}
for metric in experiment.metrics.keys():
try:
plot_config = plot_fitted(model=generation_strategy.model,
metric=metric,
rel=False).data
predicted_outcomes_plot = plot(plot_config,
show_link=False,
output_type='div')
me_plots[metric] = predicted_outcomes_plot
except (ValueError, KeyError) as e:
me_plots[metric] = str(e)
return me_plots
elif input_data['query_type'] == 'full_result_table':
try:
arm_table = prepare_data_for_stat_table()
arm_table = pd.pivot_table(
arm_table,
index=['trial_index', 'arm_name', 'arm_params'],
columns='metric_name',
values=['mean', 'sem', 'n'])
arm_table.columns = ['_'.join(i) for i in arm_table.columns]
arm_table = arm_table.reset_index()
return arm_table.to_html()
except:
arm_table = pd.DataFrame()
return arm_table.to_html()
elif input_data['query_type'] == 'last_trial_statistics':
try:
arm_table = prepare_data_for_stat_table()
metric_weight = {
metric.name: weight
for metric, weight in
experiment.optimization_config.objective.metric_weights
}
arm_table['weight'] = arm_table['metric_name'].replace(
metric_weight)
arm_table = arm_table.sort_values(
by='trial_index', ascending=True).drop_duplicates(
subset=['arm_name', 'metric_name'], keep='last')
arm_table['diffs'] = arm_table.groupby(
'metric_name')['mean'].transform(get_mean_diff)
arm_table['share'] = arm_table['diffs'] / (arm_table['mean'] -
arm_table['diffs'])
arm_table['type'] = np.where(arm_table['weight'] > 0, 'main',
'other')
def z_scaler(x):
return (x - np.mean(x)) / np.std(x)
arm_table_scaled = arm_table.copy()
arm_table_scaled['mean'] = arm_table_scaled.groupby(
'metric_name')['mean'].transform(z_scaler)
best_row = _get_best_row_for_scalarized_objective(
arm_table_scaled, experiment.optimization_config.objective)
arm_table['best_arm'] = np.where(
arm_table['arm_name'] == best_row['arm_name'], True, False)
arm_table[['share_control', 'confidence']] = np.nan, np.nan
if 'control' in arm_table['arm_name'].unique():
arm_table = pd.merge(
arm_table,
arm_table[arm_table['arm_name'] == 'control'][[
'metric_name', 'mean'
]],
on=['metric_name'],
how='left',
suffixes=['', '_control'])
arm_table[
'diffs2'] = arm_table['mean'] - arm_table['mean_control']
arm_table['share_control'] = arm_table['diffs2'] / (
arm_table['mean'] - arm_table['diffs2'])
for metric in arm_table['metric_name'].unique():
arm_table_cut = arm_table[
arm_table['metric_name'].str.contains(metric)]
control_index = arm_table_cut[arm_table_cut['arm_name'] ==
'control'].index[0]
best_index = arm_table_cut[
arm_table_cut['best_arm']].index[0]
confidence = get_winner_confidence(
arm_table=arm_table,
control_index=control_index,
best_index=best_index,
metric=metric)[2]
arm_table.loc[best_index, 'confidence'] = confidence
arm_table = arm_table[[
'arm_name', 'best_arm', 'confidence', 'arm_params', 'type',
'metric_name', 'mean', 'n', 'share', 'share_control'
]]
full_list = [
'arm_name', 'best_arm', 'arm_params', 'type', 'metric_name'
]
arm_table = arm_table.groupby(full_list)[[
'mean', 'n', 'share', 'share_control', 'confidence'
]].apply(lambda x: x.to_dict('records')[0]).reset_index(
name='data')
for i in range(2):
agg = full_list.pop()
arm_table = arm_table.groupby(full_list)[[agg, 'data']].apply(
lambda x: x.set_index(agg)['data'].to_dict()).reset_index(
name='data')
arm_table['data'] = arm_table.apply(
lambda row: {
**row['data'], "arm_params": json.loads(row['arm_params']),
"best_arm": row['best_arm']
},
axis=1)
arm_table = arm_table.set_index('arm_name')['data'].to_json()
return arm_table
except:
return '{}'
# for i in range(25):
# parameters, trial_index = ax_client.get_next_trial()
# # Local evaluation here can be replaced with deployment to external system.
# ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters))
# # _, trial_index = ax_client.get_next_trial()
# ax_client.log_trial_failure(trial_index=trial_index)
#
# ax_client.get_trials_data_frame().sort_values('trial_index')
# best_parameters, values = ax_client.get_best_parameters()
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
plot = plot_contour(
model=gpei,
param_x=opt_list[0],
param_y=opt_list[1],
metric_name="base",
)
render(plot)
ax_client.generation_strategy.model = gpei
init_notebook_plotting(offline=True)
# render(ax_client.get_contour_plot())
render(ax_client.get_contour_plot(param_x=opt_list[0],
param_y=opt_list[0])) #, metric_name=base))
# render(ax_client.get_optimization_trace(objective_optimum=hartmann6.fmin)) # Objective_optimum is optional.
ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument.
restored_ax_client = AxClient.load_from_json_file(
) # For custom filepath, pass `filepath` argument.
"bounds": [256, 2048],
"log_scale": False
},
{
"name": "batch_size",
"type": "choice",
"values": [32, 64, 128, 256, 512]
},
],
evaluation_function=train_evaluate,
objective_name='accuracy',
# generation_strategy=ax.models.random.sobol.SobolGenerator,
)
# import pdb; pdb.set_trace()
render(
plot_contour(model=model,
param_x='lr',
param_y='training_split',
metric_name='accuracy'))
print(best_parameters, values[0])
best_objectives = np.array(
[[trial.objective_mean * 100 for trial in experiment.trials.values()]])
best_objective_plot = optimization_trace_single_method(
y=np.maximum.accumulate(best_objectives, axis=1),
title="Model performance vs. # of iterations",
ylabel="Classification Accuracy, %",
)
render(best_objective_plot)
