def testContours(self): exp = get_branin_experiment(with_str_choice_param=True, with_batch=True) exp.trials[0].run() model = Models.BOTORCH( # Model bridge kwargs experiment=exp, data=exp.fetch_data(), ) # Assert that each type of plot can be constructed successfully plot = plot_contour_plotly(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = interact_contour_plotly(model, list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = interact_contour(model, list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig) plot = plot = plot_contour(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig) # Make sure all parameters and metrics are displayed in tooltips tooltips = list(exp.parameters.keys()) + list(exp.metrics.keys()) for d in plot.data["data"]: # Only check scatter plots hoverovers if d["type"] != "scatter": continue for text in d["text"]: for tt in tooltips: self.assertTrue(tt in text)
def testContours(self): exp = get_branin_experiment(with_batch=True) exp.trials[0].run() model = Models.BOTORCH( # Model bridge kwargs experiment=exp, data=exp.fetch_data(), ) # Assert that each type of plot can be constructed successfully plot = plot_contour_plotly(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = interact_contour_plotly(model, list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = plot = plot_contour(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig) plot = interact_contour(model, list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig)
def plot_all(model, objectives, name="", rend=False): for o in objectives: plot = plot_contour( model=model, param_x="roll-p", param_y="roll-d", metric_name=o, ) plot[0]['layout']['title'] = o data = plot[0]['data'] lay = plot[0]['layout'] for i, d in enumerate(data): if i > 1: d['cliponaxis'] = False fig = { "data": data, "layout": lay, } go.Figure(fig).write_image(name + o + ".png") if rend: render(plot)
def get_contour_plot( self, param_x: Optional[str] = None, param_y: Optional[str] = None, metric_name: Optional[str] = None, ) -> AxPlotConfig: """Retrieves a plot configuration for a contour plot of the response surface. For response surfaces with more than two parameters, selected two parameters will appear on the axes, and remaining parameters will be affixed to the middle of their range. If contour params arguments are not provided, the first two parameters in the search space will be used. If contour metrics are not provided, objective will be used. Args: param_x: name of parameters to use on x-axis for the contour response surface plots. param_y: name of parameters to use on y-axis for the contour response surface plots. metric_name: Name of the metric, for which to plot the response surface. """ if not self.experiment.trials: raise ValueError("Cannot generate plot as there are no trials.") if len(self.experiment.parameters) < 2: raise ValueError( "Cannot create a contour plot as experiment has less than 2 " "parameters, but a contour-related argument was provided." ) if (param_x or param_y) and not (param_x and param_y): raise ValueError( "If `param_x` is provided, `param_y` is " "required as well, and vice-versa." ) objective_name = self.objective_name if not metric_name: metric_name = objective_name if not param_x or not param_y: parameter_names = list(self.experiment.parameters.keys()) param_x = parameter_names[0] param_y = parameter_names[1] if param_x not in self.experiment.parameters: raise ValueError( f'Parameter "{param_x}" not found in the optimization search space.' ) if param_y not in self.experiment.parameters: raise ValueError( f'Parameter "{param_y}" not found in the optimization search space.' ) if metric_name not in self.experiment.metrics: raise ValueError( f'Metric "{metric_name}" is not associated with this optimization.' ) if self.generation_strategy.model is not None: try: logger.info( f"Retrieving contour plot with parameter '{param_x}' on X-axis " f"and '{param_y}' on Y-axis, for metric '{metric_name}'. " "Ramaining parameters are affixed to the middle of their range." ) return plot_contour( model=not_none(self.generation_strategy.model), param_x=param_x, param_y=param_y, metric_name=metric_name, ) except NotImplementedError: # Some models don't implement '_predict', which is needed # for the contour plots. logger.info( f"Model {self.generation_strategy.model} does not implement " "`predict`, so it cannot be used to generate a response " "surface plot." ) raise ValueError( f'Could not obtain contour plot of "{metric_name}" for parameters ' f'"{param_x}" and "{param_y}", as a model with predictive ability, ' "such as a Gaussian Process, has not yet been trained in the course " "of this optimization." )
def hyperparam_plot(exp_model, param_x, param_y): render(plot_contour(exp_model, param_x, param_y, metric_name='-reward'))
def mems_exp(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") search_space = gen_search_space(cfg.problem) outcome_con = gen_outcome_constraints(cfg.problem) exp = SimpleExperiment( name=cfg.problem.name, search_space=SearchSpace(search_space), evaluation_function=jumper, objective_name="Energy_(uJ)", minimize=False, outcome_constraints=outcome_con, ) optimization_config = OptimizationConfig(objective=Objective( metric=MEMsMetric(name="Energy_(uJ)"), minimize=False, ), ) class MyRunner(Runner): def run(self, trial): return {"name": str(trial.index)} exp.runner = MyRunner() exp.optimization_config = optimization_config from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour print(f"Running {cfg.bo.random} Sobol initialization trials...") sobol = Models.SOBOL(exp.search_space) num_search = cfg.bo.random for i in range(num_search): exp.new_trial(generator_run=sobol.gen(1)) exp.trials[len(exp.trials) - 1].run() # data = exp.fetch_data() num_opt = cfg.bo.optimized for i in range(num_opt): if (i % 5) == 0 and cfg.plot_during: plot = plot_contour( model=gpei, param_x="N", param_y="L", metric_name="Energy_(uJ)", ) data = plot[0]['data'] lay = plot[0]['layout'] render(plot) print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...") # Reinitialize GP+EI model at each step with updated data. batch = exp.new_trial(generator_run=gpei.gen(1)) gpei = Models.BOTORCH(experiment=exp, data=exp.eval()) plot_learn = plot_learning(exp, cfg) # go.Figure(plot_learn).show() save_fig([plot_learn], "optimize") from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour plot = plot_contour(model=gpei, param_x="N", param_y="L", metric_name="Energy_(uJ)", lower_is_better=cfg.metric.minimize) save_fig(plot, dir=f"N_L_Energy") # render(plot) plot = plot_contour(model=gpei, param_x="N", param_y="w", metric_name="Energy_(uJ)", lower_is_better=cfg.metric.minimize) # render(plot) save_fig(plot, dir=f"N_w_Energy") plot = plot_contour(model=gpei, param_x="w", param_y="L", metric_name="Energy_(uJ)", lower_is_better=cfg.metric.minimize) save_fig(plot, dir=f"w_L_Energy")
def mems_exp(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") raise NotImplementedError("TODO load experimental data and verify likelihood of differnet points") search_space = gen_search_space(cfg.problem) outcome_con = gen_outcome_constraints(cfg.problem) exp = SimpleExperiment( name=cfg.problem.name, search_space=SearchSpace(search_space), evaluation_function=jumper, objective_name="Energy_(uJ)", minimize=False, outcome_constraints=outcome_con, ) optimization_config = OptimizationConfig( objective=Objective( metric=MEMsMetric(name="Energy_(uJ)"), minimize=False, ), ) class MyRunner(Runner): def run(self, trial): return {"name": str(trial.index)} exp.runner = MyRunner() exp.optimization_config = optimization_config from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour print(f"Running {cfg.bo.random} Sobol initialization trials...") sobol = Models.SOBOL(exp.search_space) num_search = cfg.bo.random for i in range(num_search): exp.new_trial(generator_run=sobol.gen(1)) exp.trials[len(exp.trials) - 1].run() # data = exp.fetch_data() num_opt = cfg.bo.optimized for i in range(num_opt): if (i % 5) == 0 and cfg.plot_during: plot = plot_contour(model=gpei, param_x="N", param_y="L", metric_name="Energy_(uJ)", ) data = plot[0]['data'] lay = plot[0]['layout'] render(plot) print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...") # Reinitialize GP+EI model at each step with updated data. batch = exp.new_trial(generator_run=gpei.gen(1)) gpei = Models.BOTORCH(experiment=exp, data=exp.eval()) gpei = Models.BOTORCH(experiment=exp, data=exp.eval()) from ax.models.torch.botorch_defaults import predict_from_model import torch X = torch.Tensor([[2, 7e-4, 1e-4], [1, 5e-4, 1e-4]]).double() mean, cov = predict_from_model(gpei.model.model, X) # X(Tensor) – n x d parameters ll = log_likelihood(X, mean, cov) plot_ll(ll)
def make_stat(input_data, experiment, generation_strategy): def prepare_data_for_plots(): ## preparing data res = pd.DataFrame() for trial in experiment.data_by_trial: last_entry = list(experiment.data_by_trial[trial].keys())[-1] res = res.append(experiment.data_by_trial[trial][last_entry].df) res = res.sort_values( by=['trial_index', 'arm_name', 'metric_name']).drop_duplicates( subset=['arm_name', 'metric_name'], keep='last') generation_strategy._set_or_update_model(data=Data(df=res)) def prepare_data_for_stat_table(): res = pd.DataFrame() for trial in experiment.data_by_trial: last_entry = list(experiment.data_by_trial[trial].keys())[-1] res = res.append(experiment.data_by_trial[trial][last_entry].df) arm_dict = pd.DataFrame(experiment.arms_by_name.items()) arm_dict[1] = arm_dict[1].apply(lambda x: x.parameters) arm_dict.columns = ['arm_name', 'arm_params'] arm_table = pd.merge(res, arm_dict, on='arm_name', how='left') arm_table['arm_params'] = arm_table['arm_params'].apply(json.dumps) return arm_table def get_mean_diff(x): group_vals = np.array(x) diff = np.subtract.outer(group_vals, group_vals) diff_clean = diff[~np.eye(diff.shape[0], dtype=bool)].reshape( len(group_vals), len(group_vals) - 1) return diff_clean.mean(axis=1) def get_winner_confidence(arm_table, control_index, best_index, metric): if 'ARPU' in metric: mu1, sem1, n1 = arm_table.loc[best_index, ['mean', 'sem', 'n']].values.T mu2, sem2, n2 = arm_table.loc[control_index, ['mean', 'sem', 'n']].values.T std1 = sem1 * np.sqrt(n1) std2 = sem2 * np.sqrt(n2) satSE = np.sqrt(std1**2 / n1 + std2**2 / n2) Z = (mu1 - mu2) / satSE pval = stats.t.sf(np.abs(Z), n1 + n2 - 2) * 2 else: p1, n1 = arm_table.loc[best_index][['mean', 'n']].values.T p2, n2 = arm_table.loc[control_index][['mean', 'n']].values.T SE = np.sqrt((p1 * (1 - p1) / n1) + (p2 * (1 - p2) / n2)) Z = (p1 - p2) / SE pval = stats.norm.sf(Z) * 2 confidence = 1 - pval return Z, pval, confidence if input_data['query_type'] == 'experiment_info': exp_json = object_to_json(experiment) end_time = '' if not [ trial for stage_type in [0, 1, 4] for trial in experiment.trials_by_status[stage_type] ]: try: end_time = experiment.trials_by_status[3][ -1].time_completed.strftime('%Y-%m-%d %H:%M:%S') except: pass experiment_info = { 'experiment_name': exp_json['name'], 'experiment_description': exp_json['description'], 'status': { trial: data['status'] for trial, data in exp_json['trials'].items() }, 'tеst_started': exp_json['time_created']['value'], 'test_ended': end_time, 'ends_approx': '????' } return experiment_info # TODO: show dates on x axis for rollout plot elif input_data['query_type'] == 'rollout_plot': plot_config = plot_bandit_rollout(experiment).data plot_config['layout']['showlegend'] = True rollout_plot = plot(plot_config, show_link=False, output_type='div') return rollout_plot elif input_data['query_type'] == 'marginal_effects_plots': prepare_data_for_plots() me_plots = {} params_list = list(experiment.parameters.keys()) for metric in experiment.metrics.keys(): try: if experiment.description['module'] == 'bayesian_optimization': if len(params_list) == 1: plot_config = plot_slice( model=generation_strategy.model, param_name=params_list[0], metric_name=metric).data slice_plot = plot(plot_config, show_link=False, output_type='div') me_plots[metric] = slice_plot elif len(params_list) >= 2: for i, j in itertools.combinations(params_list, 2): plot_config = plot_contour( model=generation_strategy.model, param_x=i, param_y=j, metric_name=metric).data counter_plot = plot(plot_config, show_link=False, output_type='div') me_plots[metric + '_' + i + '_' + j] = counter_plot elif experiment.description['module'] == 'bandit': plot_config = plot_marginal_effects( model=generation_strategy.model, metric=metric).data marginal_effects_plot = plot(plot_config, show_link=False, output_type='div') me_plots[metric] = marginal_effects_plot except (ValueError, KeyError) as e: me_plots[metric] = str(e) return me_plots elif input_data['query_type'] == 'predicted_outcomes_plots': prepare_data_for_plots() me_plots = {} for metric in experiment.metrics.keys(): try: plot_config = plot_fitted(model=generation_strategy.model, metric=metric, rel=False).data predicted_outcomes_plot = plot(plot_config, show_link=False, output_type='div') me_plots[metric] = predicted_outcomes_plot except (ValueError, KeyError) as e: me_plots[metric] = str(e) return me_plots elif input_data['query_type'] == 'full_result_table': try: arm_table = prepare_data_for_stat_table() arm_table = pd.pivot_table( arm_table, index=['trial_index', 'arm_name', 'arm_params'], columns='metric_name', values=['mean', 'sem', 'n']) arm_table.columns = ['_'.join(i) for i in arm_table.columns] arm_table = arm_table.reset_index() return arm_table.to_html() except: arm_table = pd.DataFrame() return arm_table.to_html() elif input_data['query_type'] == 'last_trial_statistics': try: arm_table = prepare_data_for_stat_table() metric_weight = { metric.name: weight for metric, weight in experiment.optimization_config.objective.metric_weights } arm_table['weight'] = arm_table['metric_name'].replace( metric_weight) arm_table = arm_table.sort_values( by='trial_index', ascending=True).drop_duplicates( subset=['arm_name', 'metric_name'], keep='last') arm_table['diffs'] = arm_table.groupby( 'metric_name')['mean'].transform(get_mean_diff) arm_table['share'] = arm_table['diffs'] / (arm_table['mean'] - arm_table['diffs']) arm_table['type'] = np.where(arm_table['weight'] > 0, 'main', 'other') def z_scaler(x): return (x - np.mean(x)) / np.std(x) arm_table_scaled = arm_table.copy() arm_table_scaled['mean'] = arm_table_scaled.groupby( 'metric_name')['mean'].transform(z_scaler) best_row = _get_best_row_for_scalarized_objective( arm_table_scaled, experiment.optimization_config.objective) arm_table['best_arm'] = np.where( arm_table['arm_name'] == best_row['arm_name'], True, False) arm_table[['share_control', 'confidence']] = np.nan, np.nan if 'control' in arm_table['arm_name'].unique(): arm_table = pd.merge( arm_table, arm_table[arm_table['arm_name'] == 'control'][[ 'metric_name', 'mean' ]], on=['metric_name'], how='left', suffixes=['', '_control']) arm_table[ 'diffs2'] = arm_table['mean'] - arm_table['mean_control'] arm_table['share_control'] = arm_table['diffs2'] / ( arm_table['mean'] - arm_table['diffs2']) for metric in arm_table['metric_name'].unique(): arm_table_cut = arm_table[ arm_table['metric_name'].str.contains(metric)] control_index = arm_table_cut[arm_table_cut['arm_name'] == 'control'].index[0] best_index = arm_table_cut[ arm_table_cut['best_arm']].index[0] confidence = get_winner_confidence( arm_table=arm_table, control_index=control_index, best_index=best_index, metric=metric)[2] arm_table.loc[best_index, 'confidence'] = confidence arm_table = arm_table[[ 'arm_name', 'best_arm', 'confidence', 'arm_params', 'type', 'metric_name', 'mean', 'n', 'share', 'share_control' ]] full_list = [ 'arm_name', 'best_arm', 'arm_params', 'type', 'metric_name' ] arm_table = arm_table.groupby(full_list)[[ 'mean', 'n', 'share', 'share_control', 'confidence' ]].apply(lambda x: x.to_dict('records')[0]).reset_index( name='data') for i in range(2): agg = full_list.pop() arm_table = arm_table.groupby(full_list)[[agg, 'data']].apply( lambda x: x.set_index(agg)['data'].to_dict()).reset_index( name='data') arm_table['data'] = arm_table.apply( lambda row: { **row['data'], "arm_params": json.loads(row['arm_params']), "best_arm": row['best_arm'] }, axis=1) arm_table = arm_table.set_index('arm_name')['data'].to_json() return arm_table except: return '{}'
# for i in range(25): # parameters, trial_index = ax_client.get_next_trial() # # Local evaluation here can be replaced with deployment to external system. # ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters)) # # _, trial_index = ax_client.get_next_trial() # ax_client.log_trial_failure(trial_index=trial_index) # # ax_client.get_trials_data_frame().sort_values('trial_index') # best_parameters, values = ax_client.get_best_parameters() from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour plot = plot_contour( model=gpei, param_x=opt_list[0], param_y=opt_list[1], metric_name="base", ) render(plot) ax_client.generation_strategy.model = gpei init_notebook_plotting(offline=True) # render(ax_client.get_contour_plot()) render(ax_client.get_contour_plot(param_x=opt_list[0], param_y=opt_list[0])) #, metric_name=base)) # render(ax_client.get_optimization_trace(objective_optimum=hartmann6.fmin)) # Objective_optimum is optional. ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument. restored_ax_client = AxClient.load_from_json_file( ) # For custom filepath, pass `filepath` argument.
"bounds": [256, 2048], "log_scale": False }, { "name": "batch_size", "type": "choice", "values": [32, 64, 128, 256, 512] }, ], evaluation_function=train_evaluate, objective_name='accuracy', # generation_strategy=ax.models.random.sobol.SobolGenerator, ) # import pdb; pdb.set_trace() render( plot_contour(model=model, param_x='lr', param_y='training_split', metric_name='accuracy')) print(best_parameters, values[0]) best_objectives = np.array( [[trial.objective_mean * 100 for trial in experiment.trials.values()]]) best_objective_plot = optimization_trace_single_method( y=np.maximum.accumulate(best_objectives, axis=1), title="Model performance vs. # of iterations", ylabel="Classification Accuracy, %", ) render(best_objective_plot)