def test_pareto_frontier(self, _): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ] exp.optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds) exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys())) modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=[t1, t2], experiment=exp, data=exp.fetch_data(), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator observed_frontier = observed_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertEqual(1, len(observed_frontier)) self.assertEqual(observed_frontier[0].arm_name, "0_0") with self.assertRaises(ValueError): predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=[], ) predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=None, ) self.assertEqual(predicted_frontier[0].arm_name, "0_0") observation_features = [ ObservationFeatures(parameters={ "x1": 0.0, "x2": 1.0 }), ObservationFeatures(parameters={ "x1": 1.0, "x2": 0.0 }), ] observation_data = [ ObservationData( metric_names=["branin_b", "branin_a"], means=np.array([1.0, 2.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ObservationData( metric_names=["branin_a", "branin_b"], means=np.array([3.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ] predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(len(predicted_frontier) <= 2) self.assertIsNone(predicted_frontier[0].arm_name, None) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, ) wrapped_frontier_evaluator.assert_called_once() self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["X"], torch.tensor([[1.0, 4.0], [4.0, 1.0]]), )) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, use_model_predictions=False, ) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["Y"], torch.tensor([[9.0, 4.0], [16.0, 25.0]]), ))
def get_observed_pareto_frontiers( experiment: Experiment, data: Optional[Data] = None, rel: bool = True, ) -> List[ParetoFrontierResults]: """ Find all Pareto points from an experiment. Uses only values as observed in the data; no modeling is involved. Makes no assumption about the search space or types of parameters. If "data" is provided will use that, otherwise will use all data attached to the experiment. Uses all arms present in data; does not filter according to experiment search space. Assumes experiment has a multiobjective optimization config from which the objectives and outcome constraints will be extracted. Will generate a ParetoFrontierResults for every pair of metrics in the experiment's multiobjective optimization config. """ if data is None: data = experiment.fetch_data() if experiment.optimization_config is None: raise ValueError("Experiment must have an optimization config") mb = get_tensor_converter_model(experiment=experiment, data=data) pareto_observations = observed_pareto_frontier(modelbridge=mb) # Convert to ParetoFrontierResults metric_names = [ metric.name for metric in experiment.optimization_config.objective.metrics # pyre-ignore ] pfr_means = {name: [] for name in metric_names} pfr_sems = {name: [] for name in metric_names} for obs in pareto_observations: for i, name in enumerate(obs.data.metric_names): pfr_means[name].append(obs.data.means[i]) pfr_sems[name].append(np.sqrt(obs.data.covariance[i, i])) # Relativize as needed if rel and experiment.status_quo is not None: # Get status quo values sq_df = data.df[data.df["arm_name"] == experiment.status_quo.name # pyre-ignore ] sq_df = sq_df.to_dict(orient="list") # pyre-ignore sq_means = {} sq_sems = {} for i, metric in enumerate(sq_df["metric_name"]): sq_means[metric] = sq_df["mean"][i] sq_sems[metric] = sq_df["sem"][i] # Relativize for name in metric_names: if np.isnan(sq_sems[name]) or np.isnan(pfr_sems[name]).any(): # Just relativize means pfr_means[name] = [(mu / sq_means[name] - 1) * 100 for mu in pfr_means[name]] else: # Use delta method pfr_means[name], pfr_sems[name] = relativize( means_t=pfr_means[name], sems_t=pfr_sems[name], mean_c=sq_means[name], sem_c=sq_sems[name], as_percent=True, ) absolute_metrics = [] else: absolute_metrics = metric_names objective_thresholds = {} if experiment.optimization_config.objective_thresholds is not None: # pyre-ignore for objth in experiment.optimization_config.objective_thresholds: is_rel = objth.metric.name not in absolute_metrics if objth.relative != is_rel: raise ValueError( f"Objective threshold for {objth.metric.name} has " f"rel={objth.relative} but was specified here as rel={is_rel}" ) objective_thresholds[objth.metric.name] = objth.bound # Construct ParetoFrontResults for each pair pfr_list = [] param_dicts = [obs.features.parameters for obs in pareto_observations] arm_names = [obs.arm_name for obs in pareto_observations] for metric_a, metric_b in combinations(metric_names, 2): pfr_list.append( ParetoFrontierResults( param_dicts=param_dicts, means=pfr_means, sems=pfr_sems, primary_metric=metric_a, secondary_metric=metric_b, absolute_metrics=absolute_metrics, objective_thresholds=objective_thresholds, arm_names=arm_names, )) return pfr_list