def get_branin_multi_objective_optimization_config( has_objective_thresholds: bool = False, ) -> OptimizationConfig: objective_thresholds = ( [ ObjectiveThreshold( metric=get_branin_metric(name="branin_a"), bound=10, op=ComparisonOp.GEQ, relative=False, ), ObjectiveThreshold( metric=get_branin_metric(name="branin_b"), bound=20, op=ComparisonOp.GEQ, relative=False, ), ] if has_objective_thresholds else None ) return MultiObjectiveOptimizationConfig( objective=get_branin_multi_objective(), objective_thresholds=objective_thresholds, )
def testTransformOptimizationConfigMOO(self): m1 = Metric(name="m1", lower_is_better=False) m2 = Metric(name="m2", lower_is_better=True) mo = MultiObjective( objectives=[ Objective(metric=m1, minimize=False), Objective(metric=m2, minimize=True), ], ) objective_thresholds = [ ObjectiveThreshold(metric=m1, bound=1.234, relative=False), ObjectiveThreshold(metric=m2, bound=3.456, relative=False), ] oc = MultiObjectiveOptimizationConfig( objective=mo, objective_thresholds=objective_thresholds, ) tf = LogY( search_space=None, observation_features=None, observation_data=[self.obsd1, self.obsd2], config={"metrics": ["m1"]}, ) oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None) oc.objective_thresholds[0].bound = math.log(1.234) self.assertEqual(oc_tf, oc)
def test_pareto_frontier(self, _): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ] exp.optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds ) exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys()) ) modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=[t1, t2], experiment=exp, data=exp.fetch_data(), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator ) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator observed_frontier_data = modelbridge.observed_pareto_frontier( objective_thresholds=objective_thresholds ) wrapped_frontier_evaluator.assert_called_once() self.assertEqual(1, len(observed_frontier_data)) with self.assertRaises(ValueError): modelbridge.predicted_pareto_frontier( objective_thresholds=objective_thresholds, observation_features=[] ) observation_features = [ ObservationFeatures(parameters={"x1": 0.0, "x2": 1.0}), ObservationFeatures(parameters={"x1": 1.0, "x2": 0.0}), ] predicted_frontier_data = modelbridge.predicted_pareto_frontier( objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(len(predicted_frontier_data) <= 2)
def test_extract_objective_thresholds(self): outcomes = ["m1", "m2", "m3", "m4"] objective = MultiObjective(metrics=[Metric(name) for name in outcomes[:3]]) objective_thresholds = [ ObjectiveThreshold( metric=Metric(name), op=ComparisonOp.LEQ, bound=float(i + 2) ) for i, name in enumerate(outcomes[:3]) ] # None of no thresholds self.assertIsNone( extract_objective_thresholds( objective_thresholds=[], objective=objective, outcomes=outcomes ) ) # Working case obj_t = extract_objective_thresholds( objective_thresholds=objective_thresholds, objective=objective, outcomes=outcomes, ) self.assertTrue(np.array_equal(obj_t, np.array([2.0, 3.0, 4.0, 0.0]))) # Fails if threshold not provided for all objective metrics with self.assertRaises(ValueError): extract_objective_thresholds( objective_thresholds=objective_thresholds[:2], objective=objective, outcomes=outcomes, ) # Fails if number of thresholds doesn't equal number of objectives objective2 = Objective(Metric("m1")) with self.assertRaises(ValueError): extract_objective_thresholds( objective_thresholds=objective_thresholds, objective=objective2, outcomes=outcomes, ) # Works with a single objective, single threshold obj_t = extract_objective_thresholds( objective_thresholds=objective_thresholds[:1], objective=objective2, outcomes=outcomes, ) self.assertTrue(np.array_equal(obj_t, np.array([2.0, 0.0, 0.0, 0.0]))) # Fails if relative objective_thresholds[2] = ObjectiveThreshold( metric=Metric("m3"), op=ComparisonOp.LEQ, bound=3, relative=True ) with self.assertRaises(ValueError): extract_objective_thresholds( objective_thresholds=objective_thresholds, objective=objective, outcomes=outcomes, )
def get_branin_multi_objective_optimization_config( has_objective_thresholds: bool = False, num_objectives: int = 2, ) -> MultiObjectiveOptimizationConfig: _validate_num_objectives(num_objectives=num_objectives) if has_objective_thresholds: objective_thresholds = [ ObjectiveThreshold( metric=get_branin_metric(name="branin_a"), bound=10, op=ComparisonOp.GEQ, relative=False, ), ObjectiveThreshold( metric=get_branin_metric(name="branin_b"), bound=20, op=ComparisonOp.GEQ, relative=False, ), ] if num_objectives == 3: objective_thresholds.append( ObjectiveThreshold( metric=get_branin_metric(name="branin_c"), bound=5.0, op=ComparisonOp.GEQ, relative=False, )) else: objective_thresholds = None return MultiObjectiveOptimizationConfig( objective=get_branin_multi_objective(num_objectives=num_objectives), objective_thresholds=objective_thresholds, )
def test_MOO_with_more_outcomes_than_thresholds(self): experiment = get_branin_experiment_with_multi_objective( has_optimization_config=False) metric_c = Metric(name="c", lower_is_better=False) metric_a = Metric(name="a", lower_is_better=False) objective_thresholds = [ ObjectiveThreshold( metric=metric_c, bound=2.0, relative=False, ), ObjectiveThreshold( metric=metric_a, bound=1.0, relative=False, ), ] experiment.optimization_config = MultiObjectiveOptimizationConfig( objective=MultiObjective(objectives=[ Objective(metric=metric_a), Objective(metric=metric_c), ]), objective_thresholds=objective_thresholds, ) experiment.add_tracking_metric(Metric(name="b", lower_is_better=False)) sobol = get_sobol(search_space=experiment.search_space, ) sobol_run = sobol.gen(1) experiment.new_batch_trial().add_generator_run( sobol_run).run().mark_completed() data = Data( pd.DataFrame( data={ "arm_name": ["0_0", "0_0", "0_0"], "metric_name": ["a", "b", "c"], "mean": [1.0, 2.0, 3.0], "trial_index": [0, 0, 0], "sem": [0, 0, 0], })) test_names_to_fns = { "MOO_NEHVI": get_MOO_NEHVI, "MOO_EHVI": get_MOO_NEHVI, "MOO_PAREGO": get_MOO_PAREGO, "MOO_RS": get_MOO_RS, } for test_name, factory_fn in test_names_to_fns.items(): with self.subTest(test_name): moo_model = factory_fn( experiment=experiment, data=data, ) moo_gr = moo_model.gen(n=1) obj_t = moo_gr.gen_metadata["objective_thresholds"] self.assertEqual(obj_t[0], objective_thresholds[1]) self.assertEqual(obj_t[1], objective_thresholds[0]) self.assertEqual(len(obj_t), 2)
def test_feasible_hypervolume(self): ma = Metric(name="a", lower_is_better=False) mb = Metric(name="b", lower_is_better=True) mc = Metric(name="c", lower_is_better=False) optimization_config = MultiObjectiveOptimizationConfig( objective=MultiObjective(metrics=[ma, mb]), outcome_constraints=[ OutcomeConstraint( mc, op=ComparisonOp.GEQ, bound=0, relative=False, ) ], objective_thresholds=[ ObjectiveThreshold( ma, bound=1.0, ), ObjectiveThreshold( mb, bound=1.0, ), ], ) feas_hv = feasible_hypervolume( optimization_config, values={ "a": np.array( [ 1.0, 3.0, 2.0, 2.0, ] ), "b": np.array( [ 0.0, 1.0, 0.0, 0.0, ] ), "c": np.array( [ 0.0, -0.0, 1.0, -2.0, ] ), }, ) self.assertEqual(list(feas_hv), [0.0, 0.0, 1.0, 1.0])
def testEq(self): threshold1 = ObjectiveThreshold(metric=self.minimize_metric, bound=self.bound) threshold2 = ObjectiveThreshold(metric=self.minimize_metric, bound=self.bound) self.assertEqual(threshold1, threshold2) constraint3 = OutcomeConstraint(metric=self.minimize_metric, op=ComparisonOp.LEQ, bound=self.bound) self.assertNotEqual(threshold1, constraint3)
def setUp(self): self.metrics = { "m1": Metric(name="m1", lower_is_better=True), "m2": Metric(name="m2", lower_is_better=False), "m3": Metric(name="m3", lower_is_better=False), } self.objectives = { "o1": Objective(metric=self.metrics["m1"]), "o2": Objective(metric=self.metrics["m2"], minimize=False), "o3": Objective(metric=self.metrics["m3"], minimize=False), } self.objective = Objective(metric=self.metrics["m1"], minimize=False) self.multi_objective = MultiObjective( objectives=[self.objectives["o1"], self.objectives["o2"]]) self.multi_objective_just_m2 = MultiObjective( objectives=[self.objectives["o2"]]) self.outcome_constraint = OutcomeConstraint(metric=self.metrics["m2"], op=ComparisonOp.GEQ, bound=-0.25) self.additional_outcome_constraint = OutcomeConstraint( metric=self.metrics["m2"], op=ComparisonOp.LEQ, bound=0.25) self.outcome_constraints = [ self.outcome_constraint, self.additional_outcome_constraint, ] self.objective_thresholds = [ ObjectiveThreshold(metric=self.metrics["m1"], bound=-1.0, relative=False), ObjectiveThreshold(metric=self.metrics["m2"], bound=-1.0, relative=False), ] self.relative_objective_thresholds = [ ObjectiveThreshold(metric=self.metrics["m1"], bound=-1.0, relative=True), ObjectiveThreshold( metric=self.metrics["m2"], op=ComparisonOp.GEQ, bound=-1.0, relative=True, ), ] self.m1_constraint = OutcomeConstraint(metric=self.metrics["m1"], op=ComparisonOp.LEQ, bound=0.1, relative=True) self.m3_constraint = OutcomeConstraint(metric=self.metrics["m3"], op=ComparisonOp.GEQ, bound=0.1, relative=True)
def testObjectiveThresholdFail(self): logger_name = OUTCOME_CONSTRAINT_PATH + ".logger" with mock.patch(logger_name) as mock_warning: ObjectiveThreshold(metric=self.minimize_metric, op=ComparisonOp.GEQ, bound=self.bound) mock_warning.debug.assert_called_once_with( CONSTRAINT_WARNING_MESSAGE.format(**LOWER_BOUND_MISMATCH)) with mock.patch(logger_name) as mock_warning: ObjectiveThreshold(metric=self.maximize_metric, op=ComparisonOp.LEQ, bound=self.bound) mock_warning.debug.assert_called_once_with( CONSTRAINT_WARNING_MESSAGE.format(**UPPER_BOUND_MISMATCH))
def get_objective_threshold( metric_name: str = "m1", bound=-0.25, comparison_op: ComparisonOp = ComparisonOp.GEQ) -> ObjectiveThreshold: return ObjectiveThreshold(metric=Metric(name=metric_name), bound=bound, op=comparison_op)
def test_ST_MTGP_NEHVI(self): """Tests single type MTGP NEHVI instantiation.""" multi_obj_exp = get_branin_experiment_with_multi_objective( with_batch=True, with_status_quo=True, ) metrics = multi_obj_exp.optimization_config.objective.metrics multi_objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ), ObjectiveThreshold(metric=metrics[1], bound=0.0, relative=False, op=ComparisonOp.GEQ), ] sobol = Models.SOBOL(search_space=multi_obj_exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(2): sobol_run = sobol.gen(n=1) t = multi_obj_exp.new_batch_trial().add_generator_run(sobol_run) t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5) t.run().mark_completed() status_quo_features = ObservationFeatures( parameters=multi_obj_exp.trials[0].status_quo.parameters, trial_index=0, ) mtgp = Models.ST_MTGP_NEHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), status_quo_features=status_quo_features, objective_thresholds=multi_objective_thresholds, ) self.assertIsInstance(mtgp, TorchModelBridge) self.assertIsInstance(mtgp.model, MultiObjectiveBotorchModel) # test it can generate mtgp_run = mtgp.gen(n=1, fixed_features=ObservationFeatures(parameters={}, trial_index=1)) self.assertEqual(len(mtgp_run.arms), 1) # test a generated trial can be completed t = multi_obj_exp.new_batch_trial().add_generator_run(mtgp_run) t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5) t.run().mark_completed()
def test_MOO_EHVI(self): single_obj_exp = get_branin_experiment(with_batch=True) metrics = single_obj_exp.optimization_config.objective.metrics objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ) ] # ValueError: Multi-objective optimization requires multiple objectives. with self.assertRaises(ValueError): get_MOO_EHVI( experiment=single_obj_exp, data=single_obj_exp.fetch_data(), objective_thresholds=objective_thresholds, ) multi_obj_exp = get_branin_experiment_with_multi_objective( with_batch=True) metrics = multi_obj_exp.optimization_config.objective.metrics multi_objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ), ObjectiveThreshold(metric=metrics[1], bound=0.0, relative=False, op=ComparisonOp.GEQ), ] # ValueError: MultiObjectiveOptimization requires non-empty data. with self.assertRaises(ValueError): get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, ) multi_obj_exp.trials[0].run().mark_completed() moo_ehvi = get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, ) self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge) moo_ehvi_run = moo_ehvi.gen(n=1) self.assertEqual(len(moo_ehvi_run.arms), 1)
def setUp(self): self.minimize_metric = Metric(name="bar", lower_is_better=True) self.maximize_metric = Metric(name="baz", lower_is_better=False) self.ambiguous_metric = Metric(name="buz") self.bound = 0 self.threshold = ObjectiveThreshold(metric=self.maximize_metric, op=ComparisonOp.GEQ, bound=self.bound)
def untransform_objective_thresholds( self, objective_thresholds: Tensor, objective_weights: Tensor, bounds: List[Tuple[Union[int, float], Union[int, float]]], fixed_features: Optional[Dict[int, float]], ) -> List[ObjectiveThreshold]: objective_thresholds_np = objective_thresholds.cpu().numpy() # pyre-ignore [16] objective_indices = objective_weights.nonzero().view(-1).tolist() objective_names = [self.outcomes[i] for i in objective_indices] # create an ObservationData object for untransforming the objective thresholds observation_data = [ ObservationData( metric_names=objective_names, means=objective_thresholds_np[objective_indices].copy(), covariance=np.zeros( (len(objective_indices), len(objective_indices))), ) ] # Untransform objective thresholds. Note: there is one objective threshold # for every outcome. # Construct dummy observation features X = [bound[0] for bound in bounds] fixed_features = fixed_features or {} for i, val in fixed_features.items(): X[i] = val observation_features = parse_observation_features( X=np.array([X]), param_names=self.parameters, ) # Apply reverse transforms, in reverse order for t in reversed(self.transforms.values()): observation_data = t.untransform_observation_data( observation_data=observation_data, observation_features=observation_features, ) observation_features = t.untransform_observation_features( observation_features=observation_features, ) observation_data = observation_data[0] oc = not_none(self._optimization_config) metrics_names_to_metric = oc.metrics obj_thresholds = [] for idx, (name, bound) in enumerate( zip(observation_data.metric_names, observation_data.means)): if not np.isnan(bound): obj_weight = objective_weights[objective_indices[idx]] op = (ComparisonOp.LEQ if torch.sign(obj_weight) == -1.0 else ComparisonOp.GEQ) obj_thresholds.append( ObjectiveThreshold( metric=metrics_names_to_metric[name], bound=bound, relative=False, op=op, )) return obj_thresholds
def testRelativize(self): self.assertTrue( ObjectiveThreshold(metric=self.maximize_metric, op=ComparisonOp.LEQ, bound=self.bound).relative) self.assertTrue( ObjectiveThreshold( metric=self.maximize_metric, op=ComparisonOp.LEQ, bound=self.bound, relative=True, ).relative) self.assertFalse( ObjectiveThreshold( metric=self.maximize_metric, op=ComparisonOp.LEQ, bound=self.bound, relative=False, ).relative)
def test_MOO_EHVI(self): single_obj_exp = get_branin_experiment(with_batch=True) metrics = single_obj_exp.optimization_config.objective.metrics metrics[0].lower_is_better = True objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ) ] with self.assertRaises(ValueError): get_MOO_EHVI( experiment=single_obj_exp, data=single_obj_exp.fetch_data(), objective_thresholds=objective_thresholds, ) multi_obj_exp = get_branin_experiment_with_multi_objective( with_batch=True) metrics = multi_obj_exp.optimization_config.objective.metrics metrics[0].lower_is_better = False metrics[1].lower_is_better = True multi_objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False), ObjectiveThreshold(metric=metrics[1], bound=0.0, relative=False), ] with self.assertRaises(ValueError): get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, ) multi_obj_exp.trials[0].run() moo_ehvi = get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, ) self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge) moo_ehvi_run = moo_ehvi.gen(n=1) self.assertEqual(len(moo_ehvi_run.arms), 1)
def metric_from_sqa( self, metric_sqa: SQAMetric ) -> Union[Metric, Objective, OutcomeConstraint]: """Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint.""" metric = self.metric_from_sqa_util(metric_sqa) if metric_sqa.intent == MetricIntent.TRACKING: return metric elif metric_sqa.intent == MetricIntent.OBJECTIVE: if metric_sqa.minimize is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Objective because minimize is None." ) if metric_sqa.scalarized_objective_weight is not None: raise SQADecodeError( # pragma: no cover "The metric corresponding to regular objective does not \ have weight attribute") return Objective(metric=metric, minimize=metric_sqa.minimize) elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE ): # metric_sqa is a parent whose children are individual # metrics in MultiObjective try: metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics except DetachedInstanceError: metrics_sqa_children = _get_scalarized_objective_children_metrics( metric_id=metric_sqa.id, decoder=self) if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to MultiObjective \ because the parent metric has no children metrics.") # Extracting metric and weight for each child objectives = [ Objective( metric=self.metric_from_sqa_util(metric_sqa), minimize=metric_sqa.minimize, ) for metric_sqa in metrics_sqa_children ] multi_objective = MultiObjective(objectives=objectives) multi_objective.db_id = metric_sqa.id return multi_objective elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE ): # metric_sqa is a parent whose children are individual # metrics in Scalarized Objective if metric_sqa.minimize is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized Objective \ because minimize is None.") try: metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics except DetachedInstanceError: metrics_sqa_children = _get_scalarized_objective_children_metrics( metric_id=metric_sqa.id, decoder=self) if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized Objective \ because the parent metric has no children metrics.") # Extracting metric and weight for each child metrics, weights = zip(*[( self.metric_from_sqa_util(child), child.scalarized_objective_weight, ) for child in metrics_sqa_children]) scalarized_objective = ScalarizedObjective( metrics=list(metrics), weights=list(weights), minimize=not_none(metric_sqa.minimize), ) scalarized_objective.db_id = metric_sqa.id return scalarized_objective elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT: if (metric_sqa.bound is None or metric_sqa.op is None or metric_sqa.relative is None): raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to OutcomeConstraint because " "bound, op, or relative is None.") return OutcomeConstraint( metric=metric, bound=metric_sqa.bound, op=metric_sqa.op, relative=metric_sqa.relative, ) elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT: if (metric_sqa.bound is None or metric_sqa.op is None or metric_sqa.relative is None): raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized OutcomeConstraint because " "bound, op, or relative is None.") try: metrics_sqa_children = ( metric_sqa.scalarized_outcome_constraint_children_metrics) except DetachedInstanceError: metrics_sqa_children = ( _get_scalarized_outcome_constraint_children_metrics( metric_id=metric_sqa.id, decoder=self)) if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized OutcomeConstraint \ because the parent metric has no children metrics.") # Extracting metric and weight for each child metrics, weights = zip(*[( self.metric_from_sqa_util(child), child.scalarized_outcome_constraint_weight, ) for child in metrics_sqa_children]) scalarized_outcome_constraint = ScalarizedOutcomeConstraint( metrics=list(metrics), weights=list(weights), bound=not_none(metric_sqa.bound), op=not_none(metric_sqa.op), relative=not_none(metric_sqa.relative), ) scalarized_outcome_constraint.db_id = metric_sqa.id return scalarized_outcome_constraint elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD: if metric_sqa.bound is None or metric_sqa.relative is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to ObjectiveThreshold because " "bound, op, or relative is None.") ot = ObjectiveThreshold( metric=metric, bound=metric_sqa.bound, relative=metric_sqa.relative, op=metric_sqa.op, ) # ObjectiveThreshold constructor clones the passed-in metric, which means # the db id gets lost and so we need to reset it ot.metric._db_id = metric.db_id return ot else: raise SQADecodeError( f"Cannot decode SQAMetric because {metric_sqa.intent} " f"is an invalid intent.")
def test_pareto_frontier(self, _): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ] exp.optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds) exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys())) modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=[t1, t2], experiment=exp, data=exp.fetch_data(), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator observed_frontier = observed_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertEqual(1, len(observed_frontier)) self.assertEqual(observed_frontier[0].arm_name, "0_0") with self.assertRaises(ValueError): predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=[], ) predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=None, ) self.assertEqual(predicted_frontier[0].arm_name, "0_0") observation_features = [ ObservationFeatures(parameters={ "x1": 0.0, "x2": 1.0 }), ObservationFeatures(parameters={ "x1": 1.0, "x2": 0.0 }), ] observation_data = [ ObservationData( metric_names=["branin_b", "branin_a"], means=np.array([1.0, 2.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ObservationData( metric_names=["branin_a", "branin_b"], means=np.array([3.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ] predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(len(predicted_frontier) <= 2) self.assertIsNone(predicted_frontier[0].arm_name, None) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, ) wrapped_frontier_evaluator.assert_called_once() self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["X"], torch.tensor([[1.0, 4.0], [4.0, 1.0]]), )) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, use_model_predictions=False, ) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["Y"], torch.tensor([[9.0, 4.0], [16.0, 25.0]]), ))
def metric_from_sqa( self, metric_sqa: SQAMetric ) -> Union[Metric, Objective, OutcomeConstraint]: """Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint.""" metric = self.metric_from_sqa_util(metric_sqa) if metric_sqa.intent == MetricIntent.TRACKING: return metric elif metric_sqa.intent == MetricIntent.OBJECTIVE: if metric_sqa.minimize is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Objective because minimize is None." ) if metric_sqa.scalarized_objective_weight is not None: raise SQADecodeError( # pragma: no cover "The metric corresponding to regular objective does not \ have weight attribute") return Objective(metric=metric, minimize=metric_sqa.minimize) elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE ): # metric_sqa is a parent whose children are individual # metrics in MultiObjective if metric_sqa.minimize is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to MultiObjective \ because minimize is None.") metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to MultiObjective \ because the parent metric has no children metrics.") # Extracting metric and weight for each child metrics = [ self.metric_from_sqa_util(child) for child in metrics_sqa_children ] return MultiObjective( metrics=list(metrics), # pyre-fixme[6]: Expected `bool` for 2nd param but got `Optional[bool]`. minimize=metric_sqa.minimize, ) elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE ): # metric_sqa is a parent whose children are individual # metrics in Scalarized Objective if metric_sqa.minimize is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized Objective \ because minimize is None.") metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized Objective \ because the parent metric has no children metrics.") # Extracting metric and weight for each child metrics, weights = zip(*[( self.metric_from_sqa_util(child), child.scalarized_objective_weight, ) for child in metrics_sqa_children]) return ScalarizedObjective( metrics=list(metrics), weights=list(weights), # pyre-fixme[6]: Expected `bool` for 3nd param but got `Optional[bool]`. minimize=metric_sqa.minimize, ) elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT: if (metric_sqa.bound is None or metric_sqa.op is None or metric_sqa.relative is None): raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to OutcomeConstraint because " "bound, op, or relative is None.") return OutcomeConstraint( metric=metric, # pyre-fixme[6]: Expected `float` for 2nd param but got # `Optional[float]`. bound=metric_sqa.bound, op=metric_sqa.op, relative=metric_sqa.relative, ) elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT: if (metric_sqa.bound is None or metric_sqa.op is None or metric_sqa.relative is None): raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized OutcomeConstraint because " "bound, op, or relative is None.") metrics_sqa_children = ( metric_sqa.scalarized_outcome_constraint_children_metrics) if metrics_sqa_children is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to Scalarized OutcomeConstraint \ because the parent metric has no children metrics.") # Extracting metric and weight for each child metrics, weights = zip(*[( self.metric_from_sqa_util(child), child.scalarized_outcome_constraint_weight, ) for child in metrics_sqa_children]) return ScalarizedOutcomeConstraint( metrics=list(metrics), weights=list(weights), # pyre-fixme[6]: Expected `float` for 2nd param but got # `Optional[float]`. bound=metric_sqa.bound, op=metric_sqa.op, relative=metric_sqa.relative, ) elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD: if metric_sqa.bound is None or metric_sqa.relative is None: raise SQADecodeError( # pragma: no cover "Cannot decode SQAMetric to ObjectiveThreshold because " "bound, op, or relative is None.") return ObjectiveThreshold( metric=metric, # pyre-fixme[6]: Expected `float` for 2nd param but got # `Optional[float]`. bound=metric_sqa.bound, relative=metric_sqa.relative, op=metric_sqa.op, ) else: raise SQADecodeError( f"Cannot decode SQAMetric because {metric_sqa.intent} " f"is an invalid intent.")
def test_hypervolume(self, _, cuda=False): for num_objectives in (2, 3): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True, num_objectives=num_objectives, ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=1.0, relative=False, op=ComparisonOp.GEQ, ), ] if num_objectives == 3: objective_thresholds.append( ObjectiveThreshold( metric=metrics_dict["branin_c"], bound=2.0, relative=False, op=ComparisonOp.GEQ, ) ) optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds ) exp.attach_data( get_branin_data_multi_objective( trial_indices=exp.trials.keys(), num_objectives=num_objectives ) ) modelbridge = TorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=optimization_config, transforms=[], experiment=exp, data=exp.fetch_data(), torch_device=torch.device("cuda" if cuda else "cpu"), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator ) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator hv = observed_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds ) expected_hv = 20 if num_objectives == 2 else 60 # 5 * 4 (* 3) wrapped_frontier_evaluator.assert_called_once() self.assertEqual(expected_hv, hv) if num_objectives == 3: # Test selected_metrics hv = observed_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds, selected_metrics=["branin_a", "branin_c"], ) expected_hv = 15 # (5 - 0) * (5 - 2) self.assertEqual(expected_hv, hv) # test that non-objective outcome raises value error with self.assertRaises(ValueError): hv = observed_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds, selected_metrics=["tracking"], ) with self.assertRaises(ValueError): predicted_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=[], ) observation_features = [ ObservationFeatures(parameters={"x1": 1.0, "x2": 2.0}), ObservationFeatures(parameters={"x1": 2.0, "x2": 1.0}), ] predicted_hv = predicted_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(predicted_hv >= 0) if num_objectives == 3: # Test selected_metrics predicted_hv = predicted_hypervolume( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, selected_metrics=["branin_a", "branin_c"], ) self.assertTrue(predicted_hv >= 0)
def test_MTGP_NEHVI(self): single_obj_exp = get_branin_experiment(with_batch=True) metrics = single_obj_exp.optimization_config.objective.metrics metrics[0].lower_is_better = True objective_thresholds = [ ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False) ] with self.assertRaises(ValueError): get_MTGP_NEHVI( experiment=single_obj_exp, data=single_obj_exp.fetch_data(), objective_thresholds=objective_thresholds, ) multi_obj_exp = get_branin_experiment_with_multi_objective(with_batch=True) metrics = multi_obj_exp.optimization_config.objective.metrics multi_objective_thresholds = [ ObjectiveThreshold( metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ ), ObjectiveThreshold( metric=metrics[1], bound=0.0, relative=False, op=ComparisonOp.GEQ ), ] with self.assertRaises(ValueError): get_MTGP_NEHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, ) multi_obj_exp.trials[0].run() sobol_generator = get_sobol(search_space=multi_obj_exp.search_space) sobol_run = sobol_generator.gen(n=3) multi_obj_exp.new_batch_trial(optimize_for_power=False).add_generator_run( sobol_run ) multi_obj_exp.trials[1].run() mt_ehvi = get_MTGP_NEHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, trial_index=1, ) self.assertIsInstance(mt_ehvi, TorchModelBridge) self.assertIsInstance(mt_ehvi.model.model.models[0], MultiTaskGP) task_covar_factor = mt_ehvi.model.model.models[0].task_covar_module.covar_factor self.assertEqual(task_covar_factor.shape, torch.Size([2, 2])) mt_ehvi_run = mt_ehvi.gen( n=1, fixed_features=ObservationFeatures(parameters={}, trial_index=1) ) self.assertEqual(len(mt_ehvi_run.arms), 1) # Bad index given with self.assertRaises(ValueError): get_MTGP_NEHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), objective_thresholds=multi_objective_thresholds, trial_index=999, ) # Multi-type + multi-objective experiment multi_type_multi_obj_exp = get_multi_type_experiment_with_multi_objective( add_trials=True ) data = multi_type_multi_obj_exp.fetch_data() mt_ehvi = get_MTGP_NEHVI( experiment=multi_type_multi_obj_exp, data=data, objective_thresholds=multi_objective_thresholds, )
def testObservedParetoFrontiers(self): experiment = get_branin_experiment( with_batch=True, has_optimization_config=False, with_status_quo=True ) # Optimization config is not optional with self.assertRaises(ValueError): get_observed_pareto_frontiers(experiment=experiment, data=Data()) metrics = [ BraninMetric(name="m1", param_names=["x1", "x2"], lower_is_better=True), NegativeBraninMetric( name="m2", param_names=["x1", "x2"], lower_is_better=True ), BraninMetric(name="m3", param_names=["x1", "x2"], lower_is_better=True), ] bounds = [0, -100, 0] objective_thresholds = [ ObjectiveThreshold( metric=metric, bound=bounds[i], relative=True, op=ComparisonOp.LEQ, ) for i, metric in enumerate(metrics) ] objective = MultiObjective(metrics=metrics, minimize=True) optimization_config = MultiObjectiveOptimizationConfig( objective=objective, objective_thresholds=objective_thresholds, ) experiment.optimization_config = optimization_config experiment.trials[0].run() # For the check below, compute which arms are better than SQ df = experiment.fetch_data().df df["sem"] = np.nan data = Data(df) sq_val = df[(df["arm_name"] == "status_quo") & (df["metric_name"] == "m1")][ "mean" ].values[0] pareto_arms = sorted( df[(df["mean"] <= sq_val) & (df["metric_name"] == "m1")]["arm_name"] .unique() .tolist() ) pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data) # We have all pairs of metrics self.assertEqual(len(pfrs), 3) true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")] for i, pfr in enumerate(pfrs): self.assertEqual(pfr.primary_metric, true_pairs[i][0]) self.assertEqual(pfr.secondary_metric, true_pairs[i][1]) self.assertEqual(pfr.absolute_metrics, []) self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"]) self.assertEqual(len(pfr.means["m1"]), len(pareto_arms)) self.assertTrue(np.isnan(pfr.sems["m1"]).all()) self.assertEqual(len(pfr.arm_names), len(pareto_arms)) arm_idx = np.argsort(pfr.arm_names) for i, idx in enumerate(arm_idx): name = pareto_arms[i] self.assertEqual(pfr.arm_names[idx], name) self.assertEqual( pfr.param_dicts[idx], experiment.arms_by_name[name].parameters )
def test_get_standard_plots(self): exp = get_branin_experiment() self.assertEqual( len( get_standard_plots(experiment=exp, model=get_generation_strategy().model)), 0, ) exp = get_branin_experiment(with_batch=True, minimize=True) exp.trials[0].run() plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), ) self.assertEqual(len(plots), 6) self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots)) exp = get_branin_experiment_with_multi_objective(with_batch=True) exp.optimization_config.objective.objectives[0].minimize = False exp.optimization_config.objective.objectives[1].minimize = True exp.optimization_config._objective_thresholds = [ ObjectiveThreshold(metric=exp.metrics["branin_a"], op=ComparisonOp.GEQ, bound=-100.0), ObjectiveThreshold(metric=exp.metrics["branin_b"], op=ComparisonOp.LEQ, bound=100.0), ] exp.trials[0].run() plots = get_standard_plots(experiment=exp, model=Models.MOO(experiment=exp, data=exp.fetch_data())) self.assertEqual(len(plots), 7) # All plots are successfully created when objective thresholds are absent exp.optimization_config._objective_thresholds = [] plots = get_standard_plots(experiment=exp, model=Models.MOO(experiment=exp, data=exp.fetch_data())) self.assertEqual(len(plots), 7) exp = get_branin_experiment_with_timestamp_map_metric( with_status_quo=True) exp.new_trial().add_arm(exp.status_quo) exp.trials[0].run() exp.new_trial(generator_run=Models.SOBOL( search_space=exp.search_space).gen(n=1)) exp.trials[1].run() plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), true_objective_metric_name="branin", ) self.assertEqual(len(plots), 9) self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots)) self.assertIn( "Objective branin_map vs. True Objective Metric branin", [p.layout.title.text for p in plots], ) with self.assertRaisesRegex( ValueError, "Please add a valid true_objective_metric_name"): plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), true_objective_metric_name="not_present", )
def get_objective_threshold() -> ObjectiveThreshold: return ObjectiveThreshold( metric=Metric(name="m1"), bound=-0.25, op=ComparisonOp.GEQ )
def test_hypervolume(self): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=False) metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ] exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True) optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds) exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials)) modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=optimization_config, transforms=[t1, t2], experiment=exp, data=exp.fetch_data(), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator hv = modelbridge.observed_hypervolume( objective_thresholds=objective_thresholds) expected_hv = 25 # (5 - 0) * (5 - 0) wrapped_frontier_evaluator.assert_called_once() self.assertEqual(expected_hv, hv) with self.assertRaises(ValueError): modelbridge.predicted_hypervolume( objective_thresholds=objective_thresholds, observation_features=[]) observation_features = [ ObservationFeatures(parameters={ "x1": 1.0, "x2": 2.0 }), ObservationFeatures(parameters={ "x1": 2.0, "x2": 1.0 }), ] predicted_hv = modelbridge.predicted_hypervolume( objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(predicted_hv >= 0)