def testConstraintValidation(self):
# Cannot build with non-MultiObjective
with self.assertRaises(ValueError):
MultiObjectiveOptimizationConfig(objective=self.objective)
# Two outcome_constraints on the same metric with the same op
# should raise.
duplicate_constraint = OutcomeConstraint(
metric=self.outcome_constraint.metric,
op=self.outcome_constraint.op,
bound=self.outcome_constraint.bound + 1,
)
with self.assertRaises(ValueError):
MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=[self.outcome_constraint, duplicate_constraint],
)
# Three outcome_constraints on the same metric should raise.
opposing_constraint = OutcomeConstraint(
metric=self.outcome_constraint.metric,
op=not self.outcome_constraint.op,
bound=self.outcome_constraint.bound,
)
with self.assertRaises(ValueError):
MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=self.outcome_constraints + [opposing_constraint],
)
# Two outcome_constraints on the same metric with different ops and
# flipped bounds (lower < upper) should raise.
add_bound = 1 if self.outcome_constraint.op == ComparisonOp.LEQ else -1
opposing_constraint = OutcomeConstraint(
metric=self.outcome_constraint.metric,
op=not self.outcome_constraint.op,
bound=self.outcome_constraint.bound + add_bound,
)
with self.assertRaises(ValueError):
MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=([self.outcome_constraint, opposing_constraint]),
)
# Two outcome_constraints on the same metric with different ops and
# bounds should not raise.
opposing_constraint = OutcomeConstraint(
metric=self.outcome_constraint.metric,
op=not self.outcome_constraint.op,
bound=self.outcome_constraint.bound + 1,
)
config = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=([self.outcome_constraint, opposing_constraint]),
)
self.assertEqual(
config.outcome_constraints, [self.outcome_constraint, opposing_constraint]
)
def testCloneWithArgs(self):
config1 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective_just_m2,
outcome_constraints=[self.m1_constraint],
)
config2 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective_just_m2,
objective_thresholds=self.objective_thresholds,
outcome_constraints=[self.m1_constraint],
)
self.assertEqual(
config1.clone_with_args(objective_thresholds=self.objective_thresholds),
config2,
)
def optimization_config_from_objectives(
objectives: List[Objective],
objective_thresholds: List[ObjectiveThreshold],
outcome_constraints: List[OutcomeConstraint],
) -> OptimizationConfig:
"""Parse objectives and constraints to define optimization config.
The resulting optimization config will be regular single-objective config
if `objectives` is a list of one element and a multi-objective config
otherwise.
NOTE: If passing in multiple objectives, `objective_thresholds` must be a
non-empty list definining constraints for each objective.
"""
if len(objectives) == 1:
if objective_thresholds:
raise ValueError(
"Single-objective optimizations must not specify objective thresholds."
)
return OptimizationConfig(
objective=objectives[0],
outcome_constraints=outcome_constraints,
)
if not objective_thresholds:
logger.info(
"Due to non-specification, we will use the heuristic for selecting "
"objective thresholds.")
return MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=objectives),
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def opt_config_and_tracking_metrics_from_sqa(
self, metrics_sqa: List[SQAMetric]
) -> Tuple[Optional[OptimizationConfig], List[Metric]]:
"""Convert a list of SQLAlchemy Metrics to a a tuple of Ax OptimizationConfig
and tracking metrics.
"""
objective = None
objective_thresholds = []
outcome_constraints = []
tracking_metrics = []
for metric_sqa in metrics_sqa:
metric = self.metric_from_sqa(metric_sqa=metric_sqa)
if isinstance(metric, Objective):
objective = metric
elif isinstance(metric, ObjectiveThreshold):
objective_thresholds.append(metric)
elif isinstance(metric, OutcomeConstraint):
outcome_constraints.append(metric)
else:
tracking_metrics.append(metric)
if objective is None:
return None, tracking_metrics
if objective_thresholds or type(objective) == MultiObjective:
optimization_config = MultiObjectiveOptimizationConfig(
objective=objective,
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
else:
optimization_config = OptimizationConfig(
objective=objective, outcome_constraints=outcome_constraints)
return (optimization_config, tracking_metrics)
def testTransformOptimizationConfigMOO(self):
m1 = Metric(name="m1", lower_is_better=False)
m2 = Metric(name="m2", lower_is_better=True)
mo = MultiObjective(
objectives=[
Objective(metric=m1, minimize=False),
Objective(metric=m2, minimize=True),
],
)
objective_thresholds = [
ObjectiveThreshold(metric=m1, bound=1.234, relative=False),
ObjectiveThreshold(metric=m2, bound=3.456, relative=False),
]
oc = MultiObjectiveOptimizationConfig(
objective=mo,
objective_thresholds=objective_thresholds,
)
tf = LogY(
search_space=None,
observation_features=None,
observation_data=[self.obsd1, self.obsd2],
config={"metrics": ["m1"]},
)
oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None)
oc.objective_thresholds[0].bound = math.log(1.234)
self.assertEqual(oc_tf, oc)
def get_branin_multi_objective_optimization_config(
has_objective_thresholds: bool = False,
) -> OptimizationConfig:
objective_thresholds = (
[
ObjectiveThreshold(
metric=get_branin_metric(name="branin_a"),
bound=10,
op=ComparisonOp.GEQ,
relative=False,
),
ObjectiveThreshold(
metric=get_branin_metric(name="branin_b"),
bound=20,
op=ComparisonOp.GEQ,
relative=False,
),
]
if has_objective_thresholds
else None
)
return MultiObjectiveOptimizationConfig(
objective=get_branin_multi_objective(),
objective_thresholds=objective_thresholds,
)
def get_branin_multi_objective_optimization_config(
has_objective_thresholds: bool = False,
num_objectives: int = 2,
) -> MultiObjectiveOptimizationConfig:
_validate_num_objectives(num_objectives=num_objectives)
if has_objective_thresholds:
objective_thresholds = [
ObjectiveThreshold(
metric=get_branin_metric(name="branin_a"),
bound=10,
op=ComparisonOp.GEQ,
relative=False,
),
ObjectiveThreshold(
metric=get_branin_metric(name="branin_b"),
bound=20,
op=ComparisonOp.GEQ,
relative=False,
),
]
if num_objectives == 3:
objective_thresholds.append(
ObjectiveThreshold(
metric=get_branin_metric(name="branin_c"),
bound=5.0,
op=ComparisonOp.GEQ,
relative=False,
))
else:
objective_thresholds = None
return MultiObjectiveOptimizationConfig(
objective=get_branin_multi_objective(num_objectives=num_objectives),
objective_thresholds=objective_thresholds,
)
def testEq(self):
config1 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective, outcome_constraints=self.outcome_constraints
)
config2 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective, outcome_constraints=self.outcome_constraints
)
self.assertEqual(config1, config2)
new_outcome_constraint = OutcomeConstraint(
metric=self.metrics["m3"], op=ComparisonOp.LEQ, bound=0.5
)
config3 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=[self.outcome_constraint, new_outcome_constraint],
)
self.assertNotEqual(config1, config3)
def __init__(
self,
experiment: Experiment,
search_space: SearchSpace,
data: Data,
model: TorchModel,
transforms: List[Type[Transform]],
transform_configs: Optional[Dict[str, TConfig]] = None,
torch_dtype: Optional[torch.dtype] = None, # noqa T484
torch_device: Optional[torch.device] = None,
status_quo_name: Optional[str] = None,
status_quo_features: Optional[ObservationFeatures] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
fit_out_of_design: bool = False,
objective_thresholds: Optional[TRefPoint] = None,
default_model_gen_options: Optional[TConfig] = None,
) -> None:
if (isinstance(experiment, MultiTypeExperiment)
and objective_thresholds is not None):
raise NotImplementedError(
"Objective threshold dependent multi-objective optimization algos "
"like EHVI are not yet supported for MultiTypeExperiments. "
"Remove the objective threshold arg and use a compatible algorithm "
"like ParEGO.")
self._objective_metric_names = None
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, experiment.optimization_config)
# Extract objective_thresholds from optimization_config, or inject it.
if not mooc:
raise ValueError(
("experiment must have an existing optimization_config "
"of type MultiObjectiveOptimizationConfig "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(
MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(
objective_thresholds=objective_thresholds)
optimization_config = mooc
super().__init__(
experiment=experiment,
search_space=search_space,
data=data,
model=model,
transforms=transforms,
transform_configs=transform_configs,
torch_dtype=torch_dtype,
torch_device=torch_device,
status_quo_name=status_quo_name,
status_quo_features=status_quo_features,
optimization_config=optimization_config,
fit_out_of_design=fit_out_of_design,
default_model_gen_options=default_model_gen_options,
)
def get_multi_objective_optimization_config() -> OptimizationConfig:
objective = get_multi_objective()
outcome_constraints = [get_outcome_constraint()]
objective_thresholds = [get_objective_threshold()]
return MultiObjectiveOptimizationConfig(
objective=objective,
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def transform_optimization_config(
self,
optimization_config: OptimizationConfig,
modelbridge: Optional[modelbridge_module.base.ModelBridge],
fixed_features: ObservationFeatures,
) -> OptimizationConfig:
r"""
Change the relative flag of the given relative optimization configuration
to False. This is needed in order for the new opt config to pass ModelBridge
that requires non-relativized opt config.
Args:
opt_config: Optimization configuaration relative to status quo.
Returns:
Optimization configuration relative to status quo with relative flag
equal to false.
"""
# Getting constraints
constraints = [
constraint.clone()
for constraint in optimization_config.outcome_constraints
]
if not all(constraint.relative
for constraint in optimization_config.outcome_constraints):
raise ValueError(
"All constraints must be relative to use the Relativize transform."
)
for constraint in constraints:
constraint.relative = False
if isinstance(optimization_config, MultiObjectiveOptimizationConfig):
# Getting objective thresholds
obj_thresholds = [
obj_threshold.clone()
for obj_threshold in optimization_config.objective_thresholds
]
for obj_threshold in obj_thresholds:
if not obj_threshold.relative:
raise ValueError(
"All objective thresholds must be relative to use "
"the Relativize transform.")
obj_threshold.relative = False
new_optimization_config = MultiObjectiveOptimizationConfig(
objective=optimization_config.objective,
outcome_constraints=constraints,
objective_thresholds=obj_thresholds,
)
else:
new_optimization_config = OptimizationConfig(
objective=optimization_config.objective,
outcome_constraints=constraints,
)
return new_optimization_config
def test_feasible_hypervolume(self):
ma = Metric(name="a", lower_is_better=False)
mb = Metric(name="b", lower_is_better=True)
mc = Metric(name="c", lower_is_better=False)
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(metrics=[ma, mb]),
outcome_constraints=[
OutcomeConstraint(
mc,
op=ComparisonOp.GEQ,
bound=0,
relative=False,
)
],
objective_thresholds=[
ObjectiveThreshold(
ma,
bound=1.0,
),
ObjectiveThreshold(
mb,
bound=1.0,
),
],
)
feas_hv = feasible_hypervolume(
optimization_config,
values={
"a": np.array(
[
1.0,
3.0,
2.0,
2.0,
]
),
"b": np.array(
[
0.0,
1.0,
0.0,
0.0,
]
),
"c": np.array(
[
0.0,
-0.0,
1.0,
-2.0,
]
),
},
)
self.assertEqual(list(feas_hv), [0.0, 0.0, 1.0, 1.0])
def test_MOO_with_more_outcomes_than_thresholds(self):
experiment = get_branin_experiment_with_multi_objective(
has_optimization_config=False)
metric_c = Metric(name="c", lower_is_better=False)
metric_a = Metric(name="a", lower_is_better=False)
objective_thresholds = [
ObjectiveThreshold(
metric=metric_c,
bound=2.0,
relative=False,
),
ObjectiveThreshold(
metric=metric_a,
bound=1.0,
relative=False,
),
]
experiment.optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[
Objective(metric=metric_a),
Objective(metric=metric_c),
]),
objective_thresholds=objective_thresholds,
)
experiment.add_tracking_metric(Metric(name="b", lower_is_better=False))
sobol = get_sobol(search_space=experiment.search_space, )
sobol_run = sobol.gen(1)
experiment.new_batch_trial().add_generator_run(
sobol_run).run().mark_completed()
data = Data(
pd.DataFrame(
data={
"arm_name": ["0_0", "0_0", "0_0"],
"metric_name": ["a", "b", "c"],
"mean": [1.0, 2.0, 3.0],
"trial_index": [0, 0, 0],
"sem": [0, 0, 0],
}))
test_names_to_fns = {
"MOO_NEHVI": get_MOO_NEHVI,
"MOO_EHVI": get_MOO_NEHVI,
"MOO_PAREGO": get_MOO_PAREGO,
"MOO_RS": get_MOO_RS,
}
for test_name, factory_fn in test_names_to_fns.items():
with self.subTest(test_name):
moo_model = factory_fn(
experiment=experiment,
data=data,
)
moo_gr = moo_model.gen(n=1)
obj_t = moo_gr.gen_metadata["objective_thresholds"]
self.assertEqual(obj_t[0], objective_thresholds[1])
self.assertEqual(obj_t[1], objective_thresholds[0])
self.assertEqual(len(obj_t), 2)
def get_multi_objective_optimization_config() -> OptimizationConfig:
objective = get_multi_objective()
outcome_constraints = [get_outcome_constraint()]
objective_thresholds = [
get_objective_threshold(metric_name="m1"),
get_objective_threshold(metric_name="m3", comparison_op=ComparisonOp.LEQ),
]
return MultiObjectiveOptimizationConfig(
objective=objective,
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def _build_new_optimization_config(weights,
primary_objective,
secondary_objective,
outcome_constraints=None):
obj = ScalarizedObjective(
metrics=[primary_objective, secondary_objective],
weights=weights,
minimize=False,
)
optimization_config = MultiObjectiveOptimizationConfig(
objective=obj, outcome_constraints=outcome_constraints)
return optimization_config
def test_choose_generation_strategy(self):
with self.subTest("GPEI"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space()
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "GPEI")
with self.subTest("MOO"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(),
optimization_config=MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[])
),
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "MOO")
with self.subTest("Sobol (we can try every option)"):
sobol = choose_generation_strategy(
search_space=get_factorial_search_space(), num_trials=1000
)
self.assertEqual(sobol._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol._steps), 1)
with self.subTest("Sobol (because of too many categories)"):
sobol_large = choose_generation_strategy(
search_space=get_large_factorial_search_space()
)
self.assertEqual(sobol_large._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol_large._steps), 1)
with self.subTest("GPEI-Batched"):
sobol_gpei_batched = choose_generation_strategy(
search_space=get_branin_search_space(), use_batch_trials=3
)
self.assertEqual(sobol_gpei_batched._steps[0].num_trials, 1)
with self.subTest("BO_MIXED (purely categorical)"):
bo_mixed = choose_generation_strategy(
search_space=get_factorial_search_space()
)
self.assertEqual(bo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed._steps[0].num_trials, 5)
self.assertEqual(bo_mixed._steps[1].model.value, "BO_MIXED")
with self.subTest("BO_MIXED (mixed search space)"):
bo_mixed_2 = choose_generation_strategy(
search_space=get_branin_search_space(with_choice_parameter=True)
)
self.assertEqual(bo_mixed_2._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed_2._steps[0].num_trials, 5)
self.assertEqual(bo_mixed_2._steps[1].model.value, "BO_MIXED")
def testClone(self):
config1 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective, outcome_constraints=self.outcome_constraints
)
self.assertEqual(config1, config1.clone())
config2 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
objective_thresholds=self.objective_thresholds,
)
self.assertEqual(config2, config2.clone())
def testHasGoodOptConfigModelFit(self):
# Construct diagnostics
result = []
for i, obs in enumerate(self.training_data):
result.append(
CVResult(observed=obs, predicted=self.observation_data[i]))
diag = compute_diagnostics(result=result)
assess_model_fit_result = assess_model_fit(
diagnostics=diag,
significance_level=0.05,
)
# Test single objective
optimization_config = OptimizationConfig(objective=Objective(
metric=Metric("a")))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test multi objective
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(
metrics=[Metric("a"), Metric("b")]))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test constraints
optimization_config = OptimizationConfig(
objective=Objective(metric=Metric("a")),
outcome_constraints=[
OutcomeConstraint(metric=Metric("b"),
op=ComparisonOp.GEQ,
bound=0.1)
],
)
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
def optimization_config_from_objectives(
objectives: List[Metric],
objective_thresholds: List[ObjectiveThreshold],
outcome_constraints: List[OutcomeConstraint],
) -> OptimizationConfig:
"""Parse objectives and constraints to define optimization config.
The resulting optimization config will be regular single-objective config
if `objectives` is a list of one element and a multi-objective config
otherwise.
NOTE: If passing in multiple objectives, `objective_thresholds` must be a
non-empty list definining constraints for each objective.
"""
if len(objectives) == 1:
if objective_thresholds:
raise ValueError(
"Single-objective optimizations must not specify objective thresholds."
)
return OptimizationConfig(
objective=Objective(
metric=objectives[0],
),
outcome_constraints=outcome_constraints,
)
else:
objective_names = {m.name for m in objectives}
threshold_names = {oc.metric.name for oc in objective_thresholds}
if objective_names != threshold_names:
diff = objective_names.symmetric_difference(threshold_names)
raise ValueError(
"Multi-objective optimization requires one objective threshold "
f"per objective metric; unmatched names are {diff}"
)
return MultiObjectiveOptimizationConfig(
objective=MultiObjective(metrics=objectives),
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def _pareto_frontier(
self,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
observation_data: Optional[List[ObservationData]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[ObservationData]:
# TODO(jej): This method should be refactored to move tensor
# conversions into a separate utility, and eventually should be
# moved into base.py.
# The reason this method is currently implemented in array.py is to
# allow the broadest possible set of models to call frontier and
# hypervolume evaluation functions given the current API.
X = (self.transform_observation_features(observation_features)
if observation_features else None)
X = self._array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data:
Y, Yvar = self.transform_observation_data(observation_data)
if Y is not None and Yvar is not None:
Y, Yvar = (self._array_to_tensor(Y), self._array_to_tensor(Yvar))
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, self._optimization_config)
if not mooc:
raise ValueError(
("experiment must have an existing optimization_config "
"of type MultiObjectiveOptimizationConfig "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(
MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(
objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = self.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=self.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=self.outcomes,
)
objective_thresholds_arr = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
outcomes=self.outcomes,
)
# Transform to tensors.
obj_w, oc_c, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=self._array_to_tensor,
)
obj_t = self._array_to_tensor(objective_thresholds_arr)
frontier_evaluator = self._get_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `model` to anonymous call
f, cov = frontier_evaluator(
model=self.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone().numpy(), cov.detach().cpu().clone(
).numpy()
frontier_observation_data = array_to_observation_data(
f=f, cov=cov, outcomes=not_none(self.outcomes))
# Untransform observations
for t in reversed(self.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
return frontier_observation_data
def test_choose_generation_strategy(self):
with self.subTest("GPEI"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space())
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "GPEI")
self.assertIsNone(sobol_gpei._steps[1].model_kwargs)
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(), verbose=True)
self.assertIsNone(sobol_gpei._steps[1].model_kwargs)
with self.subTest("MOO"):
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[]))
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(),
optimization_config=optimization_config,
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "MOO")
model_kwargs = sobol_gpei._steps[1].model_kwargs
self.assertEqual(list(model_kwargs.keys()),
["transforms", "transform_configs"])
self.assertGreater(len(model_kwargs["transforms"]), 0)
transform_config_dict = {
"Winsorize": {
"optimization_config": optimization_config
}
}
self.assertEqual(model_kwargs["transform_configs"],
transform_config_dict)
with self.subTest("Sobol (we can try every option)"):
sobol = choose_generation_strategy(
search_space=get_factorial_search_space(), num_trials=1000)
self.assertEqual(sobol._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol._steps), 1)
with self.subTest("Sobol (because of too many categories)"):
ss = get_large_factorial_search_space()
sobol_large = choose_generation_strategy(
search_space=get_large_factorial_search_space(), verbose=True)
self.assertEqual(sobol_large._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol_large._steps), 1)
with self.subTest("GPEI-Batched"):
sobol_gpei_batched = choose_generation_strategy(
search_space=get_branin_search_space(), use_batch_trials=3)
self.assertEqual(sobol_gpei_batched._steps[0].num_trials, 1)
with self.subTest("BO_MIXED (purely categorical)"):
bo_mixed = choose_generation_strategy(
search_space=get_factorial_search_space())
self.assertEqual(bo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed._steps[0].num_trials, 6)
self.assertEqual(bo_mixed._steps[1].model.value, "BO_MIXED")
self.assertIsNone(bo_mixed._steps[1].model_kwargs)
with self.subTest("BO_MIXED (mixed search space)"):
ss = get_branin_search_space(with_choice_parameter=True)
ss.parameters["x2"]._is_ordered = False
bo_mixed_2 = choose_generation_strategy(search_space=ss)
self.assertEqual(bo_mixed_2._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed_2._steps[0].num_trials, 5)
self.assertEqual(bo_mixed_2._steps[1].model.value, "BO_MIXED")
self.assertIsNone(bo_mixed_2._steps[1].model_kwargs)
with self.subTest("BO_MIXED (mixed multi-objective optimization)"):
search_space = get_branin_search_space(with_choice_parameter=True)
search_space.parameters["x2"]._is_ordered = False
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[]))
moo_mixed = choose_generation_strategy(
search_space=search_space,
optimization_config=optimization_config)
self.assertEqual(moo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(moo_mixed._steps[0].num_trials, 5)
self.assertEqual(moo_mixed._steps[1].model.value, "BO_MIXED")
model_kwargs = moo_mixed._steps[1].model_kwargs
self.assertEqual(list(model_kwargs.keys()),
["transforms", "transform_configs"])
self.assertGreater(len(model_kwargs["transforms"]), 0)
transform_config_dict = {
"Winsorize": {
"optimization_config": optimization_config
}
}
self.assertEqual(model_kwargs["transform_configs"],
transform_config_dict)
with self.subTest("SAASBO"):
sobol_fullybayesian = choose_generation_strategy(
search_space=get_branin_search_space(),
use_batch_trials=True,
num_initialization_trials=3,
use_saasbo=True,
)
self.assertEqual(sobol_fullybayesian._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesian._steps[0].num_trials, 3)
self.assertEqual(sobol_fullybayesian._steps[1].model.value,
"FullyBayesian")
self.assertTrue(
sobol_fullybayesian._steps[1].model_kwargs["verbose"])
with self.subTest("SAASBO MOO"):
sobol_fullybayesianmoo = choose_generation_strategy(
search_space=get_branin_search_space(),
use_batch_trials=True,
num_initialization_trials=3,
use_saasbo=True,
optimization_config=MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[])),
)
self.assertEqual(sobol_fullybayesianmoo._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesianmoo._steps[0].num_trials, 3)
self.assertEqual(sobol_fullybayesianmoo._steps[1].model.value,
"FullyBayesianMOO")
self.assertTrue(
sobol_fullybayesianmoo._steps[1].model_kwargs["verbose"])
with self.subTest("SAASBO"):
sobol_fullybayesian_large = choose_generation_strategy(
search_space=get_large_ordinal_search_space(
n_ordinal_choice_parameters=5,
n_continuous_range_parameters=10),
use_saasbo=True,
)
self.assertEqual(sobol_fullybayesian_large._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesian_large._steps[0].num_trials,
30)
self.assertEqual(sobol_fullybayesian_large._steps[1].model.value,
"FullyBayesian")
self.assertTrue(
sobol_fullybayesian_large._steps[1].model_kwargs["verbose"])
with self.subTest("num_initialization_trials"):
ss = get_large_factorial_search_space()
for _, param in ss.parameters.items():
param._is_ordered = True
# 2 * len(ss.parameters) init trials are performed if num_trials is large
gs_12_init_trials = choose_generation_strategy(search_space=ss,
num_trials=100)
self.assertEqual(gs_12_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_12_init_trials._steps[0].num_trials, 12)
self.assertEqual(gs_12_init_trials._steps[1].model.value, "GPEI")
# at least 5 initialization trials are performed
gs_5_init_trials = choose_generation_strategy(search_space=ss,
num_trials=0)
self.assertEqual(gs_5_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_5_init_trials._steps[0].num_trials, 5)
self.assertEqual(gs_5_init_trials._steps[1].model.value, "GPEI")
# avoid spending >20% of budget on initialization trials if there are
# more than 5 initialization trials
gs_6_init_trials = choose_generation_strategy(search_space=ss,
num_trials=30)
self.assertEqual(gs_6_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_6_init_trials._steps[0].num_trials, 6)
self.assertEqual(gs_6_init_trials._steps[1].model.value, "GPEI")
def testGen(self, mock_init, mock_best_point, mock_gen):
# Test with constraints
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=True),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
ma = NumpyModelBridge()
ma.parameters = ["x", "y", "z"]
ma.outcomes = ["a", "b"]
ma.transforms = OrderedDict()
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[0][2]
self.assertEqual(gen_args["n"], 3)
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([-1.0,
0.0])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][0],
np.array([[0.0, -1.0]])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][1],
np.array([[-2]])))
self.assertTrue(
np.array_equal(
gen_args["linear_constraints"][0],
np.array([[1.0, -1, 0.0], [-1.0, 0.0, -1.0]]),
))
self.assertTrue(
np.array_equal(gen_args["linear_constraints"][1],
np.array([[0.0], [-3.5]])))
self.assertEqual(gen_args["fixed_features"], {2: 3.0})
self.assertTrue(
np.array_equal(gen_args["pending_observations"][0], np.array([])))
self.assertTrue(
np.array_equal(gen_args["pending_observations"][1],
np.array([[0.6, 1.6, 3.0]])))
self.assertEqual(gen_args["model_gen_options"], {"option": "yes"})
self.assertEqual(observation_features[0].parameters, {
"x": 1.0,
"y": 2.0,
"z": 3.0
})
self.assertEqual(observation_features[1].parameters, {
"x": 3.0,
"y": 4.0,
"z": 3.0
})
self.assertTrue(np.array_equal(weights, np.array([1.0, 2.0])))
# Test with multiple objectives.
oc2 = OptimizationConfig(objective=ScalarizedObjective(
metrics=[Metric(name="a"), Metric(name="b")], minimize=True))
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=oc2,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[1][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"],
np.array([-1.0, -1.0])))
# Test with MultiObjective (unweighted multiple objectives)
oc3 = MultiObjectiveOptimizationConfig(objective=MultiObjective(
metrics=[Metric(name="a"),
Metric(name="b", lower_is_better=True)],
minimize=True,
))
search_space = SearchSpace(self.parameters) # Unconstrained
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=search_space,
optimization_config=oc3,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[2][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([1.0,
-1.0])))
# Test with no constraints, no fixed feature, no pending observations
search_space = SearchSpace(self.parameters[:2])
optimization_config.outcome_constraints = []
ma.parameters = ["x", "y"]
ma._gen(3, search_space, {}, ObservationFeatures({}), None,
optimization_config)
gen_args = mock_gen.mock_calls[3][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertIsNone(gen_args["linear_constraints"])
self.assertIsNone(gen_args["fixed_features"])
self.assertIsNone(gen_args["pending_observations"])
# Test validation
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=False),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
optimization_config.objective.minimize = True
optimization_config.outcome_constraints[0].relative = True
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
def get_pareto_frontier_and_transformed_configs(
modelbridge: modelbridge_module.array.ArrayModelBridge,
observation_features: List[ObservationFeatures],
observation_data: Optional[List[ObservationData]] = None,
objective_thresholds: Optional[TRefPoint] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
arm_names: Optional[List[Optional[str]]] = None,
use_model_predictions: bool = True,
) -> Tuple[List[Observation], Tensor, Tensor, Optional[Tensor]]:
"""Helper that applies transforms and calls frontier_evaluator.
Returns transformed configs in addition to the Pareto observations.
Args:
modelbridge: Modelbridge used to predict metrics outcomes.
observation_features: observation features to predict, if provided and
use_model_predictions is True.
observation_data: data for computing the Pareto front, unless features
are provided and model_predictions is True.
objective_thresholds: metric values bounding the region of interest in
the objective outcome space.
optimization_config: Optimization config.
arm_names: Arm names for each observation.
use_model_predictions: If True, will use model predictions at
observation_features to compute Pareto front, if provided. If False,
will use observation_data directly to compute Pareto front, regardless
of whether observation_features are provided.
Returns:
frontier_observations: Observations of points on the pareto frontier.
f: n x m tensor representation of the Pareto frontier values where n is the
length of frontier_observations and m is the number of metrics.
obj_w: m tensor of objective weights.
obj_t: m tensor of objective thresholds corresponding to Y, or None if no
objective thresholds used.
"""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
X = (modelbridge.transform_observation_features(observation_features)
if use_model_predictions else None)
X = array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data is not None:
Y, Yvar = modelbridge.transform_observation_data(observation_data)
Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar))
if arm_names is None:
arm_names = [None] * len(observation_features)
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, modelbridge._optimization_config)
if not mooc:
raise ValueError(
("Experiment must have an existing optimization_config "
"of type `MultiObjectiveOptimizationConfig` "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = modelbridge.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=modelbridge.outcomes,
)
obj_t = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
objective=optimization_config.objective,
outcomes=modelbridge.outcomes,
)
obj_t = array_to_tensor(obj_t)
# Transform to tensors.
obj_w, oc_c, _, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=array_to_tensor,
)
frontier_evaluator = get_default_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call
f, cov, indx = frontier_evaluator(
model=modelbridge.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone(), cov.detach().cpu().clone()
indx = indx.tolist()
frontier_observation_data = array_to_observation_data(
f=f.numpy(), cov=cov.numpy(), outcomes=not_none(modelbridge.outcomes))
# Untransform observations
for t in reversed(modelbridge.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
# Construct observations
frontier_observations = []
for i, obsd in enumerate(frontier_observation_data):
frontier_observations.append(
Observation(
features=observation_features[indx[i]],
data=obsd,
arm_name=arm_names[indx[i]],
))
return frontier_observations, f, obj_w, obj_t
def pareto_frontier(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
observation_data: Optional[List[ObservationData]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[ObservationData]:
"""Helper that applies transforms and calls frontier_evaluator."""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
X = (modelbridge.transform_observation_features(observation_features)
if observation_features else None)
X = array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data:
Y, Yvar = modelbridge.transform_observation_data(observation_data)
if Y is not None and Yvar is not None:
Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar))
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, modelbridge._optimization_config)
if not mooc:
raise ValueError(
("Experiment must have an existing optimization_config "
"of type `MultiObjectiveOptimizationConfig` "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = modelbridge.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=modelbridge.outcomes,
)
objective_thresholds_arr = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
outcomes=modelbridge.outcomes,
)
# Transform to tensors.
obj_w, oc_c, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=array_to_tensor,
)
obj_t = array_to_tensor(objective_thresholds_arr)
frontier_evaluator = get_default_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call
f, cov = frontier_evaluator(
model=modelbridge.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone().numpy(), cov.detach().cpu().clone(
).numpy()
frontier_observation_data = array_to_observation_data(
f=f, cov=cov, outcomes=not_none(modelbridge.outcomes))
# Untransform observations
for t in reversed(modelbridge.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
return frontier_observation_data
def testInit(self):
config1 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
outcome_constraints=self.outcome_constraints)
self.assertEqual(str(config1), MOOC_STR)
with self.assertRaises(TypeError):
config1.objective = self.objective # Wrong objective type
# updating constraints is fine.
config1.outcome_constraints = [self.outcome_constraint]
self.assertEqual(len(config1.metrics), 2)
# objective without outcome_constraints is also supported
config2 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective)
# setting objective is fine too, if it's compatible with constraints.
config2.objective = self.multi_objective
# setting constraints on objectives is fine for MultiObjective components.
config2.outcome_constraints = [self.outcome_constraint]
self.assertEqual(config2.outcome_constraints,
[self.outcome_constraint])
# construct constraints with objective_thresholds:
config3 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
objective_thresholds=self.objective_thresholds,
)
self.assertEqual(config3.all_constraints, self.objective_thresholds)
# objective_thresholds and outcome constraints together.
config4 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
objective_thresholds=self.objective_thresholds,
outcome_constraints=[self.m3_constraint],
)
self.assertEqual(config4.all_constraints,
[self.m3_constraint] + self.objective_thresholds)
self.assertEqual(config4.outcome_constraints, [self.m3_constraint])
self.assertEqual(config4.objective_thresholds,
self.objective_thresholds)
# verify relative_objective_thresholds works:
config5 = MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
objective_thresholds=[self.outcome_constraint],
)
threshold = config5.objective_thresholds[0]
self.assertTrue(threshold.relative)
self.assertEqual(threshold.bound, -0.25)
# ValueError on wrong direction constraints
with self.assertRaises(ValueError):
MultiObjectiveOptimizationConfig(
objective=self.multi_objective,
objective_thresholds=[self.additional_outcome_constraint],
)
def get_branin_multi_objective_optimization_config() -> OptimizationConfig:
return MultiObjectiveOptimizationConfig(objective=get_branin_multi_objective())
def testObservedParetoFrontiers(self):
experiment = get_branin_experiment(
with_batch=True, has_optimization_config=False, with_status_quo=True
)
# Optimization config is not optional
with self.assertRaises(ValueError):
get_observed_pareto_frontiers(experiment=experiment, data=Data())
metrics = [
BraninMetric(name="m1", param_names=["x1", "x2"], lower_is_better=True),
NegativeBraninMetric(
name="m2", param_names=["x1", "x2"], lower_is_better=True
),
BraninMetric(name="m3", param_names=["x1", "x2"], lower_is_better=True),
]
bounds = [0, -100, 0]
objective_thresholds = [
ObjectiveThreshold(
metric=metric,
bound=bounds[i],
relative=True,
op=ComparisonOp.LEQ,
)
for i, metric in enumerate(metrics)
]
objective = MultiObjective(metrics=metrics, minimize=True)
optimization_config = MultiObjectiveOptimizationConfig(
objective=objective,
objective_thresholds=objective_thresholds,
)
experiment.optimization_config = optimization_config
experiment.trials[0].run()
# For the check below, compute which arms are better than SQ
df = experiment.fetch_data().df
df["sem"] = np.nan
data = Data(df)
sq_val = df[(df["arm_name"] == "status_quo") & (df["metric_name"] == "m1")][
"mean"
].values[0]
pareto_arms = sorted(
df[(df["mean"] <= sq_val) & (df["metric_name"] == "m1")]["arm_name"]
.unique()
.tolist()
)
pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data)
# We have all pairs of metrics
self.assertEqual(len(pfrs), 3)
true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")]
for i, pfr in enumerate(pfrs):
self.assertEqual(pfr.primary_metric, true_pairs[i][0])
self.assertEqual(pfr.secondary_metric, true_pairs[i][1])
self.assertEqual(pfr.absolute_metrics, [])
self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"])
self.assertEqual(len(pfr.means["m1"]), len(pareto_arms))
self.assertTrue(np.isnan(pfr.sems["m1"]).all())
self.assertEqual(len(pfr.arm_names), len(pareto_arms))
arm_idx = np.argsort(pfr.arm_names)
for i, idx in enumerate(arm_idx):
name = pareto_arms[i]
self.assertEqual(pfr.arm_names[idx], name)
self.assertEqual(
pfr.param_dicts[idx], experiment.arms_by_name[name].parameters
)
