def setUp(self):
self.metrics = {
"m1": Metric(name="m1", lower_is_better=True),
"m2": Metric(name="m2", lower_is_better=False),
"m3": Metric(name="m3", lower_is_better=False),
}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.multi_objective = MultiObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
self.multi_objective_just_m2 = MultiObjective(
metrics=[self.metrics["m2"]])
self.outcome_constraint = OutcomeConstraint(metric=self.metrics["m2"],
op=ComparisonOp.GEQ,
bound=-0.25)
self.additional_outcome_constraint = OutcomeConstraint(
metric=self.metrics["m2"], op=ComparisonOp.LEQ, bound=0.25)
self.outcome_constraints = [
self.outcome_constraint,
self.additional_outcome_constraint,
]
self.objective_thresholds = [
ObjectiveThreshold(metric=self.metrics["m2"],
bound=-1.0,
relative=False)
]
self.m1_constraint = OutcomeConstraint(metric=self.metrics["m1"],
op=ComparisonOp.LEQ,
bound=0.1,
relative=True)
self.m3_constraint = OutcomeConstraint(metric=self.metrics["m3"],
op=ComparisonOp.GEQ,
bound=0.1,
relative=True)
def setUp(self):
self.metrics = {
"m1": Metric(name="m1"),
"m2": Metric(name="m2", lower_is_better=True),
"m3": Metric(name="m3", lower_is_better=False),
}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.multi_objective = MultiObjective(metrics=[
self.metrics["m1"], self.metrics["m2"], self.metrics["m3"]
])
self.scalarized_objective = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
def test_extract_objective_thresholds(self):
outcomes = ["m1", "m2", "m3", "m4"]
objective = MultiObjective(metrics=[Metric(name) for name in outcomes[:3]])
objective_thresholds = [
ObjectiveThreshold(
metric=Metric(name), op=ComparisonOp.LEQ, bound=float(i + 2)
)
for i, name in enumerate(outcomes[:3])
]
# None of no thresholds
self.assertIsNone(
extract_objective_thresholds(
objective_thresholds=[], objective=objective, outcomes=outcomes
)
)
# Working case
obj_t = extract_objective_thresholds(
objective_thresholds=objective_thresholds,
objective=objective,
outcomes=outcomes,
)
self.assertTrue(np.array_equal(obj_t, np.array([2.0, 3.0, 4.0, 0.0])))
# Fails if threshold not provided for all objective metrics
with self.assertRaises(ValueError):
extract_objective_thresholds(
objective_thresholds=objective_thresholds[:2],
objective=objective,
outcomes=outcomes,
)
# Fails if number of thresholds doesn't equal number of objectives
objective2 = Objective(Metric("m1"))
with self.assertRaises(ValueError):
extract_objective_thresholds(
objective_thresholds=objective_thresholds,
objective=objective2,
outcomes=outcomes,
)
# Works with a single objective, single threshold
obj_t = extract_objective_thresholds(
objective_thresholds=objective_thresholds[:1],
objective=objective2,
outcomes=outcomes,
)
self.assertTrue(np.array_equal(obj_t, np.array([2.0, 0.0, 0.0, 0.0])))
# Fails if relative
objective_thresholds[2] = ObjectiveThreshold(
metric=Metric("m3"), op=ComparisonOp.LEQ, bound=3, relative=True
)
with self.assertRaises(ValueError):
extract_objective_thresholds(
objective_thresholds=objective_thresholds,
objective=objective,
outcomes=outcomes,
)
def optimization_config_from_objectives(
objectives: List[Objective],
objective_thresholds: List[ObjectiveThreshold],
outcome_constraints: List[OutcomeConstraint],
) -> OptimizationConfig:
"""Parse objectives and constraints to define optimization config.
The resulting optimization config will be regular single-objective config
if `objectives` is a list of one element and a multi-objective config
otherwise.
NOTE: If passing in multiple objectives, `objective_thresholds` must be a
non-empty list definining constraints for each objective.
"""
if len(objectives) == 1:
if objective_thresholds:
raise ValueError(
"Single-objective optimizations must not specify objective thresholds."
)
return OptimizationConfig(
objective=objectives[0],
outcome_constraints=outcome_constraints,
)
if not objective_thresholds:
logger.info(
"Due to non-specification, we will use the heuristic for selecting "
"objective thresholds.")
return MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=objectives),
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def testMultiObjectiveBackwardsCompatibility(self):
multi_objective = MultiObjective(metrics=[
self.metrics["m1"], self.metrics["m2"], self.metrics["m3"]
])
minimizes = [obj.minimize for obj in multi_objective.objectives]
self.assertEqual(multi_objective.metrics, list(self.metrics.values()))
self.assertEqual(minimizes, [False, True, False])
multi_objective_min = MultiObjective(
metrics=[
self.metrics["m1"], self.metrics["m2"], self.metrics["m3"]
],
minimize=True,
)
minimizes = [obj.minimize for obj in multi_objective_min.objectives]
self.assertEqual(minimizes, [True, False, True])
def setUp(self):
self.metrics = {"m1": Metric(name="m1"), "m2": Metric(name="m2")}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.alt_objective = Objective(metric=self.metrics["m2"],
minimize=False)
self.multi_objective = MultiObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
self.m2_objective = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
self.outcome_constraint = OutcomeConstraint(metric=self.metrics["m2"],
op=ComparisonOp.GEQ,
bound=-0.25)
self.additional_outcome_constraint = OutcomeConstraint(
metric=self.metrics["m2"], op=ComparisonOp.LEQ, bound=0.25)
self.scalarized_outcome_constraint = ScalarizedOutcomeConstraint(
metrics=[self.metrics["m1"], self.metrics["m2"]],
weights=[0.5, 0.5],
op=ComparisonOp.GEQ,
bound=-0.25,
)
self.outcome_constraints = [
self.outcome_constraint,
self.additional_outcome_constraint,
self.scalarized_outcome_constraint,
]
def testTransformOptimizationConfigMOO(self):
m1 = Metric(name="m1", lower_is_better=False)
m2 = Metric(name="m2", lower_is_better=True)
mo = MultiObjective(
objectives=[
Objective(metric=m1, minimize=False),
Objective(metric=m2, minimize=True),
],
)
objective_thresholds = [
ObjectiveThreshold(metric=m1, bound=1.234, relative=False),
ObjectiveThreshold(metric=m2, bound=3.456, relative=False),
]
oc = MultiObjectiveOptimizationConfig(
objective=mo,
objective_thresholds=objective_thresholds,
)
tf = LogY(
search_space=None,
observation_features=None,
observation_data=[self.obsd1, self.obsd2],
config={"metrics": ["m1"]},
)
oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None)
oc.objective_thresholds[0].bound = math.log(1.234)
self.assertEqual(oc_tf, oc)
def get_branin_multi_objective() -> Objective:
return MultiObjective(
objectives=[
Objective(metric=get_branin_metric(name="branin_a")),
Objective(metric=get_branin_metric(name="branin_b")),
],
)
def get_multi_objective() -> Objective:
return MultiObjective(
objectives=[
Objective(metric=Metric(name="m1")),
Objective(metric=Metric(name="m3", lower_is_better=True), minimize=True),
],
)
def get_branin_multi_objective() -> Objective:
return MultiObjective(
metrics=[
get_branin_metric(name="branin_a"),
get_branin_metric(name="branin_b"),
],
minimize=False,
)
def get_branin_multi_objective(num_objectives: int = 2) -> Objective:
_validate_num_objectives(num_objectives=num_objectives)
objectives = [
Objective(metric=get_branin_metric(name="branin_a")),
Objective(metric=get_branin_metric(name="branin_b")),
]
if num_objectives == 3:
objectives.append(Objective(metric=get_branin_metric(name="branin_c")))
return MultiObjective(objectives=objectives)
def test_feasible_hypervolume(self):
ma = Metric(name="a", lower_is_better=False)
mb = Metric(name="b", lower_is_better=True)
mc = Metric(name="c", lower_is_better=False)
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(metrics=[ma, mb]),
outcome_constraints=[
OutcomeConstraint(
mc,
op=ComparisonOp.GEQ,
bound=0,
relative=False,
)
],
objective_thresholds=[
ObjectiveThreshold(
ma,
bound=1.0,
),
ObjectiveThreshold(
mb,
bound=1.0,
),
],
)
feas_hv = feasible_hypervolume(
optimization_config,
values={
"a": np.array(
[
1.0,
3.0,
2.0,
2.0,
]
),
"b": np.array(
[
0.0,
1.0,
0.0,
0.0,
]
),
"c": np.array(
[
0.0,
-0.0,
1.0,
-2.0,
]
),
},
)
self.assertEqual(list(feas_hv), [0.0, 0.0, 1.0, 1.0])
def test_MOO_with_more_outcomes_than_thresholds(self):
experiment = get_branin_experiment_with_multi_objective(
has_optimization_config=False)
metric_c = Metric(name="c", lower_is_better=False)
metric_a = Metric(name="a", lower_is_better=False)
objective_thresholds = [
ObjectiveThreshold(
metric=metric_c,
bound=2.0,
relative=False,
),
ObjectiveThreshold(
metric=metric_a,
bound=1.0,
relative=False,
),
]
experiment.optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[
Objective(metric=metric_a),
Objective(metric=metric_c),
]),
objective_thresholds=objective_thresholds,
)
experiment.add_tracking_metric(Metric(name="b", lower_is_better=False))
sobol = get_sobol(search_space=experiment.search_space, )
sobol_run = sobol.gen(1)
experiment.new_batch_trial().add_generator_run(
sobol_run).run().mark_completed()
data = Data(
pd.DataFrame(
data={
"arm_name": ["0_0", "0_0", "0_0"],
"metric_name": ["a", "b", "c"],
"mean": [1.0, 2.0, 3.0],
"trial_index": [0, 0, 0],
"sem": [0, 0, 0],
}))
test_names_to_fns = {
"MOO_NEHVI": get_MOO_NEHVI,
"MOO_EHVI": get_MOO_NEHVI,
"MOO_PAREGO": get_MOO_PAREGO,
"MOO_RS": get_MOO_RS,
}
for test_name, factory_fn in test_names_to_fns.items():
with self.subTest(test_name):
moo_model = factory_fn(
experiment=experiment,
data=data,
)
moo_gr = moo_model.gen(n=1)
obj_t = moo_gr.gen_metadata["objective_thresholds"]
self.assertEqual(obj_t[0], objective_thresholds[1])
self.assertEqual(obj_t[1], objective_thresholds[0])
self.assertEqual(len(obj_t), 2)
def test_transform_ref_point(self, _mock_fit, _mock_predict, _mock_unwrap):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=False)
metrics = exp.optimization_config.objective.metrics
ref_point = {metrics[0].name: 0.0, metrics[1].name: 0.0}
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point=ref_point,
)
self.assertIsNone(modelbridge._transformed_ref_point)
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials))
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point=ref_point,
)
self.assertIsNotNone(modelbridge._transformed_ref_point)
self.assertEqual(2, len(modelbridge._transformed_ref_point))
mixed_objective_constraints_optimization_config = OptimizationConfig(
objective=MultiObjective(
metrics=[get_branin_metric(name="branin_b")], minimize=False),
outcome_constraints=[
OutcomeConstraint(metric=Metric(name="branin_a"),
op=ComparisonOp.LEQ,
bound=1)
],
)
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=mixed_objective_constraints_optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point={"branin_b": 0.0},
)
self.assertEqual({"branin_a", "branin_b"}, modelbridge._metric_names)
self.assertEqual(["branin_b"], modelbridge._objective_metric_names)
self.assertIsNotNone(modelbridge._transformed_ref_point)
self.assertEqual(1, len(modelbridge._transformed_ref_point))
def get_experiment(self):
""" Creates the experiment defining the metrics and the configuration"""
metric_list = [
AccuracyMetric(self.epochs,
name="error",
pruning=self.pruning,
datasets=self.datasets,
classes=self.classes,
net=self.net,
quant_scheme=self.quant_scheme,
quant_params=self.quant_params,
collate_fn=self.collate_fn,
splitter=self.splitter,
models_path=self.models_path,
cuda=self.cuda,
trainer=self.trainer),
WeightMetric(name="weight",
datasets=self.datasets,
classes=self.classes,
net=self.net,
collate_fn=self.collate_fn,
splitter=self.splitter,
trainer=self.trainer),
FeatureMapMetric(name="ram",
datasets=self.datasets,
classes=self.classes,
net=self.net,
collate_fn=self.collate_fn,
splitter=self.splitter,
trainer=self.trainer),
LatencyMetric(name="latency",
datasets=self.datasets,
classes=self.classes,
net=self.net,
flops_capacity=self.flops,
collate_fn=self.collate_fn,
splitter=self.splitter,
trainer=self.trainer),
]
experiment = Experiment(name="experiment_building_blocks",
search_space=self.search_space)
metrics = list(itemgetter(*self.objectives)(metric_list))
if len(self.objectives) > 1:
objective = MultiObjective(metrics=metrics, minimize=True)
else:
objective = Objective(metric=metrics[0], minimize=True)
optimization_config = OptimizationConfig(objective=objective)
experiment.optimization_config = optimization_config
experiment.runner = MyRunner()
return experiment
class ObjectiveTest(TestCase):
def setUp(self):
self.metrics = {
"m1": Metric(name="m1"),
"m2": Metric(name="m2", lower_is_better=True),
"m3": Metric(name="m3", lower_is_better=False),
}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.multi_objective = MultiObjective(metrics=[
self.metrics["m1"], self.metrics["m2"], self.metrics["m3"]
])
self.scalarized_objective = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
def testBadInit(self):
with self.assertRaises(ValueError):
self.scalarized_objective_weighted = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]],
weights=[1.0])
def testMultiObjective(self):
with self.assertRaises(NotImplementedError):
return self.multi_objective.metric
self.assertEqual(self.multi_objective.metrics,
list(self.metrics.values()))
weights = [mw[1] for mw in self.multi_objective.metric_weights]
self.assertEqual(weights, [1.0, -1.0, 1.0])
self.assertEqual(self.multi_objective.clone(), self.multi_objective)
self.assertEqual(
str(self.multi_objective),
"MultiObjective(metric_names=['m1', 'm2', 'm3'], minimize=False)",
)
def testScalarizedObjective(self):
with self.assertRaises(NotImplementedError):
return self.scalarized_objective.metric
self.assertEqual(self.scalarized_objective.metrics,
[self.metrics["m1"], self.metrics["m2"]])
weights = [mw[1] for mw in self.scalarized_objective.metric_weights]
self.assertEqual(weights, [1.0, 1.0])
self.assertEqual(self.scalarized_objective.clone(),
self.scalarized_objective)
self.assertEqual(
str(self.scalarized_objective),
("ScalarizedObjective(metric_names=['m1', 'm2'], weights=[1.0, 1.0], "
"minimize=False)"),
)
def test_choose_generation_strategy(self):
with self.subTest("GPEI"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space()
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "GPEI")
with self.subTest("MOO"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(),
optimization_config=MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[])
),
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "MOO")
with self.subTest("Sobol (we can try every option)"):
sobol = choose_generation_strategy(
search_space=get_factorial_search_space(), num_trials=1000
)
self.assertEqual(sobol._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol._steps), 1)
with self.subTest("Sobol (because of too many categories)"):
sobol_large = choose_generation_strategy(
search_space=get_large_factorial_search_space()
)
self.assertEqual(sobol_large._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol_large._steps), 1)
with self.subTest("GPEI-Batched"):
sobol_gpei_batched = choose_generation_strategy(
search_space=get_branin_search_space(), use_batch_trials=3
)
self.assertEqual(sobol_gpei_batched._steps[0].num_trials, 1)
with self.subTest("BO_MIXED (purely categorical)"):
bo_mixed = choose_generation_strategy(
search_space=get_factorial_search_space()
)
self.assertEqual(bo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed._steps[0].num_trials, 5)
self.assertEqual(bo_mixed._steps[1].model.value, "BO_MIXED")
with self.subTest("BO_MIXED (mixed search space)"):
bo_mixed_2 = choose_generation_strategy(
search_space=get_branin_search_space(with_choice_parameter=True)
)
self.assertEqual(bo_mixed_2._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed_2._steps[0].num_trials, 5)
self.assertEqual(bo_mixed_2._steps[1].model.value, "BO_MIXED")
def testHasGoodOptConfigModelFit(self):
# Construct diagnostics
result = []
for i, obs in enumerate(self.training_data):
result.append(
CVResult(observed=obs, predicted=self.observation_data[i]))
diag = compute_diagnostics(result=result)
assess_model_fit_result = assess_model_fit(
diagnostics=diag,
significance_level=0.05,
)
# Test single objective
optimization_config = OptimizationConfig(objective=Objective(
metric=Metric("a")))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test multi objective
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(
metrics=[Metric("a"), Metric("b")]))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test constraints
optimization_config = OptimizationConfig(
objective=Objective(metric=Metric("a")),
outcome_constraints=[
OutcomeConstraint(metric=Metric("b"),
op=ComparisonOp.GEQ,
bound=0.1)
],
)
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
def optimization_config_from_objectives(
objectives: List[Metric],
objective_thresholds: List[ObjectiveThreshold],
outcome_constraints: List[OutcomeConstraint],
) -> OptimizationConfig:
"""Parse objectives and constraints to define optimization config.
The resulting optimization config will be regular single-objective config
if `objectives` is a list of one element and a multi-objective config
otherwise.
NOTE: If passing in multiple objectives, `objective_thresholds` must be a
non-empty list definining constraints for each objective.
"""
if len(objectives) == 1:
if objective_thresholds:
raise ValueError(
"Single-objective optimizations must not specify objective thresholds."
)
return OptimizationConfig(
objective=Objective(
metric=objectives[0],
),
outcome_constraints=outcome_constraints,
)
else:
objective_names = {m.name for m in objectives}
threshold_names = {oc.metric.name for oc in objective_thresholds}
if objective_names != threshold_names:
diff = objective_names.symmetric_difference(threshold_names)
raise ValueError(
"Multi-objective optimization requires one objective threshold "
f"per objective metric; unmatched names are {diff}"
)
return MultiObjectiveOptimizationConfig(
objective=MultiObjective(metrics=objectives),
outcome_constraints=outcome_constraints,
objective_thresholds=objective_thresholds,
)
def __init__(self, serialized_filepath=None):
# Give ourselves the ability to resume this experiment later.
self.serialized_filepath = serialized_filepath
if serialized_filepath is not None and os.path.exists(
serialized_filepath):
with open(serialized_filepath, "r") as f:
serialized = json.load(f)
self.initialize_from_json_snapshot(serialized)
else:
# Create a CoreAxClient.
search_space = SearchSpace(parameters=[
RangeParameter(
"x", ParameterType.FLOAT, lower=12.2, upper=602.2),
])
optimization_config = OptimizationConfig(
objective=MultiObjective(
metrics=[
# Currently MultiObjective doesn't work with
# lower_is_better=True.
# https://github.com/facebook/Ax/issues/289
Metric(name="neg_distance17", lower_is_better=False),
Metric(name="neg_distance33", lower_is_better=False)
],
minimize=False,
), )
generation_strategy = choose_generation_strategy(
search_space,
enforce_sequential_optimization=False,
no_max_parallelism=True,
num_trials=NUM_TRIALS,
num_initialization_trials=NUM_RANDOM)
super().__init__(experiment=Experiment(
search_space=search_space,
optimization_config=optimization_config),
generation_strategy=generation_strategy,
verbose=True)
def testObservedParetoFrontiers(self):
experiment = get_branin_experiment(
with_batch=True, has_optimization_config=False, with_status_quo=True
)
# Optimization config is not optional
with self.assertRaises(ValueError):
get_observed_pareto_frontiers(experiment=experiment, data=Data())
metrics = [
BraninMetric(name="m1", param_names=["x1", "x2"], lower_is_better=True),
NegativeBraninMetric(
name="m2", param_names=["x1", "x2"], lower_is_better=True
),
BraninMetric(name="m3", param_names=["x1", "x2"], lower_is_better=True),
]
bounds = [0, -100, 0]
objective_thresholds = [
ObjectiveThreshold(
metric=metric,
bound=bounds[i],
relative=True,
op=ComparisonOp.LEQ,
)
for i, metric in enumerate(metrics)
]
objective = MultiObjective(metrics=metrics, minimize=True)
optimization_config = MultiObjectiveOptimizationConfig(
objective=objective,
objective_thresholds=objective_thresholds,
)
experiment.optimization_config = optimization_config
experiment.trials[0].run()
# For the check below, compute which arms are better than SQ
df = experiment.fetch_data().df
df["sem"] = np.nan
data = Data(df)
sq_val = df[(df["arm_name"] == "status_quo") & (df["metric_name"] == "m1")][
"mean"
].values[0]
pareto_arms = sorted(
df[(df["mean"] <= sq_val) & (df["metric_name"] == "m1")]["arm_name"]
.unique()
.tolist()
)
pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data)
# We have all pairs of metrics
self.assertEqual(len(pfrs), 3)
true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")]
for i, pfr in enumerate(pfrs):
self.assertEqual(pfr.primary_metric, true_pairs[i][0])
self.assertEqual(pfr.secondary_metric, true_pairs[i][1])
self.assertEqual(pfr.absolute_metrics, [])
self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"])
self.assertEqual(len(pfr.means["m1"]), len(pareto_arms))
self.assertTrue(np.isnan(pfr.sems["m1"]).all())
self.assertEqual(len(pfr.arm_names), len(pareto_arms))
arm_idx = np.argsort(pfr.arm_names)
for i, idx in enumerate(arm_idx):
name = pareto_arms[i]
self.assertEqual(pfr.arm_names[idx], name)
self.assertEqual(
pfr.param_dicts[idx], experiment.arms_by_name[name].parameters
)
def get_multi_objective() -> Objective:
return MultiObjective(
metrics=[Metric(name="m1"), Metric(name="m3", lower_is_better=True)],
minimize=False,
)
def get_branin_multi_objective() -> Objective:
return MultiObjective(
metrics=[get_branin_metric(), get_branin_metric()], minimize=False
)
def get_multi_objective() -> Objective:
return MultiObjective(metrics=[Metric(name="m1"),
Metric(name="m2")],
minimize=False)
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics = [
self.metric_from_sqa_util(child)
for child in metrics_sqa_children
]
return MultiObjective(
metrics=list(metrics),
# pyre-fixme[6]: Expected `bool` for 2nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
return ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `bool` for 3nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
return ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
return ObjectiveThreshold(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
def testGen(self, mock_init, mock_best_point, mock_gen):
# Test with constraints
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=True),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
ma = NumpyModelBridge()
ma.parameters = ["x", "y", "z"]
ma.outcomes = ["a", "b"]
ma.transforms = OrderedDict()
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[0][2]
self.assertEqual(gen_args["n"], 3)
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([-1.0,
0.0])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][0],
np.array([[0.0, -1.0]])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][1],
np.array([[-2]])))
self.assertTrue(
np.array_equal(
gen_args["linear_constraints"][0],
np.array([[1.0, -1, 0.0], [-1.0, 0.0, -1.0]]),
))
self.assertTrue(
np.array_equal(gen_args["linear_constraints"][1],
np.array([[0.0], [-3.5]])))
self.assertEqual(gen_args["fixed_features"], {2: 3.0})
self.assertTrue(
np.array_equal(gen_args["pending_observations"][0], np.array([])))
self.assertTrue(
np.array_equal(gen_args["pending_observations"][1],
np.array([[0.6, 1.6, 3.0]])))
self.assertEqual(gen_args["model_gen_options"], {"option": "yes"})
self.assertEqual(observation_features[0].parameters, {
"x": 1.0,
"y": 2.0,
"z": 3.0
})
self.assertEqual(observation_features[1].parameters, {
"x": 3.0,
"y": 4.0,
"z": 3.0
})
self.assertTrue(np.array_equal(weights, np.array([1.0, 2.0])))
# Test with multiple objectives.
oc2 = OptimizationConfig(objective=ScalarizedObjective(
metrics=[Metric(name="a"), Metric(name="b")], minimize=True))
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=oc2,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[1][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"],
np.array([-1.0, -1.0])))
# Test with MultiObjective (unweighted multiple objectives)
oc3 = MultiObjectiveOptimizationConfig(objective=MultiObjective(
metrics=[Metric(name="a"),
Metric(name="b", lower_is_better=True)],
minimize=True,
))
search_space = SearchSpace(self.parameters) # Unconstrained
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=search_space,
optimization_config=oc3,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[2][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([1.0,
-1.0])))
# Test with no constraints, no fixed feature, no pending observations
search_space = SearchSpace(self.parameters[:2])
optimization_config.outcome_constraints = []
ma.parameters = ["x", "y"]
ma._gen(3, search_space, {}, ObservationFeatures({}), None,
optimization_config)
gen_args = mock_gen.mock_calls[3][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertIsNone(gen_args["linear_constraints"])
self.assertIsNone(gen_args["fixed_features"])
self.assertIsNone(gen_args["pending_observations"])
# Test validation
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=False),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
optimization_config.objective.minimize = True
optimization_config.outcome_constraints[0].relative = True
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
def test_choose_generation_strategy(self):
with self.subTest("GPEI"):
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space())
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "GPEI")
self.assertIsNone(sobol_gpei._steps[1].model_kwargs)
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(), verbose=True)
self.assertIsNone(sobol_gpei._steps[1].model_kwargs)
with self.subTest("MOO"):
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[]))
sobol_gpei = choose_generation_strategy(
search_space=get_branin_search_space(),
optimization_config=optimization_config,
)
self.assertEqual(sobol_gpei._steps[0].model.value, "Sobol")
self.assertEqual(sobol_gpei._steps[0].num_trials, 5)
self.assertEqual(sobol_gpei._steps[1].model.value, "MOO")
model_kwargs = sobol_gpei._steps[1].model_kwargs
self.assertEqual(list(model_kwargs.keys()),
["transforms", "transform_configs"])
self.assertGreater(len(model_kwargs["transforms"]), 0)
transform_config_dict = {
"Winsorize": {
"optimization_config": optimization_config
}
}
self.assertEqual(model_kwargs["transform_configs"],
transform_config_dict)
with self.subTest("Sobol (we can try every option)"):
sobol = choose_generation_strategy(
search_space=get_factorial_search_space(), num_trials=1000)
self.assertEqual(sobol._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol._steps), 1)
with self.subTest("Sobol (because of too many categories)"):
ss = get_large_factorial_search_space()
sobol_large = choose_generation_strategy(
search_space=get_large_factorial_search_space(), verbose=True)
self.assertEqual(sobol_large._steps[0].model.value, "Sobol")
self.assertEqual(len(sobol_large._steps), 1)
with self.subTest("GPEI-Batched"):
sobol_gpei_batched = choose_generation_strategy(
search_space=get_branin_search_space(), use_batch_trials=3)
self.assertEqual(sobol_gpei_batched._steps[0].num_trials, 1)
with self.subTest("BO_MIXED (purely categorical)"):
bo_mixed = choose_generation_strategy(
search_space=get_factorial_search_space())
self.assertEqual(bo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed._steps[0].num_trials, 6)
self.assertEqual(bo_mixed._steps[1].model.value, "BO_MIXED")
self.assertIsNone(bo_mixed._steps[1].model_kwargs)
with self.subTest("BO_MIXED (mixed search space)"):
ss = get_branin_search_space(with_choice_parameter=True)
ss.parameters["x2"]._is_ordered = False
bo_mixed_2 = choose_generation_strategy(search_space=ss)
self.assertEqual(bo_mixed_2._steps[0].model.value, "Sobol")
self.assertEqual(bo_mixed_2._steps[0].num_trials, 5)
self.assertEqual(bo_mixed_2._steps[1].model.value, "BO_MIXED")
self.assertIsNone(bo_mixed_2._steps[1].model_kwargs)
with self.subTest("BO_MIXED (mixed multi-objective optimization)"):
search_space = get_branin_search_space(with_choice_parameter=True)
search_space.parameters["x2"]._is_ordered = False
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[]))
moo_mixed = choose_generation_strategy(
search_space=search_space,
optimization_config=optimization_config)
self.assertEqual(moo_mixed._steps[0].model.value, "Sobol")
self.assertEqual(moo_mixed._steps[0].num_trials, 5)
self.assertEqual(moo_mixed._steps[1].model.value, "BO_MIXED")
model_kwargs = moo_mixed._steps[1].model_kwargs
self.assertEqual(list(model_kwargs.keys()),
["transforms", "transform_configs"])
self.assertGreater(len(model_kwargs["transforms"]), 0)
transform_config_dict = {
"Winsorize": {
"optimization_config": optimization_config
}
}
self.assertEqual(model_kwargs["transform_configs"],
transform_config_dict)
with self.subTest("SAASBO"):
sobol_fullybayesian = choose_generation_strategy(
search_space=get_branin_search_space(),
use_batch_trials=True,
num_initialization_trials=3,
use_saasbo=True,
)
self.assertEqual(sobol_fullybayesian._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesian._steps[0].num_trials, 3)
self.assertEqual(sobol_fullybayesian._steps[1].model.value,
"FullyBayesian")
self.assertTrue(
sobol_fullybayesian._steps[1].model_kwargs["verbose"])
with self.subTest("SAASBO MOO"):
sobol_fullybayesianmoo = choose_generation_strategy(
search_space=get_branin_search_space(),
use_batch_trials=True,
num_initialization_trials=3,
use_saasbo=True,
optimization_config=MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[])),
)
self.assertEqual(sobol_fullybayesianmoo._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesianmoo._steps[0].num_trials, 3)
self.assertEqual(sobol_fullybayesianmoo._steps[1].model.value,
"FullyBayesianMOO")
self.assertTrue(
sobol_fullybayesianmoo._steps[1].model_kwargs["verbose"])
with self.subTest("SAASBO"):
sobol_fullybayesian_large = choose_generation_strategy(
search_space=get_large_ordinal_search_space(
n_ordinal_choice_parameters=5,
n_continuous_range_parameters=10),
use_saasbo=True,
)
self.assertEqual(sobol_fullybayesian_large._steps[0].model.value,
"Sobol")
self.assertEqual(sobol_fullybayesian_large._steps[0].num_trials,
30)
self.assertEqual(sobol_fullybayesian_large._steps[1].model.value,
"FullyBayesian")
self.assertTrue(
sobol_fullybayesian_large._steps[1].model_kwargs["verbose"])
with self.subTest("num_initialization_trials"):
ss = get_large_factorial_search_space()
for _, param in ss.parameters.items():
param._is_ordered = True
# 2 * len(ss.parameters) init trials are performed if num_trials is large
gs_12_init_trials = choose_generation_strategy(search_space=ss,
num_trials=100)
self.assertEqual(gs_12_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_12_init_trials._steps[0].num_trials, 12)
self.assertEqual(gs_12_init_trials._steps[1].model.value, "GPEI")
# at least 5 initialization trials are performed
gs_5_init_trials = choose_generation_strategy(search_space=ss,
num_trials=0)
self.assertEqual(gs_5_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_5_init_trials._steps[0].num_trials, 5)
self.assertEqual(gs_5_init_trials._steps[1].model.value, "GPEI")
# avoid spending >20% of budget on initialization trials if there are
# more than 5 initialization trials
gs_6_init_trials = choose_generation_strategy(search_space=ss,
num_trials=30)
self.assertEqual(gs_6_init_trials._steps[0].model.value, "Sobol")
self.assertEqual(gs_6_init_trials._steps[0].num_trials, 6)
self.assertEqual(gs_6_init_trials._steps[1].model.value, "GPEI")
class ObjectiveTest(TestCase):
def setUp(self):
self.metrics = {
"m1": Metric(name="m1"),
"m2": Metric(name="m2", lower_is_better=True),
"m3": Metric(name="m3", lower_is_better=False),
}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.multi_objective = MultiObjective(metrics=[
self.metrics["m1"], self.metrics["m2"], self.metrics["m3"]
])
self.scalarized_objective = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
def testInit(self):
with self.assertRaises(ValueError):
ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]],
weights=[1.0])
warnings.resetwarnings()
warnings.simplefilter("always", append=True)
with warnings.catch_warnings(record=True) as ws:
Objective(metric=self.metrics["m1"])
self.assertTrue(
any(issubclass(w.category, DeprecationWarning) for w in ws))
self.assertTrue(
any("Defaulting to `minimize=False`" in str(w.message)
for w in ws))
with warnings.catch_warnings(record=True) as ws:
Objective(Metric(name="m4", lower_is_better=True), minimize=False)
self.assertTrue(
any("Attempting to maximize" in str(w.message) for w in ws))
with warnings.catch_warnings(record=True) as ws:
Objective(Metric(name="m4", lower_is_better=False), minimize=True)
self.assertTrue(
any("Attempting to minimize" in str(w.message) for w in ws))
self.assertEqual(self.objective.get_unconstrainable_metrics(),
[self.metrics["m1"]])
def testMultiObjective(self):
with self.assertRaises(NotImplementedError):
return self.multi_objective.metric
self.assertEqual(self.multi_objective.metrics,
list(self.metrics.values()))
weights = [mw[1] for mw in self.multi_objective.metric_weights]
self.assertEqual(weights, [1.0, -1.0, 1.0])
self.assertEqual(self.multi_objective.clone(), self.multi_objective)
self.assertEqual(
str(self.multi_objective),
"MultiObjective(metric_names=['m1', 'm2', 'm3'], minimize=False)",
)
self.assertEqual(self.multi_objective.get_unconstrainable_metrics(),
[])
def testScalarizedObjective(self):
with self.assertRaises(NotImplementedError):
return self.scalarized_objective.metric
self.assertEqual(self.scalarized_objective.metrics,
[self.metrics["m1"], self.metrics["m2"]])
weights = [mw[1] for mw in self.scalarized_objective.metric_weights]
self.assertEqual(weights, [1.0, 1.0])
self.assertEqual(self.scalarized_objective.clone(),
self.scalarized_objective)
self.assertEqual(
str(self.scalarized_objective),
("ScalarizedObjective(metric_names=['m1', 'm2'], weights=[1.0, 1.0], "
"minimize=False)"),
)
self.assertEqual(
self.scalarized_objective.get_unconstrainable_metrics(), [])
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import warnings
from ax.core.metric import Metric
from ax.core.objective import MultiObjective, Objective, ScalarizedObjective
from ax.utils.common.testutils import TestCase
MULTI_OBJECTIVE_REPR = """
MultiObjective(objectives=
[Objective(metric_name="m1", minimize=False),
Objective(metric_name="m2", minimize=True),
Objective(metric_name="m3", minimize=False)])
"""
class ObjectiveTest(TestCase):
def setUp(self):
self.metrics = {
"m1": Metric(name="m1"),
"m2": Metric(name="m2", lower_is_better=True),
"m3": Metric(name="m3", lower_is_better=False),
}
self.objectives = {
"o1": Objective(metric=self.metrics["m1"]),
"o2": Objective(metric=self.metrics["m2"], minimize=True),
"o3": Objective(metric=self.metrics["m3"], minimize=False),
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
objectives = [
Objective(
metric=self.metric_from_sqa_util(metric_sqa),
minimize=metric_sqa.minimize,
) for metric_sqa in metrics_sqa_children
]
multi_objective = MultiObjective(objectives=objectives)
multi_objective.db_id = metric_sqa.id
return multi_objective
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
scalarized_objective = ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
minimize=not_none(metric_sqa.minimize),
)
scalarized_objective.db_id = metric_sqa.id
return scalarized_objective
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
try:
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
except DetachedInstanceError:
metrics_sqa_children = (
_get_scalarized_outcome_constraint_children_metrics(
metric_id=metric_sqa.id, decoder=self))
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
scalarized_outcome_constraint = ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
bound=not_none(metric_sqa.bound),
op=not_none(metric_sqa.op),
relative=not_none(metric_sqa.relative),
)
scalarized_outcome_constraint.db_id = metric_sqa.id
return scalarized_outcome_constraint
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
ot = ObjectiveThreshold(
metric=metric,
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
# ObjectiveThreshold constructor clones the passed-in metric, which means
# the db id gets lost and so we need to reset it
ot.metric._db_id = metric.db_id
return ot
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
