def testTransformOptimizationConfig(self):
m1 = Metric(name="m1")
m2 = Metric(name="m2")
m3 = Metric(name="m3")
objective = Objective(metric=m3, minimize=False)
cons = [
OutcomeConstraint(metric=m1,
op=ComparisonOp.GEQ,
bound=2.0,
relative=False),
OutcomeConstraint(metric=m2,
op=ComparisonOp.LEQ,
bound=3.5,
relative=False),
ScalarizedOutcomeConstraint(
metrics=[m1, m2],
weights=[0.5, 0.5],
op=ComparisonOp.LEQ,
bound=3.5,
relative=False,
),
]
oc = OptimizationConfig(objective=objective, outcome_constraints=cons)
oc = self.t.transform_optimization_config(oc, None, None)
cons_t = [
OutcomeConstraint(metric=m1,
op=ComparisonOp.GEQ,
bound=1.0,
relative=False),
OutcomeConstraint(
metric=m2,
op=ComparisonOp.LEQ,
bound=2.0 * sqrt(3), # (3.5 - 1.5) / sqrt(1/3)
relative=False,
),
ScalarizedOutcomeConstraint(
metrics=[m1, m2],
weights=[0.5 * 1.0, 0.5 * sqrt(1 / 3)],
op=ComparisonOp.LEQ,
bound=2.25, # 3.5 - (0.5 * 1.0 + 0.5 * 1.5)
relative=False,
),
]
self.assertTrue(oc.outcome_constraints == cons_t)
self.assertTrue(oc.objective == objective)
# Check fail with relative
con = OutcomeConstraint(metric=m1,
op=ComparisonOp.GEQ,
bound=2.0,
relative=True)
oc = OptimizationConfig(objective=objective, outcome_constraints=[con])
with self.assertRaises(ValueError):
oc = self.t.transform_optimization_config(oc, None, None)
def setUp(self):
self.metrics = [
Metric(name="m1", lower_is_better=True),
Metric(name="m2", lower_is_better=True),
Metric(name="m3", lower_is_better=True),
]
self.weights = [0.1, 0.3, 0.6]
self.bound = 0
self.constraint = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=self.weights,
op=ComparisonOp.GEQ,
bound=self.bound,
)
def test_extract_outcome_constraints(self):
outcomes = ["m1", "m2", "m3"]
# pass no outcome constraints
self.assertIsNone(extract_outcome_constraints([], outcomes))
outcome_constraints = [
OutcomeConstraint(metric=Metric("m1"),
op=ComparisonOp.LEQ,
bound=0)
]
res = extract_outcome_constraints(outcome_constraints, outcomes)
self.assertEqual(res[0].shape, (1, 3))
self.assertListEqual(list(res[0][0]), [1, 0, 0])
self.assertEqual(res[1][0][0], 0)
outcome_constraints = [
OutcomeConstraint(metric=Metric("m1"),
op=ComparisonOp.LEQ,
bound=0),
ScalarizedOutcomeConstraint(
metrics=[Metric("m2"), Metric("m3")],
weights=[0.5, 0.5],
op=ComparisonOp.GEQ,
bound=1,
),
]
res = extract_outcome_constraints(outcome_constraints, outcomes)
self.assertEqual(res[0].shape, (2, 3))
self.assertListEqual(list(res[0][0]), [1, 0, 0])
self.assertListEqual(list(res[0][1]), [0, -0.5, -0.5])
self.assertEqual(res[1][0][0], 0)
self.assertEqual(res[1][1][0], -1)
def testEq(self):
constraint1 = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=self.weights,
op=ComparisonOp.GEQ,
bound=self.bound,
)
self.assertEqual(constraint1, self.constraint)
constraint2 = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=[0.2, 0.2, 0.6],
op=ComparisonOp.LEQ,
bound=self.bound,
)
self.assertNotEqual(constraint2, self.constraint)
def setUp(self):
self.metrics = {"m1": Metric(name="m1"), "m2": Metric(name="m2")}
self.objective = Objective(metric=self.metrics["m1"], minimize=False)
self.alt_objective = Objective(metric=self.metrics["m2"],
minimize=False)
self.multi_objective = MultiObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
self.m2_objective = ScalarizedObjective(
metrics=[self.metrics["m1"], self.metrics["m2"]])
self.outcome_constraint = OutcomeConstraint(metric=self.metrics["m2"],
op=ComparisonOp.GEQ,
bound=-0.25)
self.additional_outcome_constraint = OutcomeConstraint(
metric=self.metrics["m2"], op=ComparisonOp.LEQ, bound=0.25)
self.scalarized_outcome_constraint = ScalarizedOutcomeConstraint(
metrics=[self.metrics["m1"], self.metrics["m2"]],
weights=[0.5, 0.5],
op=ComparisonOp.GEQ,
bound=-0.25,
)
self.outcome_constraints = [
self.outcome_constraint,
self.additional_outcome_constraint,
self.scalarized_outcome_constraint,
]
def get_scalarized_outcome_constraint() -> ScalarizedOutcomeConstraint:
return ScalarizedOutcomeConstraint(
metrics=[Metric(name="oc_m3"), Metric(name="oc_m4")],
weights=[0.2, 0.8],
op=ComparisonOp.GEQ,
bound=-0.25,
)
def test_best_point(
self,
_mock_gen,
_mock_best_point,
_mock_fit,
_mock_predict,
_mock_gen_arms,
_mock_unwrap,
_mock_obs_from_data,
):
exp = Experiment(search_space=get_search_space_for_range_value(),
name="test")
modelbridge = ArrayModelBridge(
search_space=get_search_space_for_range_value(),
model=NumpyModel(),
transforms=[t1, t2],
experiment=exp,
data=Data(),
)
self.assertEqual(list(modelbridge.transforms.keys()),
["Cast", "t1", "t2"])
# _fit is mocked, which typically sets this.
modelbridge.outcomes = ["a"]
run = modelbridge.gen(
n=1,
optimization_config=OptimizationConfig(
objective=Objective(metric=Metric("a"), minimize=False),
outcome_constraints=[],
),
)
arm, predictions = run.best_arm_predictions
self.assertEqual(arm.parameters, {})
self.assertEqual(predictions[0], {"m": 1.0})
self.assertEqual(predictions[1], {"m": {"m": 2.0}})
# test check that optimization config is required
with self.assertRaises(ValueError):
run = modelbridge.gen(n=1, optimization_config=None)
# test optimization config validation - raise error when
# ScalarizedOutcomeConstraint contains a metric that is not in the outcomes
with self.assertRaises(ValueError):
run = modelbridge.gen(
n=1,
optimization_config=OptimizationConfig(
objective=Objective(metric=Metric("a"), minimize=False),
outcome_constraints=[
ScalarizedOutcomeConstraint(
metrics=[Metric("wrong_metric_name")],
weights=[1.0],
op=ComparisonOp.LEQ,
bound=0,
)
],
),
)
def testRaiseError(self):
# set a wrong weights
with self.assertRaises(ValueError):
ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=[0.2, 0.8],
op=ComparisonOp.LEQ,
bound=self.bound,
)
with self.assertRaises(NotImplementedError):
return self.constraint.metric
with self.assertRaises(NotImplementedError):
self.constraint.metric = self.metrics[0]
def testInit(self):
self.assertListEqual(self.constraint.metrics, self.metrics)
self.assertListEqual(self.constraint.weights, self.weights)
self.assertEqual(len(list(self.constraint.metric_weights)),
len(self.metrics))
self.assertEqual(
str(self.constraint),
("ScalarizedOutcomeConstraint(metric_names=['m1', 'm2', 'm3'], "
"weights=[0.1, 0.3, 0.6], >= 0%)"),
)
# check that weights are set uniformly by default
con = ScalarizedOutcomeConstraint(
metrics=[
Metric(name="m1", lower_is_better=True),
Metric(name="m2", lower_is_better=True),
],
op=ComparisonOp.LEQ,
bound=self.bound,
)
self.assertListEqual(con.weights, [0.5, 0.5])
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics = [
self.metric_from_sqa_util(child)
for child in metrics_sqa_children
]
return MultiObjective(
metrics=list(metrics),
# pyre-fixme[6]: Expected `bool` for 2nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
return ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `bool` for 3nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
return ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
return ObjectiveThreshold(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
class ScalarizedOutcomeConstraintTest(TestCase):
def setUp(self):
self.metrics = [
Metric(name="m1", lower_is_better=True),
Metric(name="m2", lower_is_better=True),
Metric(name="m3", lower_is_better=True),
]
self.weights = [0.1, 0.3, 0.6]
self.bound = 0
self.constraint = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=self.weights,
op=ComparisonOp.GEQ,
bound=self.bound,
)
def testInit(self):
self.assertListEqual(self.constraint.metrics, self.metrics)
self.assertListEqual(self.constraint.weights, self.weights)
self.assertEqual(len(list(self.constraint.metric_weights)),
len(self.metrics))
self.assertEqual(
str(self.constraint),
("ScalarizedOutcomeConstraint(metric_names=['m1', 'm2', 'm3'], "
"weights=[0.1, 0.3, 0.6], >= 0%)"),
)
# check that weights are set uniformly by default
con = ScalarizedOutcomeConstraint(
metrics=[
Metric(name="m1", lower_is_better=True),
Metric(name="m2", lower_is_better=True),
],
op=ComparisonOp.LEQ,
bound=self.bound,
)
self.assertListEqual(con.weights, [0.5, 0.5])
def testEq(self):
constraint1 = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=self.weights,
op=ComparisonOp.GEQ,
bound=self.bound,
)
self.assertEqual(constraint1, self.constraint)
constraint2 = ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=[0.2, 0.2, 0.6],
op=ComparisonOp.LEQ,
bound=self.bound,
)
self.assertNotEqual(constraint2, self.constraint)
def testClone(self):
self.assertEqual(self.constraint, self.constraint.clone())
def testValidMutations(self):
# updating constraint metric is ok as long as lower_is_better is compatible.
self.constraint.metrics = [
Metric(name="m2"),
Metric(name="m4"),
]
self.constraint.op = ComparisonOp.LEQ
def testRaiseError(self):
# set a wrong weights
with self.assertRaises(ValueError):
ScalarizedOutcomeConstraint(
metrics=self.metrics,
weights=[0.2, 0.8],
op=ComparisonOp.LEQ,
bound=self.bound,
)
with self.assertRaises(NotImplementedError):
return self.constraint.metric
with self.assertRaises(NotImplementedError):
self.constraint.metric = self.metrics[0]
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
objectives = [
Objective(
metric=self.metric_from_sqa_util(metric_sqa),
minimize=metric_sqa.minimize,
) for metric_sqa in metrics_sqa_children
]
multi_objective = MultiObjective(objectives=objectives)
multi_objective.db_id = metric_sqa.id
return multi_objective
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
scalarized_objective = ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
minimize=not_none(metric_sqa.minimize),
)
scalarized_objective.db_id = metric_sqa.id
return scalarized_objective
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
try:
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
except DetachedInstanceError:
metrics_sqa_children = (
_get_scalarized_outcome_constraint_children_metrics(
metric_id=metric_sqa.id, decoder=self))
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
scalarized_outcome_constraint = ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
bound=not_none(metric_sqa.bound),
op=not_none(metric_sqa.op),
relative=not_none(metric_sqa.relative),
)
scalarized_outcome_constraint.db_id = metric_sqa.id
return scalarized_outcome_constraint
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
ot = ObjectiveThreshold(
metric=metric,
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
# ObjectiveThreshold constructor clones the passed-in metric, which means
# the db id gets lost and so we need to reset it
ot.metric._db_id = metric.db_id
return ot
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
def testDerelativizeTransform(self, mock_predict, mock_fit,
mock_observations_from_data):
t = Derelativize(search_space=None,
observation_features=None,
observation_data=None)
# ModelBridge with in-design status quo
search_space = SearchSpace(parameters=[
RangeParameter("x", ParameterType.FLOAT, 0, 20),
RangeParameter("y", ParameterType.FLOAT, 0, 20),
])
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_1",
)
# Test with no relative constraints
objective = Objective(Metric("c"))
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
ScalarizedOutcomeConstraint(
metrics=[Metric("a"), Metric("b")],
op=ComparisonOp.LEQ,
bound=2,
weights=[0.5, 0.5],
relative=False,
),
],
)
oc2 = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc == oc2)
# Test with relative constraint, in-design status quo
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
ScalarizedOutcomeConstraint(
metrics=[Metric("a"), Metric("b")],
weights=[0.0, 1.0],
op=ComparisonOp.LEQ,
bound=-10,
relative=True,
),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False),
ScalarizedOutcomeConstraint(
metrics=[Metric("a"), Metric("b")],
weights=[0.0, 1.0],
op=ComparisonOp.LEQ,
bound=4.5,
relative=False,
),
])
obsf = mock_predict.mock_calls[0][1][1][0]
obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0})
self.assertTrue(obsf == obsf2)
# Test with relative constraint, out-of-design status quo
mock_predict.side_effect = RuntimeError()
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_2",
)
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
ScalarizedOutcomeConstraint(
metrics=[Metric("a"), Metric("b")],
weights=[0.0, 1.0],
op=ComparisonOp.LEQ,
bound=-10,
relative=True,
),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False),
ScalarizedOutcomeConstraint(
metrics=[Metric("a"), Metric("b")],
weights=[0.0, 1.0],
op=ComparisonOp.LEQ,
bound=3.6,
relative=False,
),
])
self.assertEqual(mock_predict.call_count, 2)
# Raises error if predict fails with in-design status quo
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_1",
)
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
with self.assertRaises(RuntimeError):
oc = t.transform_optimization_config(oc, g, None)
# Bypasses error if use_raw_sq
t2 = Derelativize(
search_space=None,
observation_features=None,
observation_data=None,
config={"use_raw_status_quo": True},
)
oc2 = t2.transform_optimization_config(deepcopy(oc), g, None)
# Raises error with relative constraint, no status quo
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
)
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, g, None)
# Raises error with relative constraint, no modelbridge
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, None, None)
def testTransformOptimizationConfig(self):
# basic test
m1 = Metric(name="m1")
objective_m1 = Objective(metric=m1, minimize=False)
oc = OptimizationConfig(objective=objective_m1, outcome_constraints=[])
tf = PowerTransformY(
search_space=None,
observation_features=None,
observation_data=[self.obsd1, self.obsd2],
config={"metrics": ["m1"]},
)
oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None)
self.assertEqual(oc_tf, oc)
# Output constraint on a different metric should not transform the bound
m2 = Metric(name="m2")
for bound in [-1.234, 0, 2.345]:
oc = OptimizationConfig(
objective=objective_m1,
outcome_constraints=get_constraint(
metric=m2, bound=bound, relative=False
),
)
oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None)
self.assertEqual(oc_tf, oc)
# Output constraint on the same metric should transform the bound
objective_m2 = Objective(metric=m2, minimize=False)
for bound in [-1.234, 0, 2.345]:
oc = OptimizationConfig(
objective=objective_m2,
outcome_constraints=get_constraint(
metric=m1, bound=bound, relative=False
),
)
oc_tf = tf.transform_optimization_config(deepcopy(oc), None, None)
oc_true = deepcopy(oc)
tf_bound = (
tf.power_transforms["m1"].transform(np.array(bound, ndmin=2)).item()
)
oc_true.outcome_constraints[0].bound = tf_bound
self.assertEqual(oc_tf, oc_true)
# Relative constraints aren't supported
oc = OptimizationConfig(
objective=objective_m2,
outcome_constraints=get_constraint(metric=m1, bound=2.345, relative=True),
)
with self.assertRaisesRegex(
ValueError,
"PowerTransformY cannot be applied to metric m1 since it is "
"subject to a relative constraint.",
):
tf.transform_optimization_config(oc, None, None)
# Support for scalarized outcome constraints isn't implemented
m3 = Metric(name="m3")
oc = OptimizationConfig(
objective=objective_m2,
outcome_constraints=[
ScalarizedOutcomeConstraint(
metrics=[m1, m3], op=ComparisonOp.GEQ, bound=2.345, relative=False
)
],
)
with self.assertRaises(NotImplementedError) as cm:
tf.transform_optimization_config(oc, None, None)
self.assertEqual(
"PowerTransformY cannot be used for metric(s) {'m1'} "
"that are part of a ScalarizedOutcomeConstraint.",
str(cm.exception),
)