def testNoiselessMerge(self):
# One noiseless
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 0.0]]),
)
obsd2 = ivw_metric_merge(obsd)
np.array_equal(obsd2.means, np.array([1.0, 1.0]))
cov_true = np.array([[1.0, 0.4], [0.4, 0.0]])
self.assertTrue(np.array_equal(obsd2.covariance, cov_true))
# Conflicting noiseless, default (warn)
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 0.0, 0.8], [0.4, 0.8, 0.0]]),
)
with self.assertRaises(ValueError):
obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="wrong")
obsd2 = ivw_metric_merge(obsd)
self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 2.0])))
cov_true = np.array([[1.0, 0.2], [0.2, 0.0]])
self.assertTrue(np.array_equal(obsd2.covariance, cov_true))
# Conflicting noiseless, raise
with self.assertRaises(ValueError):
obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="raise")
def testNoRepeats(self):
obsd = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.0, 2.0]),
covariance=np.array([[1.0, 0.2], [0.2, 2.0]]),
)
obsd2 = ivw_metric_merge(obsd)
self.assertEqual(obsd2, obsd)
def transform_optimization_config(
self,
optimization_config: OptimizationConfig,
modelbridge: Optional["modelbridge_module.base.ModelBridge"],
fixed_features: ObservationFeatures,
) -> OptimizationConfig:
use_raw_sq = self.config.get("use_raw_status_quo", False)
has_relative_constraint = any(
c.relative for c in optimization_config.all_constraints
)
if not has_relative_constraint:
return optimization_config
# Else, we have at least one relative constraint.
# Estimate the value at the status quo.
if modelbridge is None:
raise ValueError("ModelBridge not supplied to transform.")
if modelbridge.status_quo is None:
raise ValueError(
"Optimization config has relative constraint, but model was "
"not fit with status quo."
)
try:
# pyre-fixme[16]: `Optional` has no attribute `features`.
f, _ = modelbridge.predict([modelbridge.status_quo.features])
except Exception:
# Check if it is out-of-design.
if use_raw_sq or not modelbridge.model_space.check_membership(
modelbridge.status_quo.features.parameters
):
# Out-of-design: use the raw observation
sq_data = ivw_metric_merge(
# pyre-fixme[16]: `Optional` has no attribute `data`.
obsd=modelbridge.status_quo.data,
conflicting_noiseless="raise",
)
f, _ = unwrap_observation_data([sq_data])
else:
# Should have worked.
raise # pragma: no cover
# Plug in the status quo value to each relative constraint.
for c in optimization_config.all_constraints:
if c.relative:
if isinstance(c, ScalarizedOutcomeConstraint):
# The sq_val of scalarized outcome is the weighted
# sum of its component metrics
sq_val = np.sum(
[
c.weights[i] * f[metric.name][0]
for i, metric in enumerate(c.metrics)
]
)
else:
sq_val = f[c.metric.name][0]
c.bound = (1 + c.bound / 100.0) * sq_val
c.relative = False
return optimization_config
def testMerge(self):
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]),
)
obsd2 = ivw_metric_merge(obsd)
self.assertEqual(obsd2.metric_names, ["m1", "m2"])
self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 0.6 * 2 + 0.4])))
cov12 = 0.2 * 0.6 + 0.4 * 0.4
# var(w1*y1 + w2*y2) =
# w1 ** 2 * var(y1) + w2 ** 2 * var(y2) + 2 * w1 * w2 * cov(y1, y2)
cov22 = 0.6 ** 2 * 2.0 + 0.4 ** 2 * 3 + 2 * 0.6 * 0.4 * 0.8
cov_true = np.array([[1.0, cov12], [cov12, cov22]])
discrep = np.max(np.abs(obsd2.covariance - cov_true))
self.assertTrue(discrep < 1e-8)
def transform_optimization_config(
self,
optimization_config: OptimizationConfig,
modelbridge: Optional["modelbridge_module.base.ModelBridge"],
fixed_features: ObservationFeatures,
) -> OptimizationConfig:
has_relative_constraint = any(
c.relative for c in optimization_config.outcome_constraints
)
if not has_relative_constraint:
return optimization_config
# Else, we have at least one relative constraint.
# Estimate the value at the status quo.
if modelbridge is None:
raise ValueError("ModelBridge not supplied to transform.")
if modelbridge.status_quo is None:
raise ValueError(
"Optimization config has relative constraint, but model was "
"not fit with status quo."
)
try:
# pyre-fixme[16]: `Optional` has no attribute `features`.
f, _ = modelbridge.predict([modelbridge.status_quo.features])
except Exception:
# Check if it is out-of-design.
if not modelbridge.model_space.check_membership(
modelbridge.status_quo.features.parameters
):
# Out-of-design: use the raw observation
sq_data = ivw_metric_merge(
# pyre-fixme[16]: `Optional` has no attribute `data`.
obsd=modelbridge.status_quo.data,
conflicting_noiseless="raise",
)
f, _ = unwrap_observation_data([sq_data])
else:
# Should have worked.
raise # pragma: no cover
# Plug in the status quo value to each relative constraint.
for c in optimization_config.outcome_constraints:
if c.relative:
# Compute new bound.
c.bound = (1 + c.bound / 100.0) * f[c.metric.name][0]
c.relative = False
return optimization_config
