def test_relativize_data(self):
data = Data(
df=pd.DataFrame(
[
{
"mean": 2,
"sem": 0,
"metric_name": "foobar",
"arm_name": "status_quo",
},
{
"mean": 5,
"sem": 0,
"metric_name": "foobaz",
"arm_name": "status_quo",
},
{"mean": 1, "sem": 0, "metric_name": "foobar", "arm_name": "0_0"},
{"mean": 10, "sem": 0, "metric_name": "foobaz", "arm_name": "0_0"},
]
)
)
expected_relativized_data = Data(
df=pd.DataFrame(
[
{
"mean": -0.5,
"sem": 0,
"metric_name": "foobar",
"arm_name": "0_0",
},
{"mean": 1, "sem": 0, "metric_name": "foobaz", "arm_name": "0_0"},
]
)
)
expected_relativized_data_with_sq = Data(
df=pd.DataFrame(
[
{
"mean": 0,
"sem": 0,
"metric_name": "foobar",
"arm_name": "status_quo",
},
{
"mean": -0.5,
"sem": 0,
"metric_name": "foobar",
"arm_name": "0_0",
},
{
"mean": 0,
"sem": 0,
"metric_name": "foobaz",
"arm_name": "status_quo",
},
{"mean": 1, "sem": 0, "metric_name": "foobaz", "arm_name": "0_0"},
]
)
)
actual_relativized_data = relativize_data(data=data)
self.assertEqual(expected_relativized_data, actual_relativized_data)
actual_relativized_data_with_sq = relativize_data(data=data, include_sq=True)
self.assertEqual(
expected_relativized_data_with_sq, actual_relativized_data_with_sq
)
def _filter_feasible_rows(
df: pd.DataFrame,
optimization_config: OptimizationConfig,
status_quo: Optional[Arm],
) -> pd.DataFrame:
"""Filter out arms that do not satisfy outcome constraints
Looks at all arm data collected and removes rows corresponding to arms in
which one or more of their associated metrics' 95% confidence interval
falls outside of any outcome constraint's bounds (i.e. we are 95% sure the
bound is not satisfied).
"""
if len(optimization_config.outcome_constraints) < 1:
return df
name = df["metric_name"]
# When SEM is NaN we should treat it as if it were 0
sems = not_none(df["sem"].fillna(0))
# Bounds computed for 95% confidence interval on Normal distribution
lower_bound = df["mean"] - sems * 1.96
upper_bound = df["mean"] + sems * 1.96
# Only compute relativization if some constraints are relative
rel_df = None
rel_lower_bound = None
rel_upper_bound = None
if status_quo is not None and any(
oc.relative for oc in optimization_config.outcome_constraints):
# relativize_data expects all arms to come from the same trial, we need to
# format the data as if it was.
to_relativize = df.copy()
to_relativize["trial_index"] = 0
rel_df = relativize_data(data=Data(to_relativize),
status_quo_name=status_quo.name).df.append(
{
"arm_name": "status_quo",
"metric_name": status_quo.name,
"mean": 0,
"sem": 0,
},
ignore_index=True,
)
rel_sems = not_none(rel_df["sem"].fillna(0))
rel_lower_bound = rel_df["mean"] - rel_sems * 1.96
rel_upper_bound = rel_df["mean"] + rel_sems * 1.96
# Nested function from OC -> Mask for consumption in later map/reduce from
# [OC] -> Mask. Constraint relativity is handled inside so long as relative bounds
# are set in surrounding closure (which will occur in proper experiment setup).
def oc_mask(oc: OutcomeConstraint) -> pd.Series:
name_match_mask = name == oc.metric.name
if oc.relative:
if rel_lower_bound is None or rel_upper_bound is None:
logger.warning(
f"No status quo provided; relative constraint {oc} ignored."
)
return pd.Series(True, index=df.index)
observed_lower_bound = rel_lower_bound
observed_upper_bound = rel_upper_bound
else:
observed_lower_bound = lower_bound
observed_upper_bound = upper_bound
# Return True if metrics are different, or whether the confidence
# interval is entirely not within the bound
if oc.op == ComparisonOp.GEQ:
return ~name_match_mask | observed_upper_bound > oc.bound
else:
return ~name_match_mask | observed_lower_bound < oc.bound
mask = reduce(
lambda left, right: left & right,
map(oc_mask, optimization_config.outcome_constraints),
)
bad_arm_names = (df[~mask]["arm_name"].tolist()
if rel_df is None else rel_df[~mask]["arm_name"].tolist())
feasible = df.loc[df["arm_name"].apply(lambda x: x not in bad_arm_names)]
if feasible.empty:
raise ValueError(
"No points satisfied all outcome constraints within 95 percent" +
"confidence interval")
return feasible
def test_multitask_data(self):
experiment = get_branin_with_multi_task()
data = experiment.fetch_data()
observations = observations_from_data(
experiment=experiment,
data=data,
)
relative_observations = observations_from_data(
experiment=experiment,
data=relativize_data(
data=data,
status_quo_name="status_quo",
as_percent=True,
include_sq=True,
),
)
status_quo_row = data.df.loc[
(data.df["arm_name"] == "status_quo") & (data.df["trial_index"] == 1)
]
modelbridge = Mock(
status_quo=Observation(
data=ObservationData(
metric_names=status_quo_row["metric_name"].values,
means=status_quo_row["mean"].values,
covariance=np.array([status_quo_row["sem"].values ** 2]),
),
features=ObservationFeatures(
parameters=experiment.status_quo.parameters
),
)
)
obs_features = [obs.features for obs in observations]
obs_data = [obs.data for obs in observations]
expected_obs_data = [obs.data for obs in relative_observations]
transform = Relativize(
search_space=None,
observation_features=obs_features,
observation_data=obs_data,
modelbridge=modelbridge,
)
relative_obs_data = transform.transform_observation_data(obs_data, obs_features)
self.maxDiff = None
# this assertion just checks that order is the same, which
# is only important for the purposes of this test
self.assertEqual(
[datum.metric_names for datum in relative_obs_data],
[datum.metric_names for datum in expected_obs_data],
)
means = [
np.array([datum.means for datum in relative_obs_data]),
np.array([datum.means for datum in expected_obs_data]),
]
# `self.assertAlmostEqual(relative_obs_data, expected_obs_data)`
# fails 1% of the time, so we check with numpy.
self.assertTrue(
all(np.isclose(means[0], means[1])),
means,
)
covariances = [
np.array([datum.covariance for datum in expected_obs_data]),
np.array([datum.covariance for datum in relative_obs_data]),
]
self.assertTrue(
all(np.isclose(covariances[0], covariances[1])),
covariances,
)