def testSeparateObservations(self):
obs = Observation(
features=ObservationFeatures(parameters={"x": 20}),
data=ObservationData(
means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]
),
arm_name="0_0",
)
obs_feats, obs_data = separate_observations(observations=[obs])
self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(
means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]
),
)
obs_feats, obs_data = separate_observations(observations=[obs], copy=True)
self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(
means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]
),
)
def get_observation2(first_metric_name: str = "a",
second_metric_name="b") -> Observation:
return Observation(
features=ObservationFeatures(parameters={
"x": 3.0,
"y": 2.0
},
trial_index=np.int64(1)),
data=ObservationData(
means=np.array([2.0, 1.0]),
covariance=np.array([[2.0, 3.0], [4.0, 5.0]]),
metric_names=[first_metric_name, second_metric_name],
),
arm_name="1_1",
)
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 8.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 5.0, 2.0, 1.0]),
covariance=np.array(
[
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]
),
)
self.search_space = SearchSpace(
parameters=[
RangeParameter(
name="x", parameter_type=ParameterType.FLOAT, lower=0, upper=10
),
ChoiceParameter(
name="z", parameter_type=ParameterType.STRING, values=["a", "b"]
),
]
)
self.obsf1 = ObservationFeatures({"x": 2, "z": "a"})
self.obsf2 = ObservationFeatures({"x": 5, "z": "b"})
self.t = StratifiedStandardizeY(
search_space=self.search_space,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
config={"parameter_name": "z"},
)
def testTransformObservations(self):
obsd1_t = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, sqrt(3 / 4), -sqrt(3 / 4)]),
covariance=np.array([
[1.0, 0.2 * sqrt(3), 0.4 * sqrt(3)],
[0.2 * sqrt(3), 6.0, 2.4],
[0.4 * sqrt(3), 2.4, 9.0],
], ),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(obsd2, [])
self.assertTrue(osd_allclose(obsd2[0], obsd1_t))
obsd2 = self.t.untransform_observation_data(obsd2, [])
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))
def get_observation1trans(first_metric_name: str = "a",
second_metric_name="b") -> Observation:
return Observation(
features=ObservationFeatures(parameters={
"x": 9.0,
"y": 10.0
},
trial_index=np.int64(0)),
data=ObservationData(
means=np.array([9.0, 25.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=[first_metric_name, second_metric_name],
),
arm_name="1_1",
)
def testUnwrapObservationData(self):
observation_data = [get_observation1().data, get_observation2().data]
f, cov = unwrap_observation_data(observation_data)
self.assertEqual(f["a"], [2.0, 2.0])
self.assertEqual(f["b"], [4.0, 1.0])
self.assertEqual(cov["a"]["a"], [1.0, 2.0])
self.assertEqual(cov["b"]["b"], [4.0, 5.0])
self.assertEqual(cov["a"]["b"], [2.0, 3.0])
self.assertEqual(cov["b"]["a"], [3.0, 4.0])
# Check that errors if metric mismatch
od3 = ObservationData(metric_names=["a"],
means=np.array([2.0]),
covariance=np.array([[4.0]]))
with self.assertRaises(ValueError):
unwrap_observation_data(observation_data + [od3])
def testMerge(self):
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]),
)
obsd2 = ivw_metric_merge(obsd)
self.assertEqual(obsd2.metric_names, ["m1", "m2"])
self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 0.6 * 2 + 0.4])))
cov12 = 0.2 * 0.6 + 0.4 * 0.4
# var(w1*y1 + w2*y2) =
# w1 ** 2 * var(y1) + w2 ** 2 * var(y2) + 2 * w1 * w2 * cov(y1, y2)
cov22 = 0.6 ** 2 * 2.0 + 0.4 ** 2 * 3 + 2 * 0.6 * 0.4 * 0.8
cov_true = np.array([[1.0, cov12], [cov12, cov22]])
discrep = np.max(np.abs(obsd2.covariance - cov_true))
self.assertTrue(discrep < 1e-8)
def observation_status_quo1() -> Observation:
return Observation(
features=ObservationFeatures(parameters={
"w": 0.85,
"x": 1,
"y": "baz",
"z": False
},
trial_index=1),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="0_0",
)
def testObservationData(self):
attrs = {
"metric_names": ["a", "b"],
"means": np.array([4.0, 5.0]),
"covariance": np.array([[1.0, 4.0], [3.0, 6.0]]),
}
obsd = ObservationData(**attrs)
self.assertEqual(obsd.metric_names, attrs["metric_names"])
self.assertTrue(np.array_equal(obsd.means, attrs["means"]))
self.assertTrue(np.array_equal(obsd.covariance, attrs["covariance"]))
# use legacy printing for numpy (<= 1.13 add spaces in front of floats;
# to get around tests failing on older versions, peg version to 1.13)
if np.__version__ >= "1.14":
np.set_printoptions(legacy="1.13")
printstr = "ObservationData(metric_names=['a', 'b'], means=[ 4. 5.], "
printstr += "covariance=[[ 1. 4.]\n [ 3. 6.]])"
self.assertEqual(repr(obsd), printstr)
def testUpdate(self, mock_init):
ma = DiscreteModelBridge()
ma._training_data = self.observations
model = mock.create_autospec(DiscreteModel, instance=True)
ma._fit(model, self.search_space, self.observation_features,
self.observation_data)
new_feat = ObservationFeatures(parameters={
"x": 0,
"y": "bar",
"z": True
})
new_data = ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]]))
ma._update([new_feat], [new_data])
self.assertEqual(ma.parameters, ["x", "y", "z"])
self.assertEqual(sorted(ma.outcomes), ["a", "b"])
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.0, 0.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.0, 5.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd3 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([2.0, 25.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd4 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([3.0, 125.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd_mid = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.5, 50.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd_extreme = ObservationData(
metric_names=["m1", "m2"],
means=np.array([-1.0, 126.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.t = PercentileY(
search_space=None,
observation_features=None,
observation_data=[
deepcopy(self.obsd1),
deepcopy(self.obsd2),
deepcopy(self.obsd3),
deepcopy(self.obsd4),
],
)
self.t_with_winsorization = PercentileY(
search_space=None,
observation_features=None,
observation_data=[
deepcopy(self.obsd1),
deepcopy(self.obsd2),
deepcopy(self.obsd3),
deepcopy(self.obsd4),
],
config={"winsorize": True},
)
def _transform_ref_point(
self, ref_point: Dict[str, float],
padding_obs_data: ObservationData) -> Dict[str, float]:
"""Transform ref_point using same transforms as those applied to data.
Args:
ref_point: Reference point to transform.
padding_obs_data: Data used to add dummy outcomes that aren't part
of the reference point. This is necessary to apply transforms.
Return:
A transformed reference point.
"""
metric_names = list(self._metric_names or [])
objective_metric_names = list(self._objective_metric_names or [])
num_metrics = len(metric_names)
# Create synthetic ObservationData representing the reference point.
# Pad with non-objective outcomes from existing data.
# Should always have existing data with BO.
padding_obs_data
padded_ref_dict: Dict[str, float] = dict(
zip(padding_obs_data.metric_names, padding_obs_data.means))
padded_ref_dict.update(ref_point)
ref_obs_data = [
ObservationData(
metric_names=list(padded_ref_dict.keys()),
means=np.array(list(padded_ref_dict.values())),
covariance=np.zeros((num_metrics, num_metrics)),
)
]
ref_obs_feats = []
# Apply initialized transforms to reference point.
for t in self.transforms.values():
ref_obs_data = t.transform_observation_data(
ref_obs_data, ref_obs_feats)
transformed_ref_obsd = ref_obs_data.pop()
transformed_ref_dict = dict(
zip(transformed_ref_obsd.metric_names, transformed_ref_obsd.means))
transformed_ref_point = {
objective_metric_name: transformed_ref_dict[objective_metric_name]
for objective_metric_name in objective_metric_names
}
return transformed_ref_point
def get_observation_status_quo1(first_metric_name: str = "a",
second_metric_name="b") -> Observation:
return Observation(
features=ObservationFeatures(
parameters={
"w": 0.85,
"x": 1,
"y": "baz",
"z": False
},
trial_index=np.int64(1),
),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=[first_metric_name, second_metric_name],
),
arm_name="0_0",
)
def test_relativize_transform_requires_a_modelbridge_to_have_status_quo_data(self):
sobol = Models.SOBOL(search_space=get_search_space())
self.assertIsNone(sobol.status_quo)
with self.assertRaisesRegex(ValueError, "status quo data"):
Relativize(
search_space=None,
observation_features=[],
observation_data=[],
modelbridge=sobol,
).transform_observation_data(
observation_data=[
ObservationData(
metric_names=["foo"],
means=np.array([2]),
covariance=np.array([[0.1]]),
)
],
observation_features=[ObservationFeatures(parameters={"x": 1})],
)
def array_to_observation_data(f: np.ndarray, cov: np.ndarray,
outcomes: List[str]) -> List[ObservationData]:
"""Convert arrays of model predictions to a list of ObservationData.
Args:
f: An (n x m) array
cov: An (n x m x m) array
outcomes: A list of d outcome names
Returns: A list of n ObservationData
"""
observation_data = []
for i in range(f.shape[0]):
observation_data.append(
ObservationData(
metric_names=list(outcomes),
means=f[i, :].copy(),
covariance=cov[i, :, :].copy(),
))
return observation_data
def _untransform_objective_thresholds(
self,
objective_thresholds: Tensor,
objective_weights: Tensor,
bounds: List[Tuple[Union[int, float], Union[int, float]]],
fixed_features: Optional[Dict[int, float]],
) -> List[ObjectiveThreshold]:
objective_thresholds_np = objective_thresholds.cpu().numpy()
objective_indices = objective_weights.nonzero().view(-1).tolist()
objective_names = [self.outcomes[i] for i in objective_indices]
# Create an ObservationData object for untransforming the objective thresholds.
observation_data = [
ObservationData(
metric_names=objective_names,
means=objective_thresholds_np[objective_indices].copy(),
covariance=np.zeros((len(objective_indices), len(objective_indices))),
)
]
# Untransform objective thresholds. Note: there is one objective threshold
# for every outcome.
# Construct dummy observation features.
X = [bound[0] for bound in bounds]
fixed_features = fixed_features or {}
for i, val in fixed_features.items():
X[i] = val
observation_features = parse_observation_features(
X=np.array([X]),
param_names=self.parameters,
)
# Apply reverse transforms, in reverse order.
for t in reversed(self.transforms.values()):
observation_data = t.untransform_observation_data(
observation_data=observation_data,
observation_features=observation_features,
)
observation_features = t.untransform_observation_features(
observation_features=observation_features,
)
observation_data = observation_data[0]
oc = not_none(self._optimization_config)
metrics_names_to_metric = oc.metrics
obj_thresholds = []
for idx, (name, bound) in enumerate(
zip(observation_data.metric_names, observation_data.means)
):
if not np.isnan(bound):
obj_weight = objective_weights[objective_indices[idx]]
op = (
ComparisonOp.LEQ
if torch.sign(obj_weight) == -1.0
else ComparisonOp.GEQ
)
obj_thresholds.append(
ObjectiveThreshold(
metric=metrics_names_to_metric[name],
bound=bound,
relative=False,
op=op,
)
)
return obj_thresholds
def test_multitask_data(self):
experiment = get_branin_with_multi_task()
data = experiment.fetch_data()
observations = observations_from_data(
experiment=experiment,
data=data,
)
relative_observations = observations_from_data(
experiment=experiment,
data=relativize_data(
data=data,
status_quo_name="status_quo",
as_percent=True,
include_sq=True,
),
)
status_quo_row = data.df.loc[
(data.df["arm_name"] == "status_quo") & (data.df["trial_index"] == 1)
]
modelbridge = Mock(
status_quo=Observation(
data=ObservationData(
metric_names=status_quo_row["metric_name"].values,
means=status_quo_row["mean"].values,
covariance=np.array([status_quo_row["sem"].values ** 2]),
),
features=ObservationFeatures(
parameters=experiment.status_quo.parameters
),
)
)
obs_features = [obs.features for obs in observations]
obs_data = [obs.data for obs in observations]
expected_obs_data = [obs.data for obs in relative_observations]
transform = Relativize(
search_space=None,
observation_features=obs_features,
observation_data=obs_data,
modelbridge=modelbridge,
)
relative_obs_data = transform.transform_observation_data(obs_data, obs_features)
self.maxDiff = None
# this assertion just checks that order is the same, which
# is only important for the purposes of this test
self.assertEqual(
[datum.metric_names for datum in relative_obs_data],
[datum.metric_names for datum in expected_obs_data],
)
means = [
np.array([datum.means for datum in relative_obs_data]),
np.array([datum.means for datum in expected_obs_data]),
]
# `self.assertAlmostEqual(relative_obs_data, expected_obs_data)`
# fails 1% of the time, so we check with numpy.
self.assertTrue(
all(np.isclose(means[0], means[1])),
means,
)
covariances = [
np.array([datum.covariance for datum in expected_obs_data]),
np.array([datum.covariance for datum in relative_obs_data]),
]
self.assertTrue(
all(np.isclose(covariances[0], covariances[1])),
covariances,
)
def test_pareto_frontier(self, _):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
metrics_dict = exp.optimization_config.metrics
objective_thresholds = [
ObjectiveThreshold(
metric=metrics_dict["branin_a"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
ObjectiveThreshold(
metric=metrics_dict["branin_b"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
]
exp.optimization_config = exp.optimization_config.clone_with_args(
objective_thresholds=objective_thresholds)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials.keys()))
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator
observed_frontier = observed_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds)
wrapped_frontier_evaluator.assert_called_once()
self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"])
self.assertEqual(1, len(observed_frontier))
self.assertEqual(observed_frontier[0].arm_name, "0_0")
with self.assertRaises(ValueError):
predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=[],
)
predicted_frontier = predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=None,
)
self.assertEqual(predicted_frontier[0].arm_name, "0_0")
observation_features = [
ObservationFeatures(parameters={
"x1": 0.0,
"x2": 1.0
}),
ObservationFeatures(parameters={
"x1": 1.0,
"x2": 0.0
}),
]
observation_data = [
ObservationData(
metric_names=["branin_b", "branin_a"],
means=np.array([1.0, 2.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
),
ObservationData(
metric_names=["branin_a", "branin_b"],
means=np.array([3.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
),
]
predicted_frontier = predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
)
self.assertTrue(len(predicted_frontier) <= 2)
self.assertIsNone(predicted_frontier[0].arm_name, None)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
observed_frontier = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
)
wrapped_frontier_evaluator.assert_called_once()
self.assertTrue(
torch.equal(
wrapped_frontier_evaluator.call_args.kwargs["X"],
torch.tensor([[1.0, 4.0], [4.0, 1.0]]),
))
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
observed_frontier = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
use_model_predictions=False,
)
wrapped_frontier_evaluator.assert_called_once()
self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"])
self.assertTrue(
torch.equal(
wrapped_frontier_evaluator.call_args.kwargs["Y"],
torch.tensor([[9.0, 4.0], [16.0, 25.0]]),
))
def setUp(self):
x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1)
y = RangeParameter("y",
ParameterType.FLOAT,
lower=1,
upper=2,
is_fidelity=True,
target_value=2)
z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5)
self.parameters = [x, y, z]
parameter_constraints = [
OrderConstraint(x, y),
SumConstraint([x, z], False, 3.5),
]
self.search_space = SearchSpace(self.parameters, parameter_constraints)
self.observation_features = [
ObservationFeatures(parameters={
"x": 0.2,
"y": 1.2,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.4,
"y": 1.4,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
}),
]
self.observation_data = [
ObservationData(
metric_names=["a", "b"],
means=np.array([1.0, -1.0]),
covariance=np.array([[1.0, 4.0], [4.0, 6.0]]),
),
ObservationData(
metric_names=["a", "b"],
means=np.array([2.0, -2.0]),
covariance=np.array([[2.0, 5.0], [5.0, 7.0]]),
),
ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]])),
]
self.observations = [
Observation(
features=self.observation_features[i],
data=self.observation_data[i],
arm_name=str(i),
) for i in range(3)
]
self.pending_observations = {
"b":
[ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
})]
}
self.model_gen_options = {"option": "yes"}
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, 0.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
[0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 2.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
self.t = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_upper": 0.2},
)
self.t1 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_upper": 0.8},
)
self.t2 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_lower": 0.2},
)
self.t3 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={
"winsorization_upper": 0.6,
"percentile_bounds": {
"m1": (None, None),
"m2": (None, 1.9),
},
},
)
self.t4 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={
"winsorization_lower": 0.8,
"percentile_bounds": {
"m1": (None, None),
"m2": (0.3, None),
},
},
)
self.obsd3 = ObservationData(
metric_names=["m3", "m3", "m3", "m3"],
means=np.array([0.0, 1.0, 5.0, 3.0]),
covariance=np.eye(4),
)
self.t5 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[
deepcopy(self.obsd1),
deepcopy(self.obsd2),
deepcopy(self.obsd3),
],
config={
"winsorization_lower": {
"m2": 0.4
},
"winsorization_upper": {
"m1": 0.6
},
},
)
self.t6 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={
"winsorization_lower": {
"m2": 0.4
},
"winsorization_upper": {
"m1": 0.6
},
"percentile_bounds": {
"m1": (None, None),
"m2": (0.0, None), # This should leave m2 untouched
},
},
)
class DerelativizeTransformTest(TestCase):
def setUp(self):
m = mock.patch.object(ModelBridge, "__abstractmethods__", frozenset())
self.addCleanup(m.stop)
m.start()
@mock.patch(
"ax.modelbridge.base.observations_from_data",
autospec=True,
return_value=([
Observation(
features=ObservationFeatures(parameters={
"x": 2.0,
"y": 10.0
}),
data=ObservationData(
means=np.array([1.0, 2.0, 6.0]),
covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0],
[0.0, 0.0, 4.0]]),
metric_names=["a", "b", "b"],
),
arm_name="1_1",
),
Observation(
features=ObservationFeatures(parameters={
"x": None,
"y": None
}),
data=ObservationData(
means=np.array([1.0, 2.0, 6.0]),
covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0],
[0.0, 0.0, 4.0]]),
metric_names=["a", "b", "b"],
),
arm_name="1_2",
),
]),
)
@mock.patch("ax.modelbridge.base.ModelBridge._fit", autospec=True)
@mock.patch(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
return_value=([
ObservationData(
means=np.array([3.0, 5.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
metric_names=["a", "b"],
)
]),
)
def testDerelativizeTransform(self, mock_predict, mock_fit,
mock_observations_from_data):
t = Derelativize(search_space=None,
observation_features=None,
observation_data=None)
# ModelBridge with in-design status quo
search_space = SearchSpace(parameters=[
RangeParameter("x", ParameterType.FLOAT, 0, 20),
RangeParameter("y", ParameterType.FLOAT, 0, 20),
])
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_1",
)
# Test with no relative constraints
objective = Objective(Metric("c"))
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False)
],
)
oc2 = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc == oc2)
# Test with relative constraint, in-design status quo
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False),
])
obsf = mock_predict.mock_calls[0][1][1][0]
obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0})
self.assertTrue(obsf == obsf2)
# Test with relative constraint, out-of-design status quo
mock_predict.side_effect = Exception()
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_2",
)
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False),
])
self.assertEqual(mock_predict.call_count, 2)
# Raises error if predict fails with in-design status quo
g = ModelBridge(search_space, None, [], status_quo_name="1_1")
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
with self.assertRaises(Exception):
oc = t.transform_optimization_config(oc, g, None)
# Raises error with relative constraint, no status quo
exp = Experiment(search_space, "name")
g = ModelBridge(search_space, None, [], exp)
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, g, None)
# Raises error with relative constraint, no modelbridge
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, None, None)
def testErrors(self):
t = Derelativize(search_space=None,
observation_features=None,
observation_data=None)
oc = OptimizationConfig(
objective=Objective(Metric("c")),
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=True)
],
)
search_space = SearchSpace(
parameters=[RangeParameter("x", ParameterType.FLOAT, 0, 20)])
g = ModelBridge(search_space, None, [])
with self.assertRaises(ValueError):
t.transform_optimization_config(oc, None, None)
with self.assertRaises(ValueError):
t.transform_optimization_config(oc, g, None)
def ivw_metric_merge(obsd: ObservationData,
conflicting_noiseless: str = "warn") -> ObservationData:
"""Merge multiple observations of a metric with inverse variance weighting.
Correctly updates the covariance of the new merged estimates:
ybar1 = Sum_i w_i * y_i
ybar2 = Sum_j w_j * y_j
cov[ybar1, ybar2] = Sum_i Sum_j w_i * w_j * cov[y_i, y_j]
w_i will be infinity if any variance is 0. If one variance is 0., then
the IVW estimate is the corresponding mean. If there are multiple
measurements with 0 variance but means are all the same, then IVW estimate
is that mean. If there are multiple measurements and means differ, behavior
depends on argument conflicting_noiseless. "ignore" and "warn" will use
the first of the measurements as the IVW estimate. "warn" will additionally
log a warning. "raise" will raise an exception.
Args:
obsd: An ObservationData object
conflicting_noiseless: "warn", "ignore", or "raise"
"""
if len(obsd.metric_names) == len(set(obsd.metric_names)):
return obsd
if conflicting_noiseless not in {"warn", "ignore", "raise"}:
raise ValueError(
'conflicting_noiseless should be "warn", "ignore", or "raise".')
# Get indicies and weights for each metric.
# weights is a map from metric name to a vector of the weights for each
# measurement of that metric. indicies gives the corresponding index in
# obsd.means for each measurement.
weights: Dict[str, np.ndarray] = {}
indicies: Dict[str, List[int]] = {}
for metric_name in set(obsd.metric_names):
indcs = [
i for i, mn in enumerate(obsd.metric_names) if mn == metric_name
]
indicies[metric_name] = indcs
# Extract variances for observations of this metric
sigma2s = obsd.covariance[indcs, indcs]
# Check for noiseless observations
idx_noiseless = np.where(sigma2s == 0.0)[0]
if len(idx_noiseless) == 0:
# Weight is inverse of variance, normalized
# Expected `np.ndarray` for 3rd anonymous parameter to call
# `dict.__setitem__` but got `float`.
# pyre-fixme[6]:
weights[metric_name] = 1.0 / sigma2s
weights[metric_name] /= np.sum(weights[metric_name])
else:
# Check if there are conflicting means for the noiseless observations
means_noiseless = obsd.means[indcs][idx_noiseless]
_check_conflicting_means(means_noiseless, metric_name,
conflicting_noiseless)
# The first observation gets all the weight.
weights[metric_name] = np.zeros_like(sigma2s)
weights[metric_name][idx_noiseless[0]] = 1.0
# Compute the new values
metric_names = sorted(set(obsd.metric_names))
means = np.zeros(len(metric_names))
covariance = np.zeros((len(metric_names), len(metric_names)))
for i, metric_name in enumerate(metric_names):
ys = obsd.means[indicies[metric_name]]
means[i] = np.sum(weights[metric_name] * ys)
# Calculate covariances with metric_name
for j, metric_name2 in enumerate(metric_names[i:], start=i):
for ii, idx_i in enumerate(indicies[metric_name]):
for jj, idx_j in enumerate(indicies[metric_name2]):
covariance[i, j] += (weights[metric_name][ii] *
weights[metric_name2][jj] *
obsd.covariance[idx_i, idx_j])
covariance[j, i] = covariance[i, j]
return ObservationData(metric_names=metric_names,
means=means,
covariance=covariance)
