def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 8.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
[0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 5.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
self.search_space = SearchSpace(parameters=[
RangeParameter(name="x",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=10),
ChoiceParameter(name="z",
parameter_type=ParameterType.STRING,
values=["a", "b"]),
])
self.obsf1 = ObservationFeatures({"x": 2, "z": "a"})
self.obsf2 = ObservationFeatures({"x": 5, "z": "b"})
self.t = StratifiedStandardizeY(
search_space=self.search_space,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
config={"parameter_name": "z"},
)
def testSeparateObservations(self):
obs = Observation(
features=ObservationFeatures(parameters={"x": 20}),
data=ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
arm_name="0_0",
)
obs_feats, obs_data = separate_observations(observations=[obs])
self.assertEqual(obs.features,
ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
)
obs_feats, obs_data = separate_observations(observations=[obs],
copy=True)
self.assertEqual(obs.features,
ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
)
def testObservation(self):
obs = Observation(
features=ObservationFeatures(parameters={"x": 20}),
data=ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
arm_name="0_0",
)
self.assertEqual(obs.features,
ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
)
self.assertEqual(obs.arm_name, "0_0")
obs2 = Observation(
features=ObservationFeatures(parameters={"x": 20}),
data=ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
arm_name="0_0",
)
self.assertEqual(obs, obs2)
obs3 = Observation(
features=ObservationFeatures(parameters={"x": 10}),
data=ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
arm_name="0_0",
)
self.assertNotEqual(obs, obs3)
self.assertNotEqual(obs, 1)
def setUp(self):
x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1)
y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2)
z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5)
self.parameters = [x, y, z]
parameter_constraints = [
OrderConstraint(x, y),
SumConstraint([x, z], False, 3.5),
]
self.search_space = SearchSpace(self.parameters, parameter_constraints)
self.observation_features = [
ObservationFeatures(parameters={
"x": 0.2,
"y": 1.2,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.4,
"y": 1.4,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
}),
]
self.observation_data = [
ObservationData(
metric_names=["a", "b"],
means=np.array([1.0, -1.0]),
covariance=np.array([[1.0, 4.0], [4.0, 6.0]]),
),
ObservationData(
metric_names=["a", "b"],
means=np.array([2.0, -2.0]),
covariance=np.array([[2.0, 5.0], [5.0, 7.0]]),
),
ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]])),
]
self.observations = [
Observation(
features=self.observation_features[i],
data=self.observation_data[i],
arm_name=str(i),
) for i in range(3)
]
self.pending_observations = {
"b":
[ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
})]
}
self.model_gen_options = {"option": "yes"}
def testTransformObservations(self):
obsd1_ta = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, -1.0, 1.0]),
covariance=np.array([
[1.0, 0.2 / 3, 0.4 / 3],
[0.2 / 3, 2.0 / 9, 0.8 / 9],
[0.4 / 3, 0.8 / 9, 3.0 / 9],
]),
)
obsd1_tb = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([-1.0, 1.0, 13.0]),
covariance=np.array([[0.25, 0.2, 0.4], [0.2, 8.0, 3.2],
[0.4, 3.2, 12.0]]),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})])
self.assertEqual(obsd2[0], obsd1_ta)
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})])
self.assertEqual(obsd2[0], self.obsd1)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})])
self.assertEqual(obsd2[0], obsd1_tb)
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})])
self.assertEqual(obsd2[0], self.obsd1)
def testNoiselessMerge(self):
# One noiseless
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 0.0]]),
)
obsd2 = ivw_metric_merge(obsd)
np.array_equal(obsd2.means, np.array([1.0, 1.0]))
cov_true = np.array([[1.0, 0.4], [0.4, 0.0]])
self.assertTrue(np.array_equal(obsd2.covariance, cov_true))
# Conflicting noiseless, default (warn)
obsd = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 0.0, 0.8], [0.4, 0.8, 0.0]]),
)
with self.assertRaises(ValueError):
obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="wrong")
obsd2 = ivw_metric_merge(obsd)
self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 2.0])))
cov_true = np.array([[1.0, 0.2], [0.2, 0.0]])
self.assertTrue(np.array_equal(obsd2.covariance, cov_true))
# Conflicting noiseless, raise
with self.assertRaises(ValueError):
obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="raise")
def setUp(self):
self.parameters = [
ChoiceParameter("x", ParameterType.FLOAT, values=[0, 1]),
ChoiceParameter("y", ParameterType.STRING, values=["foo", "bar"]),
FixedParameter("z", ParameterType.BOOL, value=True),
]
parameter_constraints = []
self.search_space = SearchSpace(self.parameters, parameter_constraints)
self.observation_features = [
ObservationFeatures(parameters={
"x": 0,
"y": "foo",
"z": True
}),
ObservationFeatures(parameters={
"x": 1,
"y": "foo",
"z": True
}),
ObservationFeatures(parameters={
"x": 1,
"y": "bar",
"z": True
}),
]
self.observation_data = [
ObservationData(
metric_names=["a", "b"],
means=np.array([1.0, -1.0]),
covariance=np.array([[1.0, 4.0], [4.0, 6.0]]),
),
ObservationData(
metric_names=["a", "b"],
means=np.array([2.0, -2.0]),
covariance=np.array([[2.0, 5.0], [5.0, 7.0]]),
),
ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]])),
]
self.observations = [
Observation(
features=self.observation_features[i],
data=self.observation_data[i],
arm_name=str(i),
) for i in range(3)
]
self.pending_observations = {
"b":
[ObservationFeatures(parameters={
"x": 0,
"y": "foo",
"z": True
})]
}
self.model_gen_options = {"option": "yes"}
def setUp(self):
self.training_data = [
Observation(
features=ObservationFeatures(parameters={"x": 2.0}, trial_index=0),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_1",
),
Observation(
features=ObservationFeatures(parameters={"x": 2.0}, trial_index=1),
data=ObservationData(
means=np.array([3.0, 5.0, 6.0]),
covariance=np.array(
[[1.0, 2.0, 3.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]
),
metric_names=["a", "b", "a"],
),
arm_name="1_1",
),
Observation(
features=ObservationFeatures(parameters={"x": 3.0}),
data=ObservationData(
means=np.array([7.0, 8.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_2",
),
Observation(
features=ObservationFeatures(parameters={"x": 4.0}, trial_index=2),
data=ObservationData(
means=np.array([9.0, 10.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_3",
),
]
self.observation_data = [
ObservationData(
means=np.array([2.0, 1.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
)
] * 4
self.diagnostics: List[CVDiagnostics] = [
{"Fisher exact test p": {"y_a": 0.0, "y_b": 0.4}},
{"Fisher exact test p": {"y_a": 0.1, "y_b": 0.1}},
{"Fisher exact test p": {"y_a": 0.5, "y_b": 0.6}},
]
def testObservationDataValidation(self):
with self.assertRaises(ValueError):
ObservationData(
metric_names=["a", "b"],
means=np.array([4.0]),
covariance=np.array([[1.0, 4.0], [3.0, 6.0]]),
)
with self.assertRaises(ValueError):
ObservationData(
metric_names=["a", "b"],
means=np.array([4.0, 5.0]),
covariance=np.array([1.0, 4.0]),
)
def setUp(self):
self.training_data = [
Observation(
features=ObservationFeatures(parameters={"x": 2.0},
trial_index=0),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_1",
),
Observation(
features=ObservationFeatures(parameters={"x": 2.0},
trial_index=1),
data=ObservationData(
means=np.array([3.0, 5.0, 6.0]),
covariance=np.array([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0],
[6.0, 7.0, 8.0]]),
metric_names=["a", "b", "a"],
),
arm_name="1_1",
),
Observation(
features=ObservationFeatures(parameters={"x": 3.0}),
data=ObservationData(
means=np.array([7.0, 8.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_2",
),
Observation(
features=ObservationFeatures(parameters={"x": 4.0},
trial_index=2),
data=ObservationData(
means=np.array([9.0, 10.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_3",
),
]
self.observation_data = [
ObservationData(
means=np.array([2.0, 1.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
)
] * 4
def testObservationData(self):
attrs = {
"metric_names": ["a", "b"],
"means": np.array([4.0, 5.0]),
"covariance": np.array([[1.0, 4.0], [3.0, 6.0]]),
}
obsd = ObservationData(**attrs)
self.assertEqual(obsd.metric_names, attrs["metric_names"])
self.assertTrue(np.array_equal(obsd.means, attrs["means"]))
self.assertTrue(np.array_equal(obsd.covariance, attrs["covariance"]))
# use legacy printing for numpy (<= 1.13 add spaces in front of floats;
# to get around tests failing on older versions, peg version to 1.13)
if np.__version__ >= "1.14":
np.set_printoptions(legacy="1.13")
printstr = "ObservationData(metric_names=['a', 'b'], means=[ 4. 5.], "
printstr += "covariance=[[ 1. 4.]\n [ 3. 6.]])"
self.assertEqual(repr(obsd), printstr)
self.assertEqual(obsd.means_dict, {"a": 4.0, "b": 5.0})
self.assertEqual(
obsd.covariance_matrix,
{
"a": {
"a": 1.0,
"b": 4.0
},
"b": {
"a": 3.0,
"b": 6.0
}
},
)
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m3"],
means=np.array([0.5, 1.0, 1.0]),
covariance=np.diag(np.array([1.0, 1.0, np.exp(1) - 1])),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 1.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
def test_relativize_transform_observation_data(self):
obs_data = [
ObservationData(
metric_names=["foobar", "foobaz"],
means=np.array([2, 5]),
covariance=np.array([[0.1, 0.0], [0.0, 0.2]]),
),
ObservationData(
metric_names=["foobar", "foobaz"],
means=np.array([1.0, 10.0]),
covariance=np.array([[0.3, 0.0], [0.0, 0.4]]),
),
]
obs_features = [
ObservationFeatures(parameters={"x": 1}, trial_index=0),
ObservationFeatures(parameters={"x": 2}, trial_index=0),
]
modelbridge = Mock(
status_quo=Mock(
data=obs_data[0],
features=obs_features[0],
)
)
results = Relativize(
search_space=None,
observation_features=obs_features,
observation_data=obs_data,
modelbridge=modelbridge,
).transform_observation_data(obs_data, obs_features)
self.assertEqual(results[0].metric_names, ["foobar", "foobaz"])
# status quo means must always be zero
self.assertTrue(
np.allclose(results[0].means, np.array([0.0, 0.0])), results[0].means
)
# status quo covariances must always be zero
self.assertTrue(
np.allclose(results[0].covariance, np.array([[0.0, 0.0], [0.0, 0.0]])),
results[0].covariance,
)
self.assertEqual(results[1].metric_names, ["foobar", "foobaz"])
self.assertTrue(
np.allclose(results[1].means, np.array([-51.25, 98.4])), results[1].means
)
self.assertTrue(
np.allclose(results[1].covariance, np.array([[812.5, 0.0], [0.0, 480.0]])),
results[1].covariance,
)
def testNoRepeats(self):
obsd = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.0, 2.0]),
covariance=np.array([[1.0, 0.2], [0.2, 2.0]]),
)
obsd2 = ivw_metric_merge(obsd)
self.assertEqual(obsd2, obsd)
def untransform_objective_thresholds(
self,
objective_thresholds: Tensor,
objective_weights: Tensor,
bounds: List[Tuple[Union[int, float], Union[int, float]]],
fixed_features: Optional[Dict[int, float]],
) -> List[ObjectiveThreshold]:
objective_thresholds_np = objective_thresholds.cpu().numpy()
# pyre-ignore [16]
objective_indices = objective_weights.nonzero().view(-1).tolist()
objective_names = [self.outcomes[i] for i in objective_indices]
# create an ObservationData object for untransforming the objective thresholds
observation_data = [
ObservationData(
metric_names=objective_names,
means=objective_thresholds_np[objective_indices].copy(),
covariance=np.zeros(
(len(objective_indices), len(objective_indices))),
)
]
# Untransform objective thresholds. Note: there is one objective threshold
# for every outcome.
# Construct dummy observation features
X = [bound[0] for bound in bounds]
fixed_features = fixed_features or {}
for i, val in fixed_features.items():
X[i] = val
observation_features = parse_observation_features(
X=np.array([X]),
param_names=self.parameters,
)
# Apply reverse transforms, in reverse order
for t in reversed(self.transforms.values()):
observation_data = t.untransform_observation_data(
observation_data=observation_data,
observation_features=observation_features,
)
observation_features = t.untransform_observation_features(
observation_features=observation_features, )
observation_data = observation_data[0]
oc = not_none(self._optimization_config)
metrics_names_to_metric = oc.metrics
obj_thresholds = []
for idx, (name, bound) in enumerate(
zip(observation_data.metric_names, observation_data.means)):
if not np.isnan(bound):
obj_weight = objective_weights[objective_indices[idx]]
op = (ComparisonOp.LEQ
if torch.sign(obj_weight) == -1.0 else ComparisonOp.GEQ)
obj_thresholds.append(
ObjectiveThreshold(
metric=metrics_names_to_metric[name],
bound=bound,
relative=False,
op=op,
))
return obj_thresholds
def testObservationDataEq(self):
od1 = ObservationData(
metric_names=["a", "b"],
means=np.array([4.0, 5.0]),
covariance=np.array([[1.0, 4.0], [3.0, 6.0]]),
)
od2 = ObservationData(
metric_names=["a", "b"],
means=np.array([4.0, 5.0]),
covariance=np.array([[1.0, 4.0], [3.0, 6.0]]),
)
od3 = ObservationData(
metric_names=["a", "b"],
means=np.array([4.0, 5.0]),
covariance=np.array([[2.0, 4.0], [3.0, 6.0]]),
)
self.assertEqual(od1, od2)
self.assertNotEqual(od1, od3)
self.assertFalse(od1 == 1)
def testTransformObservations(self):
std_m2_a = sqrt(2) * 3
obsd1_ta = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, -3.0 / std_m2_a, 3.0 / std_m2_a]),
covariance=np.array(
[
[1.0, 0.2 / std_m2_a, 0.4 / std_m2_a],
[0.2 / std_m2_a, 2.0 / 18, 0.8 / 18],
[0.4 / std_m2_a, 0.8 / 18, 3.0 / 18],
]
),
)
std_m1_b, std_m2_b = 2 * sqrt(2), sqrt(1 / 2)
obsd1_tb = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([-2.0 / std_m1_b, 0.5 / std_m2_b, 6.5 / std_m2_b]),
covariance=np.array(
[
[1.0 / 8, 0.2 / 2, 0.4 / 2],
[0.2 / 2, 2.0 * 2, 0.8 * 2],
[0.4 / 2, 0.8 * 2, 3.0 * 2],
]
),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})]
)
self.assertTrue(osd_allclose(obsd2[0], obsd1_ta))
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})]
)
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})]
)
self.assertTrue(osd_allclose(obsd2[0], obsd1_tb))
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})]
)
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))
def observation2trans() -> Observation:
return Observation(
features=ObservationFeatures(parameters={"x": 16.0, "y": 2.0}, trial_index=1),
data=ObservationData(
means=np.array([9.0, 4.0]),
covariance=np.array([[2.0, 3.0], [4.0, 5.0]]),
metric_names=["a", "b"],
),
arm_name="1_1",
)
def test_transform_status_quos_always_zero(
self,
sq_mean: float,
sq_sem: float,
mean: float,
sem: float,
):
assume(abs(sq_mean) >= 1e-10)
assume(abs(sq_mean) != sq_sem)
obs_data = [
ObservationData(
metric_names=["foo"],
means=np.array([sq_mean]),
covariance=np.array([[sq_sem]]),
),
ObservationData(
metric_names=["foo"],
means=np.array([mean]),
covariance=np.array([[sem]]),
),
]
obs_features = [
ObservationFeatures(parameters={"x": 1}, trial_index=0),
ObservationFeatures(parameters={"x": 2}, trial_index=0),
]
modelbridge = Mock(
status_quo=Mock(
data=obs_data[0],
features=obs_features[0],
)
)
transform = Relativize(
search_space=None,
observation_features=obs_features,
observation_data=obs_data,
modelbridge=modelbridge,
)
relative_data = transform.transform_observation_data(obs_data, obs_features)
self.assertEqual(relative_data[0].metric_names, ["foo"])
self.assertEqual(relative_data[0].means[0], 0)
self.assertEqual(relative_data[0].covariance[0][0], 0)
def setUp(self):
self.obsd_mid = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.5, 0.9]),
covariance=np.array([[0.005, 0.0], [0.0, 0.005]]),
)
self.obsd_extreme = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.0, 1.0]),
covariance=np.array([[1.0, 0.0], [0.0, 1.0]]),
)
self.obsd_nan_covars = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.5, 0.9]),
covariance=np.array([[float("nan"), float("nan")],
[float("nan"), float("nan")]]),
)
self.t = InverseGaussianCdfY(search_space=None,
observation_features=None,
observation_data=None)
def testTransformObservations(self):
obsd1_t = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, 1.0, -1.0]),
covariance=np.array([[1.0, 0.4, 0.8], [0.4, 8.0, 3.2], [0.8, 3.2, 12.0]]),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(obsd2, [])
self.assertTrue(obsd2[0] == obsd1_t)
obsd2 = self.t.untransform_observation_data(obsd2, [])
self.assertTrue(obsd2[0] == self.obsd1)
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.5, 0.9]),
covariance=np.array([[0.03, 0.0], [0.0, 0.001]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.1, 0.4]),
covariance=np.array([[0.005, 0.0], [0.0, 0.05]]),
)
self.obsd3 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.9, 0.8]),
covariance=np.array([[0.02, 0.0], [0.0, 0.01]]),
)
self.obsd_nan = ObservationData(
metric_names=["m1", "m2"],
means=np.array([0.3, 0.2]),
covariance=np.array([[float("nan"), 0.0], [0.0, float("nan")]]),
)
def get_observation() -> Observation:
return Observation(
features=ObservationFeatures(
parameters={"x": 2.0, "y": 10.0}, trial_index=np.int64(0)
),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="1_1",
)
def tensor_to_observation_data(
y_tensor: torch.Tensor,
noise_std=0.0,
) -> List[ObservationData]:
"""Convert torch Tensors to ax ObservationData."""
y_tensor = y_tensor.squeeze()
y_data = []
for y in y_tensor:
y_data_point = ObservationData(metric_names=['function'],
means=y.unsqueeze(dim=0).numpy(),
covariance=np.array([[noise_std**2]]))
y_data.append(y_data_point)
return y_data
def get_observation_status_quo1() -> Observation:
return Observation(
features=ObservationFeatures(
parameters={"w": 0.85, "x": 1, "y": "baz", "z": False},
trial_index=np.int64(1),
),
data=ObservationData(
means=np.array([2.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
metric_names=["a", "b"],
),
arm_name="0_0",
)
def testTransform(self):
obsd1_0 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
[0.4, 0.8, 3.0]]),
)
obsd1_1 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 1.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
obsd2_0 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.0, 1.6]),
covariance=np.array([[1.0, 0.28], [0.28, 1.584]]),
)
obsd2_1 = ObservationData(
metric_names=["m1", "m2"],
means=np.array([1.0, 1.6]),
covariance=np.array([[0.5, 0.14], [0.14, 1.584]]),
)
observation_data = [obsd1_0, obsd1_1]
t = IVW(None, None, None)
observation_data2 = t.transform_observation_data(observation_data, [])
observation_data2_true = [obsd2_0, obsd2_1]
for i, obsd in enumerate(observation_data2_true):
self.assertEqual(observation_data2[i].metric_names,
obsd.metric_names)
self.assertTrue(
np.array_equal(observation_data2[i].means, obsd.means))
discrep = np.max(
np.abs(observation_data2[i].covariance - obsd.covariance))
self.assertTrue(discrep < 1e-8)
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
[0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 1.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
self.t = StandardizeY(
search_space=None,
observation_features=None,
observation_data=[self.obsd1, self.obsd2],
)
def testObservationDataToArray(self):
outcomes = ["a", "b", "c"]
obsd = ObservationData(
metric_names=["c", "a", "b"],
means=np.array([1, 2, 3]),
covariance=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
)
Y, Ycov = observation_data_to_array(outcomes=outcomes,
observation_data=[obsd])
self.assertTrue(np.array_equal(Y, np.array([[2, 3, 1]])))
self.assertTrue(
np.array_equal(Ycov, np.array([[[5, 6, 4], [8, 9, 7], [2, 3,
1]]])))
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, 0.0, 1.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
[0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 2.0, 2.0, 1.0]),
covariance=np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]),
)
self.t = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_upper": 0.2},
)
self.t1 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_upper": 0.8},
)
self.t2 = Winsorize(
search_space=None,
observation_features=None,
observation_data=[deepcopy(self.obsd1),
deepcopy(self.obsd2)],
config={"winsorization_lower": 0.2},
)
def testTransformObservations(self):
# test default transform
obsd1_t = ObservationData(
metric_names=["m1", "m2", "m3"],
means=np.array([0.5, 1.0, -0.5]),
covariance=np.diag(np.array([1.0, 1.0, 1.0])),
)
tf = LogY(
search_space=None,
observation_features=None,
observation_data=[],
config={"metrics": ["m3"]},
)
obsd1 = deepcopy(self.obsd1)
obsd1_ = tf.transform_observation_data([obsd1], [])
self.assertTrue(obsd1_[0] == obsd1_t)
obsd1 = tf.untransform_observation_data(obsd1_, [])
self.assertTrue(obsd1[0] == self.obsd1)
# test raise on non-independent noise
obsd1_ = deepcopy(self.obsd1)
obsd1_.covariance[0, 2] = 0.1
obsd1_.covariance[2, 0] = 0.1
with self.assertRaises(NotImplementedError):
tf.transform_observation_data([obsd1_], [])
# test full covariance for single metric
Z = np.zeros((3, 3))
Z[0, 2] = np.sqrt(np.exp(1)) - 1
Z[2, 0] = np.sqrt(np.exp(1)) - 1
obsd1 = ObservationData(
metric_names=["m3", "m3", "m3"],
means=np.ones(3),
covariance=np.diag(np.ones(3) * (np.exp(1) - 1)) + Z,
)
obsd1_ = tf.transform_observation_data([obsd1], [])
cov_expected = np.array([[1.0, 0.0, 0.5], [0.0, 1.0, 0.0],
[0.5, 0.0, 1.0]])
self.assertTrue(np.allclose(obsd1_[0].means, -0.5 * np.ones(3)))
self.assertTrue(np.allclose(obsd1_[0].covariance, cov_expected))