def testSeparateObservations(self):
obs = Observation(
features=ObservationFeatures(parameters={"x": 20}),
data=ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
arm_name="0_0",
)
obs_feats, obs_data = separate_observations(observations=[obs])
self.assertEqual(obs.features,
ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
)
obs_feats, obs_data = separate_observations(observations=[obs],
copy=True)
self.assertEqual(obs.features,
ObservationFeatures(parameters={"x": 20}))
self.assertEqual(
obs.data,
ObservationData(means=np.array([1]),
covariance=np.array([[2]]),
metric_names=["a"]),
)
def cross_validate(
self,
cv_training_data: List[Observation],
cv_test_points: List[ObservationFeatures],
) -> List[ObservationData]:
"""Make a set of cross-validation predictions.
Args:
cv_training_data: The training data to use for cross validation.
cv_test_points: The test points at which predictions will be made.
Returns:
A list of predictions at the test points.
"""
# Apply transforms to cv_training_data and cv_test_points
cv_test_points = deepcopy(cv_test_points)
obs_feats, obs_data = separate_observations(
observations=cv_training_data, copy=True)
for t in self.transforms.values():
obs_feats = t.transform_observation_features(obs_feats)
obs_data = t.transform_observation_data(obs_data, obs_feats)
cv_test_points = t.transform_observation_features(cv_test_points)
# Apply terminal transform, and get predictions.
cv_predictions = self._cross_validate(obs_feats=obs_feats,
obs_data=obs_data,
cv_test_points=cv_test_points)
# Apply reverse transforms, in reverse order
# pyre-fixme[6]: Expected `Sequence[_T]` for 1st param but got `ValuesView[Tr...
for t in reversed(self.transforms.values()):
cv_test_points = t.untransform_observation_features(cv_test_points)
cv_predictions = t.untransform_observation_data(
cv_predictions, cv_test_points)
return cv_predictions
def _extend_training_data(
self, observations: List[Observation]
) -> Tuple[List[ObservationFeatures], List[ObservationData]]:
"""Extend and return training data, not-transformed.
If the modelbridge specifies _fit_out_of_design, all training data is
returned. Otherwise, only in design points are returned.
Args:
observations: New observations.
Returns:
observation_features: New + old observation features.
observation_data: New + old observation data.
"""
observation_features, observation_data = self._prepare_training_data(
observations=observations)
for obsd in observation_data:
for metric_name in obsd.metric_names:
if metric_name not in self._metric_names:
raise ValueError(
f"Unrecognised metric {metric_name}; cannot update "
"training data with metrics that were not in the original "
"training data.")
# Initialize with all points in design.
self._training_data.extend(deepcopy(observations))
all_observation_features, all_observation_data = separate_observations(
self.get_training_data())
return self._process_in_design(
search_space=self._model_space,
observation_features=all_observation_features,
observation_data=all_observation_data,
)
def _extend_training_data(
self, observations: List[Observation]
) -> Tuple[List[ObservationFeatures], List[ObservationData]]:
"""Extend and return training data, not-transformed.
Args:
observations: New observations.
Returns:
observation_features: New + old observation features.
observation_data: New + old observation data.
"""
observation_features, observation_data = self._prepare_training_data(
observations=observations)
for obsd in observation_data:
for metric_name in obsd.metric_names:
if metric_name not in self._metric_names:
raise ValueError(
f"Unrecognised metric {metric_name}; cannot update "
"training data with metrics that were not in the original "
"training data.")
# Initialize with all points in design.
self._training_data.extend(deepcopy(observations))
self._training_in_design.extend([True] * len(observations))
return separate_observations(self.get_training_data())
def _prepare_training_data(
self, observations: List[Observation]
) -> Tuple[List[ObservationFeatures], List[ObservationData]]:
observation_features, observation_data = separate_observations(observations)
if len(observation_features) != len(set(observation_features)):
raise ValueError(
"Observation features not unique."
"Something went wrong constructing training data..."
)
return observation_features, observation_data
def get_current_regrets(experiment: Experiment, observer: ExObserver):
observations = observations_from_data(experiment, experiment.eval())
observation_features, observation_data = separate_observations(
observations)
obs_x = observation_features_to_tensor(observation_features)
if obs_x.dim() == 1:
obs_x.unsqueeze_(-1)
obs_f = observation_data_to_tensor(observation_data)
opt_x = observer.current['optimum']['x']
opt_f = observer.current['optimum']['function']
loc_regret = torch.norm(obs_x - opt_x, dim=1).min()
fun_regret = torch.abs(obs_f - opt_f).min()
return loc_regret, fun_regret
def cross_validate(
self,
cv_training_data: List[Observation],
cv_test_points: List[ObservationFeatures],
) -> List[ObservationData]:
"""Make a set of cross-validation predictions.
Args:
cv_training_data: The training data to use for cross validation.
cv_test_points: The test points at which predictions will be made.
Returns:
A list of predictions at the test points.
"""
# Apply transforms to cv_training_data and cv_test_points
cv_test_points = deepcopy(cv_test_points)
obs_feats, obs_data = separate_observations(
observations=cv_training_data, copy=True)
search_space = self._model_space.clone()
for t in self.transforms.values():
obs_feats = t.transform_observation_features(obs_feats)
obs_data = t.transform_observation_data(
obs_data,
obs_feats,
)
cv_test_points = t.transform_observation_features(cv_test_points)
search_space = t.transform_search_space(search_space)
# Apply terminal transform, and get predictions.
cv_predictions = self._cross_validate(
search_space=search_space,
obs_feats=obs_feats,
obs_data=obs_data,
cv_test_points=cv_test_points,
)
# Apply reverse transforms, in reverse order
for t in reversed(self.transforms.values()):
cv_test_points = t.untransform_observation_features(cv_test_points)
cv_predictions = t.untransform_observation_data(
cv_predictions, cv_test_points)
return cv_predictions
