def testSeparateObservations(self): obs = Observation( features=ObservationFeatures(parameters={"x": 20}), data=ObservationData( means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"] ), arm_name="0_0", ) obs_feats, obs_data = separate_observations(observations=[obs]) self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20})) self.assertEqual( obs.data, ObservationData( means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"] ), ) obs_feats, obs_data = separate_observations(observations=[obs], copy=True) self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20})) self.assertEqual( obs.data, ObservationData( means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"] ), )
def get_observation2(first_metric_name: str = "a", second_metric_name="b") -> Observation: return Observation( features=ObservationFeatures(parameters={ "x": 3.0, "y": 2.0 }, trial_index=np.int64(1)), data=ObservationData( means=np.array([2.0, 1.0]), covariance=np.array([[2.0, 3.0], [4.0, 5.0]]), metric_names=[first_metric_name, second_metric_name], ), arm_name="1_1", )
def setUp(self): self.obsd1 = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 8.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]), ) self.obsd2 = ObservationData( metric_names=["m1", "m1", "m2", "m2"], means=np.array([1.0, 5.0, 2.0, 1.0]), covariance=np.array( [ [1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.2, 0.4], [0.0, 0.2, 2.0, 0.8], [0.0, 0.4, 0.8, 3.0], ] ), ) self.search_space = SearchSpace( parameters=[ RangeParameter( name="x", parameter_type=ParameterType.FLOAT, lower=0, upper=10 ), ChoiceParameter( name="z", parameter_type=ParameterType.STRING, values=["a", "b"] ), ] ) self.obsf1 = ObservationFeatures({"x": 2, "z": "a"}) self.obsf2 = ObservationFeatures({"x": 5, "z": "b"}) self.t = StratifiedStandardizeY( search_space=self.search_space, observation_features=[self.obsf1, self.obsf2], observation_data=[self.obsd1, self.obsd2], config={"parameter_name": "z"}, )
def testTransformObservations(self): obsd1_t = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([0.0, sqrt(3 / 4), -sqrt(3 / 4)]), covariance=np.array([ [1.0, 0.2 * sqrt(3), 0.4 * sqrt(3)], [0.2 * sqrt(3), 6.0, 2.4], [0.4 * sqrt(3), 2.4, 9.0], ], ), ) obsd2 = [deepcopy(self.obsd1)] obsd2 = self.t.transform_observation_data(obsd2, []) self.assertTrue(osd_allclose(obsd2[0], obsd1_t)) obsd2 = self.t.untransform_observation_data(obsd2, []) self.assertTrue(osd_allclose(obsd2[0], self.obsd1))
def get_observation1trans(first_metric_name: str = "a", second_metric_name="b") -> Observation: return Observation( features=ObservationFeatures(parameters={ "x": 9.0, "y": 10.0 }, trial_index=np.int64(0)), data=ObservationData( means=np.array([9.0, 25.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=[first_metric_name, second_metric_name], ), arm_name="1_1", )
def testUnwrapObservationData(self): observation_data = [get_observation1().data, get_observation2().data] f, cov = unwrap_observation_data(observation_data) self.assertEqual(f["a"], [2.0, 2.0]) self.assertEqual(f["b"], [4.0, 1.0]) self.assertEqual(cov["a"]["a"], [1.0, 2.0]) self.assertEqual(cov["b"]["b"], [4.0, 5.0]) self.assertEqual(cov["a"]["b"], [2.0, 3.0]) self.assertEqual(cov["b"]["a"], [3.0, 4.0]) # Check that errors if metric mismatch od3 = ObservationData(metric_names=["a"], means=np.array([2.0]), covariance=np.array([[4.0]])) with self.assertRaises(ValueError): unwrap_observation_data(observation_data + [od3])
def testMerge(self): obsd = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 1.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]), ) obsd2 = ivw_metric_merge(obsd) self.assertEqual(obsd2.metric_names, ["m1", "m2"]) self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 0.6 * 2 + 0.4]))) cov12 = 0.2 * 0.6 + 0.4 * 0.4 # var(w1*y1 + w2*y2) = # w1 ** 2 * var(y1) + w2 ** 2 * var(y2) + 2 * w1 * w2 * cov(y1, y2) cov22 = 0.6 ** 2 * 2.0 + 0.4 ** 2 * 3 + 2 * 0.6 * 0.4 * 0.8 cov_true = np.array([[1.0, cov12], [cov12, cov22]]) discrep = np.max(np.abs(obsd2.covariance - cov_true)) self.assertTrue(discrep < 1e-8)
def observation_status_quo1() -> Observation: return Observation( features=ObservationFeatures(parameters={ "w": 0.85, "x": 1, "y": "baz", "z": False }, trial_index=1), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="0_0", )
def testObservationData(self): attrs = { "metric_names": ["a", "b"], "means": np.array([4.0, 5.0]), "covariance": np.array([[1.0, 4.0], [3.0, 6.0]]), } obsd = ObservationData(**attrs) self.assertEqual(obsd.metric_names, attrs["metric_names"]) self.assertTrue(np.array_equal(obsd.means, attrs["means"])) self.assertTrue(np.array_equal(obsd.covariance, attrs["covariance"])) # use legacy printing for numpy (<= 1.13 add spaces in front of floats; # to get around tests failing on older versions, peg version to 1.13) if np.__version__ >= "1.14": np.set_printoptions(legacy="1.13") printstr = "ObservationData(metric_names=['a', 'b'], means=[ 4. 5.], " printstr += "covariance=[[ 1. 4.]\n [ 3. 6.]])" self.assertEqual(repr(obsd), printstr)
def testUpdate(self, mock_init): ma = DiscreteModelBridge() ma._training_data = self.observations model = mock.create_autospec(DiscreteModel, instance=True) ma._fit(model, self.search_space, self.observation_features, self.observation_data) new_feat = ObservationFeatures(parameters={ "x": 0, "y": "bar", "z": True }) new_data = ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])) ma._update([new_feat], [new_data]) self.assertEqual(ma.parameters, ["x", "y", "z"]) self.assertEqual(sorted(ma.outcomes), ["a", "b"])
def setUp(self): self.obsd1 = ObservationData( metric_names=["m1", "m2"], means=np.array([0.0, 0.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.obsd2 = ObservationData( metric_names=["m1", "m2"], means=np.array([1.0, 5.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.obsd3 = ObservationData( metric_names=["m1", "m2"], means=np.array([2.0, 25.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.obsd4 = ObservationData( metric_names=["m1", "m2"], means=np.array([3.0, 125.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.obsd_mid = ObservationData( metric_names=["m1", "m2"], means=np.array([1.5, 50.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.obsd_extreme = ObservationData( metric_names=["m1", "m2"], means=np.array([-1.0, 126.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), ) self.t = PercentileY( search_space=None, observation_features=None, observation_data=[ deepcopy(self.obsd1), deepcopy(self.obsd2), deepcopy(self.obsd3), deepcopy(self.obsd4), ], ) self.t_with_winsorization = PercentileY( search_space=None, observation_features=None, observation_data=[ deepcopy(self.obsd1), deepcopy(self.obsd2), deepcopy(self.obsd3), deepcopy(self.obsd4), ], config={"winsorize": True}, )
def _transform_ref_point( self, ref_point: Dict[str, float], padding_obs_data: ObservationData) -> Dict[str, float]: """Transform ref_point using same transforms as those applied to data. Args: ref_point: Reference point to transform. padding_obs_data: Data used to add dummy outcomes that aren't part of the reference point. This is necessary to apply transforms. Return: A transformed reference point. """ metric_names = list(self._metric_names or []) objective_metric_names = list(self._objective_metric_names or []) num_metrics = len(metric_names) # Create synthetic ObservationData representing the reference point. # Pad with non-objective outcomes from existing data. # Should always have existing data with BO. padding_obs_data padded_ref_dict: Dict[str, float] = dict( zip(padding_obs_data.metric_names, padding_obs_data.means)) padded_ref_dict.update(ref_point) ref_obs_data = [ ObservationData( metric_names=list(padded_ref_dict.keys()), means=np.array(list(padded_ref_dict.values())), covariance=np.zeros((num_metrics, num_metrics)), ) ] ref_obs_feats = [] # Apply initialized transforms to reference point. for t in self.transforms.values(): ref_obs_data = t.transform_observation_data( ref_obs_data, ref_obs_feats) transformed_ref_obsd = ref_obs_data.pop() transformed_ref_dict = dict( zip(transformed_ref_obsd.metric_names, transformed_ref_obsd.means)) transformed_ref_point = { objective_metric_name: transformed_ref_dict[objective_metric_name] for objective_metric_name in objective_metric_names } return transformed_ref_point
def get_observation_status_quo1(first_metric_name: str = "a", second_metric_name="b") -> Observation: return Observation( features=ObservationFeatures( parameters={ "w": 0.85, "x": 1, "y": "baz", "z": False }, trial_index=np.int64(1), ), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=[first_metric_name, second_metric_name], ), arm_name="0_0", )
def test_relativize_transform_requires_a_modelbridge_to_have_status_quo_data(self): sobol = Models.SOBOL(search_space=get_search_space()) self.assertIsNone(sobol.status_quo) with self.assertRaisesRegex(ValueError, "status quo data"): Relativize( search_space=None, observation_features=[], observation_data=[], modelbridge=sobol, ).transform_observation_data( observation_data=[ ObservationData( metric_names=["foo"], means=np.array([2]), covariance=np.array([[0.1]]), ) ], observation_features=[ObservationFeatures(parameters={"x": 1})], )
def array_to_observation_data(f: np.ndarray, cov: np.ndarray, outcomes: List[str]) -> List[ObservationData]: """Convert arrays of model predictions to a list of ObservationData. Args: f: An (n x m) array cov: An (n x m x m) array outcomes: A list of d outcome names Returns: A list of n ObservationData """ observation_data = [] for i in range(f.shape[0]): observation_data.append( ObservationData( metric_names=list(outcomes), means=f[i, :].copy(), covariance=cov[i, :, :].copy(), )) return observation_data
def _untransform_objective_thresholds( self, objective_thresholds: Tensor, objective_weights: Tensor, bounds: List[Tuple[Union[int, float], Union[int, float]]], fixed_features: Optional[Dict[int, float]], ) -> List[ObjectiveThreshold]: objective_thresholds_np = objective_thresholds.cpu().numpy() objective_indices = objective_weights.nonzero().view(-1).tolist() objective_names = [self.outcomes[i] for i in objective_indices] # Create an ObservationData object for untransforming the objective thresholds. observation_data = [ ObservationData( metric_names=objective_names, means=objective_thresholds_np[objective_indices].copy(), covariance=np.zeros((len(objective_indices), len(objective_indices))), ) ] # Untransform objective thresholds. Note: there is one objective threshold # for every outcome. # Construct dummy observation features. X = [bound[0] for bound in bounds] fixed_features = fixed_features or {} for i, val in fixed_features.items(): X[i] = val observation_features = parse_observation_features( X=np.array([X]), param_names=self.parameters, ) # Apply reverse transforms, in reverse order. for t in reversed(self.transforms.values()): observation_data = t.untransform_observation_data( observation_data=observation_data, observation_features=observation_features, ) observation_features = t.untransform_observation_features( observation_features=observation_features, ) observation_data = observation_data[0] oc = not_none(self._optimization_config) metrics_names_to_metric = oc.metrics obj_thresholds = [] for idx, (name, bound) in enumerate( zip(observation_data.metric_names, observation_data.means) ): if not np.isnan(bound): obj_weight = objective_weights[objective_indices[idx]] op = ( ComparisonOp.LEQ if torch.sign(obj_weight) == -1.0 else ComparisonOp.GEQ ) obj_thresholds.append( ObjectiveThreshold( metric=metrics_names_to_metric[name], bound=bound, relative=False, op=op, ) ) return obj_thresholds
def test_multitask_data(self): experiment = get_branin_with_multi_task() data = experiment.fetch_data() observations = observations_from_data( experiment=experiment, data=data, ) relative_observations = observations_from_data( experiment=experiment, data=relativize_data( data=data, status_quo_name="status_quo", as_percent=True, include_sq=True, ), ) status_quo_row = data.df.loc[ (data.df["arm_name"] == "status_quo") & (data.df["trial_index"] == 1) ] modelbridge = Mock( status_quo=Observation( data=ObservationData( metric_names=status_quo_row["metric_name"].values, means=status_quo_row["mean"].values, covariance=np.array([status_quo_row["sem"].values ** 2]), ), features=ObservationFeatures( parameters=experiment.status_quo.parameters ), ) ) obs_features = [obs.features for obs in observations] obs_data = [obs.data for obs in observations] expected_obs_data = [obs.data for obs in relative_observations] transform = Relativize( search_space=None, observation_features=obs_features, observation_data=obs_data, modelbridge=modelbridge, ) relative_obs_data = transform.transform_observation_data(obs_data, obs_features) self.maxDiff = None # this assertion just checks that order is the same, which # is only important for the purposes of this test self.assertEqual( [datum.metric_names for datum in relative_obs_data], [datum.metric_names for datum in expected_obs_data], ) means = [ np.array([datum.means for datum in relative_obs_data]), np.array([datum.means for datum in expected_obs_data]), ] # `self.assertAlmostEqual(relative_obs_data, expected_obs_data)` # fails 1% of the time, so we check with numpy. self.assertTrue( all(np.isclose(means[0], means[1])), means, ) covariances = [ np.array([datum.covariance for datum in expected_obs_data]), np.array([datum.covariance for datum in relative_obs_data]), ] self.assertTrue( all(np.isclose(covariances[0], covariances[1])), covariances, )
def test_pareto_frontier(self, _): exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() metrics_dict = exp.optimization_config.metrics objective_thresholds = [ ObjectiveThreshold( metric=metrics_dict["branin_a"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ObjectiveThreshold( metric=metrics_dict["branin_b"], bound=0.0, relative=False, op=ComparisonOp.GEQ, ), ] exp.optimization_config = exp.optimization_config.clone_with_args( objective_thresholds=objective_thresholds) exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys())) modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=[t1, t2], experiment=exp, data=exp.fetch_data(), objective_thresholds=objective_thresholds, ) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator observed_frontier = observed_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertEqual(1, len(observed_frontier)) self.assertEqual(observed_frontier[0].arm_name, "0_0") with self.assertRaises(ValueError): predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=[], ) predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=None, ) self.assertEqual(predicted_frontier[0].arm_name, "0_0") observation_features = [ ObservationFeatures(parameters={ "x1": 0.0, "x2": 1.0 }), ObservationFeatures(parameters={ "x1": 1.0, "x2": 0.0 }), ] observation_data = [ ObservationData( metric_names=["branin_b", "branin_a"], means=np.array([1.0, 2.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ObservationData( metric_names=["branin_a", "branin_b"], means=np.array([3.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), ), ] predicted_frontier = predicted_pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, ) self.assertTrue(len(predicted_frontier) <= 2) self.assertIsNone(predicted_frontier[0].arm_name, None) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, ) wrapped_frontier_evaluator.assert_called_once() self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["X"], torch.tensor([[1.0, 4.0], [4.0, 1.0]]), )) with patch( PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator: observed_frontier = pareto_frontier( modelbridge=modelbridge, objective_thresholds=objective_thresholds, observation_features=observation_features, observation_data=observation_data, use_model_predictions=False, ) wrapped_frontier_evaluator.assert_called_once() self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"]) self.assertTrue( torch.equal( wrapped_frontier_evaluator.call_args.kwargs["Y"], torch.tensor([[9.0, 4.0], [16.0, 25.0]]), ))
def setUp(self): x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1) y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2, is_fidelity=True, target_value=2) z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5) self.parameters = [x, y, z] parameter_constraints = [ OrderConstraint(x, y), SumConstraint([x, z], False, 3.5), ] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.observation_features = [ ObservationFeatures(parameters={ "x": 0.2, "y": 1.2, "z": 3 }), ObservationFeatures(parameters={ "x": 0.4, "y": 1.4, "z": 3 }), ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 }), ] self.observation_data = [ ObservationData( metric_names=["a", "b"], means=np.array([1.0, -1.0]), covariance=np.array([[1.0, 4.0], [4.0, 6.0]]), ), ObservationData( metric_names=["a", "b"], means=np.array([2.0, -2.0]), covariance=np.array([[2.0, 5.0], [5.0, 7.0]]), ), ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])), ] self.observations = [ Observation( features=self.observation_features[i], data=self.observation_data[i], arm_name=str(i), ) for i in range(3) ] self.pending_observations = { "b": [ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 })] } self.model_gen_options = {"option": "yes"}
def setUp(self): self.obsd1 = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([0.0, 0.0, 1.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]), ) self.obsd2 = ObservationData( metric_names=["m1", "m1", "m2", "m2"], means=np.array([1.0, 2.0, 2.0, 1.0]), covariance=np.array([ [1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.2, 0.4], [0.0, 0.2, 2.0, 0.8], [0.0, 0.4, 0.8, 3.0], ]), ) self.t = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={"winsorization_upper": 0.2}, ) self.t1 = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={"winsorization_upper": 0.8}, ) self.t2 = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={"winsorization_lower": 0.2}, ) self.t3 = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={ "winsorization_upper": 0.6, "percentile_bounds": { "m1": (None, None), "m2": (None, 1.9), }, }, ) self.t4 = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={ "winsorization_lower": 0.8, "percentile_bounds": { "m1": (None, None), "m2": (0.3, None), }, }, ) self.obsd3 = ObservationData( metric_names=["m3", "m3", "m3", "m3"], means=np.array([0.0, 1.0, 5.0, 3.0]), covariance=np.eye(4), ) self.t5 = Winsorize( search_space=None, observation_features=None, observation_data=[ deepcopy(self.obsd1), deepcopy(self.obsd2), deepcopy(self.obsd3), ], config={ "winsorization_lower": { "m2": 0.4 }, "winsorization_upper": { "m1": 0.6 }, }, ) self.t6 = Winsorize( search_space=None, observation_features=None, observation_data=[deepcopy(self.obsd1), deepcopy(self.obsd2)], config={ "winsorization_lower": { "m2": 0.4 }, "winsorization_upper": { "m1": 0.6 }, "percentile_bounds": { "m1": (None, None), "m2": (0.0, None), # This should leave m2 untouched }, }, )
class DerelativizeTransformTest(TestCase): def setUp(self): m = mock.patch.object(ModelBridge, "__abstractmethods__", frozenset()) self.addCleanup(m.stop) m.start() @mock.patch( "ax.modelbridge.base.observations_from_data", autospec=True, return_value=([ Observation( features=ObservationFeatures(parameters={ "x": 2.0, "y": 10.0 }), data=ObservationData( means=np.array([1.0, 2.0, 6.0]), covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0], [0.0, 0.0, 4.0]]), metric_names=["a", "b", "b"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={ "x": None, "y": None }), data=ObservationData( means=np.array([1.0, 2.0, 6.0]), covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0], [0.0, 0.0, 4.0]]), metric_names=["a", "b", "b"], ), arm_name="1_2", ), ]), ) @mock.patch("ax.modelbridge.base.ModelBridge._fit", autospec=True) @mock.patch( "ax.modelbridge.base.ModelBridge._predict", autospec=True, return_value=([ ObservationData( means=np.array([3.0, 5.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), metric_names=["a", "b"], ) ]), ) def testDerelativizeTransform(self, mock_predict, mock_fit, mock_observations_from_data): t = Derelativize(search_space=None, observation_features=None, observation_data=None) # ModelBridge with in-design status quo search_space = SearchSpace(parameters=[ RangeParameter("x", ParameterType.FLOAT, 0, 20), RangeParameter("y", ParameterType.FLOAT, 0, 20), ]) g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_1", ) # Test with no relative constraints objective = Objective(Metric("c")) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False) ], ) oc2 = t.transform_optimization_config(oc, g, None) self.assertTrue(oc == oc2) # Test with relative constraint, in-design status quo oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False), ]) obsf = mock_predict.mock_calls[0][1][1][0] obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0}) self.assertTrue(obsf == obsf2) # Test with relative constraint, out-of-design status quo mock_predict.side_effect = Exception() g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_2", ) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False), ]) self.assertEqual(mock_predict.call_count, 2) # Raises error if predict fails with in-design status quo g = ModelBridge(search_space, None, [], status_quo_name="1_1") oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) with self.assertRaises(Exception): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no status quo exp = Experiment(search_space, "name") g = ModelBridge(search_space, None, [], exp) with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no modelbridge with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, None, None) def testErrors(self): t = Derelativize(search_space=None, observation_features=None, observation_data=None) oc = OptimizationConfig( objective=Objective(Metric("c")), outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=True) ], ) search_space = SearchSpace( parameters=[RangeParameter("x", ParameterType.FLOAT, 0, 20)]) g = ModelBridge(search_space, None, []) with self.assertRaises(ValueError): t.transform_optimization_config(oc, None, None) with self.assertRaises(ValueError): t.transform_optimization_config(oc, g, None)
def ivw_metric_merge(obsd: ObservationData, conflicting_noiseless: str = "warn") -> ObservationData: """Merge multiple observations of a metric with inverse variance weighting. Correctly updates the covariance of the new merged estimates: ybar1 = Sum_i w_i * y_i ybar2 = Sum_j w_j * y_j cov[ybar1, ybar2] = Sum_i Sum_j w_i * w_j * cov[y_i, y_j] w_i will be infinity if any variance is 0. If one variance is 0., then the IVW estimate is the corresponding mean. If there are multiple measurements with 0 variance but means are all the same, then IVW estimate is that mean. If there are multiple measurements and means differ, behavior depends on argument conflicting_noiseless. "ignore" and "warn" will use the first of the measurements as the IVW estimate. "warn" will additionally log a warning. "raise" will raise an exception. Args: obsd: An ObservationData object conflicting_noiseless: "warn", "ignore", or "raise" """ if len(obsd.metric_names) == len(set(obsd.metric_names)): return obsd if conflicting_noiseless not in {"warn", "ignore", "raise"}: raise ValueError( 'conflicting_noiseless should be "warn", "ignore", or "raise".') # Get indicies and weights for each metric. # weights is a map from metric name to a vector of the weights for each # measurement of that metric. indicies gives the corresponding index in # obsd.means for each measurement. weights: Dict[str, np.ndarray] = {} indicies: Dict[str, List[int]] = {} for metric_name in set(obsd.metric_names): indcs = [ i for i, mn in enumerate(obsd.metric_names) if mn == metric_name ] indicies[metric_name] = indcs # Extract variances for observations of this metric sigma2s = obsd.covariance[indcs, indcs] # Check for noiseless observations idx_noiseless = np.where(sigma2s == 0.0)[0] if len(idx_noiseless) == 0: # Weight is inverse of variance, normalized # Expected `np.ndarray` for 3rd anonymous parameter to call # `dict.__setitem__` but got `float`. # pyre-fixme[6]: weights[metric_name] = 1.0 / sigma2s weights[metric_name] /= np.sum(weights[metric_name]) else: # Check if there are conflicting means for the noiseless observations means_noiseless = obsd.means[indcs][idx_noiseless] _check_conflicting_means(means_noiseless, metric_name, conflicting_noiseless) # The first observation gets all the weight. weights[metric_name] = np.zeros_like(sigma2s) weights[metric_name][idx_noiseless[0]] = 1.0 # Compute the new values metric_names = sorted(set(obsd.metric_names)) means = np.zeros(len(metric_names)) covariance = np.zeros((len(metric_names), len(metric_names))) for i, metric_name in enumerate(metric_names): ys = obsd.means[indicies[metric_name]] means[i] = np.sum(weights[metric_name] * ys) # Calculate covariances with metric_name for j, metric_name2 in enumerate(metric_names[i:], start=i): for ii, idx_i in enumerate(indicies[metric_name]): for jj, idx_j in enumerate(indicies[metric_name2]): covariance[i, j] += (weights[metric_name][ii] * weights[metric_name2][jj] * obsd.covariance[idx_i, idx_j]) covariance[j, i] = covariance[i, j] return ObservationData(metric_names=metric_names, means=means, covariance=covariance)