def testObservation(self): obs = Observation( features=ObservationFeatures(parameters={"x": 20}), data=ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), arm_name="0_0", ) self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20})) self.assertEqual( obs.data, ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), ) self.assertEqual(obs.arm_name, "0_0") obs2 = Observation( features=ObservationFeatures(parameters={"x": 20}), data=ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), arm_name="0_0", ) self.assertEqual(obs, obs2) obs3 = Observation( features=ObservationFeatures(parameters={"x": 10}), data=ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), arm_name="0_0", ) self.assertNotEqual(obs, obs3) self.assertNotEqual(obs, 1)
def setUp(self): self.training_data = [ Observation( features=ObservationFeatures(parameters={"x": 2.0}, trial_index=0), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={"x": 2.0}, trial_index=1), data=ObservationData( means=np.array([3.0, 5.0, 6.0]), covariance=np.array( [[1.0, 2.0, 3.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]] ), metric_names=["a", "b", "a"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={"x": 3.0}), data=ObservationData( means=np.array([7.0, 8.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_2", ), Observation( features=ObservationFeatures(parameters={"x": 4.0}, trial_index=2), data=ObservationData( means=np.array([9.0, 10.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_3", ), ] self.observation_data = [ ObservationData( means=np.array([2.0, 1.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ) ] * 4 self.diagnostics: List[CVDiagnostics] = [ {"Fisher exact test p": {"y_a": 0.0, "y_b": 0.4}}, {"Fisher exact test p": {"y_a": 0.1, "y_b": 0.1}}, {"Fisher exact test p": {"y_a": 0.5, "y_b": 0.6}}, ]
def setUp(self): self.training_data = [ Observation( features=ObservationFeatures(parameters={"x": 2.0}, trial_index=0), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={"x": 2.0}, trial_index=1), data=ObservationData( means=np.array([3.0, 5.0, 6.0]), covariance=np.array([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]), metric_names=["a", "b", "a"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={"x": 3.0}), data=ObservationData( means=np.array([7.0, 8.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_2", ), Observation( features=ObservationFeatures(parameters={"x": 4.0}, trial_index=2), data=ObservationData( means=np.array([9.0, 10.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_3", ), ] self.observation_data = [ ObservationData( means=np.array([2.0, 1.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ) ] * 4
def testFit(self, mock_init): sq_feat = ObservationFeatures({}) sq_data = self.observation_data[0] sq_obs = Observation(features=sq_feat, data=sq_data, arm_name="status_quo") ma = DiscreteModelBridge() ma._training_data = self.observations + [sq_obs] model = mock.create_autospec(DiscreteModel, instance=True) ma._fit( model, self.search_space, self.observation_features + [sq_feat], self.observation_data + [sq_data], ) self.assertEqual(ma.parameters, ["x", "y", "z"]) self.assertEqual(sorted(ma.outcomes), ["a", "b"]) self.assertEqual(ma.training_in_design, [True, True, True, False]) Xs = { "a": [[0, "foo", True], [1, "foo", True], [1, "bar", True]], "b": [[0, "foo", True], [1, "foo", True]], } Ys = {"a": [[1.0], [2.0], [3.0]], "b": [[-1.0], [-2.0]]} Yvars = {"a": [[1.0], [2.0], [3.0]], "b": [[6.0], [7.0]]} parameter_values = [[0.0, 1.0], ["foo", "bar"], [True]] model_fit_args = model.fit.mock_calls[0][2] for i, x in enumerate(model_fit_args["Xs"]): self.assertEqual(x, Xs[ma.outcomes[i]]) for i, y in enumerate(model_fit_args["Ys"]): self.assertEqual(y, Ys[ma.outcomes[i]]) for i, v in enumerate(model_fit_args["Yvars"]): self.assertEqual(v, Yvars[ma.outcomes[i]]) self.assertEqual(model_fit_args["parameter_values"], parameter_values)
def setUp(self): x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1) y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2) z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5) self.parameters = [x, y, z] parameter_constraints = [ OrderConstraint(x, y), SumConstraint([x, z], False, 3.5), ] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.observation_features = [ ObservationFeatures(parameters={ "x": 0.2, "y": 1.2, "z": 3 }), ObservationFeatures(parameters={ "x": 0.4, "y": 1.4, "z": 3 }), ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 }), ] self.observation_data = [ ObservationData( metric_names=["a", "b"], means=np.array([1.0, -1.0]), covariance=np.array([[1.0, 4.0], [4.0, 6.0]]), ), ObservationData( metric_names=["a", "b"], means=np.array([2.0, -2.0]), covariance=np.array([[2.0, 5.0], [5.0, 7.0]]), ), ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])), ] self.observations = [ Observation( features=self.observation_features[i], data=self.observation_data[i], arm_name=str(i), ) for i in range(3) ] self.pending_observations = { "b": [ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 })] } self.model_gen_options = {"option": "yes"}
def testSeparateObservations(self): obs = Observation( features=ObservationFeatures(parameters={"x": 20}), data=ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), arm_name="0_0", ) obs_feats, obs_data = separate_observations(observations=[obs]) self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20})) self.assertEqual( obs.data, ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), ) obs_feats, obs_data = separate_observations(observations=[obs], copy=True) self.assertEqual(obs.features, ObservationFeatures(parameters={"x": 20})) self.assertEqual( obs.data, ObservationData(means=np.array([1]), covariance=np.array([[2]]), metric_names=["a"]), )
def testFitAndUpdate(self, mock_init): sq_feat = ObservationFeatures({}) sq_data = self.observation_data[2] sq_obs = Observation( features=ObservationFeatures({}), data=self.observation_data[2], arm_name="status_quo", ) ma = NumpyModelBridge() ma._training_data = self.observations + [sq_obs] model = mock.create_autospec(NumpyModel, instance=True) # No out of design points allowed in direct calls to fit. with self.assertRaises(ValueError): ma._fit( model, self.search_space, self.observation_features + [sq_feat], self.observation_data + [sq_data], ) ma._fit(model, self.search_space, self.observation_features, self.observation_data) self.assertEqual(ma.parameters, ["x", "z", "y"]) self.assertEqual(sorted(ma.outcomes), ["a", "b"]) Xs = { "a": np.array([[0.2, 3.0, 1.2], [0.4, 3.0, 1.4], [0.6, 3.0, 1.6]]), "b": np.array([[0.2, 3.0, 1.2], [0.4, 3.0, 1.4]]), } Ys = { "a": np.array([[1.0], [2.0], [3.0]]), "b": np.array([[-1.0], [-2.0]]) } Yvars = { "a": np.array([[1.0], [2.0], [3.0]]), "b": np.array([[6.0], [7.0]]) } # put fidelity parameter to the last column bounds = [(0.0, 1.0), (0.0, 5.0), (1.0, 2.0)] model_fit_args = model.fit.mock_calls[0][2] for i, x in enumerate(model_fit_args["Xs"]): self.assertTrue(np.array_equal(x, Xs[ma.outcomes[i]])) for i, y in enumerate(model_fit_args["Ys"]): self.assertTrue(np.array_equal(y, Ys[ma.outcomes[i]])) for i, v in enumerate(model_fit_args["Yvars"]): self.assertTrue(np.array_equal(v, Yvars[ma.outcomes[i]])) self.assertEqual(model_fit_args["bounds"], bounds) self.assertEqual(model_fit_args["feature_names"], ["x", "z", "y"]) # And update ma._update( observation_features=self.observation_features, observation_data=self.observation_data, ) # Calling _update requires passing ALL data. model_update_args = model.update.mock_calls[0][2] for i, x in enumerate(model_update_args["Xs"]): self.assertTrue(np.array_equal(x, Xs[ma.outcomes[i]])) for i, y in enumerate(model_update_args["Ys"]): self.assertTrue(np.array_equal(y, Ys[ma.outcomes[i]])) for i, v in enumerate(model_update_args["Yvars"]): self.assertTrue(np.array_equal(v, Yvars[ma.outcomes[i]]))
def setUp(self): self.parameters = [ ChoiceParameter("x", ParameterType.FLOAT, values=[0, 1]), ChoiceParameter("y", ParameterType.STRING, values=["foo", "bar"]), FixedParameter("z", ParameterType.BOOL, value=True), ] parameter_constraints = [] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.observation_features = [ ObservationFeatures(parameters={ "x": 0, "y": "foo", "z": True }), ObservationFeatures(parameters={ "x": 1, "y": "foo", "z": True }), ObservationFeatures(parameters={ "x": 1, "y": "bar", "z": True }), ] self.observation_data = [ ObservationData( metric_names=["a", "b"], means=np.array([1.0, -1.0]), covariance=np.array([[1.0, 4.0], [4.0, 6.0]]), ), ObservationData( metric_names=["a", "b"], means=np.array([2.0, -2.0]), covariance=np.array([[2.0, 5.0], [5.0, 7.0]]), ), ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])), ] self.observations = [ Observation( features=self.observation_features[i], data=self.observation_data[i], arm_name=str(i), ) for i in range(3) ] self.pending_observations = { "b": [ObservationFeatures(parameters={ "x": 0, "y": "foo", "z": True })] } self.model_gen_options = {"option": "yes"}
def testFitAndUpdate(self, mock_init): sq_feat = ObservationFeatures({}) sq_data = self.observation_data[2] sq_obs = Observation( features=ObservationFeatures({}), data=self.observation_data[2], arm_name="status_quo", ) ma = NumpyModelBridge() ma._training_data = self.observations + [sq_obs] model = mock.create_autospec(NumpyModel, instance=True) ma._fit( model, self.search_space, self.observation_features + [sq_feat], self.observation_data + [sq_data], ) self.assertEqual(ma.parameters, ["x", "y", "z"]) self.assertEqual(sorted(ma.outcomes), ["a", "b"]) self.assertEqual(ma.training_in_design, [True, True, True, False]) Xs = { "a": np.array([[0.2, 1.2, 3.0], [0.4, 1.4, 3.0], [0.6, 1.6, 3]]), "b": np.array([[0.2, 1.2, 3.0], [0.4, 1.4, 3.0]]), } Ys = { "a": np.array([[1.0], [2.0], [3.0]]), "b": np.array([[-1.0], [-2.0]]) } Yvars = { "a": np.array([[1.0], [2.0], [3.0]]), "b": np.array([[6.0], [7.0]]) } bounds = [(0.0, 1.0), (1.0, 2.0), (0.0, 5.0)] model_fit_args = model.fit.mock_calls[0][2] for i, x in enumerate(model_fit_args["Xs"]): self.assertTrue(np.array_equal(x, Xs[ma.outcomes[i]])) for i, y in enumerate(model_fit_args["Ys"]): self.assertTrue(np.array_equal(y, Ys[ma.outcomes[i]])) for i, v in enumerate(model_fit_args["Yvars"]): self.assertTrue(np.array_equal(v, Yvars[ma.outcomes[i]])) self.assertEqual(model_fit_args["bounds"], bounds) self.assertEqual(model_fit_args["feature_names"], ["x", "y", "z"]) # And update ma.training_in_design.extend([True, True, True, True]) ma._update( observation_features=self.observation_features + [sq_feat], observation_data=self.observation_data + [sq_data], ) self.assertEqual(ma.training_in_design, [True, True, True, False] * 2) model_update_args = model.update.mock_calls[0][2] for i, x in enumerate(model_update_args["Xs"]): self.assertTrue(np.array_equal(x, Xs[ma.outcomes[i]])) for i, y in enumerate(model_update_args["Ys"]): self.assertTrue(np.array_equal(y, Ys[ma.outcomes[i]])) for i, v in enumerate(model_update_args["Yvars"]): self.assertTrue(np.array_equal(v, Yvars[ma.outcomes[i]]))
def observation2trans() -> Observation: return Observation( features=ObservationFeatures(parameters={"x": 16.0, "y": 2.0}, trial_index=1), data=ObservationData( means=np.array([9.0, 4.0]), covariance=np.array([[2.0, 3.0], [4.0, 5.0]]), metric_names=["a", "b"], ), arm_name="1_1", )
def get_observation() -> Observation: return Observation( features=ObservationFeatures( parameters={"x": 2.0, "y": 10.0}, trial_index=np.int64(0) ), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="1_1", )
def get_observation_status_quo1() -> Observation: return Observation( features=ObservationFeatures( parameters={"w": 0.85, "x": 1, "y": "baz", "z": False}, trial_index=np.int64(1), ), data=ObservationData( means=np.array([2.0, 4.0]), covariance=np.array([[1.0, 2.0], [3.0, 4.0]]), metric_names=["a", "b"], ), arm_name="0_0", )
def get_observation2trans(first_metric_name: str = "a", second_metric_name="b") -> Observation: return Observation( features=ObservationFeatures(parameters={ "x": 16.0, "y": 2.0 }, trial_index=np.int64(1)), data=ObservationData( means=np.array([9.0, 4.0]), covariance=np.array([[2.0, 3.0], [4.0, 5.0]]), metric_names=[first_metric_name, second_metric_name], ), arm_name="1_1", )
def convert_mt_observations( observations: List[Observation], experiment: MultiTypeExperiment) -> List[Observation]: """Apply ConvertMetricNames transform to observations for a MT experiment.""" observation_data = [o.data for o in observations] observation_features = [o.features for o in observations] transform = ConvertMetricNames( search_space=None, observation_data=observation_data, observation_features=observation_features, config=tconfig_from_mt_experiment(experiment), ) transformed_observations = transform.transform_observation_data( observation_data=observation_data, observation_features=observation_features) return [ Observation( features=obs.features, data=transformed_observations[i], arm_name=obs.arm_name, ) for i, obs in enumerate(observations) ]
def get_pareto_frontier_and_configs( modelbridge: modelbridge_module.array.ArrayModelBridge, observation_features: List[ObservationFeatures], observation_data: Optional[List[ObservationData]] = None, objective_thresholds: Optional[TRefPoint] = None, optimization_config: Optional[MultiObjectiveOptimizationConfig] = None, arm_names: Optional[List[Optional[str]]] = None, use_model_predictions: bool = True, transform_outcomes_and_configs: bool = True, ) -> Tuple[List[Observation], Tensor, Tensor, Optional[Tensor]]: """Helper that applies transforms and calls ``frontier_evaluator``. Returns the ``frontier_evaluator`` configs in addition to the Pareto observations. Args: modelbridge: ``Modelbridge`` used to predict metrics outcomes. observation_features: Observation features to consider for the Pareto frontier. observation_data: Data for computing the Pareto front, unless ``observation_features`` are provided and ``model_predictions is True``. objective_thresholds: Metric values bounding the region of interest in the objective outcome space; used to override objective thresholds specified in ``optimization_config``, if necessary. optimization_config: Multi-objective optimization config. arm_names: Arm names for each observation in ``observation_features``. use_model_predictions: If ``True``, will use model predictions at ``observation_features`` to compute Pareto front. If ``False``, will use ``observation_data`` directly to compute Pareto front, ignoring ``observation_features``. transform_outcomes_and_configs: If ``True``, will transform the optimization config, observation features and observation data, before calling ``frontier_evaluator``, then will untransform all of the above before returning the observations. Returns: Four-item tuple of: - frontier_observations: Observations of points on the pareto frontier, - f: n x m tensor representation of the Pareto frontier values where n is the length of frontier_observations and m is the number of metrics, - obj_w: m tensor of objective weights, - obj_t: m tensor of objective thresholds corresponding to Y, or None if no objective thresholds used. """ array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge) X, Y, Yvar = None, None, None if use_model_predictions: X = array_to_tensor( modelbridge.transform_observation_features(observation_features)) if observation_data is not None: if transform_outcomes_and_configs: Y, Yvar = modelbridge.transform_observation_data(observation_data) else: Y, Yvar = observation_data_to_array( outcomes=modelbridge.outcomes, observation_data=observation_data) Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar)) if arm_names is None: arm_names = [None] * len(observation_features) # Extract optimization config: make sure that the problem is a MOO # problem and clone the optimization config with specified # `objective_thresholds` if those are provided. If `optimization_config` # is not specified, uses the one stored on `modelbridge`. optimization_config = _get_multiobjective_optimization_config( modelbridge=modelbridge, optimization_config=optimization_config, objective_thresholds=objective_thresholds, ) # Transform optimization config. fixed_features = ObservationFeatures(parameters={}) if transform_outcomes_and_configs: optimization_config = modelbridge.transform_optimization_config( optimization_config=optimization_config, fixed_features=fixed_features, ) else: # de-relativize outcome constraints and objective thresholds obs_feats, obs_data, _ = _get_modelbridge_training_data( modelbridge=modelbridge) tf = Derelativize( search_space=modelbridge.model_space.clone(), observation_data=obs_data, observation_features=obs_feats, config={"use_raw_status_quo": True}, ) # pyre-ignore [9] optimization_config = tf.transform_optimization_config( optimization_config=optimization_config.clone(), modelbridge=modelbridge, fixed_features=fixed_features, ) # Extract weights, constraints, and objective_thresholds objective_weights = extract_objective_weights( objective=optimization_config.objective, outcomes=modelbridge.outcomes) outcome_constraints = extract_outcome_constraints( outcome_constraints=optimization_config.outcome_constraints, outcomes=modelbridge.outcomes, ) obj_t = extract_objective_thresholds( objective_thresholds=optimization_config.objective_thresholds, objective=optimization_config.objective, outcomes=modelbridge.outcomes, ) obj_t = array_to_tensor(obj_t) # Transform to tensors. obj_w, oc_c, _, _, _ = validate_and_apply_final_transform( objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=None, pending_observations=None, final_transform=array_to_tensor, ) frontier_evaluator = get_default_frontier_evaluator() # pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call f, cov, indx = frontier_evaluator( model=modelbridge.model, X=X, Y=Y, Yvar=Yvar, objective_thresholds=obj_t, objective_weights=obj_w, outcome_constraints=oc_c, ) f, cov = f.detach().cpu().clone(), cov.detach().cpu().clone() indx = indx.tolist() frontier_observation_data = array_to_observation_data( f=f.numpy(), cov=cov.numpy(), outcomes=not_none(modelbridge.outcomes)) if use_model_predictions: # Untransform observations for t in reversed(modelbridge.transforms.values()): # noqa T484 frontier_observation_data = t.untransform_observation_data( frontier_observation_data, []) # reconstruct tensor representation of untransformed predictions Y_arr, _ = observation_data_to_array( outcomes=modelbridge.outcomes, observation_data=frontier_observation_data) f = _array_to_tensor(Y_arr) # Construct observations frontier_observations = [] for i, obsd in enumerate(frontier_observation_data): frontier_observations.append( Observation( features=observation_features[indx[i]], data=obsd, arm_name=arm_names[indx[i]], )) return frontier_observations, f, obj_w.cpu(), obj_t.cpu()
def test_multitask_data(self): experiment = get_branin_with_multi_task() data = experiment.fetch_data() observations = observations_from_data( experiment=experiment, data=data, ) relative_observations = observations_from_data( experiment=experiment, data=relativize_data( data=data, status_quo_name="status_quo", as_percent=True, include_sq=True, ), ) status_quo_row = data.df.loc[ (data.df["arm_name"] == "status_quo") & (data.df["trial_index"] == 1) ] modelbridge = Mock( status_quo=Observation( data=ObservationData( metric_names=status_quo_row["metric_name"].values, means=status_quo_row["mean"].values, covariance=np.array([status_quo_row["sem"].values ** 2]), ), features=ObservationFeatures( parameters=experiment.status_quo.parameters ), ) ) obs_features = [obs.features for obs in observations] obs_data = [obs.data for obs in observations] expected_obs_data = [obs.data for obs in relative_observations] transform = Relativize( search_space=None, observation_features=obs_features, observation_data=obs_data, modelbridge=modelbridge, ) relative_obs_data = transform.transform_observation_data(obs_data, obs_features) self.maxDiff = None # this assertion just checks that order is the same, which # is only important for the purposes of this test self.assertEqual( [datum.metric_names for datum in relative_obs_data], [datum.metric_names for datum in expected_obs_data], ) means = [ np.array([datum.means for datum in relative_obs_data]), np.array([datum.means for datum in expected_obs_data]), ] # `self.assertAlmostEqual(relative_obs_data, expected_obs_data)` # fails 1% of the time, so we check with numpy. self.assertTrue( all(np.isclose(means[0], means[1])), means, ) covariances = [ np.array([datum.covariance for datum in expected_obs_data]), np.array([datum.covariance for datum in relative_obs_data]), ] self.assertTrue( all(np.isclose(covariances[0], covariances[1])), covariances, )
def get_pareto_frontier_and_transformed_configs( modelbridge: modelbridge_module.array.ArrayModelBridge, observation_features: List[ObservationFeatures], observation_data: Optional[List[ObservationData]] = None, objective_thresholds: Optional[TRefPoint] = None, optimization_config: Optional[MultiObjectiveOptimizationConfig] = None, arm_names: Optional[List[Optional[str]]] = None, use_model_predictions: bool = True, ) -> Tuple[List[Observation], Tensor, Tensor, Optional[Tensor]]: """Helper that applies transforms and calls frontier_evaluator. Returns transformed configs in addition to the Pareto observations. Args: modelbridge: Modelbridge used to predict metrics outcomes. observation_features: observation features to predict, if provided and use_model_predictions is True. observation_data: data for computing the Pareto front, unless features are provided and model_predictions is True. objective_thresholds: metric values bounding the region of interest in the objective outcome space. optimization_config: Optimization config. arm_names: Arm names for each observation. use_model_predictions: If True, will use model predictions at observation_features to compute Pareto front, if provided. If False, will use observation_data directly to compute Pareto front, regardless of whether observation_features are provided. Returns: frontier_observations: Observations of points on the pareto frontier. f: n x m tensor representation of the Pareto frontier values where n is the length of frontier_observations and m is the number of metrics. obj_w: m tensor of objective weights. obj_t: m tensor of objective thresholds corresponding to Y, or None if no objective thresholds used. """ array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge) X = (modelbridge.transform_observation_features(observation_features) if use_model_predictions else None) X = array_to_tensor(X) if X is not None else None Y, Yvar = (None, None) if observation_data is not None: Y, Yvar = modelbridge.transform_observation_data(observation_data) Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar)) if arm_names is None: arm_names = [None] * len(observation_features) # Optimization_config mooc = optimization_config or checked_cast_optional( MultiObjectiveOptimizationConfig, modelbridge._optimization_config) if not mooc: raise ValueError( ("Experiment must have an existing optimization_config " "of type `MultiObjectiveOptimizationConfig` " "or `optimization_config` must be passed as an argument.")) if not isinstance(mooc, MultiObjectiveOptimizationConfig): mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc)) if objective_thresholds: mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds) optimization_config = mooc # Transform OptimizationConfig. optimization_config = modelbridge.transform_optimization_config( optimization_config=optimization_config, fixed_features=ObservationFeatures(parameters={}), ) # Extract weights, constraints, and objective_thresholds objective_weights = extract_objective_weights( objective=optimization_config.objective, outcomes=modelbridge.outcomes) outcome_constraints = extract_outcome_constraints( outcome_constraints=optimization_config.outcome_constraints, outcomes=modelbridge.outcomes, ) obj_t = extract_objective_thresholds( objective_thresholds=optimization_config.objective_thresholds, objective=optimization_config.objective, outcomes=modelbridge.outcomes, ) obj_t = array_to_tensor(obj_t) # Transform to tensors. obj_w, oc_c, _, _, _ = validate_and_apply_final_transform( objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=None, pending_observations=None, final_transform=array_to_tensor, ) frontier_evaluator = get_default_frontier_evaluator() # pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call f, cov, indx = frontier_evaluator( model=modelbridge.model, X=X, Y=Y, Yvar=Yvar, objective_thresholds=obj_t, objective_weights=obj_w, outcome_constraints=oc_c, ) f, cov = f.detach().cpu().clone(), cov.detach().cpu().clone() indx = indx.tolist() frontier_observation_data = array_to_observation_data( f=f.numpy(), cov=cov.numpy(), outcomes=not_none(modelbridge.outcomes)) # Untransform observations for t in reversed(modelbridge.transforms.values()): # noqa T484 frontier_observation_data = t.untransform_observation_data( frontier_observation_data, []) # Construct observations frontier_observations = [] for i, obsd in enumerate(frontier_observation_data): frontier_observations.append( Observation( features=observation_features[indx[i]], data=obsd, arm_name=arm_names[indx[i]], )) return frontier_observations, f, obj_w, obj_t
class DerelativizeTransformTest(TestCase): def setUp(self): m = mock.patch.object(ModelBridge, "__abstractmethods__", frozenset()) self.addCleanup(m.stop) m.start() @mock.patch( "ax.modelbridge.base.observations_from_data", autospec=True, return_value=([ Observation( features=ObservationFeatures(parameters={ "x": 2.0, "y": 10.0 }), data=ObservationData( means=np.array([1.0, 2.0, 6.0]), covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0], [0.0, 0.0, 4.0]]), metric_names=["a", "b", "b"], ), arm_name="1_1", ), Observation( features=ObservationFeatures(parameters={ "x": None, "y": None }), data=ObservationData( means=np.array([1.0, 2.0, 6.0]), covariance=np.array([[1.0, 2.0, 0.0], [3.0, 4.0, 0.0], [0.0, 0.0, 4.0]]), metric_names=["a", "b", "b"], ), arm_name="1_2", ), ]), ) @mock.patch("ax.modelbridge.base.ModelBridge._fit", autospec=True) @mock.patch( "ax.modelbridge.base.ModelBridge._predict", autospec=True, return_value=([ ObservationData( means=np.array([3.0, 5.0]), covariance=np.array([[1.0, 0.0], [0.0, 1.0]]), metric_names=["a", "b"], ) ]), ) def testDerelativizeTransform(self, mock_predict, mock_fit, mock_observations_from_data): t = Derelativize(search_space=None, observation_features=None, observation_data=None) # ModelBridge with in-design status quo search_space = SearchSpace(parameters=[ RangeParameter("x", ParameterType.FLOAT, 0, 20), RangeParameter("y", ParameterType.FLOAT, 0, 20), ]) g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_1", ) # Test with no relative constraints objective = Objective(Metric("c")) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False) ], ) oc2 = t.transform_optimization_config(oc, g, None) self.assertTrue(oc == oc2) # Test with relative constraint, in-design status quo oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False), ]) obsf = mock_predict.mock_calls[0][1][1][0] obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0}) self.assertTrue(obsf == obsf2) # Test with relative constraint, out-of-design status quo mock_predict.side_effect = Exception() g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_2", ) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False), ]) self.assertEqual(mock_predict.call_count, 2) # Raises error if predict fails with in-design status quo g = ModelBridge(search_space, None, [], status_quo_name="1_1") oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) with self.assertRaises(Exception): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no status quo exp = Experiment(search_space, "name") g = ModelBridge(search_space, None, [], exp) with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no modelbridge with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, None, None) def testErrors(self): t = Derelativize(search_space=None, observation_features=None, observation_data=None) oc = OptimizationConfig( objective=Objective(Metric("c")), outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=True) ], ) search_space = SearchSpace( parameters=[RangeParameter("x", ParameterType.FLOAT, 0, 20)]) g = ModelBridge(search_space, None, []) with self.assertRaises(ValueError): t.transform_optimization_config(oc, None, None) with self.assertRaises(ValueError): t.transform_optimization_config(oc, g, None)
def pareto_frontier( modelbridge: modelbridge_module.array.ArrayModelBridge, observation_features: List[ObservationFeatures], observation_data: Optional[List[ObservationData]] = None, objective_thresholds: Optional[TRefPoint] = None, optimization_config: Optional[MultiObjectiveOptimizationConfig] = None, arm_names: Optional[List[Optional[str]]] = None, use_model_predictions: bool = True, ) -> List[Observation]: """Helper that applies transforms and calls frontier_evaluator.""" array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge) X = ( modelbridge.transform_observation_features(observation_features) if use_model_predictions else None ) X = array_to_tensor(X) if X is not None else None Y, Yvar = (None, None) if observation_data is not None: Y, Yvar = modelbridge.transform_observation_data(observation_data) Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar)) if arm_names is None: arm_names = [None] * len(observation_features) # Optimization_config mooc = optimization_config or checked_cast_optional( MultiObjectiveOptimizationConfig, modelbridge._optimization_config ) if not mooc: raise ValueError( ( "Experiment must have an existing optimization_config " "of type `MultiObjectiveOptimizationConfig` " "or `optimization_config` must be passed as an argument." ) ) if not isinstance(mooc, MultiObjectiveOptimizationConfig): mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc)) if objective_thresholds: mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds) optimization_config = mooc # Transform OptimizationConfig. optimization_config = modelbridge.transform_optimization_config( optimization_config=optimization_config, fixed_features=ObservationFeatures(parameters={}), ) # Extract weights, constraints, and objective_thresholds objective_weights = extract_objective_weights( objective=optimization_config.objective, outcomes=modelbridge.outcomes ) outcome_constraints = extract_outcome_constraints( outcome_constraints=optimization_config.outcome_constraints, outcomes=modelbridge.outcomes, ) objective_thresholds_arr = extract_objective_thresholds( objective_thresholds=optimization_config.objective_thresholds, outcomes=modelbridge.outcomes, ) # Transform to tensors. obj_w, oc_c, _, _ = validate_and_apply_final_transform( objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=None, pending_observations=None, final_transform=array_to_tensor, ) obj_t = array_to_tensor(objective_thresholds_arr) frontier_evaluator = get_default_frontier_evaluator() # pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call f, cov, indx = frontier_evaluator( model=modelbridge.model, X=X, Y=Y, Yvar=Yvar, objective_thresholds=obj_t, objective_weights=obj_w, outcome_constraints=oc_c, ) f, cov = f.detach().cpu().clone().numpy(), cov.detach().cpu().clone().numpy() indx = indx.tolist() frontier_observation_data = array_to_observation_data( f=f, cov=cov, outcomes=not_none(modelbridge.outcomes) ) # Untransform observations for t in reversed(modelbridge.transforms.values()): # noqa T484 frontier_observation_data = t.untransform_observation_data( frontier_observation_data, [] ) # Construct observations frontier_observations = [] for i, obsd in enumerate(frontier_observation_data): frontier_observations.append( Observation( features=observation_features[indx[i]], data=obsd, arm_name=arm_names[indx[i]], ) ) return frontier_observations