def testTorchModelCrossValidate(self): torch_model = TorchModel() with self.assertRaises(NotImplementedError): torch_model.cross_validate( Xs_train=[np.array([1])], Ys_train=[np.array([1])], Yvars_train=[np.array([1])], X_test=np.array([1]), )
def testTorchModelFit(self): torch_model = TorchModel() torch_model.fit( Xs=[np.array(0)], Ys=[np.array(0)], Yvars=[np.array(1)], bounds=[(0, 1)], task_features=[], feature_names=["x1"], )
def testTorchModelUpdate(self): model = TorchModel() with self.assertRaises(NotImplementedError): model.update( Xs=[np.array(0)], Ys=[np.array(0)], Yvars=[np.array(1)], search_space_digest=SearchSpaceDigest(feature_names=[], bounds=[]), metric_names=[], )
def testTorchModelCrossValidate(self): torch_model = TorchModel() with self.assertRaises(NotImplementedError): torch_model.cross_validate( Xs_train=[np.array([1])], Ys_train=[np.array([1])], Yvars_train=[np.array([1])], X_test=np.array([1]), search_space_digest=SearchSpaceDigest(feature_names=[], bounds=[]), metric_names=[], )
def testTorchModelFit(self): torch_model = TorchModel() torch_model.fit( Xs=[np.array(0)], Ys=[np.array(0)], Yvars=[np.array(1)], search_space_digest=SearchSpaceDigest( feature_names=["x1"], bounds=[(0, 1)], ), metric_names=["y"], )
def testTorchModelUpdate(self): numpy_model = TorchModel() with self.assertRaises(NotImplementedError): numpy_model.update( Xs=[np.array(0)], Ys=[np.array(0)], Yvars=[np.array(1)], bounds=[], task_features=[], fidelity_features=[], feature_names=[], metric_names=[], )
def testTorchModelCrossValidate(self): torch_model = TorchModel() with self.assertRaises(NotImplementedError): torch_model.cross_validate( Xs_train=[np.array([1])], Ys_train=[np.array([1])], Yvars_train=[np.array([1])], X_test=np.array([1]), bounds=[], task_features=[], fidelity_features=[], feature_names=[], metric_names=[], )
def get_tensor_converter_model(experiment: Experiment, data: Data) -> TorchModelBridge: """ Constructs a minimal model for converting things to tensors. Model fitting will instantiate all of the transforms but will not do any expensive (i.e. GP) fitting beyond that. The model will raise an error if it is used for predicting or generating. Will work for any search space regardless of types of parameters. Args: experiment: Experiment. data: Data for fitting the model. Returns: A torch modelbridge with transforms set. """ # Transforms is the minimal set that will work for converting any search # space to tensors. return TorchModelBridge( experiment=experiment, search_space=experiment.search_space, data=data, model=TorchModel(), transforms=[Derelativize, SearchSpaceToChoice, OrderedChoiceEncode, IntToFloat], transform_configs={ "Derelativize": {"use_raw_status_quo": True}, "SearchSpaceToChoice": {"use_ordered": True}, }, fit_out_of_design=True, )
def testNumpyTorchBestPoint(self): torch_model = TorchModel() x = torch_model.best_point(bounds=[(0, 1)], objective_weights=np.array([1])) self.assertIsNone(x)
def testTorchModelGen(self): torch_model = TorchModel() with self.assertRaises(NotImplementedError): torch_model.gen(n=1, bounds=[(0, 1)], objective_weights=np.array([1]))
def testTorchModelPredict(self): torch_model = TorchModel() with self.assertRaises(NotImplementedError): torch_model.predict(np.array([0]))
def pareto_frontier_evaluator( model: TorchModel, objective_weights: Tensor, objective_thresholds: Optional[Tensor] = None, X: Optional[Tensor] = None, Y: Optional[Tensor] = None, Yvar: Optional[Tensor] = None, outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, ) -> Tuple[Tensor, Tensor, Tensor]: """Return outcomes predicted to lie on a pareto frontier. Given a model and a points to evaluate use the model to predict which points lie on the pareto frontier. Args: model: Model used to predict outcomes. objective_weights: A `m` tensor of values indicating the weight to put on different outcomes. For pareto frontiers only the sign matters. objective_thresholds: A tensor containing thresholds forming a reference point from which to calculate pareto frontier hypervolume. Points that do not dominate the objective_thresholds contribute nothing to hypervolume. X: A `n x d` tensor of features to evaluate. Y: A `n x m` tensor of outcomes to use instead of predictions. Yvar: A `n x m x m` tensor of input covariances (NaN if unobserved). outcome_constraints: A tuple of (A, b). For k outcome constraints and m outputs at f(x), A is (k x m) and b is (k x 1) such that A f(x) <= b. Returns: 3-element tuple containing - A `j x m` tensor of outcome on the pareto frontier. j is the number of frontier points. - A `j x m x m` tensor of predictive covariances. cov[j, m1, m2] is Cov[m1@j, m2@j]. - A `j` tensor of the index of each frontier point in the input Y. """ if X is not None: Y, Yvar = model.predict(X) elif Y is None or Yvar is None: raise ValueError( "Requires `X` to predict or both `Y` and `Yvar` to select a subset of " "points on the pareto frontier.") # Apply objective_weights to outcomes and objective_thresholds. # If objective_thresholds is not None use a dummy tensor of zeros. ( obj, weighted_objective_thresholds, ) = get_weighted_mc_objective_and_objective_thresholds( objective_weights=objective_weights, objective_thresholds=(objective_thresholds if objective_thresholds is not None else torch.zeros(objective_weights.shape)), ) Y_obj = obj(Y) indx_frontier = torch.arange(Y.shape[0], dtype=torch.long, device=Y.device) # Filter Y, Yvar, Y_obj to items that dominate all objective thresholds if objective_thresholds is not None: objective_thresholds_mask = (Y_obj >= weighted_objective_thresholds).all(dim=1) Y = Y[objective_thresholds_mask] Yvar = Yvar[objective_thresholds_mask] Y_obj = Y_obj[objective_thresholds_mask] indx_frontier = indx_frontier[objective_thresholds_mask] # Get feasible points that do not violate outcome_constraints if outcome_constraints is not None: cons_tfs = get_outcome_constraint_transforms(outcome_constraints) # pyre-ignore [16] feas = torch.stack([c(Y) <= 0 for c in cons_tfs], dim=-1).all(dim=-1) Y = Y[feas] Yvar = Yvar[feas] Y_obj = Y_obj[feas] indx_frontier = indx_frontier[feas] if Y.shape[0] == 0: # if there are no feasible points that are better than the reference point # return empty tensors return Y, Yvar, indx_frontier # calculate pareto front with only objective outcomes: frontier_mask = is_non_dominated(Y_obj) # Apply masks Y_frontier = Y[frontier_mask] Yvar_frontier = Yvar[frontier_mask] indx_frontier = indx_frontier[frontier_mask] return Y_frontier, Yvar_frontier, indx_frontier
def recommend_best_out_of_sample_point( model: TorchModel, bounds: List[Tuple[float, float]], objective_weights: Tensor, outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, model_gen_options: Optional[TConfig] = None, target_fidelities: Optional[Dict[int, float]] = None, ) -> Optional[Tensor]: """ Identify the current best point by optimizing the posterior mean of the model. This is "out-of-sample" because it considers un-observed designs as well. Return None if no such point can be identified. Args: model: A TorchModel. bounds: A list of (lower, upper) tuples for each column of X. objective_weights: The objective is to maximize a weighted sum of the columns of f(x). These are the weights. outcome_constraints: A tuple of (A, b). For k outcome constraints and m outputs at f(x), A is (k x m) and b is (k x 1) such that A f(x) <= b. linear_constraints: A tuple of (A, b). For k linear constraints on d-dimensional x, A is (k x d) and b is (k x 1) such that A x <= b. fixed_features: A map {feature_index: value} for features that should be fixed to a particular value in the best point. model_gen_options: A config dictionary that can contain model-specific options. target_fidelities: A map {feature_index: value} of fidelity feature column indices to their respective target fidelities. Used for multi-fidelity optimization. Returns: A d-array of the best point, or None if no feasible point exists. """ options = model_gen_options or {} fixed_features = fixed_features or {} acf_options = options.get("acquisition_function_kwargs", {}) optimizer_options = options.get("optimizer_kwargs", {}) X_observed = get_observed( Xs=model.Xs, # pyre-ignore: [16] objective_weights=objective_weights, outcome_constraints=outcome_constraints, ) if hasattr(model, "_get_best_point_acqf"): acq_function, non_fixed_idcs = model._get_best_point_acqf( # pyre-ignore: [16] X_observed=X_observed, objective_weights=objective_weights, mc_samples=acf_options.get("mc_samples", 512), fixed_features=fixed_features, target_fidelities=target_fidelities, outcome_constraints=outcome_constraints, seed_inner=acf_options.get("seed_inner", None), qmc=acf_options.get("qmc", True), ) else: raise RuntimeError("The model should implement _get_best_point_acqf.") inequality_constraints = _to_inequality_constraints(linear_constraints) # TODO: update optimizers to handle inequality_constraints # (including transforming constraints b/c of fixed features) if inequality_constraints is not None: raise UnsupportedError("Inequality constraints are not supported!") return_best_only = optimizer_options.get("return_best_only", True) bounds_ = torch.tensor(bounds, dtype=model.dtype, device=model.device) bounds_ = bounds_.transpose(-1, -2) if non_fixed_idcs is not None: bounds_ = bounds_[..., non_fixed_idcs] candidates, _ = optimize_acqf( acq_function=acq_function, bounds=bounds_, q=1, num_restarts=optimizer_options.get("num_restarts", 60), raw_samples=optimizer_options.get("raw_samples", 1024), inequality_constraints=inequality_constraints, fixed_features=None, # handled inside the acquisition function options={ "batch_limit": optimizer_options.get("batch_limit", 8), "maxiter": optimizer_options.get("maxiter", 200), "nonnegative": optimizer_options.get("nonnegative", False), "method": "L-BFGS-B", }, return_best_only=return_best_only, ) rec_point = candidates.detach().cpu() if isinstance(acq_function, FixedFeatureAcquisitionFunction): rec_point = acq_function._construct_X_full(rec_point) if return_best_only: rec_point = rec_point.view(-1) return rec_point
def testTorchModelUpdate(self): numpy_model = TorchModel() with self.assertRaises(NotImplementedError): numpy_model.update(Xs=[np.array(0)], Ys=[np.array(0)], Yvars=[np.array(1)])
def get_observed_pareto_frontiers( experiment: Experiment, data: Optional[Data] = None, rel: bool = True, ) -> List[ParetoFrontierResults]: """ Find all Pareto points from an experiment. Uses only values as observed in the data; no modeling is involved. Makes no assumption about the search space or types of parameters. If "data" is provided will use that, otherwise will use all data attached to the experiment. Uses all arms present in data; does not filter according to experiment search space. Assumes experiment has a multiobjective optimization config from which the objectives and outcome constraints will be extracted. Will generate a ParetoFrontierResults for every pair of metrics in the experiment's multiobjective optimization config. """ if data is None: data = experiment.fetch_data() if experiment.optimization_config is None: raise ValueError("Experiment must have an optimization config") # Make a dummy model for converting things to tensors. # Transforms is the minimal set that will work for converting any search # space to tensors. mb = TorchModelBridge( experiment=experiment, search_space=experiment.search_space, data=data, model=TorchModel(), transforms=[Derelativize, SearchSpaceToChoice, OneHot], transform_configs={"Derelativize": { "use_raw_status_quo": True }}, fit_out_of_design=True, ) pareto_observations = observed_pareto_frontier(modelbridge=mb) # Convert to ParetoFrontierResults metric_names = [ metric.name for metric in experiment.optimization_config.objective.metrics # pyre-ignore ] pfr_means = {name: [] for name in metric_names} pfr_sems = {name: [] for name in metric_names} for obs in pareto_observations: for i, name in enumerate(obs.data.metric_names): pfr_means[name].append(obs.data.means[i]) pfr_sems[name].append(np.sqrt(obs.data.covariance[i, i])) # Relativize as needed if rel and experiment.status_quo is not None: # Get status quo values sq_df = data.df[data.df["arm_name"] == experiment.status_quo.name # pyre-ignore ] sq_df = sq_df.to_dict(orient="list") # pyre-ignore sq_means = {} sq_sems = {} for i, metric in enumerate(sq_df["metric_name"]): sq_means[metric] = sq_df["mean"][i] sq_sems[metric] = sq_df["sem"][i] # Relativize for name in metric_names: if np.isnan(sq_sems[name]) or np.isnan(pfr_sems[name]).any(): # Just relativize means pfr_means[name] = [(mu / sq_means[name] - 1) * 100 for mu in pfr_means[name]] else: # Use delta method pfr_means[name], pfr_sems[name] = relativize( means_t=pfr_means[name], sems_t=pfr_sems[name], mean_c=sq_means[name], sem_c=sq_sems[name], as_percent=True, ) absolute_metrics = [] else: absolute_metrics = metric_names objective_thresholds = {} if experiment.optimization_config.objective_thresholds is not None: # pyre-ignore for objth in experiment.optimization_config.objective_thresholds: is_rel = objth.metric.name not in absolute_metrics if objth.relative != is_rel: raise ValueError( f"Objective threshold for {objth.metric.name} has " f"rel={objth.relative} but was specified here as rel={is_rel}" ) objective_thresholds[objth.metric.name] = objth.bound # Construct ParetoFrontResults for each pair pfr_list = [] param_dicts = [obs.features.parameters for obs in pareto_observations] arm_names = [obs.arm_name for obs in pareto_observations] for metric_a, metric_b in combinations(metric_names, 2): pfr_list.append( ParetoFrontierResults( param_dicts=param_dicts, means=pfr_means, sems=pfr_sems, primary_metric=metric_a, secondary_metric=metric_b, absolute_metrics=absolute_metrics, objective_thresholds=objective_thresholds, arm_names=arm_names, )) return pfr_list