def testTorchModelCrossValidate(self):
torch_model = TorchModel()
with self.assertRaises(NotImplementedError):
torch_model.cross_validate(
Xs_train=[np.array([1])],
Ys_train=[np.array([1])],
Yvars_train=[np.array([1])],
X_test=np.array([1]),
)
def testTorchModelFit(self):
torch_model = TorchModel()
torch_model.fit(
Xs=[np.array(0)],
Ys=[np.array(0)],
Yvars=[np.array(1)],
bounds=[(0, 1)],
task_features=[],
feature_names=["x1"],
)
def testTorchModelUpdate(self):
model = TorchModel()
with self.assertRaises(NotImplementedError):
model.update(
Xs=[np.array(0)],
Ys=[np.array(0)],
Yvars=[np.array(1)],
search_space_digest=SearchSpaceDigest(feature_names=[],
bounds=[]),
metric_names=[],
)
def testTorchModelCrossValidate(self):
torch_model = TorchModel()
with self.assertRaises(NotImplementedError):
torch_model.cross_validate(
Xs_train=[np.array([1])],
Ys_train=[np.array([1])],
Yvars_train=[np.array([1])],
X_test=np.array([1]),
search_space_digest=SearchSpaceDigest(feature_names=[],
bounds=[]),
metric_names=[],
)
def testTorchModelFit(self):
torch_model = TorchModel()
torch_model.fit(
Xs=[np.array(0)],
Ys=[np.array(0)],
Yvars=[np.array(1)],
search_space_digest=SearchSpaceDigest(
feature_names=["x1"],
bounds=[(0, 1)],
),
metric_names=["y"],
)
def testTorchModelUpdate(self):
numpy_model = TorchModel()
with self.assertRaises(NotImplementedError):
numpy_model.update(
Xs=[np.array(0)],
Ys=[np.array(0)],
Yvars=[np.array(1)],
bounds=[],
task_features=[],
fidelity_features=[],
feature_names=[],
metric_names=[],
)
def testTorchModelCrossValidate(self):
torch_model = TorchModel()
with self.assertRaises(NotImplementedError):
torch_model.cross_validate(
Xs_train=[np.array([1])],
Ys_train=[np.array([1])],
Yvars_train=[np.array([1])],
X_test=np.array([1]),
bounds=[],
task_features=[],
fidelity_features=[],
feature_names=[],
metric_names=[],
)
def get_tensor_converter_model(experiment: Experiment, data: Data) -> TorchModelBridge:
"""
Constructs a minimal model for converting things to tensors.
Model fitting will instantiate all of the transforms but will not do any
expensive (i.e. GP) fitting beyond that. The model will raise an error if
it is used for predicting or generating.
Will work for any search space regardless of types of parameters.
Args:
experiment: Experiment.
data: Data for fitting the model.
Returns: A torch modelbridge with transforms set.
"""
# Transforms is the minimal set that will work for converting any search
# space to tensors.
return TorchModelBridge(
experiment=experiment,
search_space=experiment.search_space,
data=data,
model=TorchModel(),
transforms=[Derelativize, SearchSpaceToChoice, OrderedChoiceEncode, IntToFloat],
transform_configs={
"Derelativize": {"use_raw_status_quo": True},
"SearchSpaceToChoice": {"use_ordered": True},
},
fit_out_of_design=True,
)
def testNumpyTorchBestPoint(self):
torch_model = TorchModel()
x = torch_model.best_point(bounds=[(0, 1)],
objective_weights=np.array([1]))
self.assertIsNone(x)
def testTorchModelGen(self):
torch_model = TorchModel()
with self.assertRaises(NotImplementedError):
torch_model.gen(n=1,
bounds=[(0, 1)],
objective_weights=np.array([1]))
def testTorchModelPredict(self):
torch_model = TorchModel()
with self.assertRaises(NotImplementedError):
torch_model.predict(np.array([0]))
def pareto_frontier_evaluator(
model: TorchModel,
objective_weights: Tensor,
objective_thresholds: Optional[Tensor] = None,
X: Optional[Tensor] = None,
Y: Optional[Tensor] = None,
Yvar: Optional[Tensor] = None,
outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None,
) -> Tuple[Tensor, Tensor, Tensor]:
"""Return outcomes predicted to lie on a pareto frontier.
Given a model and a points to evaluate use the model to predict which points
lie on the pareto frontier.
Args:
model: Model used to predict outcomes.
objective_weights: A `m` tensor of values indicating the weight to put
on different outcomes. For pareto frontiers only the sign matters.
objective_thresholds: A tensor containing thresholds forming a reference point
from which to calculate pareto frontier hypervolume. Points that do not
dominate the objective_thresholds contribute nothing to hypervolume.
X: A `n x d` tensor of features to evaluate.
Y: A `n x m` tensor of outcomes to use instead of predictions.
Yvar: A `n x m x m` tensor of input covariances (NaN if unobserved).
outcome_constraints: A tuple of (A, b). For k outcome constraints
and m outputs at f(x), A is (k x m) and b is (k x 1) such that
A f(x) <= b.
Returns:
3-element tuple containing
- A `j x m` tensor of outcome on the pareto frontier. j is the number
of frontier points.
- A `j x m x m` tensor of predictive covariances.
cov[j, m1, m2] is Cov[m1@j, m2@j].
- A `j` tensor of the index of each frontier point in the input Y.
"""
if X is not None:
Y, Yvar = model.predict(X)
elif Y is None or Yvar is None:
raise ValueError(
"Requires `X` to predict or both `Y` and `Yvar` to select a subset of "
"points on the pareto frontier.")
# Apply objective_weights to outcomes and objective_thresholds.
# If objective_thresholds is not None use a dummy tensor of zeros.
(
obj,
weighted_objective_thresholds,
) = get_weighted_mc_objective_and_objective_thresholds(
objective_weights=objective_weights,
objective_thresholds=(objective_thresholds
if objective_thresholds is not None else
torch.zeros(objective_weights.shape)),
)
Y_obj = obj(Y)
indx_frontier = torch.arange(Y.shape[0], dtype=torch.long, device=Y.device)
# Filter Y, Yvar, Y_obj to items that dominate all objective thresholds
if objective_thresholds is not None:
objective_thresholds_mask = (Y_obj >=
weighted_objective_thresholds).all(dim=1)
Y = Y[objective_thresholds_mask]
Yvar = Yvar[objective_thresholds_mask]
Y_obj = Y_obj[objective_thresholds_mask]
indx_frontier = indx_frontier[objective_thresholds_mask]
# Get feasible points that do not violate outcome_constraints
if outcome_constraints is not None:
cons_tfs = get_outcome_constraint_transforms(outcome_constraints)
# pyre-ignore [16]
feas = torch.stack([c(Y) <= 0 for c in cons_tfs], dim=-1).all(dim=-1)
Y = Y[feas]
Yvar = Yvar[feas]
Y_obj = Y_obj[feas]
indx_frontier = indx_frontier[feas]
if Y.shape[0] == 0:
# if there are no feasible points that are better than the reference point
# return empty tensors
return Y, Yvar, indx_frontier
# calculate pareto front with only objective outcomes:
frontier_mask = is_non_dominated(Y_obj)
# Apply masks
Y_frontier = Y[frontier_mask]
Yvar_frontier = Yvar[frontier_mask]
indx_frontier = indx_frontier[frontier_mask]
return Y_frontier, Yvar_frontier, indx_frontier
def recommend_best_out_of_sample_point(
model: TorchModel,
bounds: List[Tuple[float, float]],
objective_weights: Tensor,
outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None,
linear_constraints: Optional[Tuple[Tensor, Tensor]] = None,
fixed_features: Optional[Dict[int, float]] = None,
model_gen_options: Optional[TConfig] = None,
target_fidelities: Optional[Dict[int, float]] = None,
) -> Optional[Tensor]:
"""
Identify the current best point by optimizing the posterior mean of the model.
This is "out-of-sample" because it considers un-observed designs as well.
Return None if no such point can be identified.
Args:
model: A TorchModel.
bounds: A list of (lower, upper) tuples for each column of X.
objective_weights: The objective is to maximize a weighted sum of
the columns of f(x). These are the weights.
outcome_constraints: A tuple of (A, b). For k outcome constraints
and m outputs at f(x), A is (k x m) and b is (k x 1) such that
A f(x) <= b.
linear_constraints: A tuple of (A, b). For k linear constraints on
d-dimensional x, A is (k x d) and b is (k x 1) such that
A x <= b.
fixed_features: A map {feature_index: value} for features that
should be fixed to a particular value in the best point.
model_gen_options: A config dictionary that can contain
model-specific options.
target_fidelities: A map {feature_index: value} of fidelity feature
column indices to their respective target fidelities. Used for
multi-fidelity optimization.
Returns:
A d-array of the best point, or None if no feasible point exists.
"""
options = model_gen_options or {}
fixed_features = fixed_features or {}
acf_options = options.get("acquisition_function_kwargs", {})
optimizer_options = options.get("optimizer_kwargs", {})
X_observed = get_observed(
Xs=model.Xs, # pyre-ignore: [16]
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
)
if hasattr(model, "_get_best_point_acqf"):
acq_function, non_fixed_idcs = model._get_best_point_acqf( # pyre-ignore: [16]
X_observed=X_observed,
objective_weights=objective_weights,
mc_samples=acf_options.get("mc_samples", 512),
fixed_features=fixed_features,
target_fidelities=target_fidelities,
outcome_constraints=outcome_constraints,
seed_inner=acf_options.get("seed_inner", None),
qmc=acf_options.get("qmc", True),
)
else:
raise RuntimeError("The model should implement _get_best_point_acqf.")
inequality_constraints = _to_inequality_constraints(linear_constraints)
# TODO: update optimizers to handle inequality_constraints
# (including transforming constraints b/c of fixed features)
if inequality_constraints is not None:
raise UnsupportedError("Inequality constraints are not supported!")
return_best_only = optimizer_options.get("return_best_only", True)
bounds_ = torch.tensor(bounds, dtype=model.dtype, device=model.device)
bounds_ = bounds_.transpose(-1, -2)
if non_fixed_idcs is not None:
bounds_ = bounds_[..., non_fixed_idcs]
candidates, _ = optimize_acqf(
acq_function=acq_function,
bounds=bounds_,
q=1,
num_restarts=optimizer_options.get("num_restarts", 60),
raw_samples=optimizer_options.get("raw_samples", 1024),
inequality_constraints=inequality_constraints,
fixed_features=None, # handled inside the acquisition function
options={
"batch_limit": optimizer_options.get("batch_limit", 8),
"maxiter": optimizer_options.get("maxiter", 200),
"nonnegative": optimizer_options.get("nonnegative", False),
"method": "L-BFGS-B",
},
return_best_only=return_best_only,
)
rec_point = candidates.detach().cpu()
if isinstance(acq_function, FixedFeatureAcquisitionFunction):
rec_point = acq_function._construct_X_full(rec_point)
if return_best_only:
rec_point = rec_point.view(-1)
return rec_point
def testTorchModelUpdate(self):
numpy_model = TorchModel()
with self.assertRaises(NotImplementedError):
numpy_model.update(Xs=[np.array(0)],
Ys=[np.array(0)],
Yvars=[np.array(1)])
def get_observed_pareto_frontiers(
experiment: Experiment,
data: Optional[Data] = None,
rel: bool = True,
) -> List[ParetoFrontierResults]:
"""
Find all Pareto points from an experiment.
Uses only values as observed in the data; no modeling is involved. Makes no
assumption about the search space or types of parameters. If "data" is provided will
use that, otherwise will use all data attached to the experiment.
Uses all arms present in data; does not filter according to experiment
search space.
Assumes experiment has a multiobjective optimization config from which the
objectives and outcome constraints will be extracted.
Will generate a ParetoFrontierResults for every pair of metrics in the experiment's
multiobjective optimization config.
"""
if data is None:
data = experiment.fetch_data()
if experiment.optimization_config is None:
raise ValueError("Experiment must have an optimization config")
# Make a dummy model for converting things to tensors.
# Transforms is the minimal set that will work for converting any search
# space to tensors.
mb = TorchModelBridge(
experiment=experiment,
search_space=experiment.search_space,
data=data,
model=TorchModel(),
transforms=[Derelativize, SearchSpaceToChoice, OneHot],
transform_configs={"Derelativize": {
"use_raw_status_quo": True
}},
fit_out_of_design=True,
)
pareto_observations = observed_pareto_frontier(modelbridge=mb)
# Convert to ParetoFrontierResults
metric_names = [
metric.name for metric in
experiment.optimization_config.objective.metrics # pyre-ignore
]
pfr_means = {name: [] for name in metric_names}
pfr_sems = {name: [] for name in metric_names}
for obs in pareto_observations:
for i, name in enumerate(obs.data.metric_names):
pfr_means[name].append(obs.data.means[i])
pfr_sems[name].append(np.sqrt(obs.data.covariance[i, i]))
# Relativize as needed
if rel and experiment.status_quo is not None:
# Get status quo values
sq_df = data.df[data.df["arm_name"] ==
experiment.status_quo.name # pyre-ignore
]
sq_df = sq_df.to_dict(orient="list") # pyre-ignore
sq_means = {}
sq_sems = {}
for i, metric in enumerate(sq_df["metric_name"]):
sq_means[metric] = sq_df["mean"][i]
sq_sems[metric] = sq_df["sem"][i]
# Relativize
for name in metric_names:
if np.isnan(sq_sems[name]) or np.isnan(pfr_sems[name]).any():
# Just relativize means
pfr_means[name] = [(mu / sq_means[name] - 1) * 100
for mu in pfr_means[name]]
else:
# Use delta method
pfr_means[name], pfr_sems[name] = relativize(
means_t=pfr_means[name],
sems_t=pfr_sems[name],
mean_c=sq_means[name],
sem_c=sq_sems[name],
as_percent=True,
)
absolute_metrics = []
else:
absolute_metrics = metric_names
objective_thresholds = {}
if experiment.optimization_config.objective_thresholds is not None: # pyre-ignore
for objth in experiment.optimization_config.objective_thresholds:
is_rel = objth.metric.name not in absolute_metrics
if objth.relative != is_rel:
raise ValueError(
f"Objective threshold for {objth.metric.name} has "
f"rel={objth.relative} but was specified here as rel={is_rel}"
)
objective_thresholds[objth.metric.name] = objth.bound
# Construct ParetoFrontResults for each pair
pfr_list = []
param_dicts = [obs.features.parameters for obs in pareto_observations]
arm_names = [obs.arm_name for obs in pareto_observations]
for metric_a, metric_b in combinations(metric_names, 2):
pfr_list.append(
ParetoFrontierResults(
param_dicts=param_dicts,
means=pfr_means,
sems=pfr_sems,
primary_metric=metric_a,
secondary_metric=metric_b,
absolute_metrics=absolute_metrics,
objective_thresholds=objective_thresholds,
arm_names=arm_names,
))
return pfr_list
