def observed_hypervolume(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> float:
"""Calculate hypervolume of a pareto frontier based on observed data.
Given observed data, return the hypervolume of the pareto frontier formed from
those outcomes.
Args:
modelbridge: Modelbridge that holds previous training data.
objective_thresholds: point defining the origin of hyperrectangles that
can contribute to hypervolume.
observation_features: observation features to predict. Model's training
data used by default if unspecified.
optimization_config: Optimization config
Returns:
(float) calculated hypervolume.
"""
# Get observation_data from current training data.
observation_data = [obs.data for obs in modelbridge.get_training_data()]
observation_features = [
obs.features for obs in modelbridge.get_training_data()
]
return hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
optimization_config=optimization_config,
use_model_predictions=False,
)
def observed_pareto_frontier(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[ObservationData]:
"""Generate a pareto frontier based on observed data.
Given observed data, return those outcomes in the pareto frontier.
Args:
modelbridge: Modelbridge that holds previous training data.
objective_thresholds: metric values bounding the region of interest in
the objective outcome space.
optimization_config: Optimization config
Returns:
Data representing points on the pareto frontier.
"""
# Get observation_data from current training data
observation_data = [obs.data for obs in modelbridge.get_training_data()]
return pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_data=observation_data,
optimization_config=optimization_config,
)
def hypervolume(
modelbridge: modelbridge_module.array.ArrayModelBridge,
observation_features: List[ObservationFeatures],
objective_thresholds: Optional[TRefPoint] = None,
observation_data: Optional[List[ObservationData]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
use_model_predictions: bool = True,
) -> float:
"""Helper function that computes hypervolume of a given list of outcomes."""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
# Extract a tensor of outcome means from observation data.
observations = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
optimization_config=optimization_config,
use_model_predictions=use_model_predictions,
)
observation_data = [obs.data for obs in observations]
if not observation_data:
# The hypervolume of an empty set is always 0.
return 0
means, _ = modelbridge.transform_observation_data(observation_data)
# Extract objective_weights and objective_thresholds
if optimization_config is None:
optimization_config = (
# pyre-ignore[16]: Optional has undefined attr `_optimization_config`
modelbridge._optimization_config.clone()
if modelbridge._optimization_config is not None
else None
)
else:
# pyre-ignore[9]: declared as type `Optional[...]` but used as type `...`
optimization_config = optimization_config.clone()
if objective_thresholds is not None:
optimization_config = optimization_config.clone_with_args(
objective_thresholds=objective_thresholds
)
obj_w = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes
)
obj_w = array_to_tensor(obj_w)
objective_thresholds_arr = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
outcomes=modelbridge.outcomes,
)
obj_t = array_to_tensor(objective_thresholds_arr)
obj, obj_t = get_weighted_mc_objective_and_objective_thresholds(
objective_weights=obj_w, objective_thresholds=obj_t
)
means = obj(means)
hv = Hypervolume(ref_point=obj_t)
return hv.compute(means)
def _get_modelbridge_training_data(
modelbridge: modelbridge_module.array.ArrayModelBridge,
) -> Tuple[List[ObservationFeatures], List[ObservationData],
List[Optional[str]]]:
obs = modelbridge.get_training_data()
obs_feats, obs_data, arm_names = [], [], []
for ob in obs:
obs_feats.append(ob.features)
obs_data.append(ob.data)
arm_names.append(ob.arm_name)
return obs_feats, obs_data, arm_names
def predicted_pareto_frontier(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[Observation]:
"""Generate a pareto frontier based on the posterior means of given
observation features.
Given a model and features to evaluate use the model to predict which points
lie on the pareto frontier.
Args:
modelbridge: Modelbridge used to predict metrics outcomes.
objective_thresholds: metric values bounding the region of interest in
the objective outcome space.
observation_features: observation features to predict. Model's training
data used by default if unspecified.
optimization_config: Optimization config
Returns:
Observations representing points on the pareto frontier.
"""
if observation_features is None:
observation_features = []
arm_names = []
for obs in modelbridge.get_training_data():
observation_features.append(obs.features)
arm_names.append(obs.arm_name)
else:
arm_names = None
if not observation_features:
raise ValueError(
"Must receive observation_features as input or the model must "
"have training data."
)
pareto_observations = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
optimization_config=optimization_config,
arm_names=arm_names,
)
return pareto_observations
def predicted_hypervolume(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> float:
"""Calculate hypervolume of a pareto frontier based on the posterior means of
given observation features.
Given a model and features to evaluate calculate the hypervolume of the pareto
frontier formed from their predicted outcomes.
Args:
modelbridge: Modelbridge used to predict metrics outcomes.
objective_thresholds: point defining the origin of hyperrectangles that
can contribute to hypervolume.
observation_features: observation features to predict. Model's training
data used by default if unspecified.
optimization_config: Optimization config
Returns:
calculated hypervolume.
"""
observation_features = (
observation_features
if observation_features is not None
else [obs.features for obs in modelbridge.get_training_data()]
)
if not observation_features:
raise ValueError(
"Must receive observation_features as input or the model must "
"have training data."
)
return hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
optimization_config=optimization_config,
)
def get_pareto_frontier_and_transformed_configs(
modelbridge: modelbridge_module.array.ArrayModelBridge,
observation_features: List[ObservationFeatures],
observation_data: Optional[List[ObservationData]] = None,
objective_thresholds: Optional[TRefPoint] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
arm_names: Optional[List[Optional[str]]] = None,
use_model_predictions: bool = True,
) -> Tuple[List[Observation], Tensor, Tensor, Optional[Tensor]]:
"""Helper that applies transforms and calls frontier_evaluator.
Returns transformed configs in addition to the Pareto observations.
Args:
modelbridge: Modelbridge used to predict metrics outcomes.
observation_features: observation features to predict, if provided and
use_model_predictions is True.
observation_data: data for computing the Pareto front, unless features
are provided and model_predictions is True.
objective_thresholds: metric values bounding the region of interest in
the objective outcome space.
optimization_config: Optimization config.
arm_names: Arm names for each observation.
use_model_predictions: If True, will use model predictions at
observation_features to compute Pareto front, if provided. If False,
will use observation_data directly to compute Pareto front, regardless
of whether observation_features are provided.
Returns:
frontier_observations: Observations of points on the pareto frontier.
f: n x m tensor representation of the Pareto frontier values where n is the
length of frontier_observations and m is the number of metrics.
obj_w: m tensor of objective weights.
obj_t: m tensor of objective thresholds corresponding to Y, or None if no
objective thresholds used.
"""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
X = (modelbridge.transform_observation_features(observation_features)
if use_model_predictions else None)
X = array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data is not None:
Y, Yvar = modelbridge.transform_observation_data(observation_data)
Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar))
if arm_names is None:
arm_names = [None] * len(observation_features)
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, modelbridge._optimization_config)
if not mooc:
raise ValueError(
("Experiment must have an existing optimization_config "
"of type `MultiObjectiveOptimizationConfig` "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = modelbridge.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=modelbridge.outcomes,
)
obj_t = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
objective=optimization_config.objective,
outcomes=modelbridge.outcomes,
)
obj_t = array_to_tensor(obj_t)
# Transform to tensors.
obj_w, oc_c, _, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=array_to_tensor,
)
frontier_evaluator = get_default_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call
f, cov, indx = frontier_evaluator(
model=modelbridge.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone(), cov.detach().cpu().clone()
indx = indx.tolist()
frontier_observation_data = array_to_observation_data(
f=f.numpy(), cov=cov.numpy(), outcomes=not_none(modelbridge.outcomes))
# Untransform observations
for t in reversed(modelbridge.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
# Construct observations
frontier_observations = []
for i, obsd in enumerate(frontier_observation_data):
frontier_observations.append(
Observation(
features=observation_features[indx[i]],
data=obsd,
arm_name=arm_names[indx[i]],
))
return frontier_observations, f, obj_w, obj_t
def pareto_frontier(
modelbridge: modelbridge_module.array.ArrayModelBridge,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
observation_data: Optional[List[ObservationData]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[ObservationData]:
"""Helper that applies transforms and calls frontier_evaluator."""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
X = (modelbridge.transform_observation_features(observation_features)
if observation_features else None)
X = array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data:
Y, Yvar = modelbridge.transform_observation_data(observation_data)
if Y is not None and Yvar is not None:
Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar))
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, modelbridge._optimization_config)
if not mooc:
raise ValueError(
("Experiment must have an existing optimization_config "
"of type `MultiObjectiveOptimizationConfig` "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = modelbridge.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=modelbridge.outcomes,
)
objective_thresholds_arr = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
outcomes=modelbridge.outcomes,
)
# Transform to tensors.
obj_w, oc_c, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=array_to_tensor,
)
obj_t = array_to_tensor(objective_thresholds_arr)
frontier_evaluator = get_default_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call
f, cov = frontier_evaluator(
model=modelbridge.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone().numpy(), cov.detach().cpu().clone(
).numpy()
frontier_observation_data = array_to_observation_data(
f=f, cov=cov, outcomes=not_none(modelbridge.outcomes))
# Untransform observations
for t in reversed(modelbridge.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
return frontier_observation_data
def get_pareto_frontier_and_configs(
modelbridge: modelbridge_module.array.ArrayModelBridge,
observation_features: List[ObservationFeatures],
observation_data: Optional[List[ObservationData]] = None,
objective_thresholds: Optional[TRefPoint] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
arm_names: Optional[List[Optional[str]]] = None,
use_model_predictions: bool = True,
transform_outcomes_and_configs: bool = True,
) -> Tuple[List[Observation], Tensor, Tensor, Optional[Tensor]]:
"""Helper that applies transforms and calls ``frontier_evaluator``.
Returns the ``frontier_evaluator`` configs in addition to the Pareto
observations.
Args:
modelbridge: ``Modelbridge`` used to predict metrics outcomes.
observation_features: Observation features to consider for the Pareto
frontier.
observation_data: Data for computing the Pareto front, unless
``observation_features`` are provided and ``model_predictions is True``.
objective_thresholds: Metric values bounding the region of interest in
the objective outcome space; used to override objective thresholds
specified in ``optimization_config``, if necessary.
optimization_config: Multi-objective optimization config.
arm_names: Arm names for each observation in ``observation_features``.
use_model_predictions: If ``True``, will use model predictions at
``observation_features`` to compute Pareto front. If ``False``,
will use ``observation_data`` directly to compute Pareto front, ignoring
``observation_features``.
transform_outcomes_and_configs: If ``True``, will transform the optimization
config, observation features and observation data, before calling
``frontier_evaluator``, then will untransform all of the above before
returning the observations.
Returns: Four-item tuple of:
- frontier_observations: Observations of points on the pareto frontier,
- f: n x m tensor representation of the Pareto frontier values where n is the
length of frontier_observations and m is the number of metrics,
- obj_w: m tensor of objective weights,
- obj_t: m tensor of objective thresholds corresponding to Y, or None if no
objective thresholds used.
"""
array_to_tensor = partial(_array_to_tensor, modelbridge=modelbridge)
X, Y, Yvar = None, None, None
if use_model_predictions:
X = array_to_tensor(
modelbridge.transform_observation_features(observation_features))
if observation_data is not None:
if transform_outcomes_and_configs:
Y, Yvar = modelbridge.transform_observation_data(observation_data)
else:
Y, Yvar = observation_data_to_array(
outcomes=modelbridge.outcomes,
observation_data=observation_data)
Y, Yvar = (array_to_tensor(Y), array_to_tensor(Yvar))
if arm_names is None:
arm_names = [None] * len(observation_features)
# Extract optimization config: make sure that the problem is a MOO
# problem and clone the optimization config with specified
# `objective_thresholds` if those are provided. If `optimization_config`
# is not specified, uses the one stored on `modelbridge`.
optimization_config = _get_multiobjective_optimization_config(
modelbridge=modelbridge,
optimization_config=optimization_config,
objective_thresholds=objective_thresholds,
)
# Transform optimization config.
fixed_features = ObservationFeatures(parameters={})
if transform_outcomes_and_configs:
optimization_config = modelbridge.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=fixed_features,
)
else:
# de-relativize outcome constraints and objective thresholds
obs_feats, obs_data, _ = _get_modelbridge_training_data(
modelbridge=modelbridge)
tf = Derelativize(
search_space=modelbridge.model_space.clone(),
observation_data=obs_data,
observation_features=obs_feats,
config={"use_raw_status_quo": True},
)
# pyre-ignore [9]
optimization_config = tf.transform_optimization_config(
optimization_config=optimization_config.clone(),
modelbridge=modelbridge,
fixed_features=fixed_features,
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=modelbridge.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=modelbridge.outcomes,
)
obj_t = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
objective=optimization_config.objective,
outcomes=modelbridge.outcomes,
)
obj_t = array_to_tensor(obj_t)
# Transform to tensors.
obj_w, oc_c, _, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=array_to_tensor,
)
frontier_evaluator = get_default_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `modelbridge` to anonymous call
f, cov, indx = frontier_evaluator(
model=modelbridge.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone(), cov.detach().cpu().clone()
indx = indx.tolist()
frontier_observation_data = array_to_observation_data(
f=f.numpy(), cov=cov.numpy(), outcomes=not_none(modelbridge.outcomes))
if use_model_predictions:
# Untransform observations
for t in reversed(modelbridge.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
# reconstruct tensor representation of untransformed predictions
Y_arr, _ = observation_data_to_array(
outcomes=modelbridge.outcomes,
observation_data=frontier_observation_data)
f = _array_to_tensor(Y_arr)
# Construct observations
frontier_observations = []
for i, obsd in enumerate(frontier_observation_data):
frontier_observations.append(
Observation(
features=observation_features[indx[i]],
data=obsd,
arm_name=arm_names[indx[i]],
))
return frontier_observations, f, obj_w.cpu(), obj_t.cpu()
def _get_modelbridge_training_data(
modelbridge: modelbridge_module.array.ArrayModelBridge,
) -> Tuple[List[ObservationFeatures], List[ObservationData],
List[Optional[str]]]:
obs = modelbridge.get_training_data()
return _unpack_observations(obs=obs)