def test_extract_outcome_constraints(self):
outcomes = ["m1", "m2", "m3"]
# pass no outcome constraints
self.assertIsNone(extract_outcome_constraints([], outcomes))
outcome_constraints = [
OutcomeConstraint(metric=Metric("m1"),
op=ComparisonOp.LEQ,
bound=0)
]
res = extract_outcome_constraints(outcome_constraints, outcomes)
self.assertEqual(res[0].shape, (1, 3))
self.assertListEqual(list(res[0][0]), [1, 0, 0])
self.assertEqual(res[1][0][0], 0)
outcome_constraints = [
OutcomeConstraint(metric=Metric("m1"),
op=ComparisonOp.LEQ,
bound=0),
ScalarizedOutcomeConstraint(
metrics=[Metric("m2"), Metric("m3")],
weights=[0.5, 0.5],
op=ComparisonOp.GEQ,
bound=1,
),
]
res = extract_outcome_constraints(outcome_constraints, outcomes)
self.assertEqual(res[0].shape, (2, 3))
self.assertListEqual(list(res[0][0]), [1, 0, 0])
self.assertListEqual(list(res[0][1]), [0, -0.5, -0.5])
self.assertEqual(res[1][0][0], 0)
self.assertEqual(res[1][1][0], -1)
def _get_transformed_model_gen_args(
self,
search_space: SearchSpace,
pending_observations: Dict[str, List[ObservationFeatures]],
fixed_features: ObservationFeatures,
model_gen_options: Optional[TConfig] = None,
optimization_config: Optional[OptimizationConfig] = None,
) -> ArrayModelGenArgs:
# Validation
if not self.parameters: # pragma: no cover
raise ValueError(FIT_MODEL_ERROR.format(action="_gen"))
# Extract search space info
search_space_digest = extract_search_space_digest(
search_space=search_space, param_names=self.parameters
)
if optimization_config is None:
raise ValueError(
"ArrayModelBridge requires an OptimizationConfig to be specified"
)
if self.outcomes is None or len(self.outcomes) == 0: # pragma: no cover
raise ValueError("No outcomes found during model fit--data are missing.")
validate_optimization_config(optimization_config, self.outcomes)
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=self.outcomes
)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=self.outcomes,
)
extra_model_gen_kwargs = self._get_extra_model_gen_kwargs(
optimization_config=optimization_config
)
linear_constraints = extract_parameter_constraints(
search_space.parameter_constraints, self.parameters
)
fixed_features_dict = get_fixed_features(fixed_features, self.parameters)
pending_array = pending_observations_as_array(
pending_observations, self.outcomes, self.parameters
)
return ArrayModelGenArgs(
search_space_digest=search_space_digest,
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=linear_constraints,
fixed_features=fixed_features_dict,
pending_observations=pending_array,
rounding_func=transform_callback(self.parameters, self.transforms),
extra_model_gen_kwargs=extra_model_gen_kwargs,
)
def _pareto_frontier(
self,
objective_thresholds: Optional[TRefPoint] = None,
observation_features: Optional[List[ObservationFeatures]] = None,
observation_data: Optional[List[ObservationData]] = None,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
) -> List[ObservationData]:
# TODO(jej): This method should be refactored to move tensor
# conversions into a separate utility, and eventually should be
# moved into base.py.
# The reason this method is currently implemented in array.py is to
# allow the broadest possible set of models to call frontier and
# hypervolume evaluation functions given the current API.
X = (self.transform_observation_features(observation_features)
if observation_features else None)
X = self._array_to_tensor(X) if X is not None else None
Y, Yvar = (None, None)
if observation_data:
Y, Yvar = self.transform_observation_data(observation_data)
if Y is not None and Yvar is not None:
Y, Yvar = (self._array_to_tensor(Y), self._array_to_tensor(Yvar))
# Optimization_config
mooc = optimization_config or checked_cast_optional(
MultiObjectiveOptimizationConfig, self._optimization_config)
if not mooc:
raise ValueError(
("experiment must have an existing optimization_config "
"of type MultiObjectiveOptimizationConfig "
"or `optimization_config` must be passed as an argument."))
if not isinstance(mooc, MultiObjectiveOptimizationConfig):
mooc = not_none(
MultiObjectiveOptimizationConfig.from_opt_conf(mooc))
if objective_thresholds:
mooc = mooc.clone_with_args(
objective_thresholds=objective_thresholds)
optimization_config = mooc
# Transform OptimizationConfig.
optimization_config = self.transform_optimization_config(
optimization_config=optimization_config,
fixed_features=ObservationFeatures(parameters={}),
)
# Extract weights, constraints, and objective_thresholds
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=self.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=self.outcomes,
)
objective_thresholds_arr = extract_objective_thresholds(
objective_thresholds=optimization_config.objective_thresholds,
outcomes=self.outcomes,
)
# Transform to tensors.
obj_w, oc_c, _, _ = validate_and_apply_final_transform(
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=None,
pending_observations=None,
final_transform=self._array_to_tensor,
)
obj_t = self._array_to_tensor(objective_thresholds_arr)
frontier_evaluator = self._get_frontier_evaluator()
# pyre-ignore[28]: Unexpected keyword `model` to anonymous call
f, cov = frontier_evaluator(
model=self.model,
X=X,
Y=Y,
Yvar=Yvar,
objective_thresholds=obj_t,
objective_weights=obj_w,
outcome_constraints=oc_c,
)
f, cov = f.detach().cpu().clone().numpy(), cov.detach().cpu().clone(
).numpy()
frontier_observation_data = array_to_observation_data(
f=f, cov=cov, outcomes=not_none(self.outcomes))
# Untransform observations
for t in reversed(self.transforms.values()): # noqa T484
frontier_observation_data = t.untransform_observation_data(
frontier_observation_data, [])
return frontier_observation_data
def _gen(
self,
n: int,
search_space: SearchSpace,
pending_observations: Dict[str, List[ObservationFeatures]],
fixed_features: ObservationFeatures,
model_gen_options: Optional[TConfig] = None,
optimization_config: Optional[OptimizationConfig] = None,
) -> Tuple[List[ObservationFeatures], List[float],
Optional[ObservationFeatures], TGenMetadata, ]:
"""Generate new candidates according to search_space and
optimization_config.
The outcome constraints should be transformed to no longer be relative.
"""
# Validation
if not self.parameters: # pragma: no cover
raise ValueError(FIT_MODEL_ERROR.format(action="_gen"))
# Extract bounds
bounds, _, target_fidelities = get_bounds_and_task(
search_space=search_space, param_names=self.parameters)
target_fidelities = {
i: float(v)
for i, v in target_fidelities.items() # pyre-ignore [6]
}
if optimization_config is None:
raise ValueError(
"ArrayModelBridge requires an OptimizationConfig to be specified"
)
if self.outcomes is None or len(
self.outcomes) == 0: # pragma: no cover
raise ValueError(
"No outcomes found during model fit--data are missing.")
validate_optimization_config(optimization_config, self.outcomes)
objective_weights = extract_objective_weights(
objective=optimization_config.objective, outcomes=self.outcomes)
outcome_constraints = extract_outcome_constraints(
outcome_constraints=optimization_config.outcome_constraints,
outcomes=self.outcomes,
)
extra_model_gen_kwargs = self._get_extra_model_gen_kwargs(
optimization_config=optimization_config)
linear_constraints = extract_parameter_constraints(
search_space.parameter_constraints, self.parameters)
fixed_features_dict = get_fixed_features(fixed_features,
self.parameters)
pending_array = pending_observations_as_array(pending_observations,
self.outcomes,
self.parameters)
# Generate the candidates
X, w, gen_metadata, candidate_metadata = self._model_gen(
n=n,
bounds=bounds,
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=linear_constraints,
fixed_features=fixed_features_dict,
pending_observations=pending_array,
model_gen_options=model_gen_options,
rounding_func=transform_callback(self.parameters, self.transforms),
target_fidelities=target_fidelities,
**extra_model_gen_kwargs,
)
# Transform array to observations
observation_features = parse_observation_features(
X=X,
param_names=self.parameters,
candidate_metadata=candidate_metadata)
xbest = self._model_best_point(
bounds=bounds,
objective_weights=objective_weights,
outcome_constraints=outcome_constraints,
linear_constraints=linear_constraints,
fixed_features=fixed_features_dict,
model_gen_options=model_gen_options,
target_fidelities=target_fidelities,
)
best_obsf = (None if xbest is None else ObservationFeatures(
parameters={
p: float(xbest[i])
for i, p in enumerate(self.parameters)
}))
return observation_features, w.tolist(), best_obsf, gen_metadata
