def testGenArms(self):
p1 = {"x": 0, "y": 1}
p2 = {"x": 4, "y": 8}
observation_features = [
ObservationFeatures(parameters=p1),
ObservationFeatures(parameters=p2),
]
arms, candidate_metadata = gen_arms(
observation_features=observation_features)
self.assertEqual(arms[0].parameters, p1)
self.assertIsNone(candidate_metadata)
arm = Arm(name="1_1", parameters=p1)
arms_by_signature = {arm.signature: arm}
observation_features[0].metadata = {"some_key": "some_val_0"}
observation_features[1].metadata = {"some_key": "some_val_1"}
arms, candidate_metadata = gen_arms(
observation_features=observation_features,
arms_by_signature=arms_by_signature,
)
self.assertEqual(arms[0].name, "1_1")
self.assertEqual(
candidate_metadata,
{
arms[0].signature: {
"some_key": "some_val_0"
},
arms[1].signature: {
"some_key": "some_val_1"
},
},
)
def testGenArms(self):
p1 = {"x": 0, "y": 1}
p2 = {"x": 4, "y": 8}
observation_features = [
ObservationFeatures(parameters=p1),
ObservationFeatures(parameters=p2),
]
arms = gen_arms(observation_features=observation_features)
self.assertEqual(arms[0].parameters, p1)
arm = Arm(name="1_1", parameters=p1)
arms_by_signature = {arm.signature: arm}
arms = gen_arms(
observation_features=observation_features,
arms_by_signature=arms_by_signature,
)
self.assertEqual(arms[0].name, "1_1")
def evaluation_function(x: List[float]) -> float:
# Check if we have exhuasted the evaluation budget
if len(experiment.trials) >= max_trials:
raise ValueError(
f"Evaluation budget ({max_trials} trials) exhuasted.")
# Create an ObservationFeatures
param_dict = {
pname: x[i]
for i, pname in enumerate(problem.search_space.parameters.keys())
}
obsf = ObservationFeatures(parameters=param_dict) # pyre-ignore
# Get the time since last call
num_trials = len(experiment.trials)
if num_trials == 0:
gen_time = None
else:
previous_ts = experiment.trials[num_trials -
1].time_created.timestamp()
gen_time = time.time() - previous_ts
# Create a GR
arms, candidate_metadata_by_arm_signature = gen_arms(
observation_features=[obsf],
arms_by_signature=experiment.arms_by_signature)
gr = GeneratorRun(
arms=arms,
gen_time=gen_time,
candidate_metadata_by_arm_signature=
candidate_metadata_by_arm_signature,
)
# Add it as a trial
trial = experiment.new_trial().add_generator_run(gr).run()
# Evaluate function
df = trial.fetch_data().df
if len(df) > 1:
raise Exception(
"Does not support multiple outcomes") # pragma: no cover
obj = float(df["mean"].values[0])
if not problem.optimization_config.objective.minimize:
obj = -obj
return obj
def evaluation_function(self, x: List[float]) -> float:
"""
An interface for directly calling the benchmark problem evaluation
function. Tracks each call as a Trial. Only works for unconstrained
problems with a batch size of 1.
Args:
x: A vector of the point to evaluate
Returns: Value of the objective at x
"""
if len(self.trials) >= self.total_iterations:
raise Exception("Evaluation budget exhausted.")
assert self.batch_size == 1
# Create an ObservationFeatures
param_dict = {
pname: x[i]
for i, pname in enumerate(self.search_space.parameters.keys())
}
obsf = ObservationFeatures(parameters=param_dict) # pyre-ignore
# Get the time since last call
if len(self.trials) == 0:
gen_time = None
else:
gen_time = (
time.time() -
self.trials[len(self.trials) - 1].time_created.timestamp())
# Create a GR
gr = GeneratorRun(
arms=gen_arms(observation_features=[obsf],
arms_by_signature=self.arms_by_signature),
gen_time=gen_time,
)
# Add it as a trial
trial = self.new_trial().add_generator_run(gr).run()
# Evaluate function
df = trial.fetch_data().df
if len(df) > 1:
raise Exception(
"Does not support multiple outcomes") # pragma: no cover
return float(df["mean"].values[0])
def model_best_point(
self,
search_space: Optional[SearchSpace] = None,
optimization_config: Optional[OptimizationConfig] = None,
pending_observations: Optional[Dict[str, List[ObservationFeatures]]] = None,
fixed_features: Optional[ObservationFeatures] = None,
model_gen_options: Optional[TConfig] = None,
) -> Optional[Tuple[Arm, Optional[TModelPredictArm]]]:
# Get modifiable versions
if search_space is None:
search_space = self._model_space
search_space = search_space.clone()
base_gen_args = self._get_transformed_gen_args(
search_space=search_space,
optimization_config=optimization_config,
pending_observations=pending_observations,
fixed_features=fixed_features,
)
array_model_gen_args = self._get_transformed_model_gen_args(
search_space=base_gen_args.search_space,
pending_observations=base_gen_args.pending_observations,
fixed_features=base_gen_args.fixed_features,
model_gen_options=None,
optimization_config=base_gen_args.optimization_config,
)
search_space_digest = array_model_gen_args.search_space_digest
xbest = self._model_best_point(
bounds=search_space_digest.bounds,
objective_weights=array_model_gen_args.objective_weights,
outcome_constraints=array_model_gen_args.outcome_constraints,
linear_constraints=array_model_gen_args.linear_constraints,
fixed_features=array_model_gen_args.fixed_features,
model_gen_options=model_gen_options,
target_fidelities=search_space_digest.target_fidelities,
)
if xbest is None:
return None
best_obsf = ObservationFeatures(
parameters={p: float(xbest[i]) for i, p in enumerate(self.parameters)}
)
for t in reversed(self.transforms.values()): # noqa T484
best_obsf = t.untransform_observation_features([best_obsf])[0]
best_point_predictions = extract_arm_predictions(
model_predictions=self.predict([best_obsf]), arm_idx=0
)
best_arms, _ = gen_arms(
observation_features=[best_obsf],
arms_by_signature=self._arms_by_signature,
)
best_arm = best_arms[0]
return best_arm, best_point_predictions
