def fetch_trial_data(self, trial: BaseTrial, **kwargs: Any) -> Data:
if not isinstance(trial, BatchTrial):
raise ValueError(
"Factorial metric can only fetch data for batch trials.")
if not trial.status.expecting_data:
raise ValueError("Can only fetch data if trial is expecting data.")
data = []
normalized_arm_weights = trial.normalized_arm_weights()
for name, arm in trial.arms_by_name.items():
weight = normalized_arm_weights[arm]
mean, sem = evaluation_function(
parameterization=arm.parameters,
weight=weight,
coefficients=self.coefficients,
batch_size=self.batch_size,
noise_var=self.noise_var,
)
n = np.random.binomial(self.batch_size, weight)
data.append({
"arm_name": name,
"metric_name": self.name,
"mean": mean,
"sem": sem,
"trial_index": trial.index,
"n": n,
"frac_nonnull": mean,
})
return Data(df=pd.DataFrame(data))
def _get_weights_by_arm(
self, trial: BaseTrial) -> Iterable[Tuple[Arm, Optional[float]]]:
if isinstance(trial, Trial):
if trial.arm is not None:
return [(not_none(trial.arm), None)]
return []
elif isinstance(trial, BatchTrial):
return trial.normalized_arm_weights().items()
else:
raise UserInputError(f"Invalid trial type: {type(trial)}")
def eval_trial(self, trial: BaseTrial) -> Data:
"""
Evaluate trial arms with the evaluation function of this
experiment.
Args:
trial: trial, whose arms to evaluate.
"""
cached_data = self.lookup_data_for_trial(trial.index)
if not cached_data.df.empty:
return cached_data
evaluations = {}
if not self.has_evaluation_function:
raise ValueError( # pragma: no cover
f"Cannot evaluate trial {trial.index} as no attached data was "
"found and no evaluation function is set on this `SimpleExperiment.`"
"`SimpleExperiment` is geared to synchronous and sequential cases "
"where each trial is evaluated before more trials are created. "
"For all other cases, use `Experiment`.")
if isinstance(trial, Trial):
if not trial.arm:
return Data() # pragma: no cover
trial.mark_running()
evaluations[not_none(
trial.arm).name] = self.evaluation_function_outer(
not_none(trial.arm).parameters, None)
elif isinstance(trial, BatchTrial):
if not trial.arms:
return Data() # pragma: no cover
trial.mark_running()
for arm, weight in trial.normalized_arm_weights().items():
arm_parameters: TParameterization = arm.parameters
evaluations[arm.name] = self.evaluation_function_outer(
arm_parameters, weight)
data = Data.from_evaluations(evaluations, trial.index)
self.attach_data(data)
return data
