def testFromFidelityEvaluations(self):
data = Data.from_fidelity_evaluations(
evaluations={
"0_1": [
({
"f1": 1.0,
"f2": 0.5
}, {
"b": (3.7, 0.5)
}),
({
"f1": 1.0,
"f2": 0.75
}, {
"b": (3.8, 0.5)
}),
]
},
trial_index=0,
sample_sizes={"0_1": 2},
start_time=current_timestamp_in_millis(),
end_time=current_timestamp_in_millis(),
)
self.assertEqual(len(data.df), 2)
self.assertIn("start_time", data.df)
self.assertIn("end_time", data.df)
def complete_trial(
self,
trial_index: int,
raw_data: TEvaluationOutcome,
metadata: Optional[Dict[str, str]] = None,
sample_size: Optional[int] = None,
) -> None:
"""
Completes the trial with given metric values and adds optional metadata
to it.
Args:
trial_index: Index of trial within the experiment.
raw_data: Evaluation data for the trial. Can be a mapping from
metric name to a tuple of mean and SEM, just a tuple of mean and
SEM if only one metric in optimization, or just the mean if there
is no SEM. Can also be a list of (fidelities, mapping from
metric name to a tuple of mean and SEM).
metadata: Additional metadata to track about this run.
"""
assert isinstance(
trial_index, int
), f"Trial index must be an int, got: {trial_index}." # pragma: no cover
trial = self.experiment.trials[trial_index]
if not isinstance(trial, Trial):
raise NotImplementedError(
"Batch trial functionality is not yet available through Service API."
)
if metadata is not None:
trial._run_metadata = metadata
arm_name = not_none(trial.arm).name
objective_name = self.experiment.optimization_config.objective.metric.name
evaluations = {
arm_name:
raw_data_to_evaluation(raw_data=raw_data,
objective_name=objective_name)
}
sample_sizes = {arm_name: sample_size} if sample_size else {}
# evaluations[arm_name] is either a trial evaluation
# {metric_name -> (mean, SEM)} or a fidelity trial evaluation
# [(fidelities, {metric_name -> (mean, SEM)})]
if isinstance(evaluations[arm_name], dict):
data = Data.from_evaluations(
evaluations=cast(Dict[str, TTrialEvaluation], evaluations),
trial_index=trial.index,
sample_sizes=sample_sizes,
)
else:
data = Data.from_fidelity_evaluations(
evaluations=cast(Dict[str, TFidelityTrialEvaluation],
evaluations),
trial_index=trial.index,
sample_sizes=sample_sizes,
)
trial.mark_completed()
self.experiment.attach_data(data)
self._updated_trials.append(trial_index)
self._save_experiment_and_generation_strategy_if_possible()
def testFromFidelityEvaluations(self):
data = Data.from_fidelity_evaluations(
evaluations={
"0_1": [
({"f1": 1.0, "f2": 0.5}, {"b": (3.7, 0.5)}),
({"f1": 1.0, "f2": 0.75}, {"b": (3.8, 0.5)}),
]
},
trial_index=0,
sample_sizes={"0_1": 2},
)
self.assertEqual(len(data.df), 2)
def data_from_evaluations(
evaluations: Dict[str, TEvaluationOutcome],
trial_index: int,
sample_sizes: Dict[str, int],
start_time: Optional[int] = None,
end_time: Optional[int] = None,
) -> Data:
"""Transforms evaluations into Ax Data.
Each evaluation is either a trial evaluation: {metric_name -> (mean, SEM)}
or a fidelity trial evaluation for multi-fidelity optimizations:
[(fidelities, {metric_name -> (mean, SEM)})].
Args:
evalutions: Mapping from arm name to evaluation.
trial_index: Index of the trial, for which the evaluations are.
sample_sizes: Number of samples collected for each arm, may be empty
if unavailable.
start_time: Optional start time of run of the trial that produced this
data, in milliseconds.
end_time: Optional end time of run of the trial that produced this
data, in milliseconds.
"""
if all(isinstance(evaluations[x], dict) for x in evaluations.keys()):
# All evaluations are no-fidelity evaluations.
data = Data.from_evaluations(
evaluations=cast(Dict[str, TTrialEvaluation], evaluations),
trial_index=trial_index,
sample_sizes=sample_sizes,
start_time=start_time,
end_time=end_time,
)
elif all(isinstance(evaluations[x], list) for x in evaluations.keys()):
# All evaluations are with-fidelity evaluations.
data = Data.from_fidelity_evaluations(
evaluations=cast(Dict[str, TFidelityTrialEvaluation], evaluations),
trial_index=trial_index,
sample_sizes=sample_sizes,
start_time=start_time,
end_time=end_time,
)
else:
raise ValueError( # pragma: no cover
"Evaluations included a mixture of no-fidelity and with-fidelity "
"evaluations, which is not currently supported."
)
return data