def test_raw_data_is_not_dict_of_dicts(self):
with self.assertRaises(ValueError):
raw_data_to_evaluation(
raw_data={"arm_0": {
"objective_a": 6
}},
metric_names=["objective_a"],
)
def test_it_turns_a_tuple_into_a_dict(self):
raw_data = (1.4, None)
result = raw_data_to_evaluation(
raw_data=raw_data,
metric_names=["objective_a"],
)
self.assertEqual(result["objective_a"], raw_data)
def test_it_accepts_a_list_for_single_objectives(self):
raw_data = [({"arm__0": {}}, {"objective_a": (1.4, None)})]
result = raw_data_to_evaluation(
raw_data=raw_data,
metric_names=["objective_a"],
)
self.assertEqual(raw_data, result)
def complete_trial(
self,
trial_index: int,
raw_data: TEvaluationOutcome,
metadata: Optional[Dict[str, str]] = None,
sample_size: Optional[int] = None,
) -> None:
"""
Completes the trial with given metric values and adds optional metadata
to it.
Args:
trial_index: Index of trial within the experiment.
raw_data: Evaluation data for the trial. Can be a mapping from
metric name to a tuple of mean and SEM, just a tuple of mean and
SEM if only one metric in optimization, or just the mean if there
is no SEM. Can also be a list of (fidelities, mapping from
metric name to a tuple of mean and SEM).
metadata: Additional metadata to track about this run.
"""
assert isinstance(
trial_index, int
), f"Trial index must be an int, got: {trial_index}." # pragma: no cover
trial = self.experiment.trials[trial_index]
if not isinstance(trial, Trial):
raise NotImplementedError(
"Batch trial functionality is not yet available through Service API."
)
if metadata is not None:
trial._run_metadata = metadata
arm_name = not_none(trial.arm).name
objective_name = self.experiment.optimization_config.objective.metric.name
evaluations = {
arm_name:
raw_data_to_evaluation(raw_data=raw_data,
objective_name=objective_name)
}
sample_sizes = {arm_name: sample_size} if sample_size else {}
# evaluations[arm_name] is either a trial evaluation
# {metric_name -> (mean, SEM)} or a fidelity trial evaluation
# [(fidelities, {metric_name -> (mean, SEM)})]
if isinstance(evaluations[arm_name], dict):
data = Data.from_evaluations(
evaluations=cast(Dict[str, TTrialEvaluation], evaluations),
trial_index=trial.index,
sample_sizes=sample_sizes,
)
else:
data = Data.from_fidelity_evaluations(
evaluations=cast(Dict[str, TFidelityTrialEvaluation],
evaluations),
trial_index=trial.index,
sample_sizes=sample_sizes,
)
trial.mark_completed()
self.experiment.attach_data(data)
self._updated_trials.append(trial_index)
self._save_experiment_and_generation_strategy_if_possible()
def _make_evaluations_and_data(
self,
trial: BaseTrial,
raw_data: Union[TEvaluationOutcome, Dict[str, TEvaluationOutcome]],
metadata: Optional[Dict[str, Union[str, int]]],
sample_sizes: Optional[Dict[str, int]] = None,
) -> Tuple[Dict[str, TEvaluationOutcome], Data]:
"""Formats given raw data as Ax evaluations and `Data`.
Args:
trial: Trial within the experiment.
raw_data: Metric outcomes for 1-arm trials, map from arm name to
metric outcomes for batched trials.
sample_size: Integer sample size for 1-arm trials, dict from arm
name to sample size for batched trials. Optional.
metadata: Additional metadata to track about this run.
data_is_for_batched_trials: Whether making evaluations and data for
a batched trial or a 1-arm trial.
"""
if isinstance(trial, BatchTrial):
assert isinstance( # pragma: no cover
raw_data, dict
), "Raw data must be a dict for batched trials."
elif isinstance(trial, Trial):
arm_name = not_none(trial.arm).name
raw_data = {arm_name: raw_data} # pyre-ignore[9]
else: # pragma: no cover
raise ValueError(f"Unexpected trial type: {type(trial)}.")
assert isinstance(raw_data, dict)
not_trial_arm_names = set(raw_data.keys()) - set(trial.arms_by_name.keys())
if not_trial_arm_names:
raise ValueError(
f"Arms {not_trial_arm_names} are not part of trial #{trial.index}."
)
evaluations = {
arm_name: raw_data_to_evaluation(
raw_data=raw_data[arm_name], objective_name=self.objective_name
)
for arm_name in raw_data
}
data = data_from_evaluations(
evaluations=evaluations,
trial_index=trial.index,
sample_sizes=sample_sizes or {},
start_time=(
checked_cast_optional(int, metadata.get("start_time"))
if metadata is not None
else None
),
end_time=(
checked_cast_optional(int, metadata.get("end_time"))
if metadata is not None
else None
),
)
return evaluations, data
def complete_trial(
self,
trial_index: int,
raw_data: TEvaluationOutcome,
metadata: Optional[Dict[str, Union[str, int]]] = None,
sample_size: Optional[int] = None,
) -> None:
"""
Completes the trial with given metric values and adds optional metadata
to it.
Args:
trial_index: Index of trial within the experiment.
raw_data: Evaluation data for the trial. Can be a mapping from
metric name to a tuple of mean and SEM, just a tuple of mean and
SEM if only one metric in optimization, or just the mean if there
is no SEM. Can also be a list of (fidelities, mapping from
metric name to a tuple of mean and SEM).
metadata: Additional metadata to track about this run.
sample_size: Number of samples collected for the underlying arm,
optional.
"""
assert isinstance(
trial_index, int
), f"Trial index must be an int, got: {trial_index}." # pragma: no cover
trial = self._get_trial(trial_index=trial_index)
if metadata is not None:
trial._run_metadata = metadata
arm_name = not_none(trial.arm).name
evaluations = {
arm_name:
raw_data_to_evaluation(raw_data=raw_data,
objective_name=self.objective_name)
}
sample_sizes = {arm_name: sample_size} if sample_size else {}
data = data_from_evaluations(
evaluations=evaluations,
trial_index=trial.index,
sample_sizes=sample_sizes,
start_time=(checked_cast_optional(int, metadata.get("start_time"))
if metadata is not None else None),
end_time=(checked_cast_optional(int, metadata.get("end_time"))
if metadata is not None else None),
)
# In service API, a trial may be completed multiple times (for multiple
# metrics, for example).
trial.mark_completed(allow_repeat_completion=True)
self.experiment.attach_data(data)
data_for_logging = _round_floats_for_logging(
item=evaluations[next(iter(evaluations.keys()))])
logger.info(f"Completed trial {trial_index} with data: "
f"{_round_floats_for_logging(item=data_for_logging)}.")
self._updated_trials.append(trial_index)
self._save_experiment_and_generation_strategy_to_db_if_possible()
def test_it_converts_to_floats_in_dict_and_leaves_tuples(self):
result = raw_data_to_evaluation(
raw_data={
"objective_a": 6,
"objective_b": 1.0,
"objective_c": ("some", "tuple"),
},
metric_names=["objective_a", "objective_b"],
)
self.assertEqual(result["objective_a"], (6.0, None))
self.assertEqual(result["objective_b"], (1.0, None))
self.assertEqual(result["objective_c"], ("some", "tuple"))
def complete_trial(
self,
trial_index: int,
raw_data: TEvaluationOutcome,
metadata: Optional[Dict[str, str]] = None,
sample_size: Optional[int] = None,
) -> None:
"""
Completes the trial with given metric values and adds optional metadata
to it.
Args:
trial_index: Index of trial within the experiment.
raw_data: Evaluation data for the trial. Can be a mapping from
metric name to a tuple of mean and SEM, just a tuple of mean and
SEM if only one metric in optimization, or just the mean if there
is no SEM. Can also be a list of (fidelities, mapping from
metric name to a tuple of mean and SEM).
metadata: Additional metadata to track about this run.
"""
assert isinstance(
trial_index, int
), f"Trial index must be an int, got: {trial_index}." # pragma: no cover
trial = self.experiment.trials[trial_index]
if not isinstance(trial, Trial):
raise NotImplementedError(
"The Service API only supports `Trial`, not `BatchTrial`.")
if metadata is not None:
trial._run_metadata = metadata
arm_name = not_none(trial.arm).name
evaluations = {
arm_name:
raw_data_to_evaluation(raw_data=raw_data,
objective_name=self.objective_name)
}
sample_sizes = {arm_name: sample_size} if sample_size else {}
data = data_from_evaluations(evaluations=evaluations,
trial_index=trial.index,
sample_sizes=sample_sizes)
# In service API, a trial may be completed multiple times (for multiple
# metrics, for example).
trial.mark_completed(allow_repeat_completion=True)
self.experiment.attach_data(data)
self._updated_trials.append(trial_index)
self._save_experiment_and_generation_strategy_to_db_if_possible()
def test_it_raises_for_unexpected_types(self):
with self.assertRaises(ValueError):
raw_data_to_evaluation(
raw_data="1.6",
metric_names=["objective_a"],
)
def test_it_turns_a_float_into_a_dict_of_tuple(self):
result = raw_data_to_evaluation(
raw_data=1.6,
metric_names=["objective_a"],
)
self.assertEqual(result["objective_a"], (1.6, None))
def test_it_requires_a_dict_for_multi_objectives(self):
with self.assertRaises(ValueError):
raw_data_to_evaluation(
raw_data=(6.0, None),
metric_names=["objective_a", "objective_b"],
)
def test_dict_entries_must_be_int_float_or_tuple(self):
with self.assertRaises(ValueError):
raw_data_to_evaluation(
raw_data={"objective_a": [6.0, None]},
metric_names=["objective_a"],
)