def test_get_pending_observation_features_based_on_trial_status(self):
# Pending observations should be none if there aren't any as trial is
# candidate.
self.assertTrue(self.trial.status.is_candidate)
self.assertIsNone(get_pending_observation_features(self.experiment))
self.trial.mark_staged()
# Now that the trial is staged, it should become a pending trial on the
# experiment and appear as pending for all metrics.
self.assertEqual(
get_pending_observation_features(self.experiment),
{"tracking": [self.obs_feat], "m2": [self.obs_feat], "m1": [self.obs_feat]},
)
# Same should be true for running trial.
# NOTE: Can't mark a staged trial running unless it uses a runner that
# specifically requires staging; hacking around that here since the marking
# logic does not matter for this test.
self.trial._status = TrialStatus.RUNNING
# Now that the trial is staged, it should become a pending trial on the
# experiment and appear as pending for all metrics.
self.assertEqual(
get_pending_observation_features(self.experiment),
{"tracking": [self.obs_feat], "m2": [self.obs_feat], "m1": [self.obs_feat]},
)
# When a trial is marked failed, it should no longer appear in pending.
self.trial.mark_failed()
self.assertIsNone(get_pending_observation_features(self.experiment))
# And if the trial is abandoned, it should always appear in pending features.
self.trial._status = TrialStatus.ABANDONED # Cannot re-mark a failed trial.
self.assertEqual(
get_pending_observation_features(
self.experiment, include_failed_as_pending=True
),
{"tracking": [self.obs_feat], "m2": [self.obs_feat], "m1": [self.obs_feat]},
)
def test_pending_observations_as_array(self):
# Mark a trial dispatched so that there are pending observations.
self.trial.mark_running(no_runner_required=True)
# If outcome names are respected, unlisted metrics should be filtered out.
self.assertEqual(
[
x.tolist() for x in pending_observations_as_array(
pending_observations=get_pending_observation_features(
self.experiment),
outcome_names=["m2", "m1"],
param_names=["x", "y", "z", "w"],
)
],
[[["1", "foo", "True", "4"]], [["1", "foo", "True", "4"]]],
)
self.experiment.attach_data(
Data.from_evaluations({self.trial.arm.name: {
"m2": (1, 0)
}},
trial_index=self.trial.index))
# There should be no pending observations for metric m2 now, since the
# only trial there is, has been updated with data for it.
self.assertEqual(
[
x.tolist() for x in pending_observations_as_array(
pending_observations=get_pending_observation_features(
self.experiment),
outcome_names=["m2", "m1"],
param_names=["x", "y", "z", "w"],
)
],
[[], [["1", "foo", "True", "4"]]],
)
def test_get_pending_observation_features(self):
# Pending observations should be none if there aren't any.
self.assertIsNone(get_pending_observation_features(self.experiment))
self.trial.mark_dispatched()
# Now that the trial is deployed, it should become a pending trial on the
# experiment and appear as pending for all metrics.
self.assertEqual(
get_pending_observation_features(self.experiment),
{"tracking": [self.obs_feat], "m2": [self.obs_feat], "m1": [self.obs_feat]},
)
self.experiment.attach_data(
Data.from_evaluations(
{self.trial.arm.name: {"m2": (1, 0)}}, trial_index=self.trial.index
)
)
# Not m2 should have empty pending features, since the trial was updated
# for m2.
self.assertEqual(
get_pending_observation_features(self.experiment),
{"tracking": [self.obs_feat], "m2": [], "m1": [self.obs_feat]},
)
# When a trial is marked failed, it should no longer appear in pending...
self.trial.mark_failed()
self.assertIsNone(get_pending_observation_features(self.experiment))
# ... unless specified to include failed trials in pending observations.
self.assertEqual(
get_pending_observation_features(
self.experiment, include_failed_as_pending=True
),
{"tracking": [self.obs_feat], "m2": [self.obs_feat], "m1": [self.obs_feat]},
)
def run_trial(self) -> None:
"""Run a single step of the optimization plan."""
if self.current_trial >= self.total_trials:
raise ValueError(
f"Optimization is complete, cannot run another trial.")
logger.info(f"Running optimization trial {self.current_trial + 1}...")
arms_per_trial = self.arms_per_trial
dat = (self.experiment._fetch_trial_data(self.current_trial - 1)
if self.current_trial > 0 else None)
if arms_per_trial == 1:
trial = self.experiment.new_trial(
generator_run=self.generation_strategy.gen(
experiment=self.experiment,
new_data=dat,
pending_observations=get_pending_observation_features(
experiment=self.experiment),
))
elif arms_per_trial > 1:
trial = self.experiment.new_batch_trial(
generator_run=self.generation_strategy.gen(
experiment=self.experiment, new_data=dat,
n=arms_per_trial))
else: # pragma: no cover
raise ValueError(
f"Invalid number of arms per trial: {arms_per_trial}")
trial.fetch_data()
self.current_trial += 1
def test_get_pending_observation_features_batch_trial(self):
# Check the same functionality for batched trials.
self.assertIsNone(get_pending_observation_features(self.experiment_2))
self.batch_trial.mark_running(no_runner_required=True)
sq_obs_feat = ObservationFeatures.from_arm(
self.batch_trial.arms_by_name.get("status_quo"),
trial_index=self.batch_trial.index,
)
self.assertEqual(
get_pending_observation_features(self.experiment_2),
{
"tracking": [self.obs_feat, sq_obs_feat],
"m2": [self.obs_feat, sq_obs_feat],
"m1": [self.obs_feat, sq_obs_feat],
},
)
def test_get_pending_observation_features(self):
# Pending observations should be none if there aren't any.
self.assertIsNone(get_pending_observation_features(self.experiment))
self.trial.mark_running(no_runner_required=True)
# Now that the trial is deployed, it should become a pending trial on the
# experiment and appear as pending for all metrics.
self.assertEqual(
get_pending_observation_features(self.experiment),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat]
},
)
# With `fetch_data` on trial returning data for metric "m2", that metric
# should no longer have pending observation features.
with patch.object(
self.trial,
"fetch_data",
return_value=Data.from_evaluations(
{self.trial.arm.name: {
"m2": (1, 0)
}},
trial_index=self.trial.index),
):
self.assertEqual(
get_pending_observation_features(self.experiment),
{
"tracking": [self.obs_feat],
"m2": [],
"m1": [self.obs_feat]
},
)
# When a trial is marked failed, it should no longer appear in pending...
self.trial.mark_failed()
self.assertIsNone(get_pending_observation_features(self.experiment))
# ... unless specified to include failed trials in pending observations.
self.assertEqual(
get_pending_observation_features(self.experiment,
include_failed_as_pending=True),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat]
},
)
def _get_new_trial(self) -> BaseTrial:
if self.arms_per_trial == 1:
return self.experiment.new_trial(
generator_run=self.generation_strategy.gen(
experiment=self.experiment,
pending_observations=get_pending_observation_features(
experiment=self.experiment),
))
elif self.arms_per_trial > 1:
return self.experiment.new_batch_trial(
generator_run=self.generation_strategy.gen(
experiment=self.experiment, n=self.arms_per_trial))
else:
raise UserInputError(
f"Invalid number of arms per trial: {self.arms_per_trial}")
def _suggest_new_trial(self) -> Trial:
"""
Suggest new candidate for this experiment.
Args:
n: Number of candidates to generate.
Returns:
Trial with candidate.
"""
new_data = self._get_new_data()
generator_run = not_none(self.generation_strategy).gen(
experiment=self.experiment,
new_data=new_data,
pending_observations=get_pending_observation_features(
experiment=self.experiment),
)
return self.experiment.new_trial(generator_run=generator_run)
def _gen_new_generator_run(self, n: int = 1) -> GeneratorRun:
"""Generate new generator run for this experiment.
Args:
n: Number of arms to generate.
"""
# If random seed is not set for this optimization, context manager does
# nothing; otherwise, it sets the random seed for torch, but only for the
# scope of this call. This is important because torch seed is set globally,
# so if we just set the seed without the context manager, it can have
# serious negative impact on the performance of the models that employ
# stochasticity.
with manual_seed(seed=self._random_seed) and warnings.catch_warnings():
# Filter out GPYTorch warnings to avoid confusing users.
warnings.simplefilter("ignore")
return not_none(self.generation_strategy).gen(
experiment=self.experiment,
n=n,
pending_observations=get_pending_observation_features(
experiment=self.experiment),
)
def test_get_pending_observation_features(self):
# Pending observations should be none if there aren't any.
self.assertIsNone(get_pending_observation_features(self.experiment))
self.trial.mark_running(no_runner_required=True)
# Now that the trial is deployed, it should become a pending trial on the
# experiment and appear as pending for all metrics.
self.assertEqual(
get_pending_observation_features(self.experiment),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat]
},
)
# With `fetch_data` on trial returning data for metric "m2", that metric
# should no longer have pending observation features.
with patch.object(
self.trial,
"lookup_data",
return_value=Data.from_evaluations(
{self.trial.arm.name: {
"m2": (1, 0)
}},
trial_index=self.trial.index),
):
self.assertEqual(
get_pending_observation_features(self.experiment),
{
"tracking": [self.obs_feat],
"m2": [],
"m1": [self.obs_feat]
},
)
# When a trial is marked failed, it should no longer appear in pending...
self.trial.mark_failed()
self.assertIsNone(get_pending_observation_features(self.experiment))
# ... unless specified to include failed trials in pending observations.
self.assertEqual(
get_pending_observation_features(self.experiment,
include_failed_as_pending=True),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat]
},
)
# When a trial is abandoned, it should appear in pending features whether
# or not there is data for it.
self.trial._status = TrialStatus.ABANDONED # Cannot re-mark a failed trial.
self.assertEqual(
get_pending_observation_features(self.experiment,
include_failed_as_pending=True),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat]
},
)
# When an arm is abandoned, it should appear in pending features whether
# or not there is data for it.
self.batch_trial.mark_arm_abandoned(arm_name="0_0")
# Checking with data for all metrics.
with patch.object(
self.batch_trial,
"fetch_data",
return_value=Data.from_evaluations(
{
self.batch_trial.arms[0].name: {
"m1": (1, 0),
"m2": (1, 0),
"tracking": (1, 0),
}
},
trial_index=self.trial.index,
),
):
self.assertEqual(
get_pending_observation_features(
self.experiment, include_failed_as_pending=True),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat],
},
)
# Checking with data for all metrics.
with patch.object(
self.trial,
"fetch_data",
return_value=Data.from_evaluations(
{
self.trial.arm.name: {
"m1": (1, 0),
"m2": (1, 0),
"tracking": (1, 0)
}
},
trial_index=self.trial.index,
),
):
self.assertEqual(
get_pending_observation_features(self.experiment),
{
"tracking": [self.obs_feat],
"m2": [self.obs_feat],
"m1": [self.obs_feat],
},
)
def test_get_pending_observation_features_hss(self):
# Pending observations should be none if there aren't any.
self.assertIsNone(get_pending_observation_features(self.hss_exp))
self.hss_trial.mark_running(no_runner_required=True)
# Now that the trial is deployed, it should become a pending trial on the
# experiment and appear as pending for all metrics.
pending = get_pending_observation_features(self.hss_exp)
self.assertEqual(
pending,
{
"m1": [self.hss_obs_feat],
"m2": [self.hss_obs_feat],
},
)
# Check that transforming observation features works correctly since this
# is applying `Cast` transform, it should inject full parameterization into
# resulting obs.feats.). Therefore, transforming the extracted pending features
# and observation features made from full parameterization should be the same.
self.assertEqual(
self.hss_sobol._transform_data(
obs_feats=pending["m1"],
obs_data=[],
search_space=self.hss_exp.search_space,
transforms=self.hss_sobol._raw_transforms,
transform_configs=None,
),
self.hss_sobol._transform_data(
obs_feats=[self.hss_obs_feat_all_params.clone()],
obs_data=[],
search_space=self.hss_exp.search_space,
transforms=self.hss_sobol._raw_transforms,
transform_configs=None,
),
)
# With `fetch_data` on trial returning data for metric "m2", that metric
# should no longer have pending observation features.
with patch.object(
self.hss_trial,
"lookup_data",
return_value=Data.from_evaluations(
{self.hss_trial.arm.name: {
"m2": (1, 0)
}},
trial_index=self.hss_trial.index,
),
):
self.assertEqual(
get_pending_observation_features(self.hss_exp),
{
"m2": [],
"m1": [self.hss_obs_feat]
},
)
# When a trial is marked failed, it should no longer appear in pending...
self.hss_trial.mark_failed()
self.assertIsNone(get_pending_observation_features(self.hss_exp))
# ... unless specified to include failed trials in pending observations.
self.assertEqual(
get_pending_observation_features(self.hss_exp,
include_failed_as_pending=True),
{
"m1": [self.hss_obs_feat],
"m2": [self.hss_obs_feat],
},
)
# When an arm is abandoned, it should appear in pending features whether
# or not there is data for it.
hss_exp = get_hierarchical_search_space_experiment()
hss_batch_trial = hss_exp.new_batch_trial(generator_run=self.hss_gr)
hss_batch_trial.mark_arm_abandoned(hss_batch_trial.arms[0].name)
# Checking with data for all metrics.
with patch.object(
hss_batch_trial,
"fetch_data",
return_value=Data.from_evaluations(
{
hss_batch_trial.arms[0].name: {
"m1": (1, 0),
"m2": (1, 0),
}
},
trial_index=hss_batch_trial.index,
),
):
pending = get_pending_observation_features(
hss_exp, include_failed_as_pending=True)
self.assertEqual(
pending,
{
"m1": [self.hss_obs_feat],
"m2": [self.hss_obs_feat],
},
)
# Check that candidate metadata is property propagated for abandoned arm.
self.assertEqual(
self.hss_sobol._transform_data(
obs_feats=pending["m1"],
obs_data=[],
search_space=hss_exp.search_space,
transforms=self.hss_sobol._raw_transforms,
transform_configs=None,
),
self.hss_sobol._transform_data(
obs_feats=[self.hss_obs_feat_all_params.clone()],
obs_data=[],
search_space=hss_exp.search_space,
transforms=self.hss_sobol._raw_transforms,
transform_configs=None,
),
)
# Checking with data for all metrics.
with patch.object(
hss_batch_trial,
"fetch_data",
return_value=Data.from_evaluations(
{
hss_batch_trial.arms[0].name: {
"m1": (1, 0),
"m2": (1, 0),
}
},
trial_index=hss_batch_trial.index,
),
):
self.assertEqual(
get_pending_observation_features(hss_exp),
{
"m2": [self.hss_obs_feat],
"m1": [self.hss_obs_feat],
},
)