def test_interruption(self) -> None:
ax_client = AxClient()
ax_client.create_experiment(
name="test",
parameters=[ # pyre-fixme[6]: expected union that should include
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
objective_name="branin",
minimize=True,
)
for i in range(6):
parameterization, trial_index = ax_client.get_next_trial()
self.assertFalse( # There should be non-complete trials.
all(t.status.is_terminal for t in ax_client.experiment.trials.values())
)
x1, x2 = parameterization.get("x1"), parameterization.get("x2")
ax_client.complete_trial(
trial_index,
raw_data=checked_cast(
float, branin(checked_cast(float, x1), checked_cast(float, x2))
),
)
old_client = ax_client
serialized = ax_client.to_json_snapshot()
ax_client = AxClient.from_json_snapshot(serialized)
self.assertEqual(len(ax_client.experiment.trials.keys()), i + 1)
self.assertIsNot(ax_client, old_client)
self.assertTrue( # There should be no non-complete trials.
all(t.status.is_terminal for t in ax_client.experiment.trials.values())
)
def test_init_position_saved(self):
ax_client = AxClient(random_seed=239)
ax_client.create_experiment(
parameters=[
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
name="sobol_init_position_test",
)
for _ in range(4):
# For each generated trial, snapshot the client before generating it,
# then recreate client, regenerate the trial and compare the trial
# generated before and after snapshotting. If the state of Sobol is
# recorded correctly, the newly generated trial will be the same as
# the one generated before the snapshotting.
serialized = ax_client.to_json_snapshot()
params, idx = ax_client.get_next_trial()
ax_client = AxClient.from_json_snapshot(serialized)
with self.subTest(ax=ax_client, params=params, idx=idx):
new_params, new_idx = ax_client.get_next_trial()
self.assertEqual(params, new_params)
self.assertEqual(idx, new_idx)
self.assertEqual(
ax_client.experiment.trials[idx]._generator_run._model_kwargs[
"init_position"
],
idx + 1,
)
ax_client.complete_trial(idx, branin(params.get("x1"), params.get("x2")))
def get_client(self):
"""Instantiate a new AxClient from previous snapshot"""
if self._client_state is not None:
# Copy client state because `from_json_snapshot` modifies it...
client = AxClient.from_json_snapshot(
copy.deepcopy(self._client_state))
else:
client = AxClient(
random_seed=self.seed,
enforce_sequential_optimization=False,
verbose_logging=False,
)
client.create_experiment(
parameters=orion_space_to_axoptimizer_space(self.space),
choose_generation_strategy_kwargs={
"num_initialization_trials": self.n_initial_trials,
"max_parallelism_override": self.max_trials,
},
objectives={
"objective": ObjectiveProperties(minimize=True),
**{
o: ObjectiveProperties(minimize=True)
for o in self.extra_objectives
},
},
outcome_constraints=self.constraints,
)
yield client
self._client_state = client.to_json_snapshot()
def test_unnamed_experiment_snapshot(self):
ax_client = AxClient(random_seed=239)
ax_client.create_experiment(
parameters=[
{"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "y", "type": "range", "bounds": [0.0, 15.0]},
]
)
serialized = ax_client.to_json_snapshot()
ax_client = AxClient.from_json_snapshot(serialized)
self.assertIsNone(ax_client.experiment._name)
"train_size >= batch_size"
],
minimize=False,
objective_name="objective",
outcome_constraints=None,
name="Test")
# print ("axclient",ax_client.experiment.trials )
# for loop in range(n_loops):
for loop in range(n_loops):
print(f"Running trial {loop}/{n_loops}...")
parameters, trial_index = ax_client.get_next_trial()
print("trial_index", trial_index)
time.sleep(2)
# parameters["n_epochs"] = 5
# Local evaluation here can be replaced with deployment to external system.
ax_client.complete_trial(trial_index=trial_index,
raw_data=train_evaluate(parameters))
print("Best params", ax_client.get_best_parameters())
# periodic save
if loop % save_loop == (save_loop - 1):
optim_result = ax_client.get_best_parameters()
# print("best_parameters", optim_result)
# print("was I saved?", ax._save_experiment_and_generation_strategy_if_possible())
hyper = {}
hyper["best_params"] = optim_result
hyper["axclient"] = ax_client.to_json_snapshot()
# print("optim_result", optim_result)
with open(f"hyperparameters_{run_mode}.pl", "wb") as handle:
pickle.dump(hyper, handle, protocol=pickle.HIGHEST_PROTOCOL)
