def test_interruption(self) -> None: ax_client = AxClient() ax_client.create_experiment( name="test", parameters=[ # pyre-fixme[6]: expected union that should include {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], objective_name="branin", minimize=True, ) for i in range(6): parameterization, trial_index = ax_client.get_next_trial() self.assertFalse( # There should be non-complete trials. all(t.status.is_terminal for t in ax_client.experiment.trials.values()) ) x1, x2 = parameterization.get("x1"), parameterization.get("x2") ax_client.complete_trial( trial_index, raw_data=checked_cast( float, branin(checked_cast(float, x1), checked_cast(float, x2)) ), ) old_client = ax_client serialized = ax_client.to_json_snapshot() ax_client = AxClient.from_json_snapshot(serialized) self.assertEqual(len(ax_client.experiment.trials.keys()), i + 1) self.assertIsNot(ax_client, old_client) self.assertTrue( # There should be no non-complete trials. all(t.status.is_terminal for t in ax_client.experiment.trials.values()) )
def test_init_position_saved(self): ax_client = AxClient(random_seed=239) ax_client.create_experiment( parameters=[ {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], name="sobol_init_position_test", ) for _ in range(4): # For each generated trial, snapshot the client before generating it, # then recreate client, regenerate the trial and compare the trial # generated before and after snapshotting. If the state of Sobol is # recorded correctly, the newly generated trial will be the same as # the one generated before the snapshotting. serialized = ax_client.to_json_snapshot() params, idx = ax_client.get_next_trial() ax_client = AxClient.from_json_snapshot(serialized) with self.subTest(ax=ax_client, params=params, idx=idx): new_params, new_idx = ax_client.get_next_trial() self.assertEqual(params, new_params) self.assertEqual(idx, new_idx) self.assertEqual( ax_client.experiment.trials[idx]._generator_run._model_kwargs[ "init_position" ], idx + 1, ) ax_client.complete_trial(idx, branin(params.get("x1"), params.get("x2")))
def get_client(self): """Instantiate a new AxClient from previous snapshot""" if self._client_state is not None: # Copy client state because `from_json_snapshot` modifies it... client = AxClient.from_json_snapshot( copy.deepcopy(self._client_state)) else: client = AxClient( random_seed=self.seed, enforce_sequential_optimization=False, verbose_logging=False, ) client.create_experiment( parameters=orion_space_to_axoptimizer_space(self.space), choose_generation_strategy_kwargs={ "num_initialization_trials": self.n_initial_trials, "max_parallelism_override": self.max_trials, }, objectives={ "objective": ObjectiveProperties(minimize=True), **{ o: ObjectiveProperties(minimize=True) for o in self.extra_objectives }, }, outcome_constraints=self.constraints, ) yield client self._client_state = client.to_json_snapshot()
def test_unnamed_experiment_snapshot(self): ax_client = AxClient(random_seed=239) ax_client.create_experiment( parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ] ) serialized = ax_client.to_json_snapshot() ax_client = AxClient.from_json_snapshot(serialized) self.assertIsNone(ax_client.experiment._name)
"train_size >= batch_size" ], minimize=False, objective_name="objective", outcome_constraints=None, name="Test") # print ("axclient",ax_client.experiment.trials ) # for loop in range(n_loops): for loop in range(n_loops): print(f"Running trial {loop}/{n_loops}...") parameters, trial_index = ax_client.get_next_trial() print("trial_index", trial_index) time.sleep(2) # parameters["n_epochs"] = 5 # Local evaluation here can be replaced with deployment to external system. ax_client.complete_trial(trial_index=trial_index, raw_data=train_evaluate(parameters)) print("Best params", ax_client.get_best_parameters()) # periodic save if loop % save_loop == (save_loop - 1): optim_result = ax_client.get_best_parameters() # print("best_parameters", optim_result) # print("was I saved?", ax._save_experiment_and_generation_strategy_if_possible()) hyper = {} hyper["best_params"] = optim_result hyper["axclient"] = ax_client.to_json_snapshot() # print("optim_result", optim_result) with open(f"hyperparameters_{run_mode}.pl", "wb") as handle: pickle.dump(hyper, handle, protocol=pickle.HIGHEST_PROTOCOL)