def test_storage_error_handling(self, mock_save_fails): """Check that if `suppress_storage_errors` is True, AxClient won't visibly fail if encountered storage errors. """ init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) ax_client = AxClient(db_settings=db_settings, suppress_storage_errors=True) ax_client.create_experiment( name="test_experiment", parameters=[ { "name": "x", "type": "range", "bounds": [-5.0, 10.0] }, { "name": "y", "type": "range", "bounds": [0.0, 15.0] }, ], minimize=True, ) for _ in range(3): parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial(trial_index=trial_index, raw_data=branin(*parameters.values()))
def setUp(self): init_test_engine_and_session_factory(force_init=True) self.config = SQAConfig() self.encoder = Encoder(config=self.config) self.decoder = Decoder(config=self.config) self.experiment = get_experiment_with_batch_trial() self.dummy_parameters = [ get_range_parameter(), # w get_range_parameter2(), # x ]
def test_save_load_experiment(self): exp = get_experiment() init_test_engine_and_session_factory() db_settings = DBSettings(url="sqlite://") save_experiment(exp, db_settings) load_experiment(exp.name, db_settings) simple_experiment = get_simple_experiment() save_experiment(simple_experiment, db_settings) with self.assertRaisesRegex(ValueError, "Service API only"): load_experiment(simple_experiment.name, db_settings)
def test_sqa_storage(self): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) ax_client = AxClient(db_settings=db_settings) ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) for _ in range(5): parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) ) gs = ax_client.generation_strategy ax_client = AxClient(db_settings=db_settings) ax_client.load_experiment_from_database("test_experiment") # Trial #4 was completed after the last time the generation strategy # generated candidates, so pre-save generation strategy was not # "aware" of completion of trial #4. Post-restoration generation # strategy is aware of it, however, since it gets restored with most # up-to-date experiment data. Do adding trial #4 to the seen completed # trials of pre-storage GS to check their equality otherwise. gs._seen_trial_indices_by_status[TrialStatus.COMPLETED].add(4) self.assertEqual(gs, ax_client.generation_strategy) with self.assertRaises(ValueError): # Overwriting existing experiment. ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) with self.assertRaises(ValueError): # Overwriting existing experiment with overwrite flag with present # DB settings. This should fail as we no longer allow overwriting # experiments stored in the DB. ax_client.create_experiment( name="test_experiment", parameters=[{"name": "x", "type": "range", "bounds": [-5.0, 10.0]}], overwrite_existing_experiment=True, ) # Original experiment should still be in DB and not have been overwritten. self.assertEqual(len(ax_client.experiment.trials), 5)
def setUp(self): self.generation_strategy = get_generation_strategy( with_experiment=True) self.experiment = self.generation_strategy.experiment init_test_engine_and_session_factory(force_init=True) self.with_db_settings = WithDBSettingsBase(db_settings=DBSettings( url="sqlite://")) _save_experiment(self.experiment, encoder=self.with_db_settings.db_settings.encoder) _save_generation_strategy( generation_strategy=self.generation_strategy, encoder=self.with_db_settings.db_settings.encoder, )
def test_save_load_experiment(self): exp = get_experiment() init_test_engine_and_session_factory(force_init=True) db_settings = DBSettings( encoder=Encoder(config=SQAConfig()), decoder=Decoder(config=SQAConfig()), creator=None, ) save_experiment(exp, db_settings) load_experiment(exp.name, db_settings) simple_experiment = get_simple_experiment() save_experiment(simple_experiment, db_settings) with self.assertRaisesRegex(ValueError, "Service API only"): load_experiment(simple_experiment.name, db_settings)
def test_suppress_all_storage_errors(self, mock_save_exp, _): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) BareBonesTestScheduler( experiment=self.branin_experiment, # Has runner and metrics. generation_strategy=self.two_sobol_steps_GS, options=SchedulerOptions( init_seconds_between_polls=0.1, # Short between polls so test is fast. suppress_storage_errors_after_retries=True, ), db_settings=db_settings, ) self.assertEqual(mock_save_exp.call_count, 3)
def test_overwrite(self): init_test_engine_and_session_factory(force_init=True) ax_client = AxClient() ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) # Log a trial parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) ) with self.assertRaises(ValueError): # Overwriting existing experiment. ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) # Overwriting existing experiment with overwrite flag. ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], overwrite_existing_experiment=True, ) # There should be no trials, as we just put in a fresh experiment. self.assertEqual(len(ax_client.experiment.trials), 0) # Log a trial parameters, trial_index = ax_client.get_next_trial() self.assertIn("x1", parameters.keys()) self.assertIn("x2", parameters.keys()) ax_client.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) )
def test_sqa_storage(self): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) ax_client = AxClient(db_settings=db_settings) ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) for _ in range(5): parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) ) gs = ax_client.generation_strategy ax_client = AxClient(db_settings=db_settings) ax_client.load_experiment_from_database("test_experiment") self.assertEqual(gs, ax_client.generation_strategy) with self.assertRaises(ValueError): # Overwriting existing experiment. ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "y", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) with self.assertRaises(ValueError): # Overwriting existing experiment with overwrite flag with present # DB settings. This should fail as we no longer allow overwriting # experiments stored in the DB. ax_client.create_experiment( name="test_experiment", parameters=[{"name": "x", "type": "range", "bounds": [-5.0, 10.0]}], overwrite_existing_experiment=True, ) # Original experiment should still be in DB and not have been overwritten. self.assertEqual(len(ax_client.experiment.trials), 5)
def test_sqa_storage(self): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) ax_client = AxClient(db_settings=db_settings) ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) for _ in range(5): parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) ) gs = ax_client.generation_strategy ax_client = AxClient(db_settings=db_settings) ax_client.load_experiment_from_database("test_experiment") self.assertEqual(gs, ax_client.generation_strategy) with self.assertRaises(ValueError): # Overwriting existing experiment. ax_client.create_experiment( name="test_experiment", parameters=[ {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) # Overwriting existing experiment with overwrite flag. ax_client.create_experiment( name="test_experiment", parameters=[{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}], overwrite_existing_experiment=True, ) # There should be no trials, as we just put in a fresh experiment. self.assertEqual(len(ax_client.experiment.trials), 0)
def test_sqa_storage(self): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) ax = AxClient(db_settings=db_settings) ax.create_experiment( name="test_experiment", parameters=[ {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], minimize=True, ) for _ in range(5): parameters, trial_index = ax.get_next_trial() ax.complete_trial( trial_index=trial_index, raw_data=branin(*parameters.values()) ) gs = ax.generation_strategy ax = AxClient(db_settings=db_settings) ax.load_experiment_from_database("test_experiment") self.assertEqual(gs, ax.generation_strategy)
def test_sqa_storage(self): init_test_engine_and_session_factory(force_init=True) config = SQAConfig() encoder = Encoder(config=config) decoder = Decoder(config=config) db_settings = DBSettings(encoder=encoder, decoder=decoder) experiment = self.branin_experiment # Scheduler currently requires that the experiment be pre-saved. with self.assertRaisesRegex(ValueError, ".* must specify a name"): experiment._name = None scheduler = TestScheduler( experiment=experiment, generation_strategy=self.two_sobol_steps_GS, options=SchedulerOptions(total_trials=1), db_settings=db_settings, ) experiment._name = "test_experiment" NUM_TRIALS = 5 scheduler = TestScheduler( experiment=experiment, generation_strategy=self.two_sobol_steps_GS, options=SchedulerOptions( total_trials=NUM_TRIALS, init_seconds_between_polls= 0, # No wait between polls so test is fast. ), db_settings=db_settings, ) # Check that experiment and GS were saved. exp, gs = scheduler._load_experiment_and_generation_strategy( experiment.name) self.assertEqual(exp, experiment) self.assertEqual(gs, self.two_sobol_steps_GS) scheduler.run_all_trials() # Check that experiment and GS were saved and test reloading with reduced state. exp, gs = scheduler._load_experiment_and_generation_strategy( experiment.name, reduced_state=True) self.assertEqual(len(exp.trials), NUM_TRIALS) self.assertEqual(len(gs._generator_runs), NUM_TRIALS) # Test `from_stored_experiment`. new_scheduler = TestScheduler.from_stored_experiment( experiment_name=experiment.name, options=SchedulerOptions( total_trials=NUM_TRIALS + 1, init_seconds_between_polls= 0, # No wait between polls so test is fast. ), db_settings=db_settings, ) # Hack "resumed from storage timestamp" into `exp` to make sure all other fields # are equal, since difference in resumed from storage timestamps is expected. exp._properties[ ExperimentStatusProperties. RESUMED_FROM_STORAGE_TIMESTAMPS] = new_scheduler.experiment._properties[ ExperimentStatusProperties.RESUMED_FROM_STORAGE_TIMESTAMPS] self.assertEqual(new_scheduler.experiment, exp) self.assertEqual(new_scheduler.generation_strategy, gs) self.assertEqual( len(new_scheduler.experiment._properties[ ExperimentStatusProperties.RESUMED_FROM_STORAGE_TIMESTAMPS]), 1, )
def testDBConnectionWithoutForceInit(self): init_test_engine_and_session_factory(tier_or_path=":memory:")
def testCreationOfTestDB(self): init_test_engine_and_session_factory(tier_or_path=":memory:", force_init=True) engine = get_engine() self.assertIsNotNone(engine)
def setUp(self): self.exp = get_experiment() init_test_engine_and_session_factory(force=True) self.db_settings = DBSettings(url="sqlite://") save_experiment(self.exp, self.db_settings)
def setUp(self): init_test_engine_and_session_factory(force_init=True)