def testUpdateGenerationStrategyIncrementally(self): experiment = get_branin_experiment() generation_strategy = choose_generation_strategy( experiment.search_space) save_experiment(experiment=experiment) save_generation_strategy(generation_strategy=generation_strategy) # add generator runs, save, reload generator_runs = [] for i in range(7): data = get_branin_data() if i > 0 else None gr = generation_strategy.gen(experiment, data=data) generator_runs.append(gr) trial = experiment.new_trial(generator_run=gr).mark_running( no_runner_required=True) trial.mark_completed() save_experiment(experiment=experiment) update_generation_strategy(generation_strategy=generation_strategy, generator_runs=generator_runs) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy._curr.index, loaded_generation_strategy._curr.index, 1) self.assertEqual(len(loaded_generation_strategy._generator_runs), 7)
def testEncodeDecodeGenerationStrategy(self): # Cannot load generation strategy before it has been saved with self.assertRaises(ValueError): load_generation_strategy_by_id(gs_id=0) # Check that we can encode and decode the generation strategy *before* # it has generated some trials and been updated with some data. generation_strategy = get_generation_strategy() # Check that we can save a generation strategy without an experiment # attached. save_generation_strategy(generation_strategy=generation_strategy) # Also try restoring this generation strategy by its ID in the DB. new_generation_strategy = load_generation_strategy_by_id( gs_id=generation_strategy._db_id ) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsNone(generation_strategy._experiment) # Cannot load generation strategy before it has been saved experiment = get_branin_experiment() save_experiment(experiment) with self.assertRaises(ValueError): load_generation_strategy_by_experiment_name(experiment_name=experiment.name) # Check that we can encode and decode the generation strategy *after* # it has generated some trials and been updated with some data. generation_strategy = new_generation_strategy experiment.new_trial(generation_strategy.gen(experiment=experiment)) experiment.new_trial( generation_strategy.gen(experiment, data=get_branin_data()) ) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) # Try restoring the generation strategy using the experiment its # attached to. new_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name ) # `_seen_trial_indices_by_status` attribute of a GS is not saved in DB, # so it will be None in the restored version of the GS. # Hackily removing it from the original GS to check equality. generation_strategy._seen_trial_indices_by_status = None self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) self.assertIsInstance(new_generation_strategy.model, ModelBridge) self.assertEqual(len(new_generation_strategy._generator_runs), 2) self.assertEqual(new_generation_strategy._experiment._name, experiment._name)
def testEncodeDecodeGenerationStrategy(self): # Cannot load generation strategy before it has been saved with self.assertRaises(ValueError): load_generation_strategy_by_id(gs_id=0) # Check that we can encode and decode the generation strategy *before* # it has generated some trials and been updated with some data. generation_strategy = get_generation_strategy() # Check that we can save a generation strategy without an experiment # attached. save_generation_strategy(generation_strategy=generation_strategy) # Also try restoring this generation strategy by its ID in the DB. new_generation_strategy = load_generation_strategy_by_id( gs_id=generation_strategy._db_id) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsNone(generation_strategy._experiment) self.assertEqual(len(generation_strategy._generated), 0) self.assertEqual(len(generation_strategy._observed), 0) # Cannot load generation strategy before it has been saved experiment = get_branin_experiment() save_experiment(experiment) with self.assertRaises(ValueError): load_generation_strategy_by_experiment_name( experiment_name=experiment.name) # Check that we can encode and decode the generation strategy *after* # it has generated some trials and been updated with some data. generation_strategy = new_generation_strategy experiment.new_trial(generator_run=generation_strategy.gen(experiment)) experiment.new_trial( generation_strategy.gen(experiment, new_data=get_branin_data())) self.assertGreater(len(generation_strategy._generated), 0) self.assertGreater(len(generation_strategy._observed), 0) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) # Try restoring the generation strategy using the experiment its # attached to. new_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) self.assertIsInstance(new_generation_strategy.model, ModelBridge) self.assertEqual(len(new_generation_strategy._generator_runs), 2) self.assertEqual(new_generation_strategy._experiment._name, experiment._name)
def testUpdateGenerationStrategy(self): generation_strategy = get_generation_strategy() save_generation_strategy(generation_strategy=generation_strategy) # Add data, save, reload generation_strategy._data = Data( df=pd.DataFrame.from_records([{ "metric_name": "foo", "mean": 1, "arm_name": "bar" }])) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_id( gs_id=generation_strategy._db_id) self.assertEqual(generation_strategy, loaded_generation_strategy) experiment = get_branin_experiment() generation_strategy = get_generation_strategy() save_experiment(experiment) # add generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen(experiment)) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # add another generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen( experiment, new_data=get_branin_data())) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # make sure that we can update the experiment too experiment.description = "foobar" save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) self.assertEqual(generation_strategy._experiment.description, experiment.description) self.assertEqual( generation_strategy._experiment.description, loaded_generation_strategy._experiment.description, )
def testUpdateGenerationStrategy(self): generation_strategy = get_generation_strategy() save_generation_strategy(generation_strategy=generation_strategy) experiment = get_branin_experiment() generation_strategy = get_generation_strategy() save_experiment(experiment) # add generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen(experiment)) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name ) # `_seen_trial_indices_by_status` attribute of a GS is not saved in DB, # so it will be None in the restored version of the GS. # Hackily removing it from the original GS to check equality. generation_strategy._seen_trial_indices_by_status = None self.assertEqual(generation_strategy, loaded_generation_strategy) # add another generator run, save, reload experiment.new_trial( generator_run=generation_strategy.gen(experiment, data=get_branin_data()) ) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name ) # `_seen_trial_indices_by_status` attribute of a GS is not saved in DB, # so it will be None in the restored version of the GS. # Hackily removing it from the original GS to check equality. generation_strategy._seen_trial_indices_by_status = None self.assertEqual(generation_strategy, loaded_generation_strategy) # make sure that we can update the experiment too experiment.description = "foobar" save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name ) self.assertEqual(generation_strategy, loaded_generation_strategy) self.assertEqual( generation_strategy._experiment.description, experiment.description ) self.assertEqual( generation_strategy._experiment.description, loaded_generation_strategy._experiment.description, )
def testUpdateGenerationStrategy(self): generation_strategy = get_generation_strategy() save_generation_strategy(generation_strategy=generation_strategy) experiment = get_branin_experiment() generation_strategy = get_generation_strategy() save_experiment(experiment) # add generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen(experiment)) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # add another generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen( experiment, data=get_branin_data())) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) # During restoration of generation strategy's model from its last generator # run, we set `_seen_trial_indices_by_status` to that of the experiment, # from which we are grabbing the data to restore the model with. When the # experiment was updated more recently than the last `gen` from generation # strategy, the generation strategy prior to save might not have 'seen' # some recently added trials, so we update the mappings to match and check # that the generation strategies are equal otherwise. generation_strategy._seen_trial_indices_by_status[ TrialStatus.CANDIDATE].add(1) self.assertEqual(generation_strategy, loaded_generation_strategy) # make sure that we can update the experiment too experiment.description = "foobar" save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) self.assertEqual(generation_strategy._experiment.description, experiment.description) self.assertEqual( generation_strategy._experiment.description, loaded_generation_strategy._experiment.description, )
def testUpdateGenerationStrategy(self): generation_strategy = get_generation_strategy() save_generation_strategy(generation_strategy=generation_strategy) experiment = get_branin_experiment() generation_strategy = get_generation_strategy() save_experiment(experiment) # add generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen(experiment)) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # add another generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen( experiment, data=get_branin_data())) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # make sure that we can update the experiment too experiment.description = "foobar" save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) self.assertEqual(generation_strategy._experiment.description, experiment.description) self.assertEqual( generation_strategy._experiment.description, loaded_generation_strategy._experiment.description, )