def testWarmStartFromOldExperiment(self): # create old_experiment len_old_trials = 5 i_failed_trial = 3 old_experiment = get_branin_experiment() for i_old_trial in range(len_old_trials): sobol_run = get_sobol(search_space=old_experiment.search_space).gen(n=1) trial = old_experiment.new_trial(generator_run=sobol_run) trial.mark_running(no_runner_required=True) if i_old_trial == i_failed_trial: trial.mark_failed() else: trial.mark_completed() # make metric noiseless for exact reproducibility old_experiment.optimization_config.objective.metric.noise_sd = 0 old_experiment.fetch_data() # should fail if new_experiment has trials new_experiment = get_branin_experiment(with_trial=True) with self.assertRaisesRegex(ValueError, "Experiment.*has.*trials"): new_experiment.warm_start_from_old_experiment(old_experiment=old_experiment) # should fail if search spaces are different with self.assertRaisesRegex(ValueError, "mismatch in search space parameters"): self.experiment.warm_start_from_old_experiment( old_experiment=old_experiment ) # check that all non-failed trials are copied to new_experiment new_experiment = get_branin_experiment() # make metric noiseless for exact reproducibility new_experiment.optimization_config.objective.metric.noise_sd = 0 for _, trial in old_experiment.trials.items(): trial._run_metadata = DUMMY_RUN_METADATA new_experiment.warm_start_from_old_experiment( old_experiment=old_experiment, copy_run_metadata=True ) self.assertEqual(len(new_experiment.trials), len(old_experiment.trials) - 1) i_old_trial = 0 for _, trial in new_experiment.trials.items(): # skip failed trial i_old_trial += i_old_trial == i_failed_trial self.assertEqual( trial.arm.parameters, old_experiment.trials[i_old_trial].arm.parameters ) self.assertRegex( trial._properties["source"], "Warm start.*Experiment.*trial" ) self.assertDictEqual(trial.run_metadata, DUMMY_RUN_METADATA) i_old_trial += 1 # Check that the data was attached for correct trials old_df = old_experiment.fetch_data().df new_df = new_experiment.fetch_data().df self.assertEqual(len(new_df), len_old_trials - 1) pd.testing.assert_frame_equal( old_df.drop(["arm_name", "trial_index"], axis=1), new_df.drop(["arm_name", "trial_index"], axis=1), )
def test_MTGP(self): """Tests MTGP instantiation.""" # Test Multi-type MTGP exp = get_multi_type_experiment(add_trials=True) mtgp = get_MTGP(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(mtgp, TorchModelBridge) # Test Single-type MTGP exp = get_branin_experiment() # Check that factory generates a valid sobol modelbridge. sobol = get_sobol(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(5): sobol_run = sobol.gen(n=1) t = exp.new_batch_trial().add_generator_run(sobol_run) t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5) t.run().mark_completed() mtgp = get_MTGP(experiment=exp, data=exp.fetch_data(), trial_index=0) self.assertIsInstance(mtgp, TorchModelBridge) # mtgp_run = mtgp.gen( # n=1 # ) # TODO[T110948251]: This is broken at the ChoiceEncode level with self.assertRaises(ValueError): get_MTGP(experiment=exp, data=exp.fetch_data(), trial_index=9) exp = get_branin_experiment() sobol = get_sobol(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) sobol_run = sobol.gen(n=1) t = exp.new_batch_trial().add_generator_run(sobol_run) t.run().mark_completed() with self.assertRaises(ValueError): get_MTGP(experiment=exp, data=exp.fetch_data(), trial_index=0)
def test_min_observed(self): # We should fail to transition the next model if there is not # enough data observed. exp = get_branin_experiment(get_branin_experiment()) gs = GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_arms=5, min_arms_observed=5), GenerationStep(model=Models.GPEI, num_arms=1), ]) self.assertFalse(gs.uses_non_registered_models) for _ in range(5): gs.gen(exp) with self.assertRaises(ValueError): gs.gen(exp)
def test_sobol_GPEI_strategy_batches(self, mock_GPEI_gen, mock_GPEI_update, mock_GPEI_init): exp = get_branin_experiment() sobol_GPEI_generation_strategy = GenerationStrategy( name="Sobol+GPEI", steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=8), ], ) self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI") self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5]) exp.new_batch_trial( generator_run=sobol_GPEI_generation_strategy.gen(exp, n=2)).run() for i in range(1, 8): if i == 7: # Check completeness error message. with self.assertRaisesRegex(ValueError, "Generation strategy"): g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1), n=2) else: g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1), n=2) exp.new_batch_trial(generator_run=g).run() with self.assertRaises(ValueError): sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data()) self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def test_store_experiment(self): exp = get_branin_experiment() sobol_generation_strategy = GenerationStrategy( steps=[GenerationStep(model=Models.SOBOL, num_arms=5)]) self.assertIsNone(sobol_generation_strategy._experiment) sobol_generation_strategy.gen(exp) self.assertIsNotNone(sobol_generation_strategy._experiment)
def setUp(self): self.branin_experiment = get_branin_experiment() self.branin_experiment._properties[ Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF] = True self.branin_experiment_no_impl_metrics = Experiment( search_space=get_branin_search_space(), optimization_config=OptimizationConfig(objective=Objective( metric=Metric(name="branin"))), ) self.sobol_GPEI_GS = choose_generation_strategy( search_space=get_branin_search_space()) self.two_sobol_steps_GS = GenerationStrategy( # Contrived GS to ensure steps=[ # that `DataRequiredError` is property handled in scheduler. GenerationStep( # This error is raised when not enough trials model=Models. SOBOL, # have been observed to proceed to next num_trials=5, # geneneration step. min_trials_observed=3, max_parallelism=2, ), GenerationStep(model=Models.SOBOL, num_trials=-1, max_parallelism=3), ]) # GS to force the scheduler to poll completed trials after each ran trial. self.sobol_GS_no_parallelism = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_trials=-1, max_parallelism=1) ])
def test_enum_uniform(self): """Tests uniform random instantiation through the Models enum.""" exp = get_branin_experiment() uniform = Models.UNIFORM(exp.search_space) self.assertIsInstance(uniform, RandomModelBridge) uniform_run = uniform.gen(n=5) self.assertEqual(len(uniform_run.arms), 5)
def test_transform_callback_int(self, *_): exp = get_branin_experiment(with_batch=True) data = get_branin_data(trial_indices=exp.trials) parameters = [ RangeParameter(name="x1", parameter_type=ParameterType.INT, lower=1, upper=10), RangeParameter(name="x2", parameter_type=ParameterType.INT, lower=5, upper=15), ] gpei = TorchModelBridge( experiment=exp, data=data, search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[IntToFloat], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([5.4, 7.6]) self.assertTrue(np.allclose(transformed, [5, 8])) np_mb = ArrayModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=NumpyModel(), transforms=[IntToFloat], ) transformed = np_mb._transform_callback(np.array([5.4, 7.6])) self.assertTrue(np.allclose(transformed, [5, 8]))
def testUpdateGenerationStrategyIncrementally(self): experiment = get_branin_experiment() generation_strategy = choose_generation_strategy( experiment.search_space) save_experiment(experiment=experiment) save_generation_strategy(generation_strategy=generation_strategy) # add generator runs, save, reload generator_runs = [] for i in range(7): data = get_branin_data() if i > 0 else None gr = generation_strategy.gen(experiment, data=data) generator_runs.append(gr) trial = experiment.new_trial(generator_run=gr).mark_running( no_runner_required=True) trial.mark_completed() save_experiment(experiment=experiment) update_generation_strategy(generation_strategy=generation_strategy, generator_runs=generator_runs) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy._curr.index, loaded_generation_strategy._curr.index, 1) self.assertEqual(len(loaded_generation_strategy._generator_runs), 7)
def test_best_raw_objective_point_unsatisfiable_relative(self): exp = get_branin_experiment() # Optimization config with unsatisfiable constraint opt_conf = exp.optimization_config.clone() opt_conf.outcome_constraints.append( OutcomeConstraint( metric=get_branin_metric(), op=ComparisonOp.GEQ, bound=9999, relative=True, ) ) trial = exp.new_trial( generator_run=GeneratorRun(arms=[Arm(parameters={"x1": 5.0, "x2": 5.0})]) ).run() trial.mark_completed() with self.assertLogs(logger="ax.service.utils.best_point", level="WARN") as lg: get_best_raw_objective_point(exp, opt_conf) self.assertTrue( any("No status quo provided" in warning for warning in lg.output), msg=lg.output, ) exp.status_quo = Arm(parameters={"x1": 0, "x2": 0}, name="status_quo") sq_trial = exp.new_trial( generator_run=GeneratorRun(arms=[exp.status_quo]) ).run() sq_trial.mark_completed() with self.assertRaisesRegex(ValueError, "No points satisfied"): get_best_raw_objective_point(exp, opt_conf)
def testContours(self): exp = get_branin_experiment(with_str_choice_param=True, with_batch=True) exp.trials[0].run() model = Models.BOTORCH( # Model bridge kwargs experiment=exp, data=exp.fetch_data(), ) # Assert that each type of plot can be constructed successfully plot = plot_contour_plotly(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = interact_contour_plotly(model, list(model.metric_names)[0]) self.assertIsInstance(plot, go.Figure) plot = interact_contour(model, list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig) plot = plot = plot_contour(model, model.parameters[0], model.parameters[1], list(model.metric_names)[0]) self.assertIsInstance(plot, AxPlotConfig) # Make sure all parameters and metrics are displayed in tooltips tooltips = list(exp.parameters.keys()) + list(exp.metrics.keys()) for d in plot.data["data"]: # Only check scatter plots hoverovers if d["type"] != "scatter": continue for text in d["text"]: for tt in tooltips: self.assertTrue(tt in text)
def test_is_moo_problem(self): exp = get_branin_experiment() self.assertFalse(exp.is_moo_problem) exp = get_branin_experiment_with_multi_objective() self.assertTrue(exp.is_moo_problem) exp._optimization_config = None self.assertFalse(exp.is_moo_problem)
def test_exp_to_df(self): exp = get_branin_experiment(with_batch=True) exp.trials[0].run() df = exp_to_df(exp) self.assertIsInstance(df, pd.DataFrame) df = exp_to_df(exp, run_metadata_fields=["name"]) self.assertIn("name", df.columns)
def test_get_model_from_generator_run(self): """Tests that it is possible to restore a model from a generator run it produced, if `Models` registry was used. """ exp = get_branin_experiment() initial_sobol = Models.SOBOL(experiment=exp, seed=239) gr = initial_sobol.gen(n=1) # Restore the model as it was before generation. sobol = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data()) self.assertEqual(sobol.model.init_position, 0) self.assertEqual(sobol.model.seed, 239) # Restore the model as it was after generation (to resume generation). sobol_after_gen = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data(), after_gen=True) self.assertEqual(sobol_after_gen.model.init_position, 1) self.assertEqual(sobol_after_gen.model.seed, 239) self.assertEqual( initial_sobol.gen(n=1).arms, sobol_after_gen.gen(n=1).arms) exp.new_trial(generator_run=gr) # Check restoration of GPEI, to ensure proper restoration of callable kwargs gpei = Models.GPEI(experiment=exp, data=get_branin_data()) # Punch GPEI model + bridge kwargs into the Sobol generator run, to avoid # a slow call to `gpei.gen`. gr._model_key = "GPEI" gr._model_kwargs = gpei._model_kwargs gr._bridge_kwargs = gpei._bridge_kwargs gpei_restored = get_model_from_generator_run(gr, experiment=exp, data=get_branin_data()) for key in gpei.__dict__: self.assertIn(key, gpei_restored.__dict__) original, restored = gpei.__dict__[key], gpei_restored.__dict__[ key] # Fit times are set in instantiation so not same and model compared below if key in ["fit_time", "fit_time_since_gen", "model"]: continue # Fit times are set in instantiation so won't be same if isinstance(original, OrderedDict) and isinstance( restored, OrderedDict): original, restored = list(original.keys()), list( restored.keys()) if isinstance(original, Model) and isinstance(restored, Model): continue # Model equality is tough to compare. self.assertEqual(original, restored) for key in gpei.model.__dict__: self.assertIn(key, gpei_restored.model.__dict__) original, restored = ( gpei.model.__dict__[key], gpei_restored.model.__dict__[key], ) # Botorch model equality is tough to compare and training data # is unnecessary to compare, because data passed to model was the same if key in ["model", "warm_start_refitting", "Xs", "Ys"]: continue self.assertEqual(original, restored)
def setUp(self): self.branin_experiment = get_branin_experiment() sobol = Models.SOBOL(search_space=self.branin_experiment.search_space) sobol_run = sobol.gen(n=20) self.branin_experiment.new_batch_trial().add_generator_run( sobol_run ).run().mark_completed() data = self.branin_experiment.fetch_data() ms_gpei = ModelSpec(model_enum=Models.GPEI) ms_gpei.fit(experiment=self.branin_experiment, data=data) ms_gpkg = ModelSpec(model_enum=Models.GPKG) ms_gpkg.fit(experiment=self.branin_experiment, data=data) self.fitted_model_specs = [ms_gpei, ms_gpkg] self.model_selection_node = GenerationNode( model_specs=self.fitted_model_specs, best_model_selector=SingleDiagnosticBestModelSelector( diagnostic="Fisher exact test p", criterion=MetricAggregation.MEAN, metric_aggregation=DiagnosticCriterion.MIN, ), )
def test_use_update(self, mock_fetch_trials_data, mock_update): exp = get_branin_experiment() sobol_gs_with_update = GenerationStrategy( steps=[GenerationStep(model=Models.SOBOL, num_trials=-1, use_update=True)] ) # Try without passing data (generation strategy fetches data from experiment). trial = exp.new_trial(generator_run=sobol_gs_with_update.gen(experiment=exp)) mock_update.assert_not_called() trial._status = TrialStatus.COMPLETED for i in range(3): trial = exp.new_trial( generator_run=sobol_gs_with_update.gen(experiment=exp) ) self.assertEqual( mock_fetch_trials_data.call_args[1].get("trial_indices"), {i} ) trial._status = TrialStatus.COMPLETED # Try with passing data. sobol_gs_with_update.gen( experiment=exp, data=get_branin_data(trial_indices=range(4)) ) # Only the data for the last completed trial should be considered new and passed # to `update`. self.assertEqual( set(mock_update.call_args[1].get("new_data").df["trial_index"].values), {3} )
def test_GPKG(self): """Tests GPKG instantiation.""" exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_GPKG(experiment=exp, data=exp.fetch_data()) exp.trials[0].run() gpkg = get_GPKG(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpkg, TorchModelBridge) # test transform_configs with winsorization configs = { "Winsorize": { "winsorization_lower": 0.1, "winsorization_upper": 0.1 } } gpkg_win = get_GPKG(experiment=exp, data=exp.fetch_data(), transform_configs=configs) self.assertIsInstance(gpkg_win, TorchModelBridge) self.assertEqual(gpkg_win._transform_configs, configs) # test multi-fidelity optimization exp.parameters["x2"] = RangeParameter( name="x2", parameter_type=exp.parameters["x2"].parameter_type, lower=-5.0, upper=10.0, is_fidelity=True, target_value=10.0, ) gpkg_mf = get_GPKG(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpkg_mf, TorchModelBridge)
def testOODStatusQuo(self): # An OOD status quo arm without a trial index will raise an error experiment = get_branin_experiment() experiment.add_tracking_metric( BraninMetric(name="m2", param_names=["x1", "x2"])) metrics = list(experiment.metrics.values()) sobol = Models.SOBOL(experiment.search_space) a = sobol.gen(5) experiment.new_batch_trial(generator_run=a).run() # Experiments with batch trials must specify a trial index with self.assertRaises(UnsupportedError): compute_pareto_frontier( experiment, metrics[0], metrics[1], absolute_metrics=[m.name for m in metrics], ) compute_pareto_frontier( experiment, metrics[0], metrics[1], trial_index=0, absolute_metrics=[m.name for m in metrics], ) compute_pareto_frontier( experiment, metrics[0], metrics[1], data=experiment.fetch_data(), absolute_metrics=[m.name for m in metrics], )
def test_MOO_PAREGO(self): single_obj_exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_MOO_PAREGO(experiment=single_obj_exp, data=single_obj_exp.fetch_data()) multi_obj_exp = get_branin_experiment_with_multi_objective(with_batch=True) with self.assertRaises(ValueError): get_MOO_PAREGO(experiment=multi_obj_exp, data=multi_obj_exp.fetch_data()) multi_obj_exp.trials[0].run().mark_completed() moo_parego = get_MOO_PAREGO( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data() ) self.assertIsInstance(moo_parego, TorchModelBridge) self.assertEqual( { "acquisition_function_kwargs": { "chebyshev_scalarization": True, "sequential": True, } }, moo_parego._default_model_gen_options, ) moo_parego_run = moo_parego.gen(n=2) self.assertEqual(len(moo_parego_run.arms), 2)
def test_restore_from_generator_run(self): gs = GenerationStrategy( steps=[GenerationStep(model=Models.SOBOL, num_trials=5)]) # No generator runs on GS, so can't restore from one. with self.assertRaises(ValueError): gs._restore_model_from_generator_run() exp = get_branin_experiment(with_batch=True) gs.gen(experiment=exp) model = gs.model # Create a copy of the generation strategy and check that when # we restore from last generator run, the model will be set # correctly and that `_seen_trial_indices_by_status` is filled. new_gs = GenerationStrategy( steps=[GenerationStep(model=Models.SOBOL, num_trials=5)]) new_gs._experiment = exp new_gs._generator_runs = gs._generator_runs self.assertIsNone(new_gs._seen_trial_indices_by_status) new_gs._restore_model_from_generator_run() self.assertEqual(gs._seen_trial_indices_by_status, exp.trial_indices_by_status) # Model should be reset, but it should be the same model with same data. self.assertIsNot(model, new_gs.model) self.assertEqual(model.__class__, new_gs.model.__class__) # Model bridge. self.assertEqual(model.model.__class__, new_gs.model.model.__class__) # Model. self.assertEqual(model._training_data, new_gs.model._training_data)
def test_transform_callback_int_log(self, *_): exp = get_branin_experiment(with_batch=True) parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.INT, lower=1, upper=100, log_scale=True, ), RangeParameter( name="x2", parameter_type=ParameterType.INT, lower=1, upper=100, log_scale=True, ), ] gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[IntToFloat, Log], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([0.5, 1.5]) self.assertTrue(np.allclose(transformed, [0.47712, 1.50515]))
def test_factorial_thompson_strategy(self, _): exp = get_branin_experiment() factorial_thompson_generation_strategy = GenerationStrategy(steps=[ GenerationStep( model=Models.FACTORIAL, num_trials=1, model_kwargs=self.step_model_kwargs, ), GenerationStep( model=Models.THOMPSON, num_trials=-1, model_kwargs=self.step_model_kwargs, ), ]) self.assertEqual(factorial_thompson_generation_strategy.name, "Factorial+Thompson") self.assertEqual( factorial_thompson_generation_strategy.model_transitions, [1]) mock_model_bridge = self.mock_discrete_model_bridge.return_value # Initial factorial batch. exp.new_batch_trial( factorial_thompson_generation_strategy.gen(experiment=exp)) args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args self.assertEqual(kwargs.get("model_key"), "Factorial") # Subsequent Thompson sampling batch. exp.new_batch_trial( factorial_thompson_generation_strategy.gen(experiment=exp)) args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args self.assertEqual(kwargs.get("model_key"), "Thompson")
def test_transform_callback_log(self, *_): parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, ), RangeParameter( name="x2", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, ), ] search_space = SearchSpace(parameters=parameters) exp = get_branin_experiment(with_batch=True, search_space=search_space) gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=search_space, model=BotorchModel(), transforms=[Log], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([1.2, 2.5]) self.assertTrue(np.allclose(transformed, [1.2, 2.5]))
def test_current_generator_run_limit_unlimited_second_step(self): NUM_INIT_TRIALS = 5 SECOND_STEP_PARALLELISM = 3 NUM_ROUNDS = 4 exp = get_branin_experiment() sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_trials=NUM_INIT_TRIALS, min_trials_observed=3, ), GenerationStep( model=Models.SOBOL, num_trials=-1, max_parallelism=SECOND_STEP_PARALLELISM, ), ]) sobol_gs_with_parallelism_limits._experiment = exp could_gen = self._run_GS_for_N_rounds( gs=sobol_gs_with_parallelism_limits, exp=exp, num_rounds=NUM_ROUNDS) # We expect trials from first generation step + trials from remaining rounds in # batches limited by parallelism setting in the second step. self.assertEqual( len(exp.trials), NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM, ) self.assertTrue(all(t.status.is_completed for t in exp.trials.values())) self.assertEqual(could_gen, [NUM_INIT_TRIALS] + [SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))
def setUp(self) -> None: self.experiment = get_branin_experiment() sobol = Models.SOBOL(search_space=self.experiment.search_space) sobol_run = sobol.gen(n=20) self.experiment.new_batch_trial().add_generator_run( sobol_run).run().mark_completed() self.data = self.experiment.fetch_data()
def test_MOO_EHVI(self): single_obj_exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_MOO_EHVI( experiment=single_obj_exp, data=single_obj_exp.fetch_data(), ref_point=[0, 0], ) multi_obj_exp = get_branin_experiment_with_multi_objective( with_batch=True) metrics = multi_obj_exp.optimization_config.objective.metrics with self.assertRaises(ValueError): get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), ref_point={ metrics[0].name: 0.0, metrics[1].name: 0.0 }, ) multi_obj_exp.trials[0].run() moo_ehvi = get_MOO_EHVI( experiment=multi_obj_exp, data=multi_obj_exp.fetch_data(), ref_point={ metrics[0].name: 0.0, metrics[1].name: 0.0 }, ) self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge) moo_ehvi_run = moo_ehvi.gen(n=1) self.assertEqual(len(moo_ehvi_run.arms), 1)
def test_MOO_RS(self): single_obj_exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_MOO_RS(experiment=single_obj_exp, data=single_obj_exp.fetch_data()) multi_obj_exp = get_branin_experiment_with_multi_objective( with_batch=True) with self.assertRaises(ValueError): get_MOO_RS(experiment=multi_obj_exp, data=multi_obj_exp.fetch_data()) multi_obj_exp.trials[0].run() moo_rs = get_MOO_RS(experiment=multi_obj_exp, data=multi_obj_exp.fetch_data()) self.assertIsInstance(moo_rs, TorchModelBridge) self.assertEqual( { "acquisition_function_kwargs": { "random_scalarization": True, "sequential": True, } }, moo_rs._default_model_gen_options, ) moo_rs_run = moo_rs.gen(n=5) self.assertEqual(len(moo_rs_run.arms), 5)
def test_sobol_GPEI_strategy_batches(self): mock_GPEI_gen = self.mock_torch_model_bridge.return_value.gen mock_GPEI_gen.return_value = GeneratorRun( arms=[ Arm(parameters={"x1": 1, "x2": 2}), Arm(parameters={"x1": 3, "x2": 4}), ] ) exp = get_branin_experiment() sobol_GPEI_generation_strategy = GenerationStrategy( name="Sobol+GPEI", steps=[ GenerationStep(model=Models.SOBOL, num_trials=1), GenerationStep(model=Models.GPEI, num_trials=6), ], ) self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI") self.assertEqual(sobol_GPEI_generation_strategy.model_transitions, [1]) gr = sobol_GPEI_generation_strategy.gen(exp, n=2) exp.new_batch_trial(generator_run=gr).run() for i in range(1, 8): if i == 7: # Check completeness error message. with self.assertRaises(GenerationStrategyCompleted): g = sobol_GPEI_generation_strategy.gen(exp, n=2) else: g = sobol_GPEI_generation_strategy.gen(exp, n=2) exp.new_batch_trial(generator_run=g).run() self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def testDecodeGenerationStrategy(self): generation_strategy = get_generation_strategy() experiment = get_branin_experiment() gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json(gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertGreater(len(new_generation_strategy._steps), 0) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) # Model has not yet been initialized on this GS since it hasn't generated # anything yet. self.assertIsNone(new_generation_strategy.model) # Check that we can encode and decode the generation strategy after # it has generated some generator runs. generation_strategy = new_generation_strategy gr = generation_strategy.gen(experiment) gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json(gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) # Since this GS has now generated one generator run, model should have # been initialized and restored when decoding from JSON. self.assertIsInstance(new_generation_strategy.model, ModelBridge) # Check that we can encode and decode the generation strategy after # it has generated some trials and been updated with some data. generation_strategy = new_generation_strategy experiment.new_trial(gr) # Add previously generated GR as trial. # Make generation strategy aware of the trial's data via `gen`. generation_strategy.gen(experiment, data=get_branin_data()) gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json(gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) self.assertIsInstance(new_generation_strategy.model, ModelBridge)
def test_factorial_thompson_strategy(self, mock_update, mock_gen, mock_discrete): exp = get_branin_experiment() factorial_thompson_generation_strategy = GenerationStrategy(steps=[ GenerationStep(model=Models.FACTORIAL, num_arms=1), GenerationStep(model=Models.THOMPSON, num_arms=-1), ]) self.assertEqual(factorial_thompson_generation_strategy.name, "Factorial+Thompson") self.assertEqual( factorial_thompson_generation_strategy.generator_changes, [1]) for i in range(2): data = get_data() if i > 0 else None factorial_thompson_generation_strategy.gen(experiment=exp, new_data=data) exp.new_batch_trial().add_arm(Arm(parameters={"x1": i, "x2": i})) if i < 1: mock_discrete.assert_called() args, kwargs = mock_discrete.call_args self.assertIsInstance(kwargs.get("model"), FullFactorialGenerator) exp.new_batch_trial() else: mock_discrete.assert_called() args, kwargs = mock_discrete.call_args self.assertIsInstance( kwargs.get("model"), (ThompsonSampler, EmpiricalBayesThompsonSampler), )