def test_monotonic_single_probit_config_file(self):
config_file = "../configs/single_lse_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(isinstance(strat.strat_list[0], SobolStrategy))
self.assertTrue(isinstance(strat.strat_list[1], ModelWrapperStrategy))
self.assertTrue(
isinstance(strat.strat_list[1].modelbridge,
MonotonicSingleProbitModelbridge))
self.assertTrue(strat.strat_list[1].modelbridge.acqf is MonotonicMCLSE)
self.assertTrue(strat.strat_list[1].modelbridge.extra_acqf_args == {
"beta": 3.98,
"target": 0.75
})
self.assertTrue(strat.strat_list[1].modelbridge.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.ub == torch.Tensor([1, 1])))
def test_experiment_deprecation(self):
config_str = """
[experiment]
acqf = PairwiseMCPosteriorVariance
model = PairwiseProbitModel
"""
config = Config()
config.update(config_str=config_str)
self.assertTrue("acqf" in config["common"])
self.assertTrue("model" in config["common"])
def test_greedyepsilon_config(self):
config_str = """
[common]
acqf = MonotonicMCLSE
[EpsilonGreedyGenerator]
subgenerator = MonotonicRejectionGenerator
epsilon = .5
"""
config = Config()
config.update(config_str=config_str)
gen = EpsilonGreedyGenerator.from_config(config)
self.assertIsInstance(gen.subgenerator, MonotonicRejectionGenerator)
self.assertEqual(gen.subgenerator.acqf, MonotonicMCLSE)
self.assertEqual(gen.epsilon, 0.5)
def test_multiple_models_and_strats(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
strategy_names = [init_strat, opt_strat1, opt_strat2]
[init_strat]
generator = SobolGenerator
n_trials = 1
[opt_strat1]
generator = OptimizeAcqfGenerator
n_trials = 1
model = GPClassificationModel
acqf = MCLevelSetEstimation
[opt_strat2]
generator = MonotonicRejectionGenerator
n_trials = 1
model = MonotonicRejectionGP
acqf = MonotonicMCLSE
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].model, GPClassificationModel))
self.assertTrue(
strat.strat_list[1].generator.acqf is MCLevelSetEstimation)
self.assertTrue(
isinstance(strat.strat_list[2].generator,
MonotonicRejectionGenerator))
self.assertTrue(
isinstance(strat.strat_list[2].model, MonotonicRejectionGP))
self.assertTrue(strat.strat_list[2].generator.acqf is MonotonicMCLSE)
def test_sobol_config(self):
config_str = """
[common]
lb = [0]
ub = [1]
parnames = [par1]
[SobolGenerator]
seed=12345
"""
config = Config()
config.update(config_str=config_str)
gen = SobolGenerator.from_config(config)
npt.assert_equal(gen.lb.numpy(), np.array([0]))
npt.assert_equal(gen.ub.numpy(), np.array([1]))
self.assertEqual(gen.seed, 12345)
def test_sobol_n_trials(self):
for n_trials in [-1, 0, 1]:
config_str = f"""
[common]
lb = [0]
ub = [1]
parnames = [par1]
strategy_names = [init_strat]
[init_strat]
generator = SobolGenerator
n_trials = {n_trials}
"""
config = Config()
config.update(config_str=config_str)
strat = Strategy.from_config(config, "init_strat")
self.assertEqual(strat.n_trials, n_trials)
self.assertEqual(strat.finished, n_trials <= 0)
def make_strat_and_flatconfig(
self, config_dict: Mapping[str, str]
) -> Tuple[SequentialStrategy, Dict[str, str]]:
"""From a config dict, generate a strategy (for running) and
flattened config (for logging)
Args:
config_dict (Mapping[str, str]): A run configuration dictionary.
Returns:
Tuple[SequentialStrategy, Dict[str,str]]: A tuple containing a strategy
object and a flat config.
"""
config = Config()
config.update(config_dict=config_dict)
strat = SequentialStrategy.from_config(config)
flatconfig = self.flatten_config(config)
return strat, flatconfig
def test_monotonic_single_probit_config_file(self):
config_file = "../configs/single_lse_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator,
MonotonicRejectionGenerator))
self.assertTrue(strat.strat_list[1].generator.acqf is MonotonicMCLSE)
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].generator.
model_gen_options["raw_samples"] == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
def test_nonmonotonic_optimization_config_file(self):
config_file = "../configs/nonmonotonic_optimization_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
strat.strat_list[1].generator.acqf is qNoisyExpectedImprovement)
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"objective"})
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
def test_single_probit_config(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
[experiment]
acqf = LevelSetEstimation
modelbridge_cls = SingleProbitModelbridge
init_strat_cls = SobolStrategy
opt_strat_cls = ModelWrapperStrategy
[LevelSetEstimation]
beta = 3.98
[GPClassificationModel]
inducing_size = 10
mean_covar_factory = default_mean_covar_factory
[SingleProbitModelbridge]
restarts = 10
samps = 1000
[SobolStrategy]
n_trials = 10
[ModelWrapperStrategy]
n_trials = 20
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(isinstance(strat.strat_list[0], SobolStrategy))
self.assertTrue(isinstance(strat.strat_list[1], ModelWrapperStrategy))
self.assertTrue(
isinstance(strat.strat_list[1].modelbridge,
SingleProbitModelbridge))
self.assertTrue(
strat.strat_list[1].modelbridge.acqf is LevelSetEstimation)
# since ProbitObjective() is turned into an obj, we check for keys and then vals
self.assertTrue(
set(strat.strat_list[1].modelbridge.extra_acqf_args.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].modelbridge.extra_acqf_args["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].modelbridge.extra_acqf_args["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].modelbridge.extra_acqf_args["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].modelbridge.restarts == 10)
self.assertTrue(strat.strat_list[1].modelbridge.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.ub == torch.Tensor([1, 1])))
def make_strat_and_flatconfig(self, config_dict):
config = Config()
config.update(config_dict=config_dict)
strat = SequentialStrategy.from_config(config)
flatconfig = self.flatten_config(config)
return strat, flatconfig
def test_single_probit_config(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
strategy_names = [init_strat, opt_strat]
model = GPClassificationModel
acqf = MCLevelSetEstimation
[init_strat]
generator = SobolGenerator
n_trials = 10
[opt_strat]
generator = OptimizeAcqfGenerator
n_trials = 20
[MCLevelSetEstimation]
beta = 3.98
objective = ProbitObjective
[GPClassificationModel]
inducing_size = 10
mean_covar_factory = default_mean_covar_factory
[OptimizeAcqfGenerator]
restarts = 10
samps = 1000
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].model, GPClassificationModel))
self.assertTrue(
strat.strat_list[1].generator.acqf is MCLevelSetEstimation)
# since ProbitObjective() is turned into an obj, we check for keys and then vals
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].generator.restarts == 10)
self.assertTrue(strat.strat_list[1].generator.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
