def test_1d_query(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 150
n_opt = 1
lb = -4.0
ub = 4.0
target = 0.5
def obj(x):
return -((Normal(0, 1).cdf(x[..., 0]) - target) ** 2)
# Test sine function with period 4
def test_fun(x):
return np.sin(np.pi * x / 4)
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound,
acqf_kwargs={"beta": 1.96, "objective": GenericMCObjective(obj)},
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(test_fun(next_x)))])
# We expect the global max to be at (2, 1), the min at (-2, -1)
fmax, argmax = strat.get_max()
self.assertTrue(np.abs(fmax - 1) < 0.5)
self.assertTrue(np.abs(argmax[0] - 2) < 0.5)
fmin, argmin = strat.get_min()
self.assertTrue(np.abs(fmin + 1) < 0.5)
self.assertTrue(np.abs(argmin[0] + 2) < 0.5)
# Query at x=2 should be f=1
self.assertTrue(np.abs(strat.predict(torch.tensor([2]))[0] - 1) < 0.5)
# Inverse query at val 1 should return (1,[2])
val, loc = strat.inv_query(1.0, constraints={})
self.assertTrue(np.abs(val - 1) < 0.5)
self.assertTrue(np.abs(loc[0] - 2) < 0.5)
def test_monotonic_single_probit_config_file(self):
config_file = "../configs/single_lse_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(isinstance(strat.strat_list[0], SobolStrategy))
self.assertTrue(isinstance(strat.strat_list[1], ModelWrapperStrategy))
self.assertTrue(
isinstance(strat.strat_list[1].modelbridge,
MonotonicSingleProbitModelbridge))
self.assertTrue(strat.strat_list[1].modelbridge.acqf is MonotonicMCLSE)
self.assertTrue(strat.strat_list[1].modelbridge.extra_acqf_args == {
"beta": 3.98,
"target": 0.75
})
self.assertTrue(strat.strat_list[1].modelbridge.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.ub == torch.Tensor([1, 1])))
def run_experiment(
self,
problem: Problem,
config_dict: Dict[str, Any],
seed: int,
rep: int,
) -> Tuple[List[Dict[str, Any]], SequentialStrategy]:
"""Run one simulated experiment.
Args:
config_dict (Dict[str, Any]): AEPsych configuration to use.
seed (int): Random seed for this run.
rep (int): Index of this repetition.
Returns:
Tuple[List[Dict[str, Any]], SequentialStrategy]: A tuple containing a log of the results and the strategy as
of the end of the simulated experiment. This is ignored in large-scale benchmarks but useful for
one-off visualization.
"""
# copy things that we mutate
local_config = deepcopy(config_dict)
try:
return super().run_experiment(problem, local_config, seed, rep)
except Exception as e:
logging.error(f"Error on config {config_dict}: {e}!" +
f"Traceback follows:\n{traceback.format_exc()}")
return [], SequentialStrategy([])
def test_1d_jnd(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 150
n_opt = 1
lb = -4.0
ub = 4.0
target = 0.5
def obj(x):
return -((Normal(0, 1).cdf(x[..., 0]) - target) ** 2)
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(next_x / 1.5))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
# we expect jnd close to the target to be close to the correct
# jnd (1.5), and since this is linear model this should be true
# for both definitions of JND
jnd_step = strat.get_jnd(grid=x[:, None], method="step")
est_jnd_step = jnd_step[50]
# looser test because step-jnd is hurt more by reverting to the mean
self.assertTrue(np.abs(est_jnd_step - 1.5) < 0.5)
jnd_taylor = strat.get_jnd(grid=x[:, None], method="taylor")
est_jnd_taylor = jnd_taylor[50]
self.assertTrue(np.abs(est_jnd_taylor - 1.5) < 0.25)
def _configure(self, config):
self.parnames = config.str_to_list(config.get("experiment",
"parnames"),
element_type=str)
self.outcome_type = config.get("common",
"outcome_type",
fallback="single_probit")
self.strat = SequentialStrategy.from_config(config)
self.strat_id = self.n_strats - 1 # 0-index strats
return self.strat_id
def test_opt_strategy_single(self):
strat_list = [
SobolStrategy(lb=[-1], ub=[1], n_trials=3),
SobolStrategy(lb=[-10], ub=[-8], n_trials=5),
]
strat = SequentialStrategy(strat_list)
out = np.zeros(8)
for i in range(8):
next_x = strat.gen()
strat.add_data(next_x, [1])
out[i] = next_x
gen1 = out[:3]
gen2 = out[3:]
self.assertTrue(np.min(gen1) >= -1)
self.assertTrue(np.min(gen2) >= -10)
self.assertTrue(np.max(gen1) <= 1)
self.assertTrue(np.max(gen2) <= -8)
def test_multiple_models_and_strats(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
strategy_names = [init_strat, opt_strat1, opt_strat2]
[init_strat]
generator = SobolGenerator
n_trials = 1
[opt_strat1]
generator = OptimizeAcqfGenerator
n_trials = 1
model = GPClassificationModel
acqf = MCLevelSetEstimation
[opt_strat2]
generator = MonotonicRejectionGenerator
n_trials = 1
model = MonotonicRejectionGP
acqf = MonotonicMCLSE
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].model, GPClassificationModel))
self.assertTrue(
strat.strat_list[1].generator.acqf is MCLevelSetEstimation)
self.assertTrue(
isinstance(strat.strat_list[2].generator,
MonotonicRejectionGenerator))
self.assertTrue(
isinstance(strat.strat_list[2].model, MonotonicRejectionGP))
self.assertTrue(strat.strat_list[2].generator.acqf is MonotonicMCLSE)
def make_strat_and_flatconfig(
self, config_dict: Mapping[str, str]
) -> Tuple[SequentialStrategy, Dict[str, str]]:
"""From a config dict, generate a strategy (for running) and
flattened config (for logging)
Args:
config_dict (Mapping[str, str]): A run configuration dictionary.
Returns:
Tuple[SequentialStrategy, Dict[str,str]]: A tuple containing a strategy
object and a flat config.
"""
config = Config()
config.update(config_dict=config_dict)
strat = SequentialStrategy.from_config(config)
flatconfig = self.flatten_config(config)
return strat, flatconfig
def test_monotonic_single_probit_config_file(self):
config_file = "../configs/single_lse_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator,
MonotonicRejectionGenerator))
self.assertTrue(strat.strat_list[1].generator.acqf is MonotonicMCLSE)
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].generator.
model_gen_options["raw_samples"] == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
def test_1d_single_targeting(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 50
n_opt = 1
lb = -4.0
ub = 4.0
target = 0.75
def obj(x):
return -((Normal(0, 1).cdf(x[..., 0]) - target) ** 2)
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(next_x))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
# since target is 0.75, find the point at which f_est is 0.75
est_max = x[np.argmin((norm.cdf(zhat.detach().numpy()) - 0.75) ** 2)]
# since true z is just x, the true max is where phi(x)=0.75,
self.assertTrue(np.abs(est_max - norm.ppf(0.75)) < 0.5)
def test_1d_single_lse(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 50
n_opt = 1
lb = -4.0
ub = 4.0
# target is in z space not phi(z) space, maybe that's
# weird
extra_acqf_args = {"target": 0.75, "beta": 1.96}
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
n_trials=n_opt,
generator=OptimizeAcqfGenerator(
MCLevelSetEstimation, acqf_kwargs=extra_acqf_args
),
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(next_x))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
# since target is 0.75, find the point at which f_est is 0.75
est_max = x[np.argmin((norm.cdf(zhat.detach().numpy()) - 0.75) ** 2)]
# since true z is just x, the true max is where phi(x)=0.75,
self.assertTrue(np.abs(est_max - norm.ppf(0.75)) < 0.5)
def test_1d_single_probit_pure_exploration(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 50
n_opt = 1
lb = -4.0
ub = 4.0
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(next_x))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
# f(x) = x so we're just looking at corr between cdf(zhat) and cdf(x)
self.assertTrue(
pearsonr(norm.cdf(zhat.detach().numpy()).flatten(), norm.cdf(x).flatten())[
0
]
> 0.95
)
def test_opt_strategy_single(self):
lbs = [[-1], [-10]]
ubs = [[1], [-8]]
n = [3, 5]
strat_list = []
for lb, ub, n in zip(lbs, ubs, n):
gen = SobolGenerator(lb, ub)
strat = Strategy(n, gen, lb, ub)
strat_list.append(strat)
strat = SequentialStrategy(strat_list)
out = np.zeros(8)
for i in range(8):
next_x = strat.gen()
strat.add_data(next_x, [1])
out[i] = next_x
gen1 = out[:3]
gen2 = out[3:]
self.assertTrue(np.min(gen1) >= -1)
self.assertTrue(np.min(gen2) >= -10)
self.assertTrue(np.max(gen1) <= 1)
self.assertTrue(np.max(gen2) <= -8)
def test_nonmonotonic_optimization_config_file(self):
config_file = "../configs/nonmonotonic_optimization_example.ini"
config_file = os.path.join(os.path.dirname(__file__), config_file)
config = Config()
config.update(config_fnames=[config_file])
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(strat.strat_list[0].model is None)
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
strat.strat_list[1].generator.acqf is qNoisyExpectedImprovement)
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"objective"})
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
def test_1d_single_probit(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 15
n_opt = 20
lb = -4.0
ub = 4.0
acqf = BernoulliMCMutualInformation
extra_acqf_args = {"objective": ProbitObjective()}
model_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, dim=1, inducing_size=10),
generator=OptimizeAcqfGenerator(acqf, extra_acqf_args),
n_trials=n_opt,
),
]
strat = SequentialStrategy(model_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(f_1d(next_x))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
true = f_1d(x.detach().numpy())
est = zhat.detach().numpy()
# close enough!
self.assertTrue((((norm.cdf(est) - true) ** 2).mean()) < 0.25)
def test_1d_monotonic_single_probit(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 15
n_opt = 1
lb = -4.0
ub = 4.0
acqf = MonotonicBernoulliMCMutualInformation
acqf_kwargs = {"objective": ProbitObjective()}
model_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
n_trials=n_opt,
model=MonotonicRejectionGP(lb=lb, ub=ub, dim=1, monotonic_idxs=[0]),
generator=MonotonicRejectionGenerator(acqf, acqf_kwargs),
),
]
strat = SequentialStrategy(model_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(f_1d(next_x))])
x = torch.linspace(-4, 4, 100).reshape(-1, 1)
zhat, _ = strat.predict(x)
true = f_1d(x.detach().numpy())
est = zhat.detach().numpy()
# close enough!
self.assertTrue((((norm.cdf(est) - true) ** 2).mean()) < 0.25)
def test_2d_single_probit_pure_exploration(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 50
n_opt = 1
lb = [-1, -1]
ub = [1, 1]
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(cdf_new_novel_det(next_x))])
xy = np.mgrid[-1:1:30j, -1:1:30j].reshape(2, -1).T
post_mean, _ = strat.predict(torch.Tensor(xy))
phi_post_mean = norm.cdf(post_mean.reshape(30, 30).detach().numpy())
phi_post_true = cdf_new_novel_det(xy)
self.assertTrue(
pearsonr(phi_post_mean.flatten(), phi_post_true.flatten())[0] > 0.9
)
def test_1d_single_probit_new_interface(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 50
n_opt = 1
lb = -4.0
ub = 4.0
model_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(model_list)
while not strat.finished:
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(f_1d(next_x))])
self.assertTrue(strat.y.shape[0] == n_init + n_opt)
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
# true max is 0, very loose test
self.assertTrue(np.abs(x[np.argmax(zhat.detach().numpy())]) < 0.5)
def test_1d_monotonic_single_probit(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 15
n_opt = 1
lb = -4.0
ub = 4.0
acqf = MonotonicBernoulliMCMutualInformation
extra_acqf_args = {"objective": ProbitObjective()}
model = MonotonicRejectionGP(likelihood="probit-bernoulli", monotonic_idxs=[0])
model_list = [
SobolStrategy(lb=lb, ub=ub, seed=seed, n_trials=n_init),
ModelWrapperStrategy(
modelbridge=MonotonicSingleProbitModelbridge(
lb=lb,
ub=ub,
dim=1,
acqf=acqf,
extra_acqf_args=extra_acqf_args,
model=model,
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(model_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(f_1d(next_x))])
x = torch.linspace(-4, 4, 100)
zhat, _ = strat.predict(x)
true = f_1d(x.detach().numpy())
est = zhat.detach().numpy()
# close enough!
self.assertTrue((((norm.cdf(est) - true) ** 2).mean()) < 0.25)
def test_extra_ask_warns(self):
# test that when we ask more times than we have models, we warn but keep going
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 3
n_opt = 1
lb = -4.0
ub = 4.0
model_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=10),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(model_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(norm.cdf(f_1d(next_x)))])
with self.assertWarns(RuntimeWarning):
strat.gen()
def test_2d_single_probit(self):
seed = 1
torch.manual_seed(seed)
np.random.seed(seed)
n_init = 150
n_opt = 1
lb = [-1, -1]
ub = [1, 1]
strat_list = [
Strategy(
lb=lb,
ub=ub,
n_trials=n_init,
generator=SobolGenerator(lb=lb, ub=ub, seed=seed),
),
Strategy(
lb=lb,
ub=ub,
model=GPClassificationModel(lb=lb, ub=ub, inducing_size=20),
generator=OptimizeAcqfGenerator(
qUpperConfidenceBound, acqf_kwargs={"beta": 1.96}
),
n_trials=n_opt,
),
]
strat = SequentialStrategy(strat_list)
for _i in range(n_init + n_opt):
next_x = strat.gen()
strat.add_data(next_x, [bernoulli.rvs(f_2d(next_x[None, :]))])
xy = np.mgrid[-1:1:30j, -1:1:30j].reshape(2, -1).T
zhat, _ = strat.predict(torch.Tensor(xy))
self.assertTrue(np.all(np.abs(xy[np.argmax(zhat.detach().numpy())]) < 0.5))
def make_strat_and_flatconfig(self, config_dict):
config = Config()
config.update(config_dict=config_dict)
strat = SequentialStrategy.from_config(config)
flatconfig = self.flatten_config(config)
return strat, flatconfig
def test_single_probit_config(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
strategy_names = [init_strat, opt_strat]
model = GPClassificationModel
acqf = MCLevelSetEstimation
[init_strat]
generator = SobolGenerator
n_trials = 10
[opt_strat]
generator = OptimizeAcqfGenerator
n_trials = 20
[MCLevelSetEstimation]
beta = 3.98
objective = ProbitObjective
[GPClassificationModel]
inducing_size = 10
mean_covar_factory = default_mean_covar_factory
[OptimizeAcqfGenerator]
restarts = 10
samps = 1000
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(
isinstance(strat.strat_list[0].generator, SobolGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].generator, OptimizeAcqfGenerator))
self.assertTrue(
isinstance(strat.strat_list[1].model, GPClassificationModel))
self.assertTrue(
strat.strat_list[1].generator.acqf is MCLevelSetEstimation)
# since ProbitObjective() is turned into an obj, we check for keys and then vals
self.assertTrue(
set(strat.strat_list[1].generator.acqf_kwargs.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].generator.acqf_kwargs["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].generator.acqf_kwargs["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].generator.restarts == 10)
self.assertTrue(strat.strat_list[1].generator.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(strat.strat_list[1].model.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(strat.strat_list[1].model.ub == torch.Tensor([1, 1])))
def test_single_probit_config(self):
config_str = """
[common]
lb = [0, 0]
ub = [1, 1]
outcome_type = single_probit
parnames = [par1, par2]
[experiment]
acqf = LevelSetEstimation
modelbridge_cls = SingleProbitModelbridge
init_strat_cls = SobolStrategy
opt_strat_cls = ModelWrapperStrategy
[LevelSetEstimation]
beta = 3.98
[GPClassificationModel]
inducing_size = 10
mean_covar_factory = default_mean_covar_factory
[SingleProbitModelbridge]
restarts = 10
samps = 1000
[SobolStrategy]
n_trials = 10
[ModelWrapperStrategy]
n_trials = 20
"""
config = Config()
config.update(config_str=config_str)
strat = SequentialStrategy.from_config(config)
self.assertTrue(isinstance(strat.strat_list[0], SobolStrategy))
self.assertTrue(isinstance(strat.strat_list[1], ModelWrapperStrategy))
self.assertTrue(
isinstance(strat.strat_list[1].modelbridge,
SingleProbitModelbridge))
self.assertTrue(
strat.strat_list[1].modelbridge.acqf is LevelSetEstimation)
# since ProbitObjective() is turned into an obj, we check for keys and then vals
self.assertTrue(
set(strat.strat_list[1].modelbridge.extra_acqf_args.keys()) ==
{"beta", "target", "objective"})
self.assertTrue(
strat.strat_list[1].modelbridge.extra_acqf_args["target"] == 0.75)
self.assertTrue(
strat.strat_list[1].modelbridge.extra_acqf_args["beta"] == 3.98)
self.assertTrue(
isinstance(
strat.strat_list[1].modelbridge.extra_acqf_args["objective"],
ProbitObjective,
))
self.assertTrue(strat.strat_list[1].modelbridge.restarts == 10)
self.assertTrue(strat.strat_list[1].modelbridge.samps == 1000)
self.assertTrue(strat.strat_list[0].n_trials == 10)
self.assertTrue(strat.strat_list[0].outcome_type == "single_probit")
self.assertTrue(strat.strat_list[1].n_trials == 20)
self.assertTrue(
torch.all(strat.strat_list[0].lb == strat.strat_list[1].lb))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.lb == torch.Tensor([0, 0])))
self.assertTrue(
torch.all(strat.strat_list[0].ub == strat.strat_list[1].ub))
self.assertTrue(
torch.all(
strat.strat_list[1].modelbridge.ub == torch.Tensor([1, 1])))
def plot_acquisition_examples(sobol_trials, opt_trials, target_level=0.75):
### Same model, different acqf figure ####
configs = {
"common": {
"pairwise": False,
"target": target_level,
"lb": "[-3]",
"ub": "[3]",
},
"experiment": {
"acqf": [
"MonotonicMCPosteriorVariance",
"MonotonicBernoulliMCMutualInformation",
"MonotonicMCLSE",
],
"modelbridge_cls":
"MonotonicSingleProbitModelbridge",
"init_strat_cls":
"SobolStrategy",
"opt_strat_cls":
"ModelWrapperStrategy",
"model":
"MonotonicRejectionGP",
"parnames":
"[intensity]",
},
"MonotonicMCLSE": {
"target": target_level,
"beta": 3.98,
},
"MonotonicRejectionGP": {
"inducing_size": 100,
"mean_covar_factory": "monotonic_mean_covar_factory",
"monotonic_idxs": "[0]",
"uniform_idxs": "[]",
},
"MonotonicSingleProbitModelbridge": {
"restarts": 10,
"samps": 1000
},
"SobolStrategy": {
"n_trials": sobol_trials
},
"ModelWrapperStrategy": {
"n_trials": opt_trials,
"refit_every": refit_every,
},
}
def true_testfun(x):
return norm.cdf(3 * x)
class SimpleLinearProblem(Problem):
def f(self, x):
return norm.ppf(true_testfun(x))
lb = [-3]
ub = [3]
logger = BenchmarkLogger()
problem = SimpleLinearProblem(lb, ub)
bench = Benchmark(
problem=problem,
logger=logger,
configs=configs,
global_seed=global_seed,
n_reps=1,
)
# sobol_trials
# now run each for just init trials, taking care to reseed each time
strats = []
for c in bench.combinations:
np.random.seed(global_seed)
torch.manual_seed(global_seed)
s = SequentialStrategy.from_config(Config(config_dict=c))
for _ in range(sobol_trials):
next_x = s.gen()
s.add_data(next_x, [problem.sample_y(next_x)])
strats.append(s)
# get first gen from all 3
first_gens = [s.gen() for s in strats]
fig, ax = plt.subplots(2, 2)
plot_strat(
strat=strats[0],
title=f"First active trial\n (after {sobol_trials} Sobol trials)",
ax=ax[0, 0],
true_testfun=true_testfun,
target_level=target_level,
show=False,
include_legend=False)
samps = [
norm.cdf(s.sample(torch.Tensor(g), num_samples=10000))
for s, g in zip(strats, first_gens)
]
predictions = [np.mean(s) for s in samps]
names = ["First BALV sample", "First BALD sample", "First LSE sample"]
markers = ["s", "*", "^"]
for i in range(3):
ax[0, 0].scatter(
first_gens[i][0][0],
predictions[i],
label=names[i],
marker=markers[i],
color="black",
)
# now run them all for the full duration
for s in strats:
for _tr in range(opt_trials):
next_x = s.gen()
s.add_data(next_x, [problem.sample_y(next_x)])
plotting_axes = [ax[0, 1], ax[1, 0], ax[1, 1]]
titles = [
f"Monotonic RBF Model,\n BALV, after {sobol_trials+opt_trials} total trials",
f"Monotonic RBF Model,\n BALD, after {sobol_trials+opt_trials} total trials",
f"Monotonic RBF Model,\n LSE (ours) after {sobol_trials+opt_trials} total trials",
]
_ = [
plot_strat(strat=s,
title=t,
ax=a,
true_testfun=true_testfun,
target_level=target_level,
show=False,
include_legend=False)
for a, s, t in zip(plotting_axes, strats, titles)
]
fig.tight_layout()
handles, labels = ax[0, 0].get_legend_handles_labels()
lgd = fig.legend(handles,
labels,
loc="lower right",
bbox_to_anchor=(1.5, 0.25))
# return legend so savefig works correctly
return fig, lgd
