def fit_sinc(sampler, stepsize, data_seed, num_training_datapoints=20):
x_train = init_random_uniform(np.zeros(1),
np.ones(1),
num_points=num_training_datapoints,
rng=np.random.RandomState(seed=data_seed))
y_train = sinc(x_train)
x_test = np.linspace(0, 1, 100)[:, None]
y_test = sinc(x_test)
if sampler == "SGHMC":
model = Robo_BNN(sampling_method=SAMPLERS[sampler], l_rate=stepsize)
else:
from keras.losses import cosine_proximity, kullback_leibler_divergence, binary_crossentropy
model = BayesianNeuralNetwork(
optimizer=SAMPLERS[sampler],
learning_rate=stepsize,
hyperloss=lambda y_true, y_pred: kullback_leibler_divergence(
y_true=y_true, y_pred=y_pred[:, 0]))
model.train(x_train, y_train)
prediction_mean, prediction_variance = model.predict(x_test)
prediction_std = np.sqrt(prediction_variance)
return {
"prediction_mean": prediction_mean.tolist(),
"prediction_std": prediction_std.tolist(),
"x_train": x_train.tolist(),
"y_train": y_train.tolist(),
"x_test": x_test.tolist(),
"y_test": y_test.tolist()
}
def __init__(self, config_space, burnin=3000, n_iters=10000):
super(Bohamiann,
self).__init__(sacred_space_to_configspace(config_space))
self.rng = np.random.RandomState(np.random.seed())
self.n_dims = len(self.config_space.get_hyperparameters())
# All inputs are mapped to be in [0, 1]^D
self.lower = np.zeros([self.n_dims])
self.upper = np.ones([self.n_dims])
self.incumbents = []
self.X = None
self.y = None
self.model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=burnin,
n_iters=n_iters,
precondition=True,
normalize_input=True,
normalize_output=True)
self.acquisition_func = LogEI(self.model)
self.maximizer = Direct(self.acquisition_func,
self.lower,
self.upper,
verbose=False)
class Bohamiann(Optimizer):
def __init__(self, config_space, burnin=3000, n_iters=10000):
super(Bohamiann,
self).__init__(sacred_space_to_configspace(config_space))
self.rng = np.random.RandomState(np.random.seed())
self.n_dims = len(self.config_space.get_hyperparameters())
# All inputs are mapped to be in [0, 1]^D
self.lower = np.zeros([self.n_dims])
self.upper = np.ones([self.n_dims])
self.incumbents = []
self.X = None
self.y = None
self.model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=burnin,
n_iters=n_iters,
precondition=True,
normalize_input=True,
normalize_output=True)
self.acquisition_func = LogEI(self.model)
self.maximizer = Direct(self.acquisition_func,
self.lower,
self.upper,
verbose=False)
def suggest_configuration(self):
if self.X is None and self.y is None:
# No data points yet to train a model, just return a random configuration instead
new_x = init_random_uniform(self.lower,
self.upper,
n_points=1,
rng=self.rng)[0, :]
else:
# Train the model on all finished runs
self.model.train(self.X, self.y)
self.acquisition_func.update(self.model)
# Maximize the acquisition function
new_x = self.maximizer.maximize()
# Maps from [0, 1]^D space back to original space
next_config = Configuration(self.config_space, vector=new_x)
# Transform to sacred configuration
result = configspace_config_to_sacred(next_config)
return result
def fit_uci(sampler, stepsize, data_seed, burn_in_steps=5000,
num_steps=15000, num_nets=100, batch_size=32, test_split=0.1):
datasets = (BostonHousing, YachtHydrodynamics, Concrete, WineQualityRed)
results = {}
for dataset in datasets:
train_data, (x_test, y_test) = dataset.load_data(
test_split=test_split, seed=data_seed
)
had_nans = True
while had_nans:
if sampler == "sghmc":
model = Robo_BNN(
l_rate=stepsize,
sampling_method="sghmc", n_nets=num_nets, burn_in=burn_in_steps,
n_iters=num_steps, bsize=batch_size
)
elif sampler.startswith("SGHMCHD"):
# SGHMCHD approaches with different kwargs
model = KerasBayesianNeuralNetwork(
optimizer=SAMPLERS[sampler], learning_rate=stepsize,
train_callbacks=(TensorBoard(histogram_freq=1, batch_size=20, ),),
hyperloss=lambda y_true, y_pred: kullback_leibler_divergence(y_true=y_true, y_pred=y_pred[:, 0])
)
else:
raise NotImplementedError()
model.train(*train_data)
prediction_mean, prediction_variance = model.predict(x_test)
had_nans = np.isnan(prediction_mean).any() or np.isnan(prediction_variance).any()
results[dataset.__name__] = {
"x_test": x_test.tolist(),
"y_test": y_test.tolist(),
"prediction_mean": prediction_mean.tolist(),
"prediction_variance": prediction_variance.tolist()
}
return results
class TestBayesianNeuralNetwork(unittest.TestCase):
def setUp(self):
self.X = np.random.rand(10, 2)
self.y = np.sinc(self.X * 10 - 5).sum(axis=1)
self.model = BayesianNeuralNetwork(normalize_output=True,
normalize_input=True)
self.model.train(self.X, self.y)
def test_predict(self):
X_test = np.random.rand(10, 2)
m, v = self.model.predict(X_test)
assert len(m.shape) == 1
assert m.shape[0] == X_test.shape[0]
assert len(v.shape) == 1
assert v.shape[0] == X_test.shape[0]
def test_get_incumbent(self):
inc, inc_val = self.model.get_incumbent()
b = np.argmin(self.y)
np.testing.assert_almost_equal(inc, self.X[b], decimal=5)
class TestBayesianNeuralNetwork(unittest.TestCase):
def setUp(self):
self.X = np.random.rand(10, 2)
self.y = np.sinc(self.X * 10 - 5).sum(axis=1)
self.model = BayesianNeuralNetwork(normalize_output=True, normalize_input=True)
self.model.train(self.X, self.y)
def test_predict(self):
X_test = np.random.rand(10, 2)
m, v = self.model.predict(X_test)
assert len(m.shape) == 1
assert m.shape[0] == X_test.shape[0]
assert len(v.shape) == 1
assert v.shape[0] == X_test.shape[0]
def test_get_incumbent(self):
inc, inc_val = self.model.get_incumbent()
b = np.argmin(self.y)
np.testing.assert_almost_equal(inc, self.X[b], decimal=5)
def bohamiann(objective_function,
lower,
upper,
num_iterations=30,
maximizer="random",
acquisition_func="log_ei",
n_init=3,
output_path=None,
rng=None):
"""
Bohamiann uses Bayesian neural networks to model the objective function [1] inside Bayesian optimization.
Bayesian neural networks usually scale better with the number of function evaluations and the number of dimensions
than Gaussian processes.
[1] Bayesian optimization with robust Bayesian neural networks
J. T. Springenberg and A. Klein and S. Falkner and F. Hutter
Advances in Neural Information Processing Systems 29
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
maximizer: {"direct", "cmaes", "random", "scipy"}
The optimizer for the acquisition function. NOTE: "cmaes" only works in D > 1 dimensions
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=3000,
n_iters=50000,
precondition=True,
normalize_input=True,
normalize_output=True)
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
if maximizer == "cmaes":
max_func = CMAES(a, lower, upper, verbose=True, rng=rng)
elif maximizer == "direct":
max_func = Direct(a, lower, upper, verbose=True)
elif maximizer == "random":
max_func = RandomSampling(a, lower, upper, rng=rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(a, lower, upper, rng=rng)
bo = BayesianOptimization(objective_function,
lower,
upper,
a,
model,
max_func,
initial_points=n_init,
output_path=output_path,
rng=rng)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [x.tolist() for x in bo.X]
results["y"] = [y for y in bo.y]
return results
def bohamiann(objective_function,
lower,
upper,
num_iterations=30,
acquisition_func="log_ei",
n_init=3,
rng=None):
"""
General interface for Bayesian optimization for global black box optimization problems.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=3000,
n_iters=50000,
precondition=True,
normalize_input=True,
normalize_output=True)
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
max_func = Direct(a, lower, upper, verbose=False)
bo = BayesianOptimization(objective_function,
lower,
upper,
a,
model,
max_func,
initial_points=n_init,
rng=rng)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
return results
def setUp(self):
self.X = np.random.rand(10, 2)
self.y = np.sinc(self.X * 10 - 5).sum(axis=1)
self.model = BayesianNeuralNetwork(normalize_output=True,
normalize_input=True)
self.model.train(self.X, self.y)
def setUp(self):
self.X = np.random.rand(10, 2)
self.y = np.sinc(self.X * 10 - 5).sum(axis=1)
self.model = BayesianNeuralNetwork(normalize_output=True, normalize_input=True)
self.model.train(self.X, self.y)
def f(x):
return np.sinc(x * 10 - 5).sum(axis=1)
logging.basicConfig(level=logging.INFO)
rng = np.random.RandomState(42)
X = init_random_uniform(np.zeros(1), np.ones(1), 20, rng)
y = f(X)
model = BayesianNeuralNetwork(sampling_method="sgld",
l_rate=1e-4,
mdecay=0.05,
burn_in=3000,
n_iters=50000,
precondition=True,
normalize_input=True,
normalize_output=True)
model.train(X, y)
x = np.linspace(0, 1, 100)[:, None]
vals = f(x)
mean_pred, var_pred = model.predict(x)
std_pred = np.sqrt(var_pred)
plt.grid()