class BNN_PYSGMCMC(nn.Module, BNN):
"""
Wraper of the bayesian neural network provided by pysgmcmc,
with custom architecture(number of layers, acitvations)
"""
def __init__(self, dim, act=nn.ReLU(), num_hiddens=[50], conf=dict()):
nn.Module.__init__(self)
BNN.__init__(self)
self.dim = dim
self.act = act
self.num_hiddens = num_hiddens
self.lr = conf.get('lr', 1e-2)
self.steps_burnin = conf.get('steps_burnin', 3000)
self.steps = conf.get('steps', 10000)
self.keep_every = conf.get('keep_every', 100)
self.batch_size = conf.get('batch_size', 32)
self.normalize = conf.get('normalize', True)
self.bnn = BayesianNeuralNetwork(
network_architecture=lambda input_dimensionality: NoisyNN(
input_dimensionality, self.act, self.num_hiddens),
optimizer=SGLD,
lr=self.lr,
batch_size=self.batch_size,
burn_in_steps=self.steps_burnin,
num_steps=self.steps_burnin + self.steps,
keep_every=self.keep_every)
def train(self, X, y):
self.normalize_Xy(X, y, self.normalize)
_X = self.X.numpy()
_y = self.y.numpy()
self.bnn.train(_X, _y)
def sample(self, num_samples):
assert (num_samples <= len(self.bnn.sampled_weights))
return [None for i in range(num_samples)]
def sample_predict(self, nns, X):
assert (len(nns) <= len(self.bnn.sampled_weights))
X = (X - self.x_mean) / self.x_std
num_x = X.shape[0]
_, _, out = self.bnn.predict(X.numpy(),
return_individual_predictions=True)
pred = torch.zeros(len(nns), num_x)
prec = torch.zeros(len(nns), num_x)
for i in range(len(nns)):
rec = torch.FloatTensor(out[i])
pred[i] = rec[:, 0] * self.y_std + self.y_mean
prec[i] = 1 / (torch.exp(rec[:, 1]) * self.y_std**2)
return pred, prec
def report(self):
print(self.bnn.model)
print("Number of samples: %d" % len(self.bnn.sampled_weights))
def test_invalid_sampling_methods(sampling_method):
with pytest.raises(ValueError):
graph = tf.Graph()
with tf.Session(graph=graph) as session:
BayesianNeuralNetwork(
sampling_method=sampling_method, session=session
)
def fit_sinc(sampler, stepsize, data_seed, num_training_datapoints=20):
x_train = init_random_uniform(np.zeros(1),
np.ones(1),
num_points=num_training_datapoints,
rng=np.random.RandomState(seed=data_seed))
y_train = sinc(x_train)
x_test = np.linspace(0, 1, 100)[:, None]
y_test = sinc(x_test)
if sampler == "SGHMC":
model = Robo_BNN(sampling_method=SAMPLERS[sampler], l_rate=stepsize)
else:
from keras.losses import cosine_proximity, kullback_leibler_divergence, binary_crossentropy
model = BayesianNeuralNetwork(
optimizer=SAMPLERS[sampler],
learning_rate=stepsize,
hyperloss=lambda y_true, y_pred: kullback_leibler_divergence(
y_true=y_true, y_pred=y_pred[:, 0]))
model.train(x_train, y_train)
prediction_mean, prediction_variance = model.predict(x_test)
prediction_std = np.sqrt(prediction_variance)
return {
"prediction_mean": prediction_mean.tolist(),
"prediction_std": prediction_std.tolist(),
"x_train": x_train.tolist(),
"y_train": y_train.tolist(),
"x_test": x_test.tolist(),
"y_test": y_test.tolist()
}
def test_predict_before_train_error():
"""
This test asserts that calling `predict` on a bnn that has not been trained
raises a ValueError.
"""
X_test = np.linspace(0, 1, 100)[:, None]
graph = tf.Graph()
with tf.Session(graph=graph) as session:
bnn = BayesianNeuralNetwork(session=session,
dtype=tf.float64,
burn_in_steps=1000,
n_nets=10)
assert not bnn.is_trained
with pytest.raises(ValueError):
prediction_mean, prediction_variance = bnn.predict(X_test)
def __init__(self, dim, act=nn.ReLU(), num_hiddens=[50], conf=dict()):
nn.Module.__init__(self)
BNN.__init__(self)
self.dim = dim
self.act = act
self.num_hiddens = num_hiddens
self.lr = conf.get('lr', 1e-2)
self.steps_burnin = conf.get('steps_burnin', 3000)
self.steps = conf.get('steps', 10000)
self.keep_every = conf.get('keep_every', 100)
self.batch_size = conf.get('batch_size', 32)
self.normalize = conf.get('normalize', True)
self.bnn = BayesianNeuralNetwork(
network_architecture=lambda input_dimensionality: NoisyNN(
input_dimensionality, self.act, self.num_hiddens),
optimizer=SGLD,
lr=self.lr,
batch_size=self.batch_size,
burn_in_steps=self.steps_burnin,
num_steps=self.steps_burnin + self.steps,
keep_every=self.keep_every)
def test_predict_individual_predictions_flag():
"""
This test asserts that it is possible to get
pysgmcmc.models.bayesian_neural_network.BayesianNeuralNetwork
to return individual network predictions when setting
`return_invidual_predictions` to `True`.
"""
rng, n_datapoints = RandomState(randint(0, 10000)), 100
x_train = asarray([rng.uniform(0., 1., 1) for _ in range(n_datapoints)])
y_train = sinc(x_train)
X_test = np.linspace(0, 1, 100)[:, None]
y_test = sinc(X_test)
normalization_kwargs = ({
"normalize_input": boolean,
"normalize_output": boolean
} for boolean in (True, False))
n_nets = 10
for normalization in normalization_kwargs:
graph = tf.Graph()
with tf.Session(graph=graph) as session:
bnn = BayesianNeuralNetwork(
session=session,
dtype=tf.float64,
burn_in_steps=1000,
n_nets=n_nets,
**normalization,
)
bnn.train(x_train, y_train)
assert bnn.is_trained
predictions, prediction_variance = bnn.predict(
X_test, return_individual_predictions=True)
assert len(predictions) == n_nets
def test_train_predict_performance():
"""
This test asserts that training
pysgmcmc.models.bayesian_neural_network.BayesianNeuralNetwork
on data from sinc and running predict on seperate validation data
gives error close to 0.
"""
rng, n_datapoints = RandomState(randint(0, 10000)), 100
x_train = asarray([rng.uniform(0., 1., 1) for _ in range(n_datapoints)])
y_train = sinc(x_train)
X_test = np.linspace(0, 1, 100)[:, None]
y_test = sinc(X_test)
normalization_kwargs = ({
"normalize_input": boolean,
"normalize_output": boolean
} for boolean in (True, False))
for normalization in normalization_kwargs:
graph = tf.Graph()
with tf.Session(graph=graph) as session:
bnn = BayesianNeuralNetwork(
session=session,
dtype=tf.float64,
burn_in_steps=1000,
n_nets=10,
**normalization,
)
bnn.train(x_train, y_train)
assert bnn.is_trained
prediction_mean, prediction_variance = bnn.predict(X_test)
assert allclose(mean_squared_error(y_test, prediction_mean),
0.0,
atol=1e-01)
def sampler_test(
objective_function,
dimensionality,
passing_criterion, # use default for this (r2 score)
function_domain=(0., 1.),
n_train_points=100,
seed=1,
sampling_method=Sampler.SGHMC,
sampler_args=None):
if sampler_args is None:
sampler_args = dict()
data = data_for(
objective_function,
dimensionality=dimensionality,
n_train_points=n_train_points,
function_domain=function_domain,
seed=seed,
)
graph = tf.Graph()
with tf.Session(graph=graph) as session:
bnn = BayesianNeuralNetwork(sampling_method=sampling_method,
session=session,
**sampler_args)
bnn.train(*data["train"])
X_test, y_test = data["test"]
mean_prediction, variance_prediction = bnn.predict(X_test)
passing_criterion(mean_prediction=mean_prediction,
variance_prediction=variance_prediction,
labels=y_test)
def test_invalid_batch_size(batch_size):
with pytest.raises(AssertionError):
graph = tf.Graph()
with tf.Session(graph=graph) as session:
BayesianNeuralNetwork(batch_size=batch_size, session=session)
def test_invalid_sample_steps(sample_steps):
with pytest.raises(AssertionError):
graph = tf.Graph()
with tf.Session(graph=graph) as session:
BayesianNeuralNetwork(sample_steps=sample_steps, session=session)
def test_invalid_n_iters(n_iters):
with pytest.raises(AssertionError):
graph = tf.Graph()
with tf.Session(graph=graph) as session:
BayesianNeuralNetwork(n_iters=n_iters, session=session)
def fit_autompg(sampler,
stepsize,
num_steps=15000,
burn_in_steps=5000,
num_nets=100,
batch_size=32,
data_seed=1,
test_split=0.1):
DATASET_FILENAME = path_join(dirname(__file__), "pysgmcmc_development",
"datasets", "auto-mpg.data_cleaned")
categorical_columns = ("cylinders", "model year", "origin", "car name")
label_column = "mpg"
df = pd.read_csv(DATASET_FILENAME, sep="\t")
for column in categorical_columns:
df = categories(df, column)
# reorder-columns; categorical columns first
df = df[[
"cylinders", "model year", "origin", "car name", "displacement",
"horsepower", "weight", "acceleration", "mpg"
]]
test_data = df.sample(frac=test_split, random_state=data_seed)
train_data = df[~df.index.isin(test_data.index)]
train_features = train_data.drop(label_column, axis=1)
continuous_columns = tuple(column for column in train_features.columns
if column not in categorical_columns)
x_train = [
train_features[column].as_matrix() for column in categorical_columns
]
x_train.append(train_features[list(continuous_columns)].as_matrix())
y_train = np.squeeze(train_data[label_column].as_matrix())
num_datapoints, = y_train.shape
network_factory = lambda _, seed: architecture(
features_dataframe=df.drop(label_column, axis=1),
categorical_columns=categorical_columns,
seed=data_seed)
model = BayesianNeuralNetwork(
n_steps=num_steps,
num_nets=num_nets,
burn_in_steps=burn_in_steps,
batch_size=batch_size,
batch_generator=batch_generator_embedding,
network_architecture=network_factory,
optimizer=SAMPLERS[sampler],
hyperloss=lambda y_true, y_pred: cosine_proximity(y_true=y_true,
y_pred=y_pred[:, 0]),
# Normalization?
normalize_input=False,
normalize_output=True,
learning_rate=stepsize,
)
model.train(x_train, np.asarray(y_train))
test_features = test_data.drop(label_column, axis=1)
x_test = [
test_features[column].as_matrix() for column in categorical_columns
]
x_test.append(test_features[list(continuous_columns)].as_matrix())
y_test = np.squeeze(test_data[label_column].as_matrix())
prediction_mean, prediction_variance = model.predict(x_test)
return {
"x_test": listify(x_test),
"y_test": listify(y_test),
"prediction_mean": listify(prediction_mean),
"prediction_variance": listify(prediction_variance),
"samples": listify(model.sampled_weights)
}