def rbfnn(args, X, y, Xval, yval):
from sklearn.cluster import KMeans
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
args.n_clusters = min(args.n_clusters, X.shape[0])
y, y_mean, y_std = normalize_y(y)
cluster_alg = KMeans(args.n_clusters)
cluster_alg.fit(X)
c = cluster_alg.cluster_centers_
if torch.cuda.is_available() and args.cuda:
c = torch.tensor(c).to(torch.float32).to('cuda')
else:
c = torch.tensor(c).to(torch.float32)
mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1))
var = torch.nn.Sequential(RBF(None, None, c, 1.0),
PosLinear(args.n_clusters, 1, bias=False),
Reciprocal(0.1), PosLinear(1, 1, bias=False))
if torch.cuda.is_available() and args.cuda:
mean.cuda()
var.cuda()
device = torch.device('cuda')
else:
device = torch.device('cpu')
optimizer = torch.optim.Adam(chain(mean.parameters(), var.parameters()),
lr=args.lr)
it = 0
progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter')
batches = batchify(X, y, batch_size=args.batch_size, shuffel=args.shuffel)
while it < args.iters:
switch = 1.0 if it > args.iters / 2 else 0.0
optimizer.zero_grad()
data, label = next(batches)
data = torch.tensor(data).to(torch.float32).to(device)
label = torch.tensor(label).to(torch.float32).to(device)
m, v = mean(data), var(data)
v = switch * v + (1 - switch) * torch.tensor([0.02**2], device=device)
loss = normal_log_prob(label, m, v).sum()
(-loss).backward()
optimizer.step()
it += 1
progressBar.update()
progressBar.set_postfix({'loss': loss.item()})
progressBar.close()
data = torch.tensor(Xval).to(torch.float32).to(device)
label = torch.tensor(yval).to(torch.float32).to(device)
m, v = mean(data), var(data)
m = m * y_std + y_mean
v = v * y_std**2
log_px = normal_log_prob(label, m, v)
rmse = ((label - m.flatten())**2).mean().sqrt()
return log_px.mean().item(), rmse.item()
def nn(args, X, y, Xpool, ypool, Xtest, ytest):
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
y, y_mean, y_std = normalize_y(y)
mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1))
var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1),
torch.nn.Softplus())
if torch.cuda.is_available() and args.cuda:
mean.cuda()
var.cuda()
device = torch.device('cuda')
else:
device = torch.device('cpu')
optimizer = torch.optim.Adam(chain(mean.parameters(), var.parameters()),
lr=args.lr)
it = 0
progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter')
batches = batchify(X, y, batch_size=args.batch_size, shuffel=args.shuffel)
while it < args.iters:
switch = 1.0 if it > args.iters / 2 else 0.0
optimizer.zero_grad()
data, label = next(batches)
data = torch.tensor(data).to(torch.float32).to(device)
label = torch.tensor(label).to(torch.float32).to(device)
m, v = mean(data), var(data)
v = switch * v + (1 - switch) * torch.tensor([0.02**2], device=device)
loss = normal_log_prob(label, m, v).sum()
(-loss).backward()
optimizer.step()
it += 1
progressBar.update()
progressBar.set_postfix({'loss': loss.item()})
progressBar.close()
with torch.no_grad():
data = torch.tensor(Xpool).to(torch.float32).to(device)
label = torch.tensor(ypool).to(torch.float32).to(device)
m, v = mean(data), var(data)
pool_m = m * y_std + y_mean
pool_v = v * y_std**2
data = torch.tensor(Xtest).to(torch.float32).to(device)
label = torch.tensor(ytest).to(torch.float32).to(device)
m, v = mean(data), var(data)
m = m * y_std + y_mean
v = v * y_std**2
test_log_px = normal_log_prob(label, m, v)
test_rmse = ((label - m.flatten())**2).mean().sqrt()
return test_log_px.mean().item(), \
test_rmse.item(), \
pool_m.cpu().flatten().numpy(), \
pool_v.cpu().flatten().numpy()
def mcdnn(args, X, y, Xval, yval):
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.Dropout(p=0.05),
torch.nn.ReLU(),
torch.nn.Linear(n_neurons, 1),
torch.nn.Dropout(p=0.05))
if torch.cuda.is_available() and args.cuda:
mean.cuda()
device=torch.device('cuda')
else:
device=torch.device('cpu')
optimizer = torch.optim.Adam(mean.parameters(), lr=args.lr)
it = 0
progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter')
batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel)
while it < args.iters:
optimizer.zero_grad()
data, label = next(batches)
data = torch.tensor(data).to(torch.float32).to(device)
label = torch.tensor(label).to(torch.float32).to(device)
m = mean(data)
loss = (m - label).abs().pow(2.0).mean()
loss.backward()
optimizer.step()
it+=1
progressBar.update()
progressBar.set_postfix({'loss': loss.item()})
progressBar.close()
data = torch.tensor(Xval).to(torch.float32).to(device)
label = torch.tensor(yval).to(torch.float32).to(device)
samples = torch.zeros(Xval.shape[0], args.mcmc).to(device)
for i in range(args.mcmc):
samples[:,i] = mean(data).flatten()
m, v = samples.mean(dim=1), samples.var(dim=1)
log_probs = normal_log_prob(label, m, v)
rmse = ((label - m.flatten())**2).mean().sqrt()
return log_probs.mean().item(), rmse.item()
def bnn(args, X, y, Xval, yval):
import tensorflow as tf
import tensorflow_probability as tfp
from tensorflow_probability import distributions as tfd
tf.reset_default_graph()
y, y_mean, y_std = normalize_y(y)
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
def VariationalNormal(name, shape, constraint=None):
means = tf.get_variable(name+'_mean',
initializer=tf.ones([1]),
constraint=constraint)
stds = tf.get_variable(name+'_std',
initializer=-1.0*tf.ones([1]))
return tfd.Normal(loc=means, scale=tf.nn.softplus(stds))
x_p = tf.placeholder(tf.float32, shape=(None, X.shape[1]))
y_p = tf.placeholder(tf.float32, shape=(None, 1))
with tf.name_scope('model', values=[x_p]):
layer1 = tfp.layers.DenseFlipout(
units=n_neurons,
activation='relu',
kernel_posterior_fn = tfp.layers.default_mean_field_normal_fn(),
bias_posterior_fn = tfp.layers.default_mean_field_normal_fn()
)
layer2 = tfp.layers.DenseFlipout(
units=1,
activation='linear',
kernel_posterior_fn = tfp.layers.default_mean_field_normal_fn(),
bias_posterior_fn = tfp.layers.default_mean_field_normal_fn()
)
predictions = layer2(layer1(x_p))
noise = VariationalNormal('noise', [1], constraint=tf.keras.constraints.NonNeg())
pred_distribution = tfd.Normal(loc=predictions,
scale=noise.sample())
neg_log_prob = -tf.reduce_mean(pred_distribution.log_prob(y_p))
kl_div = sum(layer1.losses + layer2.losses) / X.shape[0]
elbo_loss = neg_log_prob + kl_div
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate=args.lr)
train_op = optimizer.minimize(elbo_loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
it = 0
progressBar = tqdm(desc='Training BNN', total=args.iters, unit='iter')
batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel)
while it < args.iters:
data, label = next(batches)
_, l = sess.run([train_op, elbo_loss], feed_dict={x_p: data, y_p: label.reshape(-1, 1)})
progressBar.update()
progressBar.set_postfix({'loss': l})
it+=1
progressBar.close()
W0_samples = layer1.kernel_posterior.sample(1000)
b0_samples = layer1.bias_posterior.sample(1000)
W1_samples = layer2.kernel_posterior.sample(1000)
b1_samples = layer2.bias_posterior.sample(1000)
noise_samples = noise.sample(1000)
W0, b0, W1, b1, n = sess.run([W0_samples,
b0_samples,
W1_samples,
b1_samples,
noise_samples])
def sample_net(x, W0, b0, W1, b1, n):
h = np.maximum(np.matmul(x[np.newaxis], W0) + b0[:, np.newaxis, :], 0.0)
return np.matmul(h, W1) + b1[:, np.newaxis, :] + n[:, np.newaxis, :] * np.random.randn()
samples = sample_net(Xval, W0, b0, W1, b1, n)
m = samples.mean(axis=0)
v = samples.var(axis=0)
m = m * y_std + y_mean
v = v * y_std**2
log_probs = normal_log_prob(yval, m, v)
rmse = math.sqrt(((m.flatten() - yval)**2).mean())
return log_probs.mean(), rmse
def ensnn(args, X, y, Xval, yval):
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
y, y_mean, y_std = normalize_y(y)
ms, vs = [ ], [ ]
for m in range(args.n_models): # initialize differently
mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(),
torch.nn.Linear(n_neurons, 1))
var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(),
torch.nn.Linear(n_neurons, 1),
torch.nn.Softplus())
if torch.cuda.is_available() and args.cuda:
mean.cuda(); var.cuda();
device=torch.device('cuda')
else:
device=torch.device('cpu')
optimizer = torch.optim.Adam(chain(mean.parameters(),
var.parameters()), lr=args.lr)
it = 0
progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter')
batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel)
while it < args.iters:
switch = 0.0#1.0 if it > args.iters/2 else 0.0
optimizer.zero_grad()
data, label = next(batches)
data = torch.tensor(data).to(torch.float32).to(device)
label = torch.tensor(label).to(torch.float32).to(device)
m, v = mean(data), var(data)
v = switch*v + (1-switch)*torch.tensor([0.02**2], device=device)
loss = normal_log_prob(label, m, v).sum()
(-loss).backward()
optimizer.step()
it+=1
progressBar.update()
progressBar.set_postfix({'loss': loss.item()})
progressBar.close()
data = torch.tensor(Xval).to(torch.float32).to(device)
label = torch.tensor(yval).to(torch.float32).to(device)
m, v = mean(data), var(data)
m = m * y_std + y_mean
v = v * y_std**2
ms.append(m)
vs.append(v)
ms = torch.stack(ms)
vs = torch.stack(vs)
m = ms.mean(dim=0)
v = (vs + ms**2).mean(dim=0) - m**2
log_px = normal_log_prob(label, m, v)
rmse = ((label - m.flatten())**2).mean().sqrt()
return log_px.mean().item(), rmse.item()
def nnlsmv(args, X, y, Xval, yval):
if args.dataset == 'protein' or args.dataset == 'year_prediction':
n_neurons = 100
else:
n_neurons = 50
y, y_mean, y_std = normalize_y(y)
mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1))
var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons),
torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1),
torch.nn.Softplus())
if torch.cuda.is_available() and args.cuda:
mean.cuda()
var.cuda()
device = torch.device('cuda')
else:
device = torch.device('cpu')
optimizer = torch.optim.Adam(mean.parameters(), lr=args.lr)
optimizer2 = torch.optim.Adam(var.parameters(), lr=args.lr)
it = 0
opt_switch = 0
progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter')
batches = local_batchify(X,
y,
batch_size=args.batch_size,
shuffel=args.shuffel)
while it < args.iters:
switch = 1.0 if it > args.iters / 2 else 0.0
if it % 11 == 0 and switch:
opt_switch = opt_switch + 1 # change between var and mean optimizer
data, label, mean_w, var_w = next(batches)
data = torch.tensor(data).to(torch.float32).to(device)
label = torch.tensor(label).to(torch.float32).to(device)
mean_w = torch.tensor(mean_w).to(torch.float32).to(device)
var_w = torch.tensor(var_w).to(torch.float32).to(device)
if opt_switch % 2 == 0:
optimizer.zero_grad()
m, v = mean(data), var(data)
v = switch * v + (1 - switch) * torch.tensor([0.02**2],
device=device)
loss = -(-v.log() - (m.flatten() - label)**2 / (2 * v)) / mean_w
loss = loss.sum()
loss.backward()
optimizer.step()
else:
optimizer2.zero_grad()
m, v = mean(data), var(data)
loss = -(-v.log() - (m.flatten() - label)**2 / (2 * v)) / var_w
loss = loss.sum()
loss.backward()
optimizer2.step()
it += 1
progressBar.update()
progressBar.set_postfix({'loss': loss.item()})
progressBar.close()
data = torch.tensor(Xval).to(torch.float32).to(device)
label = torch.tensor(yval).to(torch.float32).to(device)
m, v = mean(data), var(data)
m = m * y_std + y_mean
v = v * y_std**2
log_px = normal_log_prob(label, m, v)
rmse = ((label - m.flatten())**2).mean().sqrt()
return log_px.mean().item(), rmse.item()
