def rbfnn(args, X, y, Xval, yval): from sklearn.cluster import KMeans if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 args.n_clusters = min(args.n_clusters, X.shape[0]) y, y_mean, y_std = normalize_y(y) cluster_alg = KMeans(args.n_clusters) cluster_alg.fit(X) c = cluster_alg.cluster_centers_ if torch.cuda.is_available() and args.cuda: c = torch.tensor(c).to(torch.float32).to('cuda') else: c = torch.tensor(c).to(torch.float32) mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1)) var = torch.nn.Sequential(RBF(None, None, c, 1.0), PosLinear(args.n_clusters, 1, bias=False), Reciprocal(0.1), PosLinear(1, 1, bias=False)) if torch.cuda.is_available() and args.cuda: mean.cuda() var.cuda() device = torch.device('cuda') else: device = torch.device('cpu') optimizer = torch.optim.Adam(chain(mean.parameters(), var.parameters()), lr=args.lr) it = 0 progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter') batches = batchify(X, y, batch_size=args.batch_size, shuffel=args.shuffel) while it < args.iters: switch = 1.0 if it > args.iters / 2 else 0.0 optimizer.zero_grad() data, label = next(batches) data = torch.tensor(data).to(torch.float32).to(device) label = torch.tensor(label).to(torch.float32).to(device) m, v = mean(data), var(data) v = switch * v + (1 - switch) * torch.tensor([0.02**2], device=device) loss = normal_log_prob(label, m, v).sum() (-loss).backward() optimizer.step() it += 1 progressBar.update() progressBar.set_postfix({'loss': loss.item()}) progressBar.close() data = torch.tensor(Xval).to(torch.float32).to(device) label = torch.tensor(yval).to(torch.float32).to(device) m, v = mean(data), var(data) m = m * y_std + y_mean v = v * y_std**2 log_px = normal_log_prob(label, m, v) rmse = ((label - m.flatten())**2).mean().sqrt() return log_px.mean().item(), rmse.item()
def nn(args, X, y, Xpool, ypool, Xtest, ytest): if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 y, y_mean, y_std = normalize_y(y) mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1)) var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1), torch.nn.Softplus()) if torch.cuda.is_available() and args.cuda: mean.cuda() var.cuda() device = torch.device('cuda') else: device = torch.device('cpu') optimizer = torch.optim.Adam(chain(mean.parameters(), var.parameters()), lr=args.lr) it = 0 progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter') batches = batchify(X, y, batch_size=args.batch_size, shuffel=args.shuffel) while it < args.iters: switch = 1.0 if it > args.iters / 2 else 0.0 optimizer.zero_grad() data, label = next(batches) data = torch.tensor(data).to(torch.float32).to(device) label = torch.tensor(label).to(torch.float32).to(device) m, v = mean(data), var(data) v = switch * v + (1 - switch) * torch.tensor([0.02**2], device=device) loss = normal_log_prob(label, m, v).sum() (-loss).backward() optimizer.step() it += 1 progressBar.update() progressBar.set_postfix({'loss': loss.item()}) progressBar.close() with torch.no_grad(): data = torch.tensor(Xpool).to(torch.float32).to(device) label = torch.tensor(ypool).to(torch.float32).to(device) m, v = mean(data), var(data) pool_m = m * y_std + y_mean pool_v = v * y_std**2 data = torch.tensor(Xtest).to(torch.float32).to(device) label = torch.tensor(ytest).to(torch.float32).to(device) m, v = mean(data), var(data) m = m * y_std + y_mean v = v * y_std**2 test_log_px = normal_log_prob(label, m, v) test_rmse = ((label - m.flatten())**2).mean().sqrt() return test_log_px.mean().item(), \ test_rmse.item(), \ pool_m.cpu().flatten().numpy(), \ pool_v.cpu().flatten().numpy()
def mcdnn(args, X, y, Xval, yval): if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.Dropout(p=0.05), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1), torch.nn.Dropout(p=0.05)) if torch.cuda.is_available() and args.cuda: mean.cuda() device=torch.device('cuda') else: device=torch.device('cpu') optimizer = torch.optim.Adam(mean.parameters(), lr=args.lr) it = 0 progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter') batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel) while it < args.iters: optimizer.zero_grad() data, label = next(batches) data = torch.tensor(data).to(torch.float32).to(device) label = torch.tensor(label).to(torch.float32).to(device) m = mean(data) loss = (m - label).abs().pow(2.0).mean() loss.backward() optimizer.step() it+=1 progressBar.update() progressBar.set_postfix({'loss': loss.item()}) progressBar.close() data = torch.tensor(Xval).to(torch.float32).to(device) label = torch.tensor(yval).to(torch.float32).to(device) samples = torch.zeros(Xval.shape[0], args.mcmc).to(device) for i in range(args.mcmc): samples[:,i] = mean(data).flatten() m, v = samples.mean(dim=1), samples.var(dim=1) log_probs = normal_log_prob(label, m, v) rmse = ((label - m.flatten())**2).mean().sqrt() return log_probs.mean().item(), rmse.item()
def bnn(args, X, y, Xval, yval): import tensorflow as tf import tensorflow_probability as tfp from tensorflow_probability import distributions as tfd tf.reset_default_graph() y, y_mean, y_std = normalize_y(y) if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 def VariationalNormal(name, shape, constraint=None): means = tf.get_variable(name+'_mean', initializer=tf.ones([1]), constraint=constraint) stds = tf.get_variable(name+'_std', initializer=-1.0*tf.ones([1])) return tfd.Normal(loc=means, scale=tf.nn.softplus(stds)) x_p = tf.placeholder(tf.float32, shape=(None, X.shape[1])) y_p = tf.placeholder(tf.float32, shape=(None, 1)) with tf.name_scope('model', values=[x_p]): layer1 = tfp.layers.DenseFlipout( units=n_neurons, activation='relu', kernel_posterior_fn = tfp.layers.default_mean_field_normal_fn(), bias_posterior_fn = tfp.layers.default_mean_field_normal_fn() ) layer2 = tfp.layers.DenseFlipout( units=1, activation='linear', kernel_posterior_fn = tfp.layers.default_mean_field_normal_fn(), bias_posterior_fn = tfp.layers.default_mean_field_normal_fn() ) predictions = layer2(layer1(x_p)) noise = VariationalNormal('noise', [1], constraint=tf.keras.constraints.NonNeg()) pred_distribution = tfd.Normal(loc=predictions, scale=noise.sample()) neg_log_prob = -tf.reduce_mean(pred_distribution.log_prob(y_p)) kl_div = sum(layer1.losses + layer2.losses) / X.shape[0] elbo_loss = neg_log_prob + kl_div with tf.name_scope("train"): optimizer = tf.train.AdamOptimizer(learning_rate=args.lr) train_op = optimizer.minimize(elbo_loss) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) it = 0 progressBar = tqdm(desc='Training BNN', total=args.iters, unit='iter') batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel) while it < args.iters: data, label = next(batches) _, l = sess.run([train_op, elbo_loss], feed_dict={x_p: data, y_p: label.reshape(-1, 1)}) progressBar.update() progressBar.set_postfix({'loss': l}) it+=1 progressBar.close() W0_samples = layer1.kernel_posterior.sample(1000) b0_samples = layer1.bias_posterior.sample(1000) W1_samples = layer2.kernel_posterior.sample(1000) b1_samples = layer2.bias_posterior.sample(1000) noise_samples = noise.sample(1000) W0, b0, W1, b1, n = sess.run([W0_samples, b0_samples, W1_samples, b1_samples, noise_samples]) def sample_net(x, W0, b0, W1, b1, n): h = np.maximum(np.matmul(x[np.newaxis], W0) + b0[:, np.newaxis, :], 0.0) return np.matmul(h, W1) + b1[:, np.newaxis, :] + n[:, np.newaxis, :] * np.random.randn() samples = sample_net(Xval, W0, b0, W1, b1, n) m = samples.mean(axis=0) v = samples.var(axis=0) m = m * y_std + y_mean v = v * y_std**2 log_probs = normal_log_prob(yval, m, v) rmse = math.sqrt(((m.flatten() - yval)**2).mean()) return log_probs.mean(), rmse
def ensnn(args, X, y, Xval, yval): if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 y, y_mean, y_std = normalize_y(y) ms, vs = [ ], [ ] for m in range(args.n_models): # initialize differently mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1)) var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1), torch.nn.Softplus()) if torch.cuda.is_available() and args.cuda: mean.cuda(); var.cuda(); device=torch.device('cuda') else: device=torch.device('cpu') optimizer = torch.optim.Adam(chain(mean.parameters(), var.parameters()), lr=args.lr) it = 0 progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter') batches = batchify(X, y, batch_size = args.batch_size, shuffel=args.shuffel) while it < args.iters: switch = 0.0#1.0 if it > args.iters/2 else 0.0 optimizer.zero_grad() data, label = next(batches) data = torch.tensor(data).to(torch.float32).to(device) label = torch.tensor(label).to(torch.float32).to(device) m, v = mean(data), var(data) v = switch*v + (1-switch)*torch.tensor([0.02**2], device=device) loss = normal_log_prob(label, m, v).sum() (-loss).backward() optimizer.step() it+=1 progressBar.update() progressBar.set_postfix({'loss': loss.item()}) progressBar.close() data = torch.tensor(Xval).to(torch.float32).to(device) label = torch.tensor(yval).to(torch.float32).to(device) m, v = mean(data), var(data) m = m * y_std + y_mean v = v * y_std**2 ms.append(m) vs.append(v) ms = torch.stack(ms) vs = torch.stack(vs) m = ms.mean(dim=0) v = (vs + ms**2).mean(dim=0) - m**2 log_px = normal_log_prob(label, m, v) rmse = ((label - m.flatten())**2).mean().sqrt() return log_px.mean().item(), rmse.item()
def nnlsmv(args, X, y, Xval, yval): if args.dataset == 'protein' or args.dataset == 'year_prediction': n_neurons = 100 else: n_neurons = 50 y, y_mean, y_std = normalize_y(y) mean = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1)) var = torch.nn.Sequential(torch.nn.Linear(X.shape[1], n_neurons), torch.nn.ReLU(), torch.nn.Linear(n_neurons, 1), torch.nn.Softplus()) if torch.cuda.is_available() and args.cuda: mean.cuda() var.cuda() device = torch.device('cuda') else: device = torch.device('cpu') optimizer = torch.optim.Adam(mean.parameters(), lr=args.lr) optimizer2 = torch.optim.Adam(var.parameters(), lr=args.lr) it = 0 opt_switch = 0 progressBar = tqdm(desc='Training nn', total=args.iters, unit='iter') batches = local_batchify(X, y, batch_size=args.batch_size, shuffel=args.shuffel) while it < args.iters: switch = 1.0 if it > args.iters / 2 else 0.0 if it % 11 == 0 and switch: opt_switch = opt_switch + 1 # change between var and mean optimizer data, label, mean_w, var_w = next(batches) data = torch.tensor(data).to(torch.float32).to(device) label = torch.tensor(label).to(torch.float32).to(device) mean_w = torch.tensor(mean_w).to(torch.float32).to(device) var_w = torch.tensor(var_w).to(torch.float32).to(device) if opt_switch % 2 == 0: optimizer.zero_grad() m, v = mean(data), var(data) v = switch * v + (1 - switch) * torch.tensor([0.02**2], device=device) loss = -(-v.log() - (m.flatten() - label)**2 / (2 * v)) / mean_w loss = loss.sum() loss.backward() optimizer.step() else: optimizer2.zero_grad() m, v = mean(data), var(data) loss = -(-v.log() - (m.flatten() - label)**2 / (2 * v)) / var_w loss = loss.sum() loss.backward() optimizer2.step() it += 1 progressBar.update() progressBar.set_postfix({'loss': loss.item()}) progressBar.close() data = torch.tensor(Xval).to(torch.float32).to(device) label = torch.tensor(yval).to(torch.float32).to(device) m, v = mean(data), var(data) m = m * y_std + y_mean v = v * y_std**2 log_px = normal_log_prob(label, m, v) rmse = ((label - m.flatten())**2).mean().sqrt() return log_px.mean().item(), rmse.item()