def evaluate_per_depth_regression(net,
X_train,
y_train,
X_test,
y_test,
batch_size=None,
cuda=True):
n_layers = net.model.n_layers
bm = torch.ones((n_layers, n_layers)).tril()
bin_mat = np.concatenate([np.zeros((1, n_layers)), bm], axis=0)
fnll = net.f_neg_loglike
if cuda:
y_train = torch.from_numpy(y_train).cuda().long()
y_test = torch.from_numpy(y_test).cuda().long()
else:
y_train = torch.from_numpy(y_train).long()
y_test = torch.from_numpy(y_test).long()
if batch_size is None:
prob_mtx_pervec = net.vec_predict(X_train, bin_mat)
prob_mtx_pervec_test = net.vec_predict(X_test, bin_mat)
else:
pervecs = []
for batch in generate_ind_batch(len(X_train), batch_size, False):
pervecs.append(net.vec_predict(X_train[batch], bin_mat))
prob_mtx_pervec = torch.cat(pervecs, 1)
pervecs = []
for batch in generate_ind_batch(len(X_test), batch_size, False):
pervecs.append(net.vec_predict(X_test[batch], bin_mat))
prob_mtx_pervec_test = torch.cat(pervecs, 1)
train_err_vec = []
train_ce_vec = []
test_err_vec = []
test_ce_vec = []
for d in range(n_layers + 1):
minus_loglike = fnll(prob_mtx_pervec[d, :, :],
y_train).mean(dim=0).item()
err = rms(prob_mtx_pervec[d, :, :], y_train)
train_err_vec.append(err)
train_ce_vec.append(minus_loglike)
for d in range(n_layers + 1):
minus_loglike = fnll(prob_mtx_pervec_test[d, :, :],
y_test).mean(dim=0).item()
err = rms(prob_mtx_pervec[d, :, :], y_test)
test_err_vec.append(err)
test_ce_vec.append(minus_loglike)
train_err_vec = np.array(train_err_vec)
train_ce_vec = np.array(train_ce_vec)
test_err_vec = np.array(test_err_vec)
test_ce_vec = np.array(test_ce_vec)
return train_err_vec, train_ce_vec, test_err_vec, test_ce_vec
def eval(self, x, y):
# TODO: make computationally stable with logsoftmax and nll loss -> would require making a log prediction method
self.model.eval()
with torch.no_grad():
x, y = to_variable(var=(x, y), cuda=self.cuda)
if not self.regression:
y = y.long()
act_vec = self.model.forward(x)
if self.regression:
means, model_stds = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression),
get_std=True)
mean_pred_negloglike = self.f_neg_loglike(
means, y, model_std=model_stds).mean(dim=0).data
err = rms(means, y).item()
else:
probs = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression))
mean_pred_negloglike = self.f_neg_loglike_test(
torch.log(probs), y).mean(dim=0).data
pred = probs.max(
dim=1,
keepdim=False)[1] # get the index of the max probability
err = pred.ne(y.data).sum().item() / y.shape[0]
return mean_pred_negloglike.item(), err
def fit(self, x, y):
"""Optimise stchastically estimated marginal joint of parameters and weights"""
self.model.train()
x, y = to_variable(var=(x, y), cuda=self.cuda)
self.optimizer.zero_grad()
sample_means = self.model.forward(x, self.train_samples)
batch_size = x.shape[0]
repeat_dims = [self.train_samples] + [
1 for i in range(1, len(y.shape))
] # Repeat batchwise without interleave
y_expand = y.repeat(
*repeat_dims) # targets are same across acts -> interleave
sample_means_flat = sample_means.view(
self.train_samples * batch_size,
-1) # flattening results in batch_n changing first
E_NLL = self.f_neg_loglike(sample_means_flat,
y_expand).view(self.train_samples,
batch_size).mean(dim=(0, 1))
minus_E_ELBO = E_NLL + self.model.get_KL() / self.N_train
minus_E_ELBO.backward()
self.optimizer.step()
err = rms(sample_means.mean(dim=0), y).item()
return -minus_E_ELBO.data.item(), E_NLL.data.item(), err
def eval(self, x, y):
self.model.eval()
x, y = to_variable(var=(x, y), cuda=self.cuda)
mean, model_std = self.model.forward_predict(x, self.MC_samples)
mean_pred_loglike = self.f_neg_loglike(
mean, y, model_std=model_std).mean(dim=0).data
err = rms(mean, y).item()
return mean_pred_loglike.item(), err
def fit(self, x, y):
"""Standard training loop: MC dropout and Ensembles"""
self.model.train()
x, y = to_variable(var=(x, y), cuda=self.cuda)
self.optimizer.zero_grad()
mean = self.model.forward(x)
NLL = self.f_neg_loglike(mean, y).mean(dim=0)
NLL.backward()
self.optimizer.step()
err = rms(mean, y).item()
return -NLL.data.item(), NLL.data.item(), err
def fit(self, x, y):
"""Optimise stchastically estimated marginal joint of parameters and weights"""
self.model.train()
x, y = to_variable(var=(x, y), cuda=self.cuda)
if not self.regression:
y = y.long()
self.optimizer.zero_grad()
act_vec = self.model.forward(x)
prior_loglikes = self.model.get_w_prior_loglike(k=None)
joint_loglike_per_depth = self.prob_model.get_w_joint_loglike(
prior_loglikes, act_vec, y, self.f_neg_loglike,
self.N_train) # size(depth)
log_marginal_over_depth = self.prob_model.get_marg_loglike(
joint_loglike_per_depth)
loss = -log_marginal_over_depth / self.N_train
loss.backward()
self.optimizer.step()
# Note this posterior is 1 it behind the parameter settings as it is estimated with acts before optim step
log_depth_posteriors = self.prob_model.get_depth_log_posterior(
joint_loglike_per_depth, log_marginal_over_depth)
self.prob_model.current_posterior = log_depth_posteriors.exp()
if self.regression:
means, model_stds = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression),
get_std=True)
mean_pred_negloglike = self.f_neg_loglike(
means, y, model_std=model_stds).mean(dim=0).data
err = rms(means, y).item()
else:
probs = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression))
mean_pred_negloglike = self.f_neg_loglike_test(
torch.log(probs), y).mean(dim=0).data
pred = probs.max(
dim=1,
keepdim=False)[1] # get the index of the max probability
err = pred.ne(y.data).sum().item() / y.shape[0]
return log_marginal_over_depth.data.item(), mean_pred_negloglike.item(
), err
def fit(self, x, y):
"""Optimise stchastically estimated marginal joint of parameters and weights"""
self.set_mode_train(train=True)
x, y = to_variable(var=(x, y), cuda=self.cuda)
if not self.regression:
y = y.long()
self.optimizer.zero_grad()
act_vec = self.model.forward(x)
prior_loglikes = self.model.get_w_prior_loglike(k=None)
ELBO = self.prob_model.estimate_ELBO(prior_loglikes,
act_vec,
y,
self.f_neg_loglike,
self.N_train,
Beta=1)
loss = -ELBO / self.N_train
loss.backward()
self.optimizer.step()
self.prob_model.current_posterior = self.prob_model.get_q_probs()
if self.regression:
means, model_stds = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression),
get_std=True)
mean_pred_negloglike = self.f_neg_loglike(
means, y, model_std=model_stds).mean(dim=0).data
err = rms(means, y).item()
else:
probs = depth_categorical.marginalise_d_predict(
act_vec.data,
self.prob_model.current_posterior,
depth=None,
softmax=(not self.regression))
mean_pred_negloglike = self.f_neg_loglike_test(
torch.log(probs), y).mean(dim=0).data
pred = probs.max(
dim=1,
keepdim=False)[1] # get the index of the max probability
err = pred.ne(y.data).sum().item() / y.shape[0]
# print(ELBO.shape, mean_pred_loglike.shape, err.shape)
return ELBO.data.item(), mean_pred_negloglike.item(), err