def evaluate_per_depth_regression(net, X_train, y_train, X_test, y_test, batch_size=None, cuda=True): n_layers = net.model.n_layers bm = torch.ones((n_layers, n_layers)).tril() bin_mat = np.concatenate([np.zeros((1, n_layers)), bm], axis=0) fnll = net.f_neg_loglike if cuda: y_train = torch.from_numpy(y_train).cuda().long() y_test = torch.from_numpy(y_test).cuda().long() else: y_train = torch.from_numpy(y_train).long() y_test = torch.from_numpy(y_test).long() if batch_size is None: prob_mtx_pervec = net.vec_predict(X_train, bin_mat) prob_mtx_pervec_test = net.vec_predict(X_test, bin_mat) else: pervecs = [] for batch in generate_ind_batch(len(X_train), batch_size, False): pervecs.append(net.vec_predict(X_train[batch], bin_mat)) prob_mtx_pervec = torch.cat(pervecs, 1) pervecs = [] for batch in generate_ind_batch(len(X_test), batch_size, False): pervecs.append(net.vec_predict(X_test[batch], bin_mat)) prob_mtx_pervec_test = torch.cat(pervecs, 1) train_err_vec = [] train_ce_vec = [] test_err_vec = [] test_ce_vec = [] for d in range(n_layers + 1): minus_loglike = fnll(prob_mtx_pervec[d, :, :], y_train).mean(dim=0).item() err = rms(prob_mtx_pervec[d, :, :], y_train) train_err_vec.append(err) train_ce_vec.append(minus_loglike) for d in range(n_layers + 1): minus_loglike = fnll(prob_mtx_pervec_test[d, :, :], y_test).mean(dim=0).item() err = rms(prob_mtx_pervec[d, :, :], y_test) test_err_vec.append(err) test_ce_vec.append(minus_loglike) train_err_vec = np.array(train_err_vec) train_ce_vec = np.array(train_ce_vec) test_err_vec = np.array(test_err_vec) test_ce_vec = np.array(test_ce_vec) return train_err_vec, train_ce_vec, test_err_vec, test_ce_vec
def eval(self, x, y): # TODO: make computationally stable with logsoftmax and nll loss -> would require making a log prediction method self.model.eval() with torch.no_grad(): x, y = to_variable(var=(x, y), cuda=self.cuda) if not self.regression: y = y.long() act_vec = self.model.forward(x) if self.regression: means, model_stds = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression), get_std=True) mean_pred_negloglike = self.f_neg_loglike( means, y, model_std=model_stds).mean(dim=0).data err = rms(means, y).item() else: probs = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression)) mean_pred_negloglike = self.f_neg_loglike_test( torch.log(probs), y).mean(dim=0).data pred = probs.max( dim=1, keepdim=False)[1] # get the index of the max probability err = pred.ne(y.data).sum().item() / y.shape[0] return mean_pred_negloglike.item(), err
def fit(self, x, y): """Optimise stchastically estimated marginal joint of parameters and weights""" self.model.train() x, y = to_variable(var=(x, y), cuda=self.cuda) self.optimizer.zero_grad() sample_means = self.model.forward(x, self.train_samples) batch_size = x.shape[0] repeat_dims = [self.train_samples] + [ 1 for i in range(1, len(y.shape)) ] # Repeat batchwise without interleave y_expand = y.repeat( *repeat_dims) # targets are same across acts -> interleave sample_means_flat = sample_means.view( self.train_samples * batch_size, -1) # flattening results in batch_n changing first E_NLL = self.f_neg_loglike(sample_means_flat, y_expand).view(self.train_samples, batch_size).mean(dim=(0, 1)) minus_E_ELBO = E_NLL + self.model.get_KL() / self.N_train minus_E_ELBO.backward() self.optimizer.step() err = rms(sample_means.mean(dim=0), y).item() return -minus_E_ELBO.data.item(), E_NLL.data.item(), err
def eval(self, x, y): self.model.eval() x, y = to_variable(var=(x, y), cuda=self.cuda) mean, model_std = self.model.forward_predict(x, self.MC_samples) mean_pred_loglike = self.f_neg_loglike( mean, y, model_std=model_std).mean(dim=0).data err = rms(mean, y).item() return mean_pred_loglike.item(), err
def fit(self, x, y): """Standard training loop: MC dropout and Ensembles""" self.model.train() x, y = to_variable(var=(x, y), cuda=self.cuda) self.optimizer.zero_grad() mean = self.model.forward(x) NLL = self.f_neg_loglike(mean, y).mean(dim=0) NLL.backward() self.optimizer.step() err = rms(mean, y).item() return -NLL.data.item(), NLL.data.item(), err
def fit(self, x, y): """Optimise stchastically estimated marginal joint of parameters and weights""" self.model.train() x, y = to_variable(var=(x, y), cuda=self.cuda) if not self.regression: y = y.long() self.optimizer.zero_grad() act_vec = self.model.forward(x) prior_loglikes = self.model.get_w_prior_loglike(k=None) joint_loglike_per_depth = self.prob_model.get_w_joint_loglike( prior_loglikes, act_vec, y, self.f_neg_loglike, self.N_train) # size(depth) log_marginal_over_depth = self.prob_model.get_marg_loglike( joint_loglike_per_depth) loss = -log_marginal_over_depth / self.N_train loss.backward() self.optimizer.step() # Note this posterior is 1 it behind the parameter settings as it is estimated with acts before optim step log_depth_posteriors = self.prob_model.get_depth_log_posterior( joint_loglike_per_depth, log_marginal_over_depth) self.prob_model.current_posterior = log_depth_posteriors.exp() if self.regression: means, model_stds = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression), get_std=True) mean_pred_negloglike = self.f_neg_loglike( means, y, model_std=model_stds).mean(dim=0).data err = rms(means, y).item() else: probs = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression)) mean_pred_negloglike = self.f_neg_loglike_test( torch.log(probs), y).mean(dim=0).data pred = probs.max( dim=1, keepdim=False)[1] # get the index of the max probability err = pred.ne(y.data).sum().item() / y.shape[0] return log_marginal_over_depth.data.item(), mean_pred_negloglike.item( ), err
def fit(self, x, y): """Optimise stchastically estimated marginal joint of parameters and weights""" self.set_mode_train(train=True) x, y = to_variable(var=(x, y), cuda=self.cuda) if not self.regression: y = y.long() self.optimizer.zero_grad() act_vec = self.model.forward(x) prior_loglikes = self.model.get_w_prior_loglike(k=None) ELBO = self.prob_model.estimate_ELBO(prior_loglikes, act_vec, y, self.f_neg_loglike, self.N_train, Beta=1) loss = -ELBO / self.N_train loss.backward() self.optimizer.step() self.prob_model.current_posterior = self.prob_model.get_q_probs() if self.regression: means, model_stds = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression), get_std=True) mean_pred_negloglike = self.f_neg_loglike( means, y, model_std=model_stds).mean(dim=0).data err = rms(means, y).item() else: probs = depth_categorical.marginalise_d_predict( act_vec.data, self.prob_model.current_posterior, depth=None, softmax=(not self.regression)) mean_pred_negloglike = self.f_neg_loglike_test( torch.log(probs), y).mean(dim=0).data pred = probs.max( dim=1, keepdim=False)[1] # get the index of the max probability err = pred.ne(y.data).sum().item() / y.shape[0] # print(ELBO.shape, mean_pred_loglike.shape, err.shape) return ELBO.data.item(), mean_pred_negloglike.item(), err