def test_epoch(self, epoch): # sets the model to train mode: no dropout is applied self.model.eval() # builds a confusion matrix #metric_maxvoted = ClassMetric(num_classes=self.nclasses) metric = ClassMetric(num_classes=self.nclasses) #metric_all_t = ClassMetric(num_classes=self.nclasses) with torch.no_grad(): for iteration, data in enumerate(self.validdataloader): inputs, targets = data if torch.cuda.is_available(): inputs = inputs.cuda() targets = targets.cuda() logprobabilities, deltas, pts, budget = self.model.forward( inputs.transpose(1, 2)) loss, stats = self.loss_criterion(logprobabilities, pts, targets, self.earliness_factor, self.entropy_factor, ptsepsilon=self.ptsepsilon) prediction, t_stop = self.model.predict( logprobabilities, deltas) stats = metric.add(stats) accuracy_metrics = metric.update_confmat( targets.mode(1)[0].detach().cpu().numpy(), prediction.detach().cpu().numpy()) stats["accuracy"] = accuracy_metrics["overall_accuracy"] stats["mean_accuracy"] = accuracy_metrics["accuracy"].mean() stats["mean_recall"] = accuracy_metrics["recall"].mean() stats["mean_precision"] = accuracy_metrics["precision"].mean() stats["mean_f1"] = accuracy_metrics["f1"].mean() stats["kappa"] = accuracy_metrics["kappa"] if t_stop is not None: earliness = (t_stop.float() / (inputs.shape[1] - 1)).mean() stats["earliness"] = metric.update_earliness( earliness.cpu().detach().numpy()) stats["confusion_matrix"] = metric.hist stats["targets"] = targets.cpu().numpy() stats["inputs"] = inputs.cpu().numpy() if deltas is not None: stats["deltas"] = deltas.detach().cpu().numpy() if pts is not None: stats["pts"] = pts.detach().cpu().numpy() if budget is not None: stats["budget"] = budget.detach().cpu().numpy() probas = logprobabilities.exp().transpose(0, 1) stats["probas"] = probas.detach().cpu().numpy() return stats
def train_epoch(self, epoch): # sets the model to train mode: dropout is applied self.model.train() # builds a confusion matrix metric = ClassMetric(num_classes=self.nclasses) for iteration, data in enumerate(self.traindataloader): self.optimizer.zero_grad() inputs, targets, meta = data if torch.cuda.is_available(): inputs = inputs.cuda() targets = targets.cuda() logprobabilities, deltas, pts, budget = self.model.forward( inputs.transpose(1, 2)) loss, stats = self.loss_criterion(logprobabilities, pts, targets, self.earliness_factor, self.entropy_factor, self.ptsepsilon, self.earliness_reward_power) loss.backward() if isinstance(self.optimizer, ScheduledOptim): self.optimizer.step_and_update_lr() lr = self.optimizer._optimizer.state_dict( )["param_groups"][0]["lr"] else: self.optimizer.step() lr = self.optimizer.state_dict()["param_groups"][0]["lr"] prediction, t_stop = self.model.predict(logprobabilities, deltas) stats = metric.add(stats) stats["lr"] = lr accuracy_metrics = metric.update_confmat( targets.mode(1)[0].detach().cpu().numpy(), prediction.detach().cpu().numpy()) stats["accuracy"] = accuracy_metrics["overall_accuracy"] stats["mean_accuracy"] = accuracy_metrics["accuracy"].mean() stats["mean_recall"] = accuracy_metrics["recall"].mean() stats["mean_precision"] = accuracy_metrics["precision"].mean() stats["mean_f1"] = accuracy_metrics["f1"].mean() stats["kappa"] = accuracy_metrics["kappa"] if t_stop is not None: earliness = (t_stop.float() / (inputs.shape[1] - 1)).mean() stats["earliness"] = metric.update_earliness( earliness.cpu().detach().numpy()) return stats
def test_epoch(self, dataloader, epoch=None): # sets the model to train mode: no dropout is applied self.model.eval() # builds a confusion matrix #metric_maxvoted = ClassMetric(num_classes=self.nclasses) metric = ClassMetric(num_classes=self.nclasses) #metric_all_t = ClassMetric(num_classes=self.nclasses) tstops = list() predictions = list() probas = list() ids_list = list() labels = list() with torch.no_grad(): for iteration, data in enumerate(dataloader): inputs, targets, ids = data if torch.cuda.is_available(): inputs = inputs.cuda() targets = targets.cuda() logprobabilities, deltas, pts, budget = self.model.forward( inputs.transpose(1, 2)) loss = F.nll_loss(logprobabilities, targets[:, 0]) stats = dict(loss=loss, ) prediction = self.model.predict(logprobabilities) t_stop = None ## enter numpy world prediction = prediction.detach().cpu().numpy() label = targets.mode(1)[0].detach().cpu().numpy() if t_stop is not None: t_stop = t_stop.cpu().detach().numpy() if pts is not None: pts = pts.detach().cpu().numpy() if deltas is not None: deltas = deltas.detach().cpu().numpy() if budget is not None: budget = budget.detach().cpu().numpy() if t_stop is not None: tstops.append(t_stop) predictions.append(prediction) labels.append(label) probas.append(logprobabilities.exp().detach().cpu().numpy()) ids_list.append(ids.detach().cpu().numpy()) stats = metric.add(stats) accuracy_metrics = metric.update_confmat(label, prediction) stats["accuracy"] = accuracy_metrics["overall_accuracy"] stats["mean_accuracy"] = accuracy_metrics["accuracy"].mean() #for cl in range(len(accuracy_metrics["accuracy"])): # acc = accuracy_metrics["accuracy"][cl] # stats["class_{}_accuracy".format(cl)] = acc stats["mean_recall"] = accuracy_metrics["recall"].mean() stats["mean_precision"] = accuracy_metrics["precision"].mean() stats["mean_f1"] = accuracy_metrics["f1"].mean() stats["kappa"] = accuracy_metrics["kappa"] if t_stop is not None: earliness = (t_stop.astype(float) / (inputs.shape[1] - 1)).mean() stats["earliness"] = metric.update_earliness(earliness) stats["confusion_matrix"] = copy.copy(metric.hist) stats["targets"] = targets.cpu().numpy() stats["inputs"] = inputs.cpu().numpy() if deltas is not None: stats["deltas"] = deltas if pts is not None: stats["pts"] = pts if budget is not None: stats["budget"] = budget if t_stop is not None: stats["t_stops"] = np.hstack(tstops) stats["predictions"] = np.hstack(predictions) # N stats["labels"] = np.hstack(labels) # N stats["probas"] = np.vstack(probas) # NxC stats["ids"] = np.hstack(ids_list) return stats