def log_epoch_results(self, outputs, stage): age_pred, age_true, gender_pred, gender_true = self.get_epoch_results( outputs) age_labels = [ "0-10", "11-20", "21-30", "31-40", "41-50", "51-60", "60-" ] cm_age = confusion_matrix_plot_as_array(age_true, age_pred, age_labels) cm_gender = confusion_matrix_plot_as_array(gender_true, gender_pred, ["Male", "Female"]) # Tensorboard logs age_accuracy = accuracy( torch.tensor(age_pred).to(self.device), (torch.tensor(age_true).to(self.device))).item() gender_accuracy = accuracy( torch.tensor(gender_pred).to(self.device), (torch.tensor(gender_true).to(self.device))).item() tb = self.logger.experiment tb.add_scalar(f"{stage}/Age accuracy", age_accuracy, self.current_epoch) tb.add_scalar(f"{stage}/Gender accuracy", gender_accuracy, self.current_epoch) tb.add_image(f"{stage}/Age", cm_age, self.current_epoch, dataformats="HWC") tb.add_image(f"{stage}/Gender", cm_gender, self.current_epoch, dataformats="HWC")
def test_accuracy(): pred = torch.tensor([0, 1, 2, 3]) target = torch.tensor([0, 1, 2, 2]) acc = accuracy(pred, target) assert acc.item() == 0.75 pred = torch.tensor([0, 1, 2, 2]) target = torch.tensor([0, 1, 1, 3]) acc = accuracy(pred, target) assert acc.item() == 0.50
def training_step(self, batch, batch_idx): assert self.trainer.max_epochs > self.hparams.burn_in_epochs if self.trainer.current_epoch < self.hparams.burn_in_epochs: for param in self.bert.parameters(): param.requires_grad_(False) for param in self.pooler.parameters(): param.requires_grad_(False) for param in self.company_layer.parameters(): param.requires_grad_(False) else: for param in self.parameters(): param.requires_grad_(True) input_ids, input_mask, label_ids, doc_ids = batch logits = self(batch) loss = cross_entropy(logits, label_ids, weight=self.weights.to(self.device)) preds = logits.argmax(dim=1) batch_dict = { 'loss': loss, 'logits': logits, 'preds': preds, 'labels': label_ids } self.log('train_loss', loss) self.log('train_acc', accuracy(logits, label_ids)) return batch_dict
def training_epoch_end(self, train_outputs): ''' Log all the values after the end of the epoch. ''' outputs = [x['result'] for x in train_outputs] avg_class_loss = torch.stack([x['class_loss'] for x in outputs]).mean() avg_mask_loss = torch.stack([x['mask_loss'] for x in outputs]).mean() avg_loss = torch.stack([x['total_loss'] for x in outputs]).mean() all_predictions = torch.stack([x['predictions'] for x in outputs]).flatten() all_targets = torch.stack([x['targets'] for x in outputs]).flatten() class_accuracy = accuracy(all_predictions, all_targets, num_classes=2) class_f1 = f1_score(all_predictions, all_targets, num_classes=2) self.log('train_class_loss', avg_class_loss, sync_dist=True) self.log('train_mask_loss', avg_mask_loss, sync_dist=True) self.log('train_loss', avg_loss, sync_dist=True) self.log('train_accuracy', class_accuracy, prog_bar=True, sync_dist=True) self.log('train_f1', class_f1, sync_dist=True)
def validation_step(self, batch, batch_idx): x, y = batch y_hat = self(x) val_loss = F.cross_entropy(y_hat, y) val_acc = accuracy(y_hat, y) self.log('val_loss', val_loss, prog_bar=True) self.log('val_acc', val_acc, prog_bar=True)
def training_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.cross_entropy(y_hat, y) acc = accuracy(y_hat, y) self.log('loss', loss) self.log('acc', acc, prog_bar=True) return loss
def validation_step(self, batch, batch_idx): x, y = batch output = self(x) loss = F.binary_cross_entropy(output, y.float()) acc = accuracy(torch.round(output), y.float()) self.log("val_loss", loss, logger=True) self.log("val_acc", acc, logger=True) return {"val_loss": loss, "val_acc": acc}
def validation_epoch_end(self, val_step_outputs): genre_preds, genre_ys = zip(*val_step_outputs) genre_preds, genre_ys = torch.cat(genre_preds), torch.cat(genre_ys) genre_accuracy = accuracy(genre_preds, genre_ys, num_classes=self.n_genres, class_reduction="weighted") self.log("val_genre_accuracy", genre_accuracy)
def training_step(self, batch, batch_idx): x, y = batch features = self.extract_features(x) y_hat = self.head(features[-1]) loss = F.cross_entropy(y_hat, y) acc = accuracy(y_hat, y) self.log('loss', loss) self.log('acc', acc, prog_bar=True) return loss
def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: """ Actual metric computation Args: pred: predicted labels target: ground truth labels mask: only calculate metrics where mask==1 Return: A Tensor with the classification score. """ return accuracy(pred=pred, target=target, num_classes=self.num_classes)
def validation_step(self, batch, batch_idx): inputs, targets = batch loss_function = self.loss #TODO: JOKER LOSS is probably not needed for validation output_1, output_2, output_3, output_concat = self(inputs) outputs_com = output_1 + output_2 + output_3 + output_3 + output_concat val_loss = loss_function(output_concat, targets) """ There is the individual accuracy, and combined accuracy """ _, predicted = torch.max(output_concat.data, 1) _, predicted_com = torch.max(outputs_com.data, 1) valid_acc = accuracy(predicted, targets) valid_acc_en = accuracy(predicted_com, targets) metrics = { 'val_loss': val_loss, 'val_acc': valid_acc, 'val_acc_en': valid_acc_en } self.log('val_acc', valid_acc, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val_acc_en', valid_acc_en, on_step=False, on_epoch=True, prog_bar=True, logger=True) return metrics
def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: """ Actual metric computation Args: pred: predicted labels target: ground truth labels Return: A Tensor with the classification score. """ return accuracy(pred=pred, target=target, num_classes=self.num_classes, reduction=self.reduction)
def test_multilabel_accuracy(): # Dense label indicator matrix format y1 = torch.tensor([[0, 1, 1], [1, 0, 1]]) y2 = torch.tensor([[0, 0, 1], [1, 0, 1]]) assert torch.allclose(accuracy(y1, y2, class_reduction='none'), torch.tensor([2 / 3, 1.])) assert torch.allclose(accuracy(y1, y1, class_reduction='none'), torch.tensor([1., 1.])) assert torch.allclose(accuracy(y2, y2, class_reduction='none'), torch.tensor([1., 1.])) assert torch.allclose(accuracy(y2, torch.logical_not(y2), class_reduction='none'), torch.tensor([0., 0.])) assert torch.allclose(accuracy(y1, torch.logical_not(y1), class_reduction='none'), torch.tensor([0., 0.])) with pytest.raises(RuntimeError): accuracy(y2, torch.zeros_like(y2), class_reduction='none')
def training_step(self, batch, batch_idx): array_rep, labels_list = batch atom_features = array_rep['atom_features'] bond_features = array_rep['bond_features'] atom_features = torch.tensor(atom_features, dtype=torch.float) bond_features = torch.tensor(bond_features, dtype=torch.float) if self.task == "regression": labels = torch.tensor(labels_list, dtype=torch.float) else: labels = torch.tensor(labels_list, dtype=torch.float) atom_features = atom_features.to(self.device_ext) bond_features = bond_features.to(self.device_ext) labels = labels.to(self.device_ext) y_pred = self(array_rep, atom_features, bond_features) if self.task == "regression": loss = F.mse_loss(y_pred, labels) # https://github.com/pytorch/ignite/issues/453 var_y = torch.var(labels, unbiased=False) r2 = 1.0 - F.mse_loss(y_pred, labels, reduction="mean") / var_y else: loss = F.binary_cross_entropy_with_logits(y_pred, labels) y_pred_proba = torch.sigmoid(y_pred) y_pred = y_pred_proba > 0.5 acc = accuracy(y_pred, labels) y_pred_proba = y_pred_proba.to('cpu').detach().numpy().tolist() from sklearn import metrics try: rocauc = metrics.roc_auc_score(labels_list, y_pred_proba) except Exception as es: rocauc = 0 if self.task == "regression": ret = {'loss': loss, 'train_r2': r2} else: ret = { 'loss': loss, 'train_acc': acc, 'train_rocauc': torch.tensor(rocauc, dtype=torch.float) } return ret
def test_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict: """ Runs one training step. This usually consists in the forward function followed by the loss function. :param batch: The output of your dataloader. :param batch_nb: Integer displaying which batch this is Returns: - dictionary containing the loss and the metrics to be added to the lightning logger. """ inputs, targets = batch model_out = self.forward(**inputs) loss_val = self.loss(model_out, targets) # in DP mode (default) make sure if result is scalar, there's another dim in the beginning if self.trainer.use_dp or self.trainer.use_ddp2: loss_val = loss_val.unsqueeze(0) self.log('test_loss',loss_val) y_hat=model_out['logits'] labels_hat = torch.argmax(y_hat, dim=1) y=targets['labels'] f1 = metrics.f1_score(labels_hat,y, class_reduction='weighted') prec =metrics.precision(labels_hat,y, class_reduction='weighted') recall = metrics.recall(labels_hat,y, class_reduction='weighted') acc = metrics.accuracy(labels_hat,y, class_reduction='weighted') # auroc = metrics.multiclass_auroc(labels_hat, y) self.log('test_batch_prec',prec) self.log('test_batch_f1',f1) self.log('test_batch_recall',recall) self.log('test_batch_weighted_acc', acc) # self.log('test_batch_auc_roc', auroc) from pytorch_lightning.metrics.functional import confusion_matrix # TODO CHANGE THIS # return (labels_hat, y) cm = confusion_matrix(preds = labels_hat,target=y,normalize=None, num_classes=50) # cm = confusion_matrix(preds = labels_hat,target=y,normalize=False, num_classes=len(y.unique())) self.test_conf_matrices.append(cm)
def validation_step(self, batch, batch_idx): input_ids, input_mask, label_ids, doc_ids = batch logits = self(batch) loss = cross_entropy(logits, label_ids, weight=self.weights.to(self.device)) preds = logits.argmax(dim=1) batch_dict = { 'loss': loss, 'logits': logits, 'preds': preds, 'labels': label_ids } self.log('val_loss', loss) self.log('val_acc', accuracy(logits, label_ids)) return batch_dict
def forward(self, pred: torch.Tensor, target: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: """ Actual metric computation Args: pred: predicted labels target: ground truth labels mask: only calculate metrics where mask==1 Return: A Tensor with the classification score. """ # (yuxian): much faster than pytorch-lightning's implementation if self.reduction == "elementwise_mean": correct = (pred == target).float() * mask.float() return correct.sum() / (mask.float().sum() + 1e-10) mask_fill = (1-mask).bool() pred = pred.masked_fill(mask=mask_fill, value=-1) target = target.masked_fill(mask=mask_fill, value=-1) return accuracy(pred=pred, target=target, num_classes=self.num_classes, reduction=self.reduction)
def training_step(self, batch, batch_idx): inputs, targets = batch loss_function = self.loss # step 1 (start from fine-grained jigsaw n=8) inputs1 = jigsaw_generator(inputs, 8) output_1, _, _, _ = self(inputs1) loss1 = loss_function(output_1, targets) * 1 # step 2 inputs2 = jigsaw_generator(inputs, 4) _, output_2, _, _ = self(inputs2) loss2 = loss_function(output_2, targets) * 1 # step 3 inputs3 = jigsaw_generator(inputs, 2) _, _, output_3, _ = self(inputs3) loss3 = loss_function(output_3, targets) * 1 # step 4 whole image, and vanilla loss. _, _, _, output_concat = self(inputs) concat_loss = loss_function(output_concat, targets) * 2 train_loss = loss1 + loss2 + loss3 + concat_loss # accuracy _, predicted = torch.max(output_concat.data, 1) train_acc = accuracy(predicted, targets) metrics = {'loss': train_loss, 'accuracy': train_acc} self.log('accuracy', train_acc, on_step=True, on_epoch=True, prog_bar=True, logger=True) return metrics
def log_epoch_results(self, outputs, stage): gender_pred = [] gender_true = [] for i, output in enumerate(outputs): gender_true.extend(self.tensor_to_array(output["gender"][0])) gender_pred.extend(self.tensor_to_array(output["gender"][1])) cm_gender = confusion_matrix_plot_as_array(gender_true, gender_pred, ["Male", "Female"]) # Tensorboard logs gender_accuracy = accuracy( torch.tensor(gender_pred).to(self.device), (torch.tensor(gender_true).to(self.device))).item() tb = self.logger.experiment self.log(f"{stage}/Gender accuracy", gender_accuracy) tb.add_image(f"{stage}/Gender", cm_gender, self.current_epoch, dataformats="HWC")
def validation_epoch_end(self, outputs): ''' Log all the values after the end of the epoch. ''' avg_class_loss = torch.stack([x['class_loss'] for x in outputs]).mean() avg_mask_loss = torch.stack([x['mask_loss'] for x in outputs]).mean() avg_loss = torch.stack([x['total_loss'] for x in outputs]).mean() all_predictions = torch.stack([x['predictions'] for x in outputs]).flatten() all_targets = torch.stack([x['targets'] for x in outputs]).flatten() class_accuracy = accuracy(all_predictions, all_targets, num_classes=2) class_f1 = f1_score(all_predictions, all_targets, num_classes=2) self.log('valid_class_loss', avg_class_loss) self.log('valid_mask_loss', avg_mask_loss) self.log('valid_loss', avg_loss) self.log('valid_accuracy', class_accuracy, prog_bar=True) self.log('valid_f1', class_f1)
def _get_metrics_at_optimal_cutoff( self ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Computes the ROC to find the optimal cut-off i.e. the probability threshold for which the difference between true positive rate and false positive rate is smallest. Then, computes the false positive rate, false negative rate and accuracy at this threshold (i.e. when the predicted probability is higher than the threshold the predicted label is 1 otherwise 0). :returns: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) """ preds, targets = self._get_preds_and_targets() if torch.unique(targets).numel() == 1: return torch.tensor(np.nan), torch.tensor(np.nan), torch.tensor( np.nan), torch.tensor(np.nan) fpr, tpr, thresholds = roc(preds, targets) optimal_idx = torch.argmax(tpr - fpr) optimal_threshold = thresholds[optimal_idx] acc = accuracy(preds > optimal_threshold, targets) false_negative_optimal = 1 - tpr[optimal_idx] false_positive_optimal = fpr[optimal_idx] return optimal_threshold, false_positive_optimal, false_negative_optimal, acc
def validation_step(self, batch, batch_idx): data, y = batch p = self.hparams.shot * self.hparams.test_way data_shot, data_query = data[:p], data[p:] proto = self(data_shot) proto = proto.reshape(self.hparams.shot, self.hparams.test_way, -1).mean(dim=0) label = torch.arange(self.hparams.test_way, device=self.device).repeat(self.hparams.query) logits = euclidean_metric(self(data_query), proto) loss = F.cross_entropy(logits, label) pred = torch.argmax(logits, dim=1) acc = accuracy(pred, label) self.log_dict({ 'valid_loss': loss, 'val_acc': acc }, prog_bar=True, on_step=True)
def validation_epoch_end(self, outputs): logits = torch.cat([o["logits"] for o in outputs], dim=0) labels = torch.cat([o["labels"] for o in outputs], dim=0) self.log( "val_f1", f1_score(torch.argmax(logits, dim=1), labels, class_reduction="macro").detach().cpu().item(), prog_bar=True, ) self.log( "val_acc", accuracy(torch.argmax(logits, dim=1), labels).detach().cpu().item(), prog_bar=True, ) self.log( "val_loss", self.loss(logits, labels).detach().cpu().item(), prog_bar=True, )
def validation_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict: """ Similar to the training step but with the model in eval mode. Returns: - dictionary passed to the validation_end function. """ inputs, targets = batch model_out = self.forward(**inputs) loss_val = self.loss(model_out, targets) y = targets["labels"] y_hat = model_out["logits"] # acc labels_hat = torch.argmax(y_hat, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) val_acc = torch.tensor(val_acc) if self.on_gpu: val_acc = val_acc.cuda(loss_val.device.index) # in DP mode (default) make sure if result is scalar, there's another dim in the beginning if self.trainer.use_dp or self.trainer.use_ddp2: loss_val = loss_val.unsqueeze(0) val_acc = val_acc.unsqueeze(0) self.log('val_loss',loss_val) f1 = metrics.f1_score(labels_hat, y,class_reduction='weighted') prec =metrics.precision(labels_hat, y,class_reduction='weighted') recall = metrics.recall(labels_hat, y,class_reduction='weighted') acc = metrics.accuracy(labels_hat, y,class_reduction='weighted') # auroc = metrics.multiclass_auroc(y_hat,y) self.log('val_prec',prec) self.log('val_f1',f1) self.log('val_recall',recall) self.log('val_acc_weighted', acc)
def log_epoch_results(self, outputs, stage): age_pred = [] age_true = [] for i, output in enumerate(outputs): age_true.extend(self.tensor_to_array(output["age"][0])) age_pred.extend(self.tensor_to_array(output["age"][1])) age_labels = [ "0-10", "11-20", "21-30", "31-40", "41-50", "51-60", "60-" ] cm_age = confusion_matrix_plot_as_array(age_true, age_pred, age_labels) # Tensorboard logs age_accuracy = accuracy( torch.tensor(age_pred).to(self.device), (torch.tensor(age_true).to(self.device))).item() tb = self.logger.experiment self.log(f"{stage}/Age accuracy", age_accuracy) tb.add_image(f"{stage}/Age", cm_age, self.current_epoch, dataformats="HWC")
def test_multilabel_accuracy(): # Dense label indicator matrix format y1 = torch.tensor([[0, 1, 1], [1, 0, 1]]) y2 = torch.tensor([[0, 0, 1], [1, 0, 1]]) assert torch.allclose(accuracy(y1, y2, class_reduction='none'), torch.tensor([2 / 3, 1.])) assert torch.allclose(accuracy(y1, y1, class_reduction='none'), torch.tensor([1., 1.])) assert torch.allclose(accuracy(y2, y2, class_reduction='none'), torch.tensor([1., 1.])) assert torch.allclose( accuracy(y2, torch.logical_not(y2), class_reduction='none'), torch.tensor([0., 0.])) assert torch.allclose( accuracy(y1, torch.logical_not(y1), class_reduction='none'), torch.tensor([0., 0.])) # num_classes does not match extracted number from input we expect a warning with pytest.warns(RuntimeWarning, match=r'You have set .* number of classes which is' r' different from predicted (.*) and' r' target (.*) number of classes'): _ = accuracy(y2, torch.zeros_like(y2), num_classes=3)
def accuracy_function(self, outputs, ground_truth): return dict(accuracy=FMC.accuracy( outputs, ground_truth, num_classes=self.hparams.num_classes))
def calculate_metrics(self, y, y_hat): loss = F.cross_entropy(y_hat, y) y_pred = y_hat.argmax(dim=1) acc = classification.accuracy(y_pred, y) f1_score = f1(y_pred, y, self.num_classes) return {"loss": loss, "acc": acc, "f1": f1_score}
def test_step(self, batch, batch_idx): x, y = batch output = self(x) loss = F.binary_cross_entropy(output, y.float()) acc = accuracy(torch.round(output), y.float()) return {"test_loss": loss, "test_acc": acc}