def metrics(self, pred, target, num_classes=3, remove_bg=False, is_swnet=False): if is_swnet: pred[pred == 2] = 1 target[target == 2] = 1 num_classes = 2 confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes) # acc accuracy = confusion_m.diag().sum() / len(pred) # kappa p0 = accuracy pc = 0 for i in range(confusion_m.shape[0]): pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum() pc = pc / len(pred)**2 kc = (p0 - pc) / (1 - pc) # iou if remove_bg: iou = mf.iou(pred, target, num_classes=num_classes, ignore_index=0) else: iou = mf.iou(pred, target, num_classes=num_classes) f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none') precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none') recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none') return accuracy, kc, iou, f1, precision, recall
def training_step(self, batch, batch_idx): segs, label = self(batch) seg = op.bu(segs[-1], label.size(2)) if hasattr(self, "loss_fn_layer"): segloss = loss.segloss(segs, label, self.loss_fn_layer) n_layers = len(segloss) - 1 # The last one is final segmentation total_loss = 0 for i in range(len(segloss)): # 0 ~ len(segloss) - 1 layer = 'final' if i == n_layers else f'{i}' layer_loss = segloss[i] * self.loss_layer_weight \ if i < n_layers else segloss[i] self.log(f'layer/{layer}', layer_loss) total_loss = total_loss + layer_loss else: total_loss = self.loss_fn_final(seg, label) self.log("main/total", total_loss) dt = seg.argmax(1).detach() gt = label.detach() IoU = iou(dt, gt, num_classes=self.model.n_class, ignore_index=0, absent_score=-1, reduction='none') pixelacc = (dt == gt).sum() / float(dt.shape.numel()) # pixelacc, mIoU, IoU self.train_evaluation.append([pixelacc, IoU]) return total_loss
def training_step(self, batch, batch_idx): x, y = batch output = self(x, softmax=False, am=False) sm_output = self.sm(output) ce = self.ce_loss(output, y.squeeze(1)) dice = self.dice_loss(sm_output, y) total_loss = (ce + dice) / 2 miou = iou(torch.argmax(sm_output, dim=1), y, self.num_classes) dice_s = dice_score(sm_output, y.squeeze(1), bg=True) self.log("Train MIoU", miou.item(), on_step=False, on_epoch=True, prog_bar=True) self.log("Train Dice Score", dice_s.item(), on_step=False, on_epoch=True, prog_bar=True) self.log("Train Loss", total_loss.item(), on_step=False, on_epoch=True, prog_bar=False) return {'loss': total_loss}
def metrics(self, y_sig, y): y_pred = y_sig > 0.5 return { 'acc': metricsF.accuracy(y_pred, y), 'roc': metricsF.auroc(y_sig, y), 'iou': metricsF.iou(y_pred, y), }
def test_v1_5_metric_detect(): IoU.__init__._warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): IoU(num_classes=1) target = torch.randint(0, 2, (10, 25, 25)) preds = torch.tensor(target) preds[2:5, 7:13, 9:15] = 1 - preds[2:5, 7:13, 9:15] iou._warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = iou(preds, target) assert torch.allclose(res, torch.tensor(0.9660), atol=1e-4)
def validation_epoch_end(self, outputs): preds = torch.cat([tmp['pred'] for tmp in outputs]) targets = torch.cat([tmp['target'] for tmp in outputs]) val_iou = iou(F.softmax(preds, dim=1), targets, num_classes=4, reduction='none') self.logger.experiment.add_scalars("val_iou", { cat_name: cat_iou for cat_name, cat_iou in zip(self.train_dataset.dataset.categories, val_iou) }, global_step=self.global_step)
def evaluate_batch(self, batch): x, y = batch # Forward propagation, loss calculation outputs = self.forward(x) loss = cross_entropy(outputs, y) _, labels_hat = torch.max(outputs, 1) weight = x.shape[0] return { 'loss': loss * weight, 'acc': accuracy(labels_hat, y) * weight, 'dice': dice_score(outputs, y) * weight, 'iou': iou(labels_hat, y) * weight, 'weight': weight, }
def validation_step(self, batch, batch_idx) -> object: x, y = batch output = self(x, softmax=False, am=False) sm_output = self.sm(output) ce = self.ce_loss(output, y.squeeze(1)) dice = self.dice_loss(sm_output, y.squeeze(1)) loss = (ce + dice) / 2 miou = iou(torch.argmax(sm_output, dim=1), y, self.num_classes) dice_s = dice_score(sm_output, y.squeeze(1), bg=True) return { 'v_loss': loss.item(), 'step_miou': miou.item(), 'v_dice_score': dice_s.item() }
def evaluate_predictions(target_labels, sample_labels): N, M = sample_labels.shape[:2] n_class = sample_labels.max() + 1 res = [] for i in range(N): gt = target_labels[i].cuda() for j in range(M): dt = sample_labels[i, j].cuda() IoU = iou( dt, gt, num_classes=n_class, ignore_index=0, # This will cause background to be ignored absent_score=-1, # resulting in n_class - 1 vectors reduction='none').cpu() pixelacc = (dt == gt).sum() / float(dt.shape.numel()) res.append([pixelacc, IoU]) return res
def validation_step(self, batch, batch_idx): x, y = batch # Forward propagation, loss calculation outputs = self.forward(x) loss = cross_entropy(outputs, y) _, labels_hat = torch.max(outputs, 1) weight = x.shape[0] output = OrderedDict({ 'loss': loss * weight, 'acc': accuracy(labels_hat, y) * weight, 'dice': dice_score(outputs, y) * weight, 'iou': iou(labels_hat, y) * weight, 'weight': weight, }) return output
def training_step(self, batch, batch_ix): sample_ixs, stacked_sample_ims, mask = batch out = self.forward(stacked_sample_ims) loss = F.cross_entropy(out, mask, reduction="mean") # loss = focal_loss(out, mask, alpha=1, gamma=2, reduction="mean") self.log('train_loss', loss) self.logger.experiment.add_scalars("train_iou", { cat_name: cat_iou for cat_name, cat_iou in zip( self.train_dataset.categories, iou(F.softmax(out, dim=1), mask, num_classes=self.num_classes, reduction='none')) }, global_step=self.global_step) return {'loss': loss}
def evaluate_SE(SE, G, P, resolution, num, ls='trunc-wp'): learner = SELearner(SE, G, P, resolution=resolution, latent_strategy=ls).cuda() res = [] for i in tqdm(range(num)): with torch.no_grad(): seg, label = learner(torch.randn(1, 512).cuda()) dt = seg[-1].argmax(1) gt = label IoU = iou(dt, gt, num_classes=SE.n_class, ignore_index=0, absent_score=-1, reduction='none') pixelacc = (dt == gt).sum() / float(dt.shape.numel()) res.append([pixelacc, IoU]) mIoU, c_ious = aggregate_iou(res) return mIoU, c_ious
def calculate_class_miou(output, y, num_classes): class_mious = torch.zeros(num_classes) class_samples = torch.zeros(num_classes) output = output.detach().clone() output = torch.argmax(output, dim=1).unsqueeze(1) for i in range(num_classes): a = output.detach().clone() b = y.detach().clone() a[output != i] = 0 a[output == i] = 1 b[y != i] = 0 b[y == i] = 1 class_mious[i] = iou(a[:, 0, :, :], b, 2).item() if i in torch.unique(y): class_samples[i] += 1 else: class_mious[i] = 0.0 return class_mious, class_samples
def main(*, module_type, checkpointPath, showCount, realDataPath, trainDataPath, testDataPath): # Ensure reproducibility seed_everything(42) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True # Parse model if module_type == 'mme': model = MMETrainingModule.load_from_checkpoint( checkpoint_path=checkpointPath, num_cls=4) elif module_type in ['baseline', 'sandt', 'hm', 'CycleGAN']: model = SimpleTrainModule.load_from_checkpoint( checkpoint_path=checkpointPath, num_cls=4) else: raise RuntimeError(f"Cannot recognize module type {module_type}") model.eval() print(f"Loaded {model.__class__} instance.") print(f"Model has the following hyperparameters: {model.hparams}") # Get transform function testTransform = MyTransform(augment=False) if trainDataPath is not None and realDataPath is not None: # Randomly sample showCount number of images from training and real folders train_img_paths = glob.glob(os.path.join(trainDataPath, '*.png')) train_img_paths = random.sample(train_img_paths, showCount) real_img_paths = glob.glob(os.path.join(realDataPath, '*.png')) real_img_paths = random.sample(real_img_paths, showCount) # Create samples from training and real image predictions finalResult = np.empty([0, 4 * model.width, 3], dtype=np.uint8) for train_img_path, real_img_path in zip(train_img_paths, real_img_paths): train_img = cv2.imread(train_img_path, cv2.IMREAD_COLOR) train_img = cv2.resize(train_img, (model.width, model.height), cv2.INTER_LANCZOS4) real_img = cv2.imread(real_img_path, cv2.IMREAD_COLOR) real_img = cv2.resize(real_img, (model.width, model.height), cv2.INTER_LANCZOS4) img_batch = [train_img, real_img] img_batch = torch.stack( [testTransform(img_)[0] for img_ in img_batch]) _, pred = torch.max(model.forward(img_batch), 1) pred = pred.byte() pred = [pred_.squeeze().numpy() for pred_ in pred] train_img2 = train_img.copy() train_img2[pred[0] == 1] = (0, 255, 0) # Right lane train_img2[pred[0] == 2] = (255, 0, 0) # Left lane train_img2[pred[0] == 3] = (0, 0, 255) # Obstacles real_img2 = real_img.copy() real_img2[pred[1] == 1] = (0, 255, 0) # Right lane real_img2[pred[1] == 2] = (255, 0, 0) # Left lane real_img2[pred[1] == 3] = (0, 0, 255) # Obstacles result = np.concatenate( (train_img, train_img2, real_img, real_img2), axis=1) finalResult = np.concatenate((finalResult, result), axis=0) cv2.imwrite('results/samplePredictions.png', finalResult) if testDataPath is not None: # Perform qualitative evaluation testDataset = RightLaneDataset(testDataPath, transform=testTransform, haveLabels=True) testDataLoader = DataLoader(testDataset, batch_size=32, shuffle=False, num_workers=8) if torch.cuda.is_available(): model = model.cuda() test_acc, test_dice, test_iou = 0.0, 0.0, 0.0 test_conf_matrix = torch.zeros(4, 4, device=model.device) totalWeight = 0 for batch in tqdm(testDataLoader): img, label = batch if torch.cuda.is_available(): img, label = img.cuda(), label.cuda() probas = model.forward(img) _, label_hat = torch.max(probas, 1) weight = img.shape[0] test_acc += accuracy(label_hat, label) * weight test_dice += dice_score(probas, label) * weight test_iou += iou(label_hat, label) * weight test_conf_matrix += confusion_matrix(label_hat, label, num_classes=4) totalWeight += weight assert totalWeight == len(testDataset) if len(testDataset) > 0: test_acc /= len(testDataset) test_dice /= len(testDataset) test_iou /= len(testDataset) print(f"Accuracy on test set: {test_acc * 100.0:.4f}%") print(f"Dice score on test set: {test_dice:.4f}") print(f"IoU on test set: {test_iou * 100.0:.4f}") print(f"Confusion matrix (column: prediction, row: label):") print(test_conf_matrix) print(f"Total: {torch.sum(test_conf_matrix)}")