def metrics(self, pred, target, num_classes=3, remove_bg=False, is_swnet=False):
if is_swnet:
pred[pred == 2] = 1
target[target == 2] = 1
num_classes = 2
confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes)
# acc
accuracy = confusion_m.diag().sum() / len(pred)
# kappa
p0 = accuracy
pc = 0
for i in range(confusion_m.shape[0]):
pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum()
pc = pc / len(pred)**2
kc = (p0 - pc) / (1 - pc)
# iou
if remove_bg:
iou = mf.iou(pred, target, num_classes=num_classes, ignore_index=0)
else:
iou = mf.iou(pred, target, num_classes=num_classes)
f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none')
precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none')
recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none')
return accuracy, kc, iou, f1, precision, recall
def training_step(self, batch, batch_idx):
segs, label = self(batch)
seg = op.bu(segs[-1], label.size(2))
if hasattr(self, "loss_fn_layer"):
segloss = loss.segloss(segs, label, self.loss_fn_layer)
n_layers = len(segloss) - 1 # The last one is final segmentation
total_loss = 0
for i in range(len(segloss)): # 0 ~ len(segloss) - 1
layer = 'final' if i == n_layers else f'{i}'
layer_loss = segloss[i] * self.loss_layer_weight \
if i < n_layers else segloss[i]
self.log(f'layer/{layer}', layer_loss)
total_loss = total_loss + layer_loss
else:
total_loss = self.loss_fn_final(seg, label)
self.log("main/total", total_loss)
dt = seg.argmax(1).detach()
gt = label.detach()
IoU = iou(dt,
gt,
num_classes=self.model.n_class,
ignore_index=0,
absent_score=-1,
reduction='none')
pixelacc = (dt == gt).sum() / float(dt.shape.numel())
# pixelacc, mIoU, IoU
self.train_evaluation.append([pixelacc, IoU])
return total_loss
def training_step(self, batch, batch_idx):
x, y = batch
output = self(x, softmax=False, am=False)
sm_output = self.sm(output)
ce = self.ce_loss(output, y.squeeze(1))
dice = self.dice_loss(sm_output, y)
total_loss = (ce + dice) / 2
miou = iou(torch.argmax(sm_output, dim=1), y, self.num_classes)
dice_s = dice_score(sm_output, y.squeeze(1), bg=True)
self.log("Train MIoU",
miou.item(),
on_step=False,
on_epoch=True,
prog_bar=True)
self.log("Train Dice Score",
dice_s.item(),
on_step=False,
on_epoch=True,
prog_bar=True)
self.log("Train Loss",
total_loss.item(),
on_step=False,
on_epoch=True,
prog_bar=False)
return {'loss': total_loss}
def metrics(self, y_sig, y):
y_pred = y_sig > 0.5
return {
'acc': metricsF.accuracy(y_pred, y),
'roc': metricsF.auroc(y_sig, y),
'iou': metricsF.iou(y_pred, y),
}
def test_v1_5_metric_detect():
IoU.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
IoU(num_classes=1)
target = torch.randint(0, 2, (10, 25, 25))
preds = torch.tensor(target)
preds[2:5, 7:13, 9:15] = 1 - preds[2:5, 7:13, 9:15]
iou._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
res = iou(preds, target)
assert torch.allclose(res, torch.tensor(0.9660), atol=1e-4)
def validation_epoch_end(self, outputs):
preds = torch.cat([tmp['pred'] for tmp in outputs])
targets = torch.cat([tmp['target'] for tmp in outputs])
val_iou = iou(F.softmax(preds, dim=1),
targets,
num_classes=4,
reduction='none')
self.logger.experiment.add_scalars("val_iou", {
cat_name: cat_iou
for cat_name, cat_iou in zip(self.train_dataset.dataset.categories,
val_iou)
},
global_step=self.global_step)
def evaluate_batch(self, batch):
x, y = batch
# Forward propagation, loss calculation
outputs = self.forward(x)
loss = cross_entropy(outputs, y)
_, labels_hat = torch.max(outputs, 1)
weight = x.shape[0]
return {
'loss': loss * weight,
'acc': accuracy(labels_hat, y) * weight,
'dice': dice_score(outputs, y) * weight,
'iou': iou(labels_hat, y) * weight,
'weight': weight,
}
def validation_step(self, batch, batch_idx) -> object:
x, y = batch
output = self(x, softmax=False, am=False)
sm_output = self.sm(output)
ce = self.ce_loss(output, y.squeeze(1))
dice = self.dice_loss(sm_output, y.squeeze(1))
loss = (ce + dice) / 2
miou = iou(torch.argmax(sm_output, dim=1), y, self.num_classes)
dice_s = dice_score(sm_output, y.squeeze(1), bg=True)
return {
'v_loss': loss.item(),
'step_miou': miou.item(),
'v_dice_score': dice_s.item()
}
def evaluate_predictions(target_labels, sample_labels):
N, M = sample_labels.shape[:2]
n_class = sample_labels.max() + 1
res = []
for i in range(N):
gt = target_labels[i].cuda()
for j in range(M):
dt = sample_labels[i, j].cuda()
IoU = iou(
dt,
gt,
num_classes=n_class,
ignore_index=0, # This will cause background to be ignored
absent_score=-1, # resulting in n_class - 1 vectors
reduction='none').cpu()
pixelacc = (dt == gt).sum() / float(dt.shape.numel())
res.append([pixelacc, IoU])
return res
def validation_step(self, batch, batch_idx):
x, y = batch
# Forward propagation, loss calculation
outputs = self.forward(x)
loss = cross_entropy(outputs, y)
_, labels_hat = torch.max(outputs, 1)
weight = x.shape[0]
output = OrderedDict({
'loss': loss * weight,
'acc': accuracy(labels_hat, y) * weight,
'dice': dice_score(outputs, y) * weight,
'iou': iou(labels_hat, y) * weight,
'weight': weight,
})
return output
def training_step(self, batch, batch_ix):
sample_ixs, stacked_sample_ims, mask = batch
out = self.forward(stacked_sample_ims)
loss = F.cross_entropy(out, mask, reduction="mean")
# loss = focal_loss(out, mask, alpha=1, gamma=2, reduction="mean")
self.log('train_loss', loss)
self.logger.experiment.add_scalars("train_iou", {
cat_name: cat_iou
for cat_name, cat_iou in zip(
self.train_dataset.categories,
iou(F.softmax(out, dim=1),
mask,
num_classes=self.num_classes,
reduction='none'))
},
global_step=self.global_step)
return {'loss': loss}
def evaluate_SE(SE, G, P, resolution, num, ls='trunc-wp'):
learner = SELearner(SE, G, P, resolution=resolution,
latent_strategy=ls).cuda()
res = []
for i in tqdm(range(num)):
with torch.no_grad():
seg, label = learner(torch.randn(1, 512).cuda())
dt = seg[-1].argmax(1)
gt = label
IoU = iou(dt,
gt,
num_classes=SE.n_class,
ignore_index=0,
absent_score=-1,
reduction='none')
pixelacc = (dt == gt).sum() / float(dt.shape.numel())
res.append([pixelacc, IoU])
mIoU, c_ious = aggregate_iou(res)
return mIoU, c_ious
def calculate_class_miou(output, y, num_classes):
class_mious = torch.zeros(num_classes)
class_samples = torch.zeros(num_classes)
output = output.detach().clone()
output = torch.argmax(output, dim=1).unsqueeze(1)
for i in range(num_classes):
a = output.detach().clone()
b = y.detach().clone()
a[output != i] = 0
a[output == i] = 1
b[y != i] = 0
b[y == i] = 1
class_mious[i] = iou(a[:, 0, :, :], b, 2).item()
if i in torch.unique(y):
class_samples[i] += 1
else:
class_mious[i] = 0.0
return class_mious, class_samples
def main(*, module_type, checkpointPath, showCount, realDataPath,
trainDataPath, testDataPath):
# Ensure reproducibility
seed_everything(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
# Parse model
if module_type == 'mme':
model = MMETrainingModule.load_from_checkpoint(
checkpoint_path=checkpointPath, num_cls=4)
elif module_type in ['baseline', 'sandt', 'hm', 'CycleGAN']:
model = SimpleTrainModule.load_from_checkpoint(
checkpoint_path=checkpointPath, num_cls=4)
else:
raise RuntimeError(f"Cannot recognize module type {module_type}")
model.eval()
print(f"Loaded {model.__class__} instance.")
print(f"Model has the following hyperparameters: {model.hparams}")
# Get transform function
testTransform = MyTransform(augment=False)
if trainDataPath is not None and realDataPath is not None:
# Randomly sample showCount number of images from training and real folders
train_img_paths = glob.glob(os.path.join(trainDataPath, '*.png'))
train_img_paths = random.sample(train_img_paths, showCount)
real_img_paths = glob.glob(os.path.join(realDataPath, '*.png'))
real_img_paths = random.sample(real_img_paths, showCount)
# Create samples from training and real image predictions
finalResult = np.empty([0, 4 * model.width, 3], dtype=np.uint8)
for train_img_path, real_img_path in zip(train_img_paths,
real_img_paths):
train_img = cv2.imread(train_img_path, cv2.IMREAD_COLOR)
train_img = cv2.resize(train_img, (model.width, model.height),
cv2.INTER_LANCZOS4)
real_img = cv2.imread(real_img_path, cv2.IMREAD_COLOR)
real_img = cv2.resize(real_img, (model.width, model.height),
cv2.INTER_LANCZOS4)
img_batch = [train_img, real_img]
img_batch = torch.stack(
[testTransform(img_)[0] for img_ in img_batch])
_, pred = torch.max(model.forward(img_batch), 1)
pred = pred.byte()
pred = [pred_.squeeze().numpy() for pred_ in pred]
train_img2 = train_img.copy()
train_img2[pred[0] == 1] = (0, 255, 0) # Right lane
train_img2[pred[0] == 2] = (255, 0, 0) # Left lane
train_img2[pred[0] == 3] = (0, 0, 255) # Obstacles
real_img2 = real_img.copy()
real_img2[pred[1] == 1] = (0, 255, 0) # Right lane
real_img2[pred[1] == 2] = (255, 0, 0) # Left lane
real_img2[pred[1] == 3] = (0, 0, 255) # Obstacles
result = np.concatenate(
(train_img, train_img2, real_img, real_img2), axis=1)
finalResult = np.concatenate((finalResult, result), axis=0)
cv2.imwrite('results/samplePredictions.png', finalResult)
if testDataPath is not None:
# Perform qualitative evaluation
testDataset = RightLaneDataset(testDataPath,
transform=testTransform,
haveLabels=True)
testDataLoader = DataLoader(testDataset,
batch_size=32,
shuffle=False,
num_workers=8)
if torch.cuda.is_available():
model = model.cuda()
test_acc, test_dice, test_iou = 0.0, 0.0, 0.0
test_conf_matrix = torch.zeros(4, 4, device=model.device)
totalWeight = 0
for batch in tqdm(testDataLoader):
img, label = batch
if torch.cuda.is_available():
img, label = img.cuda(), label.cuda()
probas = model.forward(img)
_, label_hat = torch.max(probas, 1)
weight = img.shape[0]
test_acc += accuracy(label_hat, label) * weight
test_dice += dice_score(probas, label) * weight
test_iou += iou(label_hat, label) * weight
test_conf_matrix += confusion_matrix(label_hat,
label,
num_classes=4)
totalWeight += weight
assert totalWeight == len(testDataset)
if len(testDataset) > 0:
test_acc /= len(testDataset)
test_dice /= len(testDataset)
test_iou /= len(testDataset)
print(f"Accuracy on test set: {test_acc * 100.0:.4f}%")
print(f"Dice score on test set: {test_dice:.4f}")
print(f"IoU on test set: {test_iou * 100.0:.4f}")
print(f"Confusion matrix (column: prediction, row: label):")
print(test_conf_matrix)
print(f"Total: {torch.sum(test_conf_matrix)}")
