def test_v1_4_0_deprecated_metrics():
from pytorch_lightning.metrics.functional.classification import stat_scores_multiple_classes
with pytest.deprecated_call(match='will be removed in v1.4'):
stat_scores_multiple_classes(pred=torch.tensor([0, 1]),
target=torch.tensor([0, 1]))
from pytorch_lightning.metrics.functional.classification import iou
with pytest.deprecated_call(match='will be removed in v1.4'):
iou(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3)))
from pytorch_lightning.metrics.functional.classification import recall
with pytest.deprecated_call(match='will be removed in v1.4'):
recall(torch.randint(0, 2, (10, 3, 3)),
torch.randint(0, 2, (10, 3, 3)))
from pytorch_lightning.metrics.functional.classification import precision
with pytest.deprecated_call(match='will be removed in v1.4'):
precision(torch.randint(0, 2, (10, 3, 3)),
torch.randint(0, 2, (10, 3, 3)))
from pytorch_lightning.metrics.functional.classification import precision_recall
with pytest.deprecated_call(match='will be removed in v1.4'):
precision_recall(torch.randint(0, 2, (10, 3, 3)),
torch.randint(0, 2, (10, 3, 3)))
# Testing deprecation of class_reduction arg in the *new* precision
from pytorch_lightning.metrics.functional import precision
with pytest.deprecated_call(match='will be removed in v1.4'):
precision(torch.randint(0, 2, (10, )),
torch.randint(0, 2, (10, )),
class_reduction='micro')
# Testing deprecation of class_reduction arg in the *new* recall
from pytorch_lightning.metrics.functional import recall
with pytest.deprecated_call(match='will be removed in v1.4'):
recall(torch.randint(0, 2, (10, )),
torch.randint(0, 2, (10, )),
class_reduction='micro')
from pytorch_lightning.metrics.functional.classification import auc
with pytest.deprecated_call(match='will be removed in v1.4'):
auc(torch.rand(10, ).sort().values, torch.rand(10, ))
from pytorch_lightning.metrics.functional.classification import auroc
with pytest.deprecated_call(match='will be removed in v1.4'):
auroc(torch.rand(10, ), torch.randint(0, 2, (10, )))
from pytorch_lightning.metrics.functional.classification import multiclass_auroc
with pytest.deprecated_call(match='will be removed in v1.4'):
multiclass_auroc(torch.rand(20, 5).softmax(dim=-1),
torch.randint(0, 5, (20, )),
num_classes=5)
from pytorch_lightning.metrics.functional.classification import auc_decorator
with pytest.deprecated_call(match='will be removed in v1.4'):
auc_decorator()
from pytorch_lightning.metrics.functional.classification import multiclass_auc_decorator
with pytest.deprecated_call(match='will be removed in v1.4'):
multiclass_auc_decorator()
def test_iou_input_check():
with pytest.raises(
ValueError,
match=r"'pred' shape (.*) must equal 'target' shape (.*)"):
_ = iou(pred=torch.randint(0, 2, (3, 4, 3)),
target=torch.randint(0, 2, (3, 3)))
with pytest.raises(ValueError,
match="'pred' must contain integer targets."):
_ = iou(pred=torch.rand((3, 3)), target=torch.randint(0, 2, (3, 3)))
def test_step_end(self, outputs):
mask = outputs["labels"] != -1
test_iou = iou(
torch.argmax(outputs["preds"], 1)[mask],
outputs["labels"][mask],
)
self.log("test_iou", test_iou, prog_bar=True)
def test_iou(half_ones, reduction, remove_bg, expected):
pred = (torch.arange(120) % 3).view(-1, 1)
target = (torch.arange(120) % 3).view(-1, 1)
if half_ones:
pred[:60] = 1
iou_val = iou(pred, target, remove_bg=remove_bg, reduction=reduction)
assert torch.allclose(iou_val, expected, atol=1e-9)
def validation_step_end(self, outputs):
mask = outputs["labels"] != -1
val_iou = iou(
torch.argmax(outputs["preds"], 1)[mask],
outputs["labels"][mask],
)
self.log("val_iou", val_iou, prog_bar=True)
def test_iou_ignore_index(pred, target, ignore_index, num_classes, reduction, expected):
iou_val = iou(
pred=torch.tensor(pred),
target=torch.tensor(target),
ignore_index=ignore_index,
num_classes=num_classes,
reduction=reduction,
)
assert torch.allclose(iou_val, torch.tensor(expected).to(iou_val))
def test_iou_absent_score(pred, target, ignore_index, absent_score, num_classes, expected):
iou_val = iou(
pred=torch.tensor(pred),
target=torch.tensor(target),
ignore_index=ignore_index,
absent_score=absent_score,
num_classes=num_classes,
reduction='none',
)
assert torch.allclose(iou_val, torch.tensor(expected).to(iou_val))
def forward(self,
y_pred: torch.Tensor,
y_true: torch.Tensor,
sample_weight: Optional[torch.Tensor] = None):
"""
Actual metric calculation.
"""
return iou(y_pred,
y_true,
remove_bg=self.remove_bg,
reduction=self.reduction)
def test_iou(half_ones, reduction, ignore_index, expected):
pred = (torch.arange(120) % 3).view(-1, 1)
target = (torch.arange(120) % 3).view(-1, 1)
if half_ones:
pred[:60] = 1
iou_val = iou(
pred=pred,
target=target,
ignore_index=ignore_index,
reduction=reduction,
)
assert torch.allclose(iou_val, expected, atol=1e-9)
def validation_step(self, batch, batch_idx):
#img, mask = batch
img, mask = batch['image'], batch['label']#torch.LongTensor(batch['label'])
img, mask = img.cuda(), mask.cuda()
img = Variable(img, requires_grad=False)
logits = self.forward(img)
#weight=self.class_weights
loss = nn.CrossEntropyLoss(
weight=self.class_weights,
ignore_index=self.IGNORE_IDX)(logits, mask)
miou = iou(logits.argmax(axis=1), mask, ignore_index=self.IGNORE_IDX)
self.log('val_loss', loss, on_epoch=True)
self.log('val_miou', miou, on_epoch=True)
def training_step(self, batch, batch_idx):
#img, mask = batch
img, mask = batch['image'], batch['label']
img, mask = img.cuda(), mask.cuda()
img, mask = Variable(img, requires_grad=True), Variable(mask, requires_grad=False)
logits = self.forward(img)
loss = nn.CrossEntropyLoss(
weight=self.class_weights, ignore_index=255)(logits, mask)
miou = iou(logits.argmax(axis=1), mask, ignore_index=self.IGNORE_IDX)
self.log('train_loss', loss, on_epoch=True)
self.log('train_miou', miou, on_epoch=True)
return loss
def forward(self,
y_pred: torch.Tensor,
y_true: torch.Tensor,
sample_weight: Optional[torch.Tensor] = None):
"""
Actual metric calculation.
"""
return iou(
pred=y_pred,
target=y_true,
ignore_index=self.ignore_index,
absent_score=self.absent_score,
num_classes=self.num_classes,
reduction=self.reduction,
)
def test_epoch_end(self, outputs):
"""
After going through the entire test set, we compute the final ROC curve accumulated overall
predictions and target masks, then compute the AUROC
"""
if self.hparams.auroc:
# Compute ROC, then compute AUROC and log the value for the whole test set
fpr, tpr, thresholds = self.roc.compute()
fpr, idx = torch.sort(fpr, descending=False)
tpr, thresholds = tpr[idx], thresholds[idx]
auroc = auc(fpr, tpr)
self.log('auroc_test', auroc)
# Divide thresholds from ROC into 100 equally separated thresholds
step_size = int(len(thresholds)/100)
thresholds = thresholds[::step_size]
# Find best best threshold based off of best IOU
best_iou = 0
best_threshold = -1
# For each threshold, compute IOU for whole test set
for i, threshold in enumerate(thresholds):
test_dataloader = self.trainer.datamodule.test_dataloader()[1]
ious = []
for batch_idx, (x, y) in enumerate(test_dataloader):
x, y = x.to(self.device), y.to(self.device)
x_rec, M, colormaps = self.forward(x)
bloc_map = self.gen_bloc_map(M, threshold)
iou_score = iou(bloc_map, y)
ious.append(iou_score.detach().cpu().item())
avg_iou = np.mean(ious)
if avg_iou > best_iou:
best_iou = avg_iou
best_threshold = threshold
self.trainer.logger.experiment.add_scalar('avg_iou', avg_iou, i)
self.trainer.logger.experiment.add_scalar('threshold', threshold, i)
# Log best iou and threshold
self.log('best_iou', best_iou)
self.log('best_threshold', best_threshold)
# Now, using best threshold, generate the binary localization maps for
# all images in the test set and log/save them
for batch_idx, (x, y) in enumerate(test_dataloader):
x, y = x.to(self.device), y.to(self.device)
x_rec, M, colormaps = self.forward(x)
bloc_map = self.gen_bloc_map(M, best_threshold)
# Save the binary localization maps
bloc_map = bloc_map.detach().cpu()
bloc_map_grid = make_grid(bloc_map).float()
save_image(bloc_map_grid, f'{self.trainer.logger.log_dir}/batch{batch_idx}-blocmaps.png')
self.trainer.logger.experiment.add_image('blocmaps', bloc_map_grid.numpy(), batch_idx)
# Save the input images
x = x.detach().cpu()
x = self.trainer.datamodule.unnormalize_batch(x)
x_grid = make_grid(x).float()
save_image(x_grid, f'{self.trainer.logger.log_dir}/batch{batch_idx}-input.png')
self.trainer.logger.experiment.add_image('input', x_grid.numpy(), batch_idx)
# Save teh target masks
y = y.detach().cpu()
y_grid = make_grid(y).float()
save_image(y_grid, f'{self.trainer.logger.log_dir}/batch{batch_idx}-targets.png')
self.trainer.logger.experiment.add_image('targets', y_grid.numpy(), batch_idx)