def get_mask(self, input_, target):
saliency_ls = []
for j in range(len(target)):
input_single = input_[j:j + 1]
target_single = target[j].item()
if self.torchray_method == "grad_cam":
from torchray.attribution.grad_cam import grad_cam
saliency = grad_cam(
model=self.original_classifier,
input=input_single,
target=target_single,
saliency_layer='layer4',
)
saliency = self.grad_cam_upsampler(saliency)
elif self.torchray_method == "guided_backprop":
from torchray.attribution.guided_backprop import guided_backprop
saliency = guided_backprop(
model=self.original_classifier,
input=input_single,
target=target_single,
resize=(224, 224),
smooth=0.02,
)
else:
raise KeyError()
if saliency.max() == saliency.min():
saliency[:] = 1
else:
saliency = (saliency - saliency.min()) / (saliency.max() -
saliency.min())
saliency_ls.append(saliency.detach())
return torch.cat(saliency_ls, dim=0)
def compute_gradcam(model, preprocessed_image, label, saliency_layer=None):
saliency = grad_cam(model, preprocessed_image, label, saliency_layer=saliency_layer)
image_shape = (preprocessed_image.shape[-2], preprocessed_image.shape[-1])
saliency = F.interpolate(saliency, image_shape, mode="bilinear", align_corners=False)
grad = saliency.detach().cpu().clone().numpy() # 1, 1, 8, 8 for cifar10_resnet8
grad = np.concatenate((grad,) * 3, axis=1).squeeze() # 3, 8, 8
return grad
def get_torchray_saliency(original_classifier, input_, target, method):
upsampler = nn.Upsample(scale_factor=32,
mode='bilinear',
align_corners=True)
input_, target = input_.to(device), target.to(device)
saliency_ls = []
for j in range(len(target)):
input_single = input_[j:j + 1]
target_single = target[j].item()
if method == "grad_cam":
from torchray.attribution.grad_cam import grad_cam
saliency = grad_cam(
model=original_classifier,
input=input_single,
target=target_single,
saliency_layer='layer4',
)
saliency = upsampler(saliency)
elif method == "guided_backprop":
from torchray.attribution.guided_backprop import guided_backprop
saliency = guided_backprop(
model=original_classifier,
input=input_single,
target=target_single,
resize=(224, 224),
smooth=0.02,
)
else:
raise KeyError()
saliency_ls.append(saliency.detach())
mask = torch.cat(saliency_ls, dim=0)
binarized_mask = binarize_mask(mask.clone())
rectangular = torch.empty_like(binarized_mask)
box_coord_ls = [BoxCoords(0, 0, 0, 0)] * len(input_)
for idx in range(mask.size(0)):
if binarized_mask[idx].sum() == 0:
continue
m = binarized_mask[idx].squeeze().cpu().numpy()
rectangular[idx], box_coord_ls[idx] = get_rectangular_mask(m)
classifier_output = original_classifier(input_, return_intermediate=False)
_, max_indexes = classifier_output.data.max(1)
is_correct = target.eq(max_indexes).long()
return (
mask.squeeze().cpu().numpy(),
binarized_mask.cpu().numpy(),
rectangular.squeeze().cpu().numpy(),
is_correct.cpu().numpy(),
box_coord_ls,
)
def get_attribution(model,
input,
target,
method,
device,
saliency_layer='features.norm5',
iba_wrapper=None):
input = input.to(device)
input.requires_grad = True
# get attribution
if method == "grad_cam":
saliency_map = grad_cam(model,
input,
target,
saliency_layer=saliency_layer)
elif method == "extremal_perturbation":
saliency_map, _ = extremal_perturbation(model, input, target)
elif method == 'ib':
assert iba_wrapper, "Please give a iba wrapper as function parameter!"
saliency_map = iba_wrapper.iba(model, input, target, device)
elif method == 'reverse_ib':
assert iba_wrapper, "Please give a iba wrapper as function parameter!"
saliency_map = iba_wrapper.iba(model,
input,
target,
device,
reverse_lambda=True)
elif method == "gradient":
saliency_map = gradient(model, input, target)
elif method == "excitation_backprop":
saliency_map = excitation_backprop(model,
input,
target,
saliency_layer=saliency_layer)
elif method == "integrated_gradients":
ig = IntegratedGradients(model)
saliency_map, _ = ig.attribute(input,
target=target,
return_convergence_delta=True)
saliency_map = saliency_map.squeeze().mean(0)
# ib heatmap already a numpy array scaled to image size
if method != 'ib' and method != 'reverse_ib':
saliency_map = saliency_map.detach().cpu().numpy().squeeze()
shape = (224, 224)
saliency_map = resize(saliency_map,
shape,
order=1,
preserve_range=True)
return saliency_map
def gcam_l3(m, x, i):
return grad_cam(m, x, i, saliency_layer=model.encoder_q.layer3)
def gcam(m, x, i):
g = grad_cam(m, x, i, saliency_layer=model.encoder_q.layer4)
return g
def __next__(self):
self._lazy_init()
x, y = next(self.data_iterator)
torch.manual_seed(self.seed)
if self.log:
from torchray.benchmark.logging import mongo_load, mongo_save, \
data_from_mongo, data_to_mongo
try:
assert len(x) == 1
x = x.to(self.device)
class_ids = self.data.as_class_ids(y[0])
image_size = self.data.as_image_size(y[0])
results = {'pointing': {}, 'pointing_difficult': {}}
info = {}
rise_saliency = None
for class_id in class_ids:
# Try to recover this result from the log.
if self.log > 0:
image_name = self.data.as_image_name(y[0])
data = mongo_load(
self.db,
self.experiment.name,
f"{image_name}-{class_id}",
)
if data is not None:
data = data_from_mongo(data)
results['pointing'][class_id] = data['pointing']
results['pointing_difficult'][class_id] = data[
'pointing_difficult']
if self.debug:
print(f'{image_name}-{class_id} loaded from log')
continue
# TODO(av): should now be obsolete
if x.grad is not None:
x.grad.data.zero_()
if self.experiment.method == "center":
w, h = image_size
point = torch.tensor([[w / 2, h / 2]])
elif self.experiment.method == "gradient":
saliency = gradient(
self.model,
x,
class_id,
resize=image_size,
smooth=0.02,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "deconvnet":
saliency = deconvnet(
self.model,
x,
class_id,
resize=image_size,
smooth=0.02,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "guided_backprop":
saliency = guided_backprop(
self.model,
x,
class_id,
resize=image_size,
smooth=0.02,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "grad_cam":
saliency = grad_cam(
self.model,
x,
class_id,
saliency_layer=self.gradcam_layer,
resize=image_size,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "excitation_backprop":
saliency = excitation_backprop(
self.model,
x,
class_id,
self.saliency_layer,
resize=image_size,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "contrastive_excitation_backprop":
saliency = contrastive_excitation_backprop(
self.model,
x,
class_id,
saliency_layer=self.saliency_layer,
contrast_layer=self.contrast_layer,
resize=image_size,
get_backward_gradient=get_pointing_gradient)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "rise":
# For RISE, compute saliency map for all classes.
if rise_saliency is None:
rise_saliency = rise(self.model,
x,
resize=image_size,
seed=self.seed)
saliency = rise_saliency[:, class_id, :, :].unsqueeze(1)
point = _saliency_to_point(saliency)
info['saliency'] = saliency
elif self.experiment.method == "extremal_perturbation":
if self.experiment.dataset == 'voc_2007':
areas = [0.025, 0.05, 0.1, 0.2]
else:
areas = [0.018, 0.025, 0.05, 0.1]
if self.experiment.boom:
raise RuntimeError("BOOM!")
mask, energy = elp.extremal_perturbation(
self.model,
x,
class_id,
areas=areas,
num_levels=8,
step=7,
sigma=7 * 3,
max_iter=800,
debug=self.debug > 0,
jitter=True,
smooth=0.09,
resize=image_size,
perturbation='blur',
reward_func=elp.simple_reward,
variant=elp.PRESERVE_VARIANT,
)
saliency = mask.sum(dim=0, keepdim=True)
point = _saliency_to_point(saliency)
info = {
'saliency': saliency,
'mask': mask,
'areas': areas,
'energy': energy
}
else:
assert False
if False:
plt.figure()
plt.subplot(1, 2, 1)
imsc(saliency[0])
plt.plot(point[0, 0], point[0, 1], 'ro')
plt.subplot(1, 2, 2)
imsc(x[0])
plt.pause(0)
results['pointing'][class_id] = self.pointing.evaluate(
y[0], class_id, point[0])
results['pointing_difficult'][
class_id] = self.pointing_difficult.evaluate(
y[0], class_id, point[0])
if self.log > 0:
image_name = self.data.as_image_name(y[0])
mongo_save(
self.db, self.experiment.name,
f"{image_name}-{class_id}",
data_to_mongo({
'image_name':
image_name,
'class_id':
class_id,
'pointing':
results['pointing'][class_id],
'pointing_difficult':
results['pointing_difficult'][class_id],
}))
if self.log > 1:
mongo_save(self.db,
str(self.experiment.name) + "-details",
f"{image_name}-{class_id}", data_to_mongo(info))
return results
except Exception as ex:
raise ProcessingError(self, self.experiment, self.model, x, y,
class_id, image_size) from ex
from torchray.attribution.grad_cam import grad_cam from torchray.benchmark import get_example_data, plot_example # Obtain example data. model, x, category_id, _ = get_example_data() # Grad-CAM backprop. saliency = grad_cam(model, x, category_id, saliency_layer='features.29') # Plots. plot_example(x, saliency, 'grad-cam backprop', category_id)
from gradual_extrapolation import GradualExtrapolator
from torchvision import models
if __name__ == "__main__":
input_batch = read_images_2_batch()
# TODO: loop over all common networks
model = models.__dict__["vgg16"](pretrained=True)
model.eval()
category_id = get_category_IDs(model, input_batch)
deconvnet(model, input_batch, category_id)
print("Processing Grad-CAM...")
saliency_grad_cam = grad_cam(
model,
input_batch,
category_id,
saliency_layer="features.28",
)
plot_example(input_batch,
saliency_grad_cam,
"Grad-CAM",
category_id,
save_path="output_Grad-CAM.jpg")
print("Processing Gradual Grad-CAM...")
GradualExtrapolator.register_hooks(model)
# we just need to process images to feed hooks
model(input_batch)
saliency_gradual_grad_cam = GradualExtrapolator.get_smooth_map(
saliency_grad_cam)
def plot_map(model, dataloader, label=None, covid=False, saliency_layer=None):
"""Plot an example.
Args:
model: trained classification model
dataloader: containing input images.
label (str): Name of Category.
covid: whether the image is from the Covid Dataset or the Chesxtray Dataset.
saliency_layer: usually output of the last convolutional layer.
"""
if not covid:
FINDINGS = [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass',
'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema',
'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia'
]
else:
FINDINGS = ['Detector01', 'Detector2', 'Detector3']
try:
if not covid:
inputs, labels, filename, bbox = next(dataloader)
bbox = bbox.type(torch.cuda.IntTensor)
else:
inputs, labels, filename = next(dataloader)
except StopIteration:
print(
"All examples exhausted - rerun cells above to generate new examples to review"
)
return None
original = inputs.clone()
inputs = inputs.to(device)
original = original.to(device)
original.requires_grad = True
# create predictions for label of interest and all labels
pred = torch.sigmoid(model(original)).data.cpu().numpy()[0]
predx = ['%.3f' % elem for elem in list(pred)]
preds_concat = pd.concat([
pd.Series(FINDINGS),
pd.Series(predx),
pd.Series(labels.numpy().astype(bool)[0])
],
axis=1)
preds = pd.DataFrame(data=preds_concat)
preds.columns = ["Finding", "Predicted Probability", "Ground Truth"]
preds.set_index("Finding", inplace=True)
preds.sort_values(by='Predicted Probability',
inplace=True,
ascending=False)
cxr = inputs.data.cpu().numpy().squeeze().transpose(1, 2, 0)
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
cxr = std * cxr + mean
cxr = np.clip(cxr, 0, 1)
if not covid:
show_next(cxr, model, label, inputs, filename, bbox)
if covid and label is None:
label = preds.loc[preds["Ground Truth"] == True].index[0]
category_id = FINDINGS.index(label)
saliency = grad_cam(model,
original,
category_id,
saliency_layer=saliency_layer)
fig, (showcxr, heatmap) = plt.subplots(ncols=2, figsize=(14, 5))
showcxr.imshow(cxr)
showcxr.axis('off')
showcxr.set_title(filename[0])
if not covid:
rect_original = patches.Rectangle((bbox[0, 0], bbox[0, 1]),
bbox[0, 2],
bbox[0, 3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2)
showcxr.add_patch(rect_original)
hmap = sns.heatmap(saliency.detach().cpu().numpy().squeeze(),
cmap='viridis',
annot=False,
zorder=2,
linewidths=0)
hmap.axis('off')
hmap.set_title('TorchRay grad cam for category {}'.format(label),
fontsize=8)
plt.show()
print(preds)
if covid:
data_brixia = pd.read_csv("model/labels/metadata_global_v2.csv",
sep=";")
data_brixia.set_index("Filename", inplace=True)
score = data_brixia.loc[filename[0].replace(".jpg", ".dcm"),
"BrixiaScore"].astype(str)
print('Brixia 6 regions Score: ', '0' * (6 - len(score)) + score)
def plot_map(self,
model,
dataloader,
label,
methods=None,
covid=False,
regression=False,
saliency_layer=None,
overlay=False,
axes_a=None):
"""Plot an example.
Args:
model: trained classification model
dataloader: containing input images.
label (str): Name of Category.
methods: list of attribution methods
covid: whether the image is from the Covid Dataset or the Chesxtray Dataset.
saliency_layer: usually output of the last convolutional layer.
overlay: if display saliency map over image
axes_a: axis of plot
"""
if not covid:
FINDINGS = [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration',
'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation',
'Edema', 'Emphysema', 'Fibrosis', 'Pleural_Thickening',
'Hernia'
]
else:
FINDINGS = ['Detector01', 'Detector2', 'Detector3']
try:
if not covid:
inputs, labels, filename, bbox = next(dataloader)
bbox = bbox.type(torch.cuda.IntTensor)
else:
inputs, labels, filename = next(dataloader)
# for consistant return value
bbox = None
except StopIteration:
print(
"All examples exhausted - rerun cells above to generate new examples to review"
)
return None
original = inputs.clone()
inputs = inputs.to(device)
original = original.to(device)
original.requires_grad = True
# create predictions for label of interest and all labels
pred = torch.sigmoid(model(original)).data.cpu().numpy()[0]
predx = ['%.3f' % elem for elem in list(pred)]
preds_concat = pd.concat([
pd.Series(FINDINGS),
pd.Series(predx),
pd.Series(labels.numpy().astype(bool)[0])
],
axis=1)
preds = pd.DataFrame(data=preds_concat)
preds.columns = ["Finding", "Predicted Probability", "Ground Truth"]
preds.set_index("Finding", inplace=True)
preds.sort_values(by='Predicted Probability',
inplace=True,
ascending=False)
# normalize image
cxr = inputs.data.cpu().numpy().squeeze().transpose(1, 2, 0)
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
cxr = std * cxr + mean
cxr = np.clip(cxr, 0, 1)
# In case we want to visualize COVID, we use the highest probability as label for the visualization
if covid and label is None:
label = preds.loc[preds["Ground Truth"] == True].index[0]
category_id = FINDINGS.index(label)
# if not covid:
# show_iba(cxr, model, label, inputs, labels, filename, bbox)
# show_iba_new(cxr, model, label, inputs, labels, filename, bbox)
# show_next(cxr, model, label, inputs, filename, bbox)
if methods is None:
methods = [
'grad-cam backprop', 'gradient', 'deconvnet',
'excitation backprop', 'guided backprop', 'linear approx',
'original IB', 'IB with reversed mask'
]
# plot original data with bounding box, else show brixia score
showcxr = axes_a.flatten()[0]
showcxr.imshow(cxr)
showcxr.axis('off')
showcxr.set_title(filename[0])
if not covid:
rect_original = patches.Rectangle((bbox[0, 0], bbox[0, 1]),
bbox[0, 2],
bbox[0, 3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2)
showcxr.add_patch(rect_original)
else:
scores = self.local_score_dataset.getScore(filename[0])
color_list = ["green", "yellow", "red", "black"]
for idx, score in enumerate(scores):
row = (1 - idx % 3 / 2) * 0.8 + 0.1
col = idx // 3 * 0.8 + 0.1
plt.text(col,
row,
score,
color="white",
fontsize=36,
bbox=dict(facecolor=color_list[score], alpha=0.7),
transform=showcxr.transAxes)
# plot visulizations
for method, hmap in zip(methods, axes_a.flatten()[1:]):
if method == 'grad-cam backprop':
saliency = grad_cam(model,
original,
category_id,
saliency_layer=saliency_layer)
elif method == 'gradient':
saliency = torchray_gradient(model, original, category_id)
elif method == 'deconvnet':
saliency = deconvnet(model, original, category_id)
elif method == 'excitation backprop':
saliency = excitation_backprop(model,
original,
category_id,
saliency_layer=saliency_layer)
elif method == 'guided backprop':
saliency = guided_backprop(model, original, category_id)
elif method == 'linear approx':
saliency = linear_approx(model,
original,
category_id,
saliency_layer=saliency_layer)
elif method == 'original IB':
saliency = self.iba(original,
labels.squeeze(),
model_loss_closure_with_target=self.
softmax_crossentropy_loss_with_target)
elif method == 'IB with reversed mask':
saliency = self.iba(original,
labels.squeeze(),
reverse_lambda=True,
model_loss_closure_with_target=self.
softmax_crossentropy_loss_with_target)
if not overlay:
sns.heatmap(saliency.detach().cpu().numpy().squeeze(),
cmap='viridis',
annot=False,
square=True,
cbar=False,
zorder=2,
linewidths=0,
ax=hmap)
else:
# resize saliancy map
if type(saliency).__module__ == np.__name__:
np_saliency = saliency
else:
np_saliency = saliency.detach().cpu().numpy().squeeze()
np_saliency = to_saliency_map(np_saliency, cxr.shape[:2])
# plot saliency map with image
plot_saliency_map(np_saliency, cxr, ax=hmap, colorbar=False)
hmap.axis('off')
hmap.set_title('{} for category {}'.format(method, label),
fontsize=12)
return inputs, labels, filename, bbox, preds