def test__call__if_loss_is_None(cnn_model):
gradcam = Gradcam(cnn_model)
try:
gradcam(None, None)
assert False
except ValueError:
assert True
def test__call__if_seed_input_is_None(cnn_model):
gradcam = Gradcam(cnn_model)
try:
gradcam(SmoothedLoss(1), None)
assert False
except ValueError:
assert True
def test__call__if_seed_input_shape_is_invalid(model):
gradcam = Gradcam(model)
try:
gradcam(CategoricalScore(1, 2), np.random.sample((8, )))
assert False
except (ValueError, tf.errors.InvalidArgumentError):
# TF became to raise InvalidArgumentError from ver.2.0.2.
assert True
def test__call__if_model_has_multiple_io(multiple_io_model):
gradcam = Gradcam(multiple_io_model)
result = gradcam(
[CategoricalScore(1, 2), lambda x: x],
[np.random.sample((1, 8, 8, 3)),
np.random.sample((1, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__if_expand_cam_is_False_and_model_has_multiple_inputs(
multiple_inputs_model):
gradcam = Gradcam(multiple_inputs_model)
result = gradcam(
CategoricalScore(1, 2),
[np.random.sample((1, 8, 8, 3)),
np.random.sample((1, 10, 10, 3))],
expand_cam=False)
assert result.shape == (1, 8, 8)
def test__call__if_penultimate_layer_is_noexist_name(cnn_model):
gradcam = Gradcam(cnn_model)
try:
gradcam(SmoothedLoss(1),
np.random.sample((1, 8, 8, 3)),
penultimate_layer='hoge')
assert False
except ValueError:
assert True
def call_grad(model, model_modifier, loss, pixel_array):
gradcam = Gradcam(model, model_modifier=model_modifier, clone=False)
# Generate heatmap with GradCAM
cam = gradcam(
loss,
pixel_array,
penultimate_layer=-1, # model.layers number
)
return normalize(cam)
def test__call__if_model_has_multiple_io_when_batchsize_is_2(
multiple_io_model):
gradcam = Gradcam(multiple_io_model)
result = gradcam(
[CategoricalScore([1, 0], 2), lambda x: x],
[np.random.sample((2, 8, 8, 3)),
np.random.sample((2, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (2, 8, 8)
assert result[1].shape == (2, 10, 10)
def model_vis_broken(model):
images = np.asarray(load_anomaly_example(anomaly=True))
# Rendering
image_titles = ['Anomaly1', 'Anomaly2', 'Anomaly3', 'Anomaly4', 'Anomaly5']
subplot_args = {
'nrows': 1,
'ncols': 5,
'figsize': (9, 3),
'subplot_kw': {
'xticks': [],
'yticks': []
}
}
f, ax = plt.subplots(**subplot_args)
for i, title in enumerate(image_titles):
ax[i].set_title(title, fontsize=14)
ax[i].imshow(images[i])
plt.tight_layout()
plt.show()
# Then, when the softmax activation function is applied to the last layer of model,
# it may obstruct generating the attention images, so you need to replace the function
# to a linear function. Here, we does so using model_modifier.
def model_modifier(m):
m.layers[-1].activation = tf.keras.activations.linear
return m
# TODO: Score_function breaks things...
def score_function(output):
return (tf.constant(1), tf.constant(1), tf.constant(1), tf.constant(1),
tf.constant(1))
# Create Gradcam object
gradcam = Gradcam(model, model_modifier=model_modifier, clone=False)
# Generate heatmap with GradCAM
cam = gradcam(
score_function,
images,
penultimate_layer=-1, # model.layers number
)
cam = normalize(cam)
f, ax = plt.subplots(**subplot_args)
for i, title in enumerate(image_titles):
heatmap = np.uint8(cm.jet(cam[i])[..., :3] * 255)
ax[i].set_title(title, fontsize=14)
ax[i].imshow(images[i])
ax[i].imshow(heatmap, cmap='jet', alpha=0.5) # overlay
plt.tight_layout()
plt.show()
def test__call__if_loss_is_None(model):
gradcam = Gradcam(model)
with pytest.raises(ValueError):
gradcam(None, None)
def test__call__if_seed_input_has_not_batch_dim(cnn_model):
gradcam = Gradcam(cnn_model)
result = gradcam(SmoothedLoss(1), np.random.sample((8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_expand_cam_is_False(model):
gradcam = Gradcam(model)
result = gradcam(CategoricalScore(1, 2),
np.random.sample((1, 8, 8, 3)),
expand_cam=False)
assert result.shape == (1, 6, 6)
def test__call__(cnn_model):
gradcam = Gradcam(cnn_model)
result = gradcam(SmoothedLoss(1), np.random.sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_model_has_multiple_outputs(multiple_outputs_model):
gradcam = Gradcam(multiple_outputs_model)
result = gradcam([CategoricalScore(1, 2), lambda x: x],
np.random.sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_model_has_only_dense_layer(dense_model):
gradcam = Gradcam(dense_model)
with pytest.raises(ValueError):
gradcam(CategoricalScore(1, 2), np.random.sample((1, 3)))
def test__call__if_penultimate_layer_is_None(cnn_model):
gradcam = Gradcam(cnn_model)
result = gradcam(SmoothedLoss(1),
np.random.sample((1, 8, 8, 3)),
penultimate_layer=None)
assert result.shape == (1, 8, 8)
def test__call__if_penultimate_layer_is_None(model):
gradcam = Gradcam(model)
result = gradcam(CategoricalScore(1, 2),
np.random.sample((1, 8, 8, 3)),
penultimate_layer=None)
assert result.shape == (1, 8, 8)
def test__call__if_seed_input_has_not_batch_dim(model):
gradcam = Gradcam(model)
result = gradcam(CategoricalScore(1, 2), np.random.sample((8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_seed_input_is_None(model):
gradcam = Gradcam(model)
with pytest.raises(ValueError):
gradcam(CategoricalScore(1, 2), None)
# X = preprocess_input(images)
X=images
print(X.shape)
subplot_args = { 'nrows': 1, 'ncols':5, 'figsize': (21, 9),
'subplot_kw': {'xticks': [], 'yticks': []} }
f, ax = plt.subplots(**subplot_args)
# from skimage.color import rgb2gray
#
# img_gray = rgb2gray(img)
# for i, title in enumerate(image_titles):
# ax[i].set_title(title, fontsize=14)
# ax[i].imshow(images[i])
# plt.savefig('X-rays.png')
# plt.show()
gradcam = Gradcam(model,
model_modifier,
clone=True)
print(model.summary())
# Generate heatmap with GradCAM++
cam = gradcam(loss,
X,
penultimate_layer=-2, # model.layers number
)
cam = normalize(cam)
f, ax = plt.subplots(**subplot_args)
for i, title in enumerate(image_titles):
heatmap = np.uint8(cm.jet(cam[i])[..., :3] * 255)
ax[i].set_title(title, fontsize=14)
ax[i].imshow(images[i])
ax[i].imshow(heatmap, cmap='jet', alpha=0.5)
def test__call__if_normalize_gradient_is_True(self, conv_model):
cam = Gradcam(conv_model)
result = cam(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
normalize_gradient=True)
assert result.shape == (1, 8, 8)
# cam_angio = normalize(cam_angio[0])
# image_titles = ['Deep', 'Avascular', 'ORCC', 'Choriocapillaris', 'Choroid']
# subplot_args = { 'nrows': 1, 'ncols': 5, 'figsize': (9, 3),
# 'subplot_kw': {'xticks': [], 'yticks': []} }
# f, ax = plt.subplots(**subplot_args)
# for i, title in enumerate(image_titles):
# heatmap = np.uint8(cm.jet(cam_angio)[0, :, :, i] * 255)
# ax[i].set_title(title, fontsize=14)
# ax[i].imshow(x_angio[0, :, :, i, :])
# ax[i].imshow(heatmap, cmap='jet', alpha=0.5) # overlay
# plt.tight_layout()
# plt.show()
gradcam_struct = Gradcam(struct_model, model_modifier=model_modifier, clone=False)
# Generate heatmap with GradCAM
cam_struct = gradcam_struct(loss, x, penultimate_layer=36, expand_cam=True)
cam_struct = normalize(cam_struct[0])
image_titles = ['Deep', 'Avascular', 'ORCC', 'Choriocapillaris', 'Choroid']
subplot_args = { 'nrows': 1, 'ncols': 5, 'figsize': (9, 3),
'subplot_kw': {'xticks': [], 'yticks': []} }
f, ax = plt.subplots(**subplot_args)
for i, title in enumerate(image_titles):
heatmap = np.uint8(cm.jet(cam_struct)[0, :, :, i] * 255)
ax[i].set_title(title, fontsize=14)
ax[i].imshow(x_angio[0, :, :, i, :])
ax[i].imshow(heatmap, cmap='jet', alpha=0.5) # overlay
plt.tight_layout()
# In[7]:
def modifier(m):
m.layers[-1].activation = tf.keras.activations.relu
return m
# In[8]:
from matplotlib import cm
from tf_keras_vis.gradcam import Gradcam
from tf_keras_vis.utils import normalize
# Create Gradcam object
gradcam = Gradcam(model, model_modifier=modifier, clone=False)
# In[9]:
subplot_args = {
'nrows': 1,
'ncols': 6,
'figsize': (20, 5),
'subplot_kw': {
'xticks': [],
'yticks': []
}
}
f, ax = plt.subplots(**subplot_args)
counter = 0
ax[0].set_title('Original image', fontsize=16)
def test__call__(model):
gradcam = Gradcam(model)
result = gradcam(CategoricalScore(1, 2), np.random.sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_penultimate_layer_is_no_exist_name(model):
gradcam = Gradcam(model)
with pytest.raises(ValueError):
gradcam(CategoricalScore(1, 2),
np.random.sample((1, 8, 8, 3)),
penultimate_layer='hoge')
def saliency(img_path, multiclass_model, name):
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import scipy.ndimage as ndimage
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.applications.densenet import preprocess_input
import numpy as np
# Preparing input data for VGG16
# X = preprocess_input(images)
# X = 'https://firebasestorage.googleapis.com/v0/b/intelrad-a680a.appspot.com/o/images%2Fxrayyyy.jpg?alt=media&token=63a8460f-5c79-422c-91d6-c5a2704d2383'
X = img_path
X = preprocess_image(X)
# Rendering
import matplotlib.pyplot as plt
from tf_keras_vis.scorecam import Scorecam
from matplotlib import cm
from tf_keras_vis.gradcam import Gradcam
from tf_keras_vis.utils.model_modifiers import ReplaceToLinear
replace2linear = ReplaceToLinear()
# Create Gradcam object
gradcam = Gradcam(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Instead of using CategoricalScore object,
from tf_keras_vis.utils.scores import CategoricalScore
# 1 is the imagenet index corresponding to Goldfish, 294 to Bear and 413 to Assault Rifle.
score = CategoricalScore(4)
from tensorflow.keras import backend as K
from tf_keras_vis.saliency import Saliency
# from tf_keras_vis.utils import normalize
# Create Saliency object.
saliency = Saliency(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Generate saliency map
X = X.tolist()
X = np.asarray(X, dtype=np.float)
# saliency_map = saliency(score, X)
# Generate saliency map with smoothing that reduce noise by adding noise
saliency_map = saliency(
score,
X,
smooth_samples=20, # The number of calculating gradients iterations.
smooth_noise=0.20) # noise spread level.
# Generate saliency map
saliency_map = saliency(score, X)
# Render
# f, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 4))
plt.figure(figsize=(8, 8))
plt.imshow(saliency_map[0], cmap='jet', alpha=0.9)
plt.axis('off')
import os
img_dir = './static/img/saliency'
if (not os.path.isdir(img_dir)):
os.mkdir(img_dir)
plt.tight_layout()
plt.savefig(f'./static/img/saliency/{name}.svg',
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.close()
return "Done"
X = image # Saliency saliency = Saliency(model, model_modifier) saliency_map = saliency(loss, X, smooth_samples=20) saliency_map = normalize(saliency_map) # plt.figure() # plt.imshow((X[0]*255).astype(np.uint8)) # plt.figure() # plt.imshow(saliency_map[0]) plt.figure() plt.imshow((X[0].mean(axis=-1)*255).astype(np.uint8)) plt.imshow(255*saliency_map[0], cmap='jet', alpha=0.5) # Grad CAM from matplotlib import cm from tf_keras_vis.gradcam import Gradcam # Create Gradcam object gradcam = Gradcam(model, model_modifier) # Generate heatmap with GradCAM cam = gradcam(loss, X) cam = normalize(cam) plt.figure() heatmap = np.uint8(cm.jet(cam[0])[..., :3] * 255) plt.imshow(X[0]) plt.imshow(heatmap, cmap='jet', alpha=0.5)