def test__call__if_loss_is_None(cnn_model):
saliency = Saliency(cnn_model)
try:
saliency(None, None)
assert False
except ValueError:
assert True
def init_salient(self, model):
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
model.summary()
self.output_names = []
for i, layer in enumerate(model.layers):
if "dropout" not in layer.name.lower(
) and "out" in layer.name.lower():
self.output_names += [layer.name]
if len(self.output_names) == 0:
print(
"Failed to find the model layer named with 'out'. Skipping salient."
)
return False
print("####################")
print("Visualizing activations on layer:", *self.output_names)
print("####################")
# Create Saliency object.
# If `clone` is True(default), the `model` will be cloned,
# so the `model` instance will be NOT modified, but it takes a machine resources.
self.saliency = Saliency(model,
model_modifier=self.model_modifier,
clone=False)
# Create GradCAM++ object, Just only repalce class name to "GradcamPlusPlus"
self.gradcampp = GradcamPlusPlus(model,
model_modifier=self.model_modifier,
clone=False)
return True
def test__call__if_seed_input_is_None(cnn_model):
saliency = Saliency(cnn_model)
try:
saliency(SmoothedLoss(1), None)
assert False
except ValueError:
assert True
def get_heatmap(self,
target_shape: Tuple,
clean_model: Model,
layers_to_cut: int = 0) -> np.ndarray:
"""
Generates a heatmap for a given batch of source files using `self.x_batch` as an input
:param target_shape: should be the same as the shape to the input layer of the model
:param clean_model: empty model (same architecture as the model in the constructor) to work with
:param layers_to_cut: the number of layers, which depend on the embedding layer
:return: a heatmap (Saliency map) for a given batch of source files using `self.x_batch` as an input
"""
def model_modifier(m: Model) -> Model:
model = clean_model
for layer_index in range(layers_to_cut, len(m.layers)):
weights = m.layers[layer_index].get_weights()
if weights: # else - dropout or normalization
model.layers[layer_index - layers_to_cut +
1].set_weights(weights)
return model
x_batch = tf.reshape(self.x_batch, target_shape)
layers = self.model.layers
for layer in layers[:layers_to_cut]:
x_batch = layer(x_batch)
modified_model = model_modifier(self.model)
saliency = Saliency(modified_model, model_modifier=None, clone=False)
saliency_map = saliency(self.loss, x_batch)
heatmap = saliency_map
assert heatmap.max() != 0.0, "the loss value is too small"
return heatmap
def Explain_tfkerasvis(self, X, model, mostCommonIndex):
"""
Generate gradCAM plusplus images, Saliency Maps and Score cam images and save them.
Seems to work very well, but it does not generate channel specific saliency maps.
"""
from matplotlib import pyplot as plt
from matplotlib import cm
import tensorflow as tf
from tf_keras_vis.utils import num_of_gpus, normalize
from tf_keras_vis.gradcam import GradcamPlusPlus, Gradcam
from tf_keras_vis.saliency import Saliency
from tf_keras_vis.scorecam import ScoreCAM
# The `output` variable refer to the output of the model,
# so, in this case, `output` shape is ` (samples, classes).
def loss(output):
# mostCommonIndex is the index determined from the inference.
# This must be in correspondance with the true label, or else the map will be wrong.
return (output[0][mostCommonIndex])
def model_modifier(model):
model.layers[-1].activation = tf.keras.activations.linear
return model
# Make prediction
output = model.predict(X)
# Create GradcamPlusPlus object
gradcamplusplus = GradcamPlusPlus(model,model_modifier=model_modifier, clone=False)
# Generate heatmap with GradCAM and normalize it
camplusplus = gradcamplusplus(loss, X, penultimate_layer=-1) # model.layers number
camplusplus = normalize(camplusplus)
# Create heat map with 4 channels
heatmap_GradCamPlusPlus = np.uint8(cm.jet(camplusplus[0,:,:])[..., :3] * 255)
# Create Saliency object.
saliency = Saliency(model, model_modifier=model_modifier, clone=False)
# Generate saliency map with smoothing that reduce noise by adding noise
saliency_map = saliency(loss,
X,
smooth_samples=20, # The number of calculating gradients iterations.
smooth_noise=0.20) # noise spread level.
# Normalize map and get rid of 1 for the batch size
saliency_map = normalize(saliency_map)
saliency_map = saliency_map[0,:,:]
# Create ScoreCAM object
scorecam = ScoreCAM(model, model_modifier, clone=False)
# This cell takes a lot of time on CPU, prefer GPU but doable on CPU.
# Generate heatmap with ScoreCAM
cam_score = scorecam(loss,
X,
penultimate_layer=-1, # model.layers number
)
cam_score = normalize(cam_score)
# Create heatmap
heatmap_CamScore = np.uint8(cm.jet(cam_score[0,:,:])[..., :3] * 255)
# Return data
return heatmap_GradCamPlusPlus, saliency_map, heatmap_CamScore
def test__call__(self, conv_model):
instance = Saliency(conv_model, model_modifier=GuidedBackpropagation())
guided_model = instance.model
assert guided_model != conv_model
assert guided_model.get_layer('conv_1').activation != conv_model.get_layer(
'conv_1').activation
assert guided_model.get_layer('dense_1').activation == conv_model.get_layer(
'dense_1').activation
instance(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
def _test_for_multiple_io(self, model):
saliency = Saliency(model)
result = saliency(
[CategoricalScore(0), BinaryScore(0)],
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__if_model_has_multiple_io(multiple_io_model):
saliency = Saliency(multiple_io_model)
result = saliency(
[CategoricalScore(1, 2), lambda x: x],
[np.random.sample((1, 8, 8, 3)),
np.random.sample((1, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__if_smoothing_is_active(model):
saliency = Saliency(model)
result = saliency(CategoricalScore(1, 2),
np.random.sample((1, 8, 8, 3)),
smooth_samples=1)
assert result.shape == (1, 8, 8)
result = saliency(CategoricalScore(1, 2),
np.random.sample((1, 8, 8, 3)),
smooth_samples=2)
assert result.shape == (1, 8, 8)
def test__call__if_smoothing_is_active(cnn_model):
saliency = Saliency(cnn_model)
result = saliency(SmoothedLoss(1),
np.random.sample((1, 8, 8, 3)),
smooth_samples=1)
assert result.shape == (1, 8, 8)
result = saliency(SmoothedLoss(1),
np.random.sample((1, 8, 8, 3)),
smooth_samples=2)
assert result.shape == (1, 8, 8)
def test__call__when_model_has_multiple_outputs(multiple_output_model):
saliency = Saliency(multiple_output_model)
def loss(output):
return K.sum(output[0])
result = saliency([loss, loss, loss],
np.random.sample((3, )),
keepdims=True)
assert result.shape == (1, 3)
def test__call__if_smoothing_is_active(self, smooth_samples,
multiple_inputs_model):
saliency = Saliency(multiple_inputs_model)
result = saliency(
CategoricalScore(0),
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))],
smooth_samples=smooth_samples)
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__if_keepdims_is_(self, keepdims, expected,
multiple_inputs_model):
saliency = Saliency(multiple_inputs_model)
result = saliency(
CategoricalScore(0),
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))],
keepdims=keepdims)
assert len(result) == 2
assert result[0].shape == expected[0]
assert result[1].shape == expected[1]
def call_smooth(model, model_modifier, loss, pixel_array):
# Create Gradcam object
# The `output` variable refer to the output of the model,
# i.e., (samples, classes). either output[0][1] or output[0][0]: output[sample_idx][class_idx]
def loss(output):
return (output[0][1]) # sample 0 class 1
saliency = Saliency(model, model_modifier=model_modifier, clone=False)
# Generate saliency map
saliency_map = saliency(
loss,
pixel_array,
smooth_samples=10, # The number of calculating gradients iterations.
smooth_noise=0.20) # noise spread level.)
capi = normalize(saliency_map)
#print("Shape normalize Cam: ", np.shape(capi))
return capi
def run(self):
"""Run Worker Thread."""
# This is the code executing in the new thread.
with open("Model_definition.pkl", 'rb') as f:
model_config = pickle.load(f)
model = tf.keras.models.Model.from_config(model_config)
model.load_weights("Model_weights.h5")
wx.CallAfter(pub.sendMessage, "update", msg="Preparing Image")
# El callback aun no funciona, pero para cuando este disponible. De momento hay 3 saltos, uno que carga el
# modelo, otro procesa las imagenes y el ultimo ordena y copia.
my_call = ProgressBar2()
img_prep = transform_img_fn(self.file)
img_prep = np.concatenate([img_prep, img_prep])
if self.info_type == 'SmoothGrad Saliency':
wx.CallAfter(pub.sendMessage,
"update",
msg="Obtaining SmoothGrad Saliency")
replace2linear = ReplaceToLinear()
score = CategoricalScore([0, 1])
saliency = Saliency(model,
model_modifier=replace2linear,
clone=True)
self.saliency_maps = saliency(score,
img_prep,
smooth_samples=50,
smooth_noise=0.10)
else:
wx.CallAfter(pub.sendMessage, "update", msg="Obtaining GradCAM++")
replace2linear = ReplaceToLinear()
score = CategoricalScore([0, 1])
gradcam = GradcamPlusPlus(model,
model_modifier=replace2linear,
clone=True)
self.gradcam_maps = gradcam(score, img_prep, penultimate_layer=-1)
wx.CallAfter(pub.sendMessage, "update", msg="")
def saliency(img_path, multiclass_model, name):
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import scipy.ndimage as ndimage
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.applications.densenet import preprocess_input
import numpy as np
# Preparing input data for VGG16
# X = preprocess_input(images)
# X = 'https://firebasestorage.googleapis.com/v0/b/intelrad-a680a.appspot.com/o/images%2Fxrayyyy.jpg?alt=media&token=63a8460f-5c79-422c-91d6-c5a2704d2383'
X = img_path
X = preprocess_image(X)
# Rendering
import matplotlib.pyplot as plt
from tf_keras_vis.scorecam import Scorecam
from matplotlib import cm
from tf_keras_vis.gradcam import Gradcam
from tf_keras_vis.utils.model_modifiers import ReplaceToLinear
replace2linear = ReplaceToLinear()
# Create Gradcam object
gradcam = Gradcam(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Instead of using CategoricalScore object,
from tf_keras_vis.utils.scores import CategoricalScore
# 1 is the imagenet index corresponding to Goldfish, 294 to Bear and 413 to Assault Rifle.
score = CategoricalScore(4)
from tensorflow.keras import backend as K
from tf_keras_vis.saliency import Saliency
# from tf_keras_vis.utils import normalize
# Create Saliency object.
saliency = Saliency(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Generate saliency map
X = X.tolist()
X = np.asarray(X, dtype=np.float)
# saliency_map = saliency(score, X)
# Generate saliency map with smoothing that reduce noise by adding noise
saliency_map = saliency(
score,
X,
smooth_samples=20, # The number of calculating gradients iterations.
smooth_noise=0.20) # noise spread level.
# Generate saliency map
saliency_map = saliency(score, X)
# Render
# f, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 4))
plt.figure(figsize=(8, 8))
plt.imshow(saliency_map[0], cmap='jet', alpha=0.9)
plt.axis('off')
import os
img_dir = './static/img/saliency'
if (not os.path.isdir(img_dir)):
os.mkdir(img_dir)
plt.tight_layout()
plt.savefig(f'./static/img/saliency/{name}.svg',
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.close()
return "Done"
def test__call__(model):
saliency = Saliency(model)
result = saliency(CategoricalScore(1, 2), np.random.sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_keepdims_is_active(dense_model):
saliency = Saliency(dense_model)
result = saliency(CategoricalScore(1, 2),
np.random.sample((3, )),
keepdims=True)
assert result.shape == (1, 3)
def test__call__if_keepdims_is_active(dense_model):
saliency = Saliency(dense_model)
result = saliency(SmoothedLoss(1), np.random.sample((3, )), keepdims=True)
assert result.shape == (1, 3)
def test__call__(cnn_model):
saliency = Saliency(cnn_model)
result = saliency(SmoothedLoss(1), np.random.sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_seed_input_has_not_batch_dim(cnn_model):
saliency = Saliency(cnn_model)
result = saliency(SmoothedLoss(1), np.random.sample((8, 8, 3)))
assert result.shape == (1, 8, 8)
def _test_for_single_io(self, model):
saliency = Saliency(model)
result = saliency(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def test__call__if_keepdims_is_(self, keepdims, expected, conv_model):
saliency = Saliency(conv_model)
result = saliency(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
keepdims=keepdims)
assert result.shape == expected
for filter_number in range(4):
def loss(output):
return output[..., filter_number]
def model_modifier(m):
m.layers[-1].activation = tf.keras.activations.linear
return m
from tf_keras_vis.saliency import Saliency
from tf_keras_vis.utils import normalize
# Create Saliency object.
# If `clone` is True(default), the `model` will be cloned,
# so the `model` instance will be NOT modified, but it takes a machine resources.
saliency = Saliency(model, model_modifier=model_modifier, clone=False)
# Generate saliency map
saliency_map = saliency(loss, Xts)
saliency_map = normalize(saliency_map)
idx0 = [i for i in range(yts.shape[0]) if yts[i] == filter_number]
i = filter_number
# Render
subplot_args = {'nrows': 1, 'ncols': 1, 'figsize': (5, 5)}
f, ax = plt.subplots(**subplot_args)
im = ax.imshow(np.mean(saliency_map[idx0, :, :], axis=0), cmap='jet')
ax.invert_yaxis()
ax.set_xlabel('channels', fontsize=12)
ax.set_ylabel('frequencies, Hz', fontsize=12)
ax.set_xlim([0, 58])
def test__call__if_smoothing_is_active(self, smooth_samples, conv_model):
saliency = Saliency(conv_model)
result = saliency(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
smooth_samples=smooth_samples)
assert result.shape == (1, 8, 8)
def saliency(
*,
conf,
model,
data,
pred,
metadata,
pdf_file,
n_examples=300,
smooth_samples=0,
aspect=12 / 16,
scale=1.2,
cmap="cool",
rasterized=False,
):
"""
Plot saliency map for each modelspec. Plot up to n_examples for each.
"""
from tf_keras_vis.saliency import Saliency
from tf_keras_vis.utils import normalize
assert n_examples >= 1
modelspecs = list(itertools.chain(*conf.tranche.values()))
modelspec_indexes = collections.defaultdict(list)
want_more = set(modelspecs)
for i in range(len(data)):
modelspec = metadata[i]["modelspec"]
if modelspec in want_more:
modelspec_indexes[modelspec].append(i)
if len(modelspec_indexes[modelspec]) == n_examples:
want_more.remove(modelspec)
if len(want_more) == 0:
break
indexes = list(itertools.chain(*modelspec_indexes.values()))
saliency = Saliency(model, model_modifier=_model_modifier)
X = np.expand_dims(data[indexes], axis=-1).astype(np.float32)
saliency_maps = saliency(_loss, X, smooth_samples=smooth_samples)
saliency_maps = normalize(saliency_maps)
_, condition_positive = list(conf.tranche.keys())
sample_counts = conf.sample_counts(haploid=conf.phased)
pop_indices = conf.pop_indices(haploid=conf.phased)
pdf = PdfPages(pdf_file)
fig_w, fig_h = plt.figaspect(aspect)
fig = plt.figure(figsize=(scale * fig_w, scale * fig_h))
nrows = len(modelspecs) + 1
gs = fig.add_gridspec(
nrows=nrows,
ncols=2,
height_ratios=[8] * (nrows - 1) + [1],
width_ratios=[40, 1],
)
axs = [fig.add_subplot(gs[i, 0]) for i in range(nrows)]
ax_cbar = fig.add_subplot(gs[:-1, 1])
axs[-1].get_shared_x_axes().join(axs[-1], axs[-2])
ax_pops = axs[-1]
i = 0
for ax, modelspec in zip(axs, modelspecs):
# Take average over maps with the same modelspec.
saliency_map = np.mean(saliency_maps[i:i + n_examples], axis=0)
i += n_examples
last_subplot = ax == axs[-2]
title = modelspec.split("/")[-2]
hap_matrix1(
saliency_map,
ax=ax,
title=title,
sample_counts=sample_counts,
pop_indices=pop_indices,
sequence_length=conf.sequence_length,
cmap=cmap,
rasterized=rasterized,
phased=conf.phased,
ploidy=conf.ploidy,
vmin=0,
vmax=1,
plot_cbar=last_subplot,
ax_cbar=ax_cbar if last_subplot else None,
cbar_label="abs(Gradient)",
plot_pop_labels=last_subplot,
ax_pops=ax_pops if last_subplot else None,
)
pdf.savefig(figure=fig)
plt.close(fig)
pdf.close()
def plot_saliency_for_windows(dev_X, predictions, num_plot=100, num_avg=1000):
if not os.path.isfile('analysis/saliency.csv'):
saliency = Saliency(model, model_modifier=model_modifier, clone=False)
saliency_map = saliency(score_function,
dev_X,
smooth_samples=20,
smooth_noise=0.20)
print(saliency_map.shape)
np.savetxt('analysis/saliency.csv', saliency_map, fmt='%f')
saliency_map = np.loadtxt('analysis/saliency.csv', dtype=float)
# Used in baseline model
baseline_saliency = np.zeros(saliency_map.shape[1])
baseline_saliency[20:26] = 1
baseline_saliency[60:68] = 1
baseline_saliency[98:100] = 1
baseline_saliency = baseline_saliency.reshape(1, (len(baseline_saliency)))
entropy_vals = get_entropy(dev_X)
# Can make plots in ranges of predicted splicing efficiency
eff_ranges = [(0, 1)] #, (0.1, 0.5), (0.5, 0.8), (0.8, 1.0)]
for (range_start, range_end) in eff_ranges:
mask = (predictions >= range_start) & (predictions <= range_end)
all_idxs = np.arange(len(predictions))
idxs = all_idxs[mask]
sample_size = min(num_plot, len(idxs))
plot_size = min(num_avg, len(idxs))
idx_sample = np.random.choice(idxs, size=plot_size, replace=False)
idx_sample = np.random.choice(idxs, size=sample_size, replace=False)
data = saliency_map[idx_sample, :]
df = pd.DataFrame(data).melt()
fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1, figsize=(12,8), \
gridspec_kw={'height_ratios':[1,5,5,10]})
# Canonical splicing sequences
sns.heatmap(baseline_saliency, cmap='Blues', ax=ax1, cbar=False)
ax1.set_title("Canonical splicing sequences")
ax1.axes.xaxis.set_visible(False)
ax1.axes.yaxis.set_visible(False)
# Per-position sequence entropy
sns.lineplot(data=entropy_vals, ax=ax2)
ax2.axes.xaxis.set_visible(False)
ax2.set_ylabel("Sequence Entropy")
ax2.set_title("Per-position sequence entropy")
# Average saliency
sns.lineplot(data=df, x="variable", y="value", ax=ax3)
ax3.axes.xaxis.set_visible(False)
ax3.set_ylabel("Saliency")
ax3.set_title("Average per-position saliency"
) #: %f to %f" % (range_start, range_end))
# Saliency heatmap for select examples
sns.heatmap(data, cmap='Blues', ax=ax4, cbar=False)
ax4.set_title(
"Saliency heatmap") #: %f to %f" % (range_start, range_end))
ax4.set_xlabel("Position")
ax4.set_ylabel("Example constructs")
plt.show()
def test__call__if_model_has_only_dense_layers(self, dense_model):
saliency = Saliency(dense_model)
result = saliency(CategoricalScore(0),
dummy_sample((8, )),
keepdims=True)
assert result.shape == (1, 8)
def test__call__if_seed_input_has_not_batch_dim(model):
saliency = Saliency(model)
result = saliency(CategoricalScore(1, 2), np.random.sample((8, 8, 3)))
assert result.shape == (1, 8, 8)
def call_vsaliency(model, model_modifier, loss, pixel_array):
saliency = Saliency(model, model_modifier=model_modifier, clone=False)
# Generate saliency map
saliency_map = saliency(loss, pixel_array)
return normalize(saliency_map)
