def test__call__if_loss_is_None(cnn_model): saliency = Saliency(cnn_model) try: saliency(None, None) assert False except ValueError: assert True
def init_salient(self, model): # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. model.summary() self.output_names = [] for i, layer in enumerate(model.layers): if "dropout" not in layer.name.lower( ) and "out" in layer.name.lower(): self.output_names += [layer.name] if len(self.output_names) == 0: print( "Failed to find the model layer named with 'out'. Skipping salient." ) return False print("####################") print("Visualizing activations on layer:", *self.output_names) print("####################") # Create Saliency object. # If `clone` is True(default), the `model` will be cloned, # so the `model` instance will be NOT modified, but it takes a machine resources. self.saliency = Saliency(model, model_modifier=self.model_modifier, clone=False) # Create GradCAM++ object, Just only repalce class name to "GradcamPlusPlus" self.gradcampp = GradcamPlusPlus(model, model_modifier=self.model_modifier, clone=False) return True
def test__call__if_seed_input_is_None(cnn_model): saliency = Saliency(cnn_model) try: saliency(SmoothedLoss(1), None) assert False except ValueError: assert True
def get_heatmap(self, target_shape: Tuple, clean_model: Model, layers_to_cut: int = 0) -> np.ndarray: """ Generates a heatmap for a given batch of source files using `self.x_batch` as an input :param target_shape: should be the same as the shape to the input layer of the model :param clean_model: empty model (same architecture as the model in the constructor) to work with :param layers_to_cut: the number of layers, which depend on the embedding layer :return: a heatmap (Saliency map) for a given batch of source files using `self.x_batch` as an input """ def model_modifier(m: Model) -> Model: model = clean_model for layer_index in range(layers_to_cut, len(m.layers)): weights = m.layers[layer_index].get_weights() if weights: # else - dropout or normalization model.layers[layer_index - layers_to_cut + 1].set_weights(weights) return model x_batch = tf.reshape(self.x_batch, target_shape) layers = self.model.layers for layer in layers[:layers_to_cut]: x_batch = layer(x_batch) modified_model = model_modifier(self.model) saliency = Saliency(modified_model, model_modifier=None, clone=False) saliency_map = saliency(self.loss, x_batch) heatmap = saliency_map assert heatmap.max() != 0.0, "the loss value is too small" return heatmap
def Explain_tfkerasvis(self, X, model, mostCommonIndex): """ Generate gradCAM plusplus images, Saliency Maps and Score cam images and save them. Seems to work very well, but it does not generate channel specific saliency maps. """ from matplotlib import pyplot as plt from matplotlib import cm import tensorflow as tf from tf_keras_vis.utils import num_of_gpus, normalize from tf_keras_vis.gradcam import GradcamPlusPlus, Gradcam from tf_keras_vis.saliency import Saliency from tf_keras_vis.scorecam import ScoreCAM # The `output` variable refer to the output of the model, # so, in this case, `output` shape is ` (samples, classes). def loss(output): # mostCommonIndex is the index determined from the inference. # This must be in correspondance with the true label, or else the map will be wrong. return (output[0][mostCommonIndex]) def model_modifier(model): model.layers[-1].activation = tf.keras.activations.linear return model # Make prediction output = model.predict(X) # Create GradcamPlusPlus object gradcamplusplus = GradcamPlusPlus(model,model_modifier=model_modifier, clone=False) # Generate heatmap with GradCAM and normalize it camplusplus = gradcamplusplus(loss, X, penultimate_layer=-1) # model.layers number camplusplus = normalize(camplusplus) # Create heat map with 4 channels heatmap_GradCamPlusPlus = np.uint8(cm.jet(camplusplus[0,:,:])[..., :3] * 255) # Create Saliency object. saliency = Saliency(model, model_modifier=model_modifier, clone=False) # Generate saliency map with smoothing that reduce noise by adding noise saliency_map = saliency(loss, X, smooth_samples=20, # The number of calculating gradients iterations. smooth_noise=0.20) # noise spread level. # Normalize map and get rid of 1 for the batch size saliency_map = normalize(saliency_map) saliency_map = saliency_map[0,:,:] # Create ScoreCAM object scorecam = ScoreCAM(model, model_modifier, clone=False) # This cell takes a lot of time on CPU, prefer GPU but doable on CPU. # Generate heatmap with ScoreCAM cam_score = scorecam(loss, X, penultimate_layer=-1, # model.layers number ) cam_score = normalize(cam_score) # Create heatmap heatmap_CamScore = np.uint8(cm.jet(cam_score[0,:,:])[..., :3] * 255) # Return data return heatmap_GradCamPlusPlus, saliency_map, heatmap_CamScore
def test__call__(self, conv_model): instance = Saliency(conv_model, model_modifier=GuidedBackpropagation()) guided_model = instance.model assert guided_model != conv_model assert guided_model.get_layer('conv_1').activation != conv_model.get_layer( 'conv_1').activation assert guided_model.get_layer('dense_1').activation == conv_model.get_layer( 'dense_1').activation instance(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
def _test_for_multiple_io(self, model): saliency = Saliency(model) result = saliency( [CategoricalScore(0), BinaryScore(0)], [dummy_sample((1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))]) assert len(result) == 2 assert result[0].shape == (1, 8, 8) assert result[1].shape == (1, 10, 10)
def test__call__if_model_has_multiple_io(multiple_io_model): saliency = Saliency(multiple_io_model) result = saliency( [CategoricalScore(1, 2), lambda x: x], [np.random.sample((1, 8, 8, 3)), np.random.sample((1, 10, 10, 3))]) assert len(result) == 2 assert result[0].shape == (1, 8, 8) assert result[1].shape == (1, 10, 10)
def test__call__if_smoothing_is_active(model): saliency = Saliency(model) result = saliency(CategoricalScore(1, 2), np.random.sample((1, 8, 8, 3)), smooth_samples=1) assert result.shape == (1, 8, 8) result = saliency(CategoricalScore(1, 2), np.random.sample((1, 8, 8, 3)), smooth_samples=2) assert result.shape == (1, 8, 8)
def test__call__if_smoothing_is_active(cnn_model): saliency = Saliency(cnn_model) result = saliency(SmoothedLoss(1), np.random.sample((1, 8, 8, 3)), smooth_samples=1) assert result.shape == (1, 8, 8) result = saliency(SmoothedLoss(1), np.random.sample((1, 8, 8, 3)), smooth_samples=2) assert result.shape == (1, 8, 8)
def test__call__when_model_has_multiple_outputs(multiple_output_model): saliency = Saliency(multiple_output_model) def loss(output): return K.sum(output[0]) result = saliency([loss, loss, loss], np.random.sample((3, )), keepdims=True) assert result.shape == (1, 3)
def test__call__if_smoothing_is_active(self, smooth_samples, multiple_inputs_model): saliency = Saliency(multiple_inputs_model) result = saliency( CategoricalScore(0), [dummy_sample((1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))], smooth_samples=smooth_samples) assert len(result) == 2 assert result[0].shape == (1, 8, 8) assert result[1].shape == (1, 10, 10)
def test__call__if_keepdims_is_(self, keepdims, expected, multiple_inputs_model): saliency = Saliency(multiple_inputs_model) result = saliency( CategoricalScore(0), [dummy_sample((1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))], keepdims=keepdims) assert len(result) == 2 assert result[0].shape == expected[0] assert result[1].shape == expected[1]
def call_smooth(model, model_modifier, loss, pixel_array): # Create Gradcam object # The `output` variable refer to the output of the model, # i.e., (samples, classes). either output[0][1] or output[0][0]: output[sample_idx][class_idx] def loss(output): return (output[0][1]) # sample 0 class 1 saliency = Saliency(model, model_modifier=model_modifier, clone=False) # Generate saliency map saliency_map = saliency( loss, pixel_array, smooth_samples=10, # The number of calculating gradients iterations. smooth_noise=0.20) # noise spread level.) capi = normalize(saliency_map) #print("Shape normalize Cam: ", np.shape(capi)) return capi
def run(self): """Run Worker Thread.""" # This is the code executing in the new thread. with open("Model_definition.pkl", 'rb') as f: model_config = pickle.load(f) model = tf.keras.models.Model.from_config(model_config) model.load_weights("Model_weights.h5") wx.CallAfter(pub.sendMessage, "update", msg="Preparing Image") # El callback aun no funciona, pero para cuando este disponible. De momento hay 3 saltos, uno que carga el # modelo, otro procesa las imagenes y el ultimo ordena y copia. my_call = ProgressBar2() img_prep = transform_img_fn(self.file) img_prep = np.concatenate([img_prep, img_prep]) if self.info_type == 'SmoothGrad Saliency': wx.CallAfter(pub.sendMessage, "update", msg="Obtaining SmoothGrad Saliency") replace2linear = ReplaceToLinear() score = CategoricalScore([0, 1]) saliency = Saliency(model, model_modifier=replace2linear, clone=True) self.saliency_maps = saliency(score, img_prep, smooth_samples=50, smooth_noise=0.10) else: wx.CallAfter(pub.sendMessage, "update", msg="Obtaining GradCAM++") replace2linear = ReplaceToLinear() score = CategoricalScore([0, 1]) gradcam = GradcamPlusPlus(model, model_modifier=replace2linear, clone=True) self.gradcam_maps = gradcam(score, img_prep, penultimate_layer=-1) wx.CallAfter(pub.sendMessage, "update", msg="")
def saliency(img_path, multiclass_model, name): import matplotlib.pyplot as plt import matplotlib.image as mpimg import scipy.ndimage as ndimage from tensorflow.keras.preprocessing.image import load_img from tensorflow.keras.applications.densenet import preprocess_input import numpy as np # Preparing input data for VGG16 # X = preprocess_input(images) # X = 'https://firebasestorage.googleapis.com/v0/b/intelrad-a680a.appspot.com/o/images%2Fxrayyyy.jpg?alt=media&token=63a8460f-5c79-422c-91d6-c5a2704d2383' X = img_path X = preprocess_image(X) # Rendering import matplotlib.pyplot as plt from tf_keras_vis.scorecam import Scorecam from matplotlib import cm from tf_keras_vis.gradcam import Gradcam from tf_keras_vis.utils.model_modifiers import ReplaceToLinear replace2linear = ReplaceToLinear() # Create Gradcam object gradcam = Gradcam(multiclass_model, model_modifier=replace2linear, clone=True) # Instead of using CategoricalScore object, from tf_keras_vis.utils.scores import CategoricalScore # 1 is the imagenet index corresponding to Goldfish, 294 to Bear and 413 to Assault Rifle. score = CategoricalScore(4) from tensorflow.keras import backend as K from tf_keras_vis.saliency import Saliency # from tf_keras_vis.utils import normalize # Create Saliency object. saliency = Saliency(multiclass_model, model_modifier=replace2linear, clone=True) # Generate saliency map X = X.tolist() X = np.asarray(X, dtype=np.float) # saliency_map = saliency(score, X) # Generate saliency map with smoothing that reduce noise by adding noise saliency_map = saliency( score, X, smooth_samples=20, # The number of calculating gradients iterations. smooth_noise=0.20) # noise spread level. # Generate saliency map saliency_map = saliency(score, X) # Render # f, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 4)) plt.figure(figsize=(8, 8)) plt.imshow(saliency_map[0], cmap='jet', alpha=0.9) plt.axis('off') import os img_dir = './static/img/saliency' if (not os.path.isdir(img_dir)): os.mkdir(img_dir) plt.tight_layout() plt.savefig(f'./static/img/saliency/{name}.svg', transparent=True, bbox_inches='tight', pad_inches=0) plt.close() return "Done"
def test__call__(model): saliency = Saliency(model) result = saliency(CategoricalScore(1, 2), np.random.sample((1, 8, 8, 3))) assert result.shape == (1, 8, 8)
def test__call__if_keepdims_is_active(dense_model): saliency = Saliency(dense_model) result = saliency(CategoricalScore(1, 2), np.random.sample((3, )), keepdims=True) assert result.shape == (1, 3)
def test__call__if_keepdims_is_active(dense_model): saliency = Saliency(dense_model) result = saliency(SmoothedLoss(1), np.random.sample((3, )), keepdims=True) assert result.shape == (1, 3)
def test__call__(cnn_model): saliency = Saliency(cnn_model) result = saliency(SmoothedLoss(1), np.random.sample((1, 8, 8, 3))) assert result.shape == (1, 8, 8)
def test__call__if_seed_input_has_not_batch_dim(cnn_model): saliency = Saliency(cnn_model) result = saliency(SmoothedLoss(1), np.random.sample((8, 8, 3))) assert result.shape == (1, 8, 8)
def _test_for_single_io(self, model): saliency = Saliency(model) result = saliency(CategoricalScore(0), dummy_sample((1, 8, 8, 3))) assert result.shape == (1, 8, 8)
def test__call__if_keepdims_is_(self, keepdims, expected, conv_model): saliency = Saliency(conv_model) result = saliency(CategoricalScore(0), dummy_sample((1, 8, 8, 3)), keepdims=keepdims) assert result.shape == expected
for filter_number in range(4): def loss(output): return output[..., filter_number] def model_modifier(m): m.layers[-1].activation = tf.keras.activations.linear return m from tf_keras_vis.saliency import Saliency from tf_keras_vis.utils import normalize # Create Saliency object. # If `clone` is True(default), the `model` will be cloned, # so the `model` instance will be NOT modified, but it takes a machine resources. saliency = Saliency(model, model_modifier=model_modifier, clone=False) # Generate saliency map saliency_map = saliency(loss, Xts) saliency_map = normalize(saliency_map) idx0 = [i for i in range(yts.shape[0]) if yts[i] == filter_number] i = filter_number # Render subplot_args = {'nrows': 1, 'ncols': 1, 'figsize': (5, 5)} f, ax = plt.subplots(**subplot_args) im = ax.imshow(np.mean(saliency_map[idx0, :, :], axis=0), cmap='jet') ax.invert_yaxis() ax.set_xlabel('channels', fontsize=12) ax.set_ylabel('frequencies, Hz', fontsize=12) ax.set_xlim([0, 58])
def test__call__if_smoothing_is_active(self, smooth_samples, conv_model): saliency = Saliency(conv_model) result = saliency(CategoricalScore(0), dummy_sample((1, 8, 8, 3)), smooth_samples=smooth_samples) assert result.shape == (1, 8, 8)
def saliency( *, conf, model, data, pred, metadata, pdf_file, n_examples=300, smooth_samples=0, aspect=12 / 16, scale=1.2, cmap="cool", rasterized=False, ): """ Plot saliency map for each modelspec. Plot up to n_examples for each. """ from tf_keras_vis.saliency import Saliency from tf_keras_vis.utils import normalize assert n_examples >= 1 modelspecs = list(itertools.chain(*conf.tranche.values())) modelspec_indexes = collections.defaultdict(list) want_more = set(modelspecs) for i in range(len(data)): modelspec = metadata[i]["modelspec"] if modelspec in want_more: modelspec_indexes[modelspec].append(i) if len(modelspec_indexes[modelspec]) == n_examples: want_more.remove(modelspec) if len(want_more) == 0: break indexes = list(itertools.chain(*modelspec_indexes.values())) saliency = Saliency(model, model_modifier=_model_modifier) X = np.expand_dims(data[indexes], axis=-1).astype(np.float32) saliency_maps = saliency(_loss, X, smooth_samples=smooth_samples) saliency_maps = normalize(saliency_maps) _, condition_positive = list(conf.tranche.keys()) sample_counts = conf.sample_counts(haploid=conf.phased) pop_indices = conf.pop_indices(haploid=conf.phased) pdf = PdfPages(pdf_file) fig_w, fig_h = plt.figaspect(aspect) fig = plt.figure(figsize=(scale * fig_w, scale * fig_h)) nrows = len(modelspecs) + 1 gs = fig.add_gridspec( nrows=nrows, ncols=2, height_ratios=[8] * (nrows - 1) + [1], width_ratios=[40, 1], ) axs = [fig.add_subplot(gs[i, 0]) for i in range(nrows)] ax_cbar = fig.add_subplot(gs[:-1, 1]) axs[-1].get_shared_x_axes().join(axs[-1], axs[-2]) ax_pops = axs[-1] i = 0 for ax, modelspec in zip(axs, modelspecs): # Take average over maps with the same modelspec. saliency_map = np.mean(saliency_maps[i:i + n_examples], axis=0) i += n_examples last_subplot = ax == axs[-2] title = modelspec.split("/")[-2] hap_matrix1( saliency_map, ax=ax, title=title, sample_counts=sample_counts, pop_indices=pop_indices, sequence_length=conf.sequence_length, cmap=cmap, rasterized=rasterized, phased=conf.phased, ploidy=conf.ploidy, vmin=0, vmax=1, plot_cbar=last_subplot, ax_cbar=ax_cbar if last_subplot else None, cbar_label="abs(Gradient)", plot_pop_labels=last_subplot, ax_pops=ax_pops if last_subplot else None, ) pdf.savefig(figure=fig) plt.close(fig) pdf.close()
def plot_saliency_for_windows(dev_X, predictions, num_plot=100, num_avg=1000): if not os.path.isfile('analysis/saliency.csv'): saliency = Saliency(model, model_modifier=model_modifier, clone=False) saliency_map = saliency(score_function, dev_X, smooth_samples=20, smooth_noise=0.20) print(saliency_map.shape) np.savetxt('analysis/saliency.csv', saliency_map, fmt='%f') saliency_map = np.loadtxt('analysis/saliency.csv', dtype=float) # Used in baseline model baseline_saliency = np.zeros(saliency_map.shape[1]) baseline_saliency[20:26] = 1 baseline_saliency[60:68] = 1 baseline_saliency[98:100] = 1 baseline_saliency = baseline_saliency.reshape(1, (len(baseline_saliency))) entropy_vals = get_entropy(dev_X) # Can make plots in ranges of predicted splicing efficiency eff_ranges = [(0, 1)] #, (0.1, 0.5), (0.5, 0.8), (0.8, 1.0)] for (range_start, range_end) in eff_ranges: mask = (predictions >= range_start) & (predictions <= range_end) all_idxs = np.arange(len(predictions)) idxs = all_idxs[mask] sample_size = min(num_plot, len(idxs)) plot_size = min(num_avg, len(idxs)) idx_sample = np.random.choice(idxs, size=plot_size, replace=False) idx_sample = np.random.choice(idxs, size=sample_size, replace=False) data = saliency_map[idx_sample, :] df = pd.DataFrame(data).melt() fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1, figsize=(12,8), \ gridspec_kw={'height_ratios':[1,5,5,10]}) # Canonical splicing sequences sns.heatmap(baseline_saliency, cmap='Blues', ax=ax1, cbar=False) ax1.set_title("Canonical splicing sequences") ax1.axes.xaxis.set_visible(False) ax1.axes.yaxis.set_visible(False) # Per-position sequence entropy sns.lineplot(data=entropy_vals, ax=ax2) ax2.axes.xaxis.set_visible(False) ax2.set_ylabel("Sequence Entropy") ax2.set_title("Per-position sequence entropy") # Average saliency sns.lineplot(data=df, x="variable", y="value", ax=ax3) ax3.axes.xaxis.set_visible(False) ax3.set_ylabel("Saliency") ax3.set_title("Average per-position saliency" ) #: %f to %f" % (range_start, range_end)) # Saliency heatmap for select examples sns.heatmap(data, cmap='Blues', ax=ax4, cbar=False) ax4.set_title( "Saliency heatmap") #: %f to %f" % (range_start, range_end)) ax4.set_xlabel("Position") ax4.set_ylabel("Example constructs") plt.show()
def test__call__if_model_has_only_dense_layers(self, dense_model): saliency = Saliency(dense_model) result = saliency(CategoricalScore(0), dummy_sample((8, )), keepdims=True) assert result.shape == (1, 8)
def test__call__if_seed_input_has_not_batch_dim(model): saliency = Saliency(model) result = saliency(CategoricalScore(1, 2), np.random.sample((8, 8, 3))) assert result.shape == (1, 8, 8)
def call_vsaliency(model, model_modifier, loss, pixel_array): saliency = Saliency(model, model_modifier=model_modifier, clone=False) # Generate saliency map saliency_map = saliency(loss, pixel_array) return normalize(saliency_map)