def visualize_filters(model, layer_name, input_data, filter_indices=None, mode="guided"): from vis.visualization import get_num_filters, visualize_saliency from vis.utils import utils from vis.input_modifiers import Jitter """ Visualize what pattern activates a filter. Helps to discover what a filter might be computing :returns tuple(List, List) containing input images and heatmaps frames from each sample is stitched into a single image """ get_num_filters inputs = [] outputs = [] # number of filters for this layer num_filters = get_num_filters(model.get_layer(layer_name)) layer_idx = utils.find_layer_idx(model, layer_name) for sample in input_data: heatmaps = visualize_saliency( model, layer_idx, filter_indices=filter_indices, seed_input=sample, backprop_modifier=mode, ) inputs.append(utils.stitch_images(sample, margin=0)) outputs.append(utils.stitch_images(heatmaps, margin=0)) return np.array(inputs), np.array(outputs)
def visualize_activation(self): from matplotlib import pyplot as plt from vis.utils import utils from vis.visualization import visualize_activation, get_num_filters vis_images = [] for i in range(self.config['output_shape'][0]): # The name of the layer we want to visualize layer_name = 'output_%d' % i layer_idx = [ idx for idx, layer in enumerate(self.net.layers) if layer.name == layer_name ][0] print('Working on %s' % layer_name) # Generate input image for each filter. Here `text` field is used to # overlay `filter_value` on top of the image. for idx in [1, 1, 1]: img = visualize_activation(self.net, layer_idx, filter_indices=idx, max_iter=500) # img = utils.draw_text(img, TAGS[i]) vis_images.append(img) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=3) plt.axis('off') plt.imshow(stitched) plt.title(self.checkpoint_name) plt.show()
def visualize_multiple_categories(): """Example to show how to visualize images that activate multiple categories """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Visualize [20] (ouzel) and [20, 71] (An ouzel-scorpion :D) indices = [20, [20, 71]] images = [ visualize_activation(model, layer_idx, filter_indices=idx, text=utils.get_imagenet_label(idx), max_iter=500) for idx in indices ] cv2.imshow('Multiple category visualization', utils.stitch_images(images)) cv2.waitKey(0)
def visualize_multiple_categories(show=True): """Example to show how to visualize images that activate multiple categories """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Visualize [20] (ouzel) and [20, 71] (An ouzel-scorpion :D) indices = [20, [20, 71]] images = [] for idx in indices: img = visualize_class_activation(model, layer_idx, filter_indices=idx, max_iter=500) img = utils.draw_text(img, utils.get_imagenet_label(idx)) images.append(img) # Easily stitch images via `utils.stitch_images` stitched = utils.stitch_images(images) if show: plt.axis('off') plt.imshow(stitched) plt.title('Multiple category visualization') plt.show()
def visualize_multiple_same_filter(num_runs=3): """Example to show how to visualize same filter multiple times via different runs. Args: num_runs: The number of times the same filter is visualized """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # 20 is the imagenet category for 'ouzel' indices = [20] * num_runs images = [ visualize_activation(model, layer_idx, filter_indices=idx, text=utils.get_imagenet_label(idx), max_iter=500) for idx in indices ] cv2.imshow('Multiple runs of ouzel', utils.stitch_images(images)) cv2.waitKey(0)
def visualize_random(num_categories=10, show=True): """Example to show how to visualize multiple filters via activation maximization. Args: num_categories: The number of random categories to visualize. (Default Value = 5) """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0] # Visualize couple random categories from imagenet. indices = np.random.permutation(1000)[:num_categories] images = [] for idx in indices: img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=500) img = utils.draw_text(img, utils.get_imagenet_label(idx)) images.append(img) # Easily stitch images via `utils.stitch_images` stitched = utils.stitch_images(images) if show: plt.axis('off') plt.imshow(stitched) plt.title('Random imagenet categories') plt.show()
def visualize_multiple_same_filter(num_runs=3, show=True): """Example to show how to visualize same filter multiple times via different runs. Args: num_runs: The number of times the same filter is visualized """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0] # 20 is the imagenet category for 'ouzel' indices = [20] * num_runs images = [] for idx in indices: img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=500) img = utils.draw_text(img, utils.get_imagenet_label(idx)) images.append(img) # Easily stitch images via `utils.stitch_images` stitched = utils.stitch_images(images) if show: plt.axis('off') plt.imshow(stitched) plt.title('Multiple runs of ouzel') plt.show()
def visualize_random(num_categories=10): """Example to show how to visualize multiple filters via activation maximization. Args: num_categories: The number of random categories to visualize. (Default Value = 5) """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Visualize couple random categories from imagenet. indices = np.random.permutation(1000)[:num_categories] images = [ visualize_activation(model, layer_idx, filter_indices=idx, text=utils.get_imagenet_label(idx), max_iter=500) for idx in indices ] # Easily stitch images via `utils.stitch_images` cv2.imshow('Random imagenet categories', utils.stitch_images(images)) cv2.waitKey(0)
def main(): model = load_model('model2.h5py') model.summary() # swap softmax activation function to linear layer_idx = -1 model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) layer_names = ['leaky_re_lu_1', 'leaky_re_lu_2', 'leaky_re_lu_3', 'leaky_re_lu_4'] for lid in range(len(layer_names)): layer_name = layer_names[lid] layer_idx = utils.find_layer_idx(model, layer_name) filters = np.arange(get_num_filters(model.layers[layer_idx])) filters = _shuffle(filters) vis_images = [] for idx in range(16): indices = filters[idx] img = visualize_activation(model, layer_idx, filter_indices=indices, tv_weight=0., input_modifiers=[Jitter(0.5)]) vis_images.append(img) #img = img.reshape((48, 48)) #plt.imshow(img, cmap="Blues") #plt.show() stitched = utils.stitch_images(vis_images, cols=8) plt.figure() plt.axis('off') shape = stitched.shape stitched = stitched.reshape((shape[0], shape[1])) plt.imshow(stitched) plt.title(layer_name) plt.tight_layout() plt.savefig('Filter_{}.png'.format(lid))
def visualize_activations(input, model, layer_idx, w, h): activations = get_activations(model, layer_idx, input)[0] filters = np.mean(activations, axis=0) vis_images = [] for i in range(0, filters.shape[0]): img = filters[i].reshape((w, h)) vis_images.append(img.reshape(w, h, 1)) stitched = utils.stitch_images(vis_images, cols=8) rs = stitched.reshape((stitched.shape[0], stitched.shape[1])) return rs
def visualize_conv_filters(output_dir, model, layer_name): layer_idx = utilsvis.find_layer_idx(model, layer_name) filters = np.arange(get_num_filters(model.layers[layer_idx])) vis_images = [] for idx in tqdm(filters): img = visualize_activation(model, layer_idx, filter_indices=idx) img = utilsvis.draw_text(img, 'Filter {}'.format(idx)) vis_images.append(img) stitched = utilsvis.stitch_images(vis_images, cols=32) path = os.path.join(output_dir, '{}.png') imsave(path, stitched)
def plt_activation(model_path, layer_idx=-1, max_iter=200, **kwargs): """ Plot activation of a given layer in a model by generating an image that maximizes the output of all `filter_indices` in the given `layer_idx`. Args: model_path: Path to the model file. layer_idx: Index of the layer to plot. max_iter: Maximum number of iterations to generate the input image. kwargs: Unused arguments. Returns: """ model = import_model(model_path) model = prediction_layer_linear_activation(model) if type(model.layers[layer_idx]) == keras.layers.Dense: img = vvis.visualize_activation(model, layer_idx, max_iter=max_iter, filter_indices=None) elif type(model.layers[layer_idx]) == keras.layers.Conv2D: filters = np.arange(vvis.get_num_filters(model.layers[layer_idx])) # Generate input image for each filter. vis_images = [] for idx in tqdm.tqdm(filters): act_img = vvis.visualize_activation(model, layer_idx, max_iter=max_iter, filter_indices=idx) vis_images.append(act_img) # Generate stitched image palette with 8 cols. img = vutils.stitch_images(vis_images, cols=8) else: raise TypeError("Invalid Layer type. model.layers[{}] is {}, " "only Dense and Conv2D layers can be used".format( str(layer_idx), str(type(model.layers[layer_idx])))) plt.axis("off") if len(img.shape) == 2 or img.shape[2] == 1: plt.imshow(img.reshape(img.shape[0:2]), cmap="gray") else: plt.imshow(img) plt.show()
def plot_activations(model, layer_name, num): layer_idx = utils.find_layer_idx(model, layer_name) filters = np.arange(get_num_filters(model.layers[layer_idx]))[:num] # Generate input image for each filter. vis_images = [] for idx in tqdm_notebook(filters, 'Generating images'): img = visualize_activation(model, layer_idx, filter_indices=idx, input_range=(0., 1.)) vis_images.append(img) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=8) plt.figure(figsize=(16, 16)) plt.axis('off') plt.imshow(stitched) plt.title(layer_name)
def visualiseDenseLayer(model, layer_name, output_classes, verbose, save=False): ''' Makes a plot for visualising the activations of a dense layer. :param model: The model to visualise. :param layer_name: The name of the dense layer to visualise. :param output_classes: The number of activations in the layer. :param verbose: Print statements of progress. :return: N/A ''' layer_index = utils.find_layer_idx(model, layer_name) model.layers[layer_index].activation = activations.linear model = utils.apply_modifications(model) vis_images = [] for filter_index in range(0, output_classes): if (verbose): print("Preparing Visualisation for class {} in layer {}".format( filter_index, layer_name)) visualisation = visualize_activation(model, layer_index, filter_index, max_iter=500, input_modifiers=[Jitter(16)]) img = utils.draw_text(visualisation, 'Class {}'.format(filter_index)) vis_images.append(img) stitched = utils.stitch_images(vis_images, cols=4) if (save): matplotlib.image.imsave("../SavedData/" + save, stitched) else: plt.imshow(stitched) plt.figure() plt.axis('off') plt.show()
def plot_conv_weights(self): PATH = os.path.dirname(__file__) cls = _get_conv_layers(self.model) i=0 for layer in cls : layer_name = layer.name print("{}/{} : {}".format(i,len(cls),layer_name)) layer_idx = utils.find_layer_idx(self.model,layer_name) filters = np.arange(get_num_filters(self.model.layers[layer_idx])) vis_images = [] for idx in filters[:64]: img = visualize_activation(self.model, layer_idx, filter_indices=idx) # Utility to overlay text on image. img = utils.draw_text(img, 'Filter {}'.format(idx)) vis_images.append(img) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=8) plt.axis('off') plt.imsave(join(PATH,'heatmaps/'+layer_name+'.jpg'),stitched) i+=1
def visualiseCovolutionLayer(model, selected_layer, selected_indices, verbose, save=False): ''' Makes a plot for visualising the activations of a convolutional layer. :param model: The model to visualise. :param selected_layer: The name of the dense layer to visualise. :param selected_indices: The indices of the filters to visualise. :param verbose: Print statements of progress. :return: N/A. ''' layer_index = utils.find_layer_idx(model, selected_layer) vis_images = [] for filter_index in selected_indices: if (verbose): print("Preparing Visualisation for filter {} in layer {}".format( filter_index, selected_layer)) img = visualize_activation(model, layer_index, filter_index) # Utility to overlay text on image. visualisation = utils.draw_text( img, 'Filter {} on layer {}'.format(filter_index, selected_layer)) vis_images.append(visualisation) # Generate stitched image palette with 5 cols so we get 2 rows. stitched = utils.stitch_images(vis_images, cols=5) plt.figure() plt.axis('off') plt.imshow(stitched) if (save): plt.savefig("../SavedData/" + save) else: plt.show()
# seed_img = utils.load_img(path, target_size=(224, 224)) # pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)]))) print('Predicted:', decode_predictions(pred)) print('Predicted:', decode_predictions(pred)[0][0][1]) # pred_class = np.argmax(model.predict(preprocess_input(np.array([img_to_array(seed_img)])))) # Here we are asking it to show attention such that prob of `pred_class` is maximized. # heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img, text=utils.get_imagenet_label(pred_class)) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img, text=utils.get_imagenet_label(pred_class)) heatmaps.append(heatmap) # Generate three different images of the same output index. # vis_images = [visualize_activation(model, layer_idx, filter_indices=idx, text=str(idx), max_iter=500) for idx in [294, 294, 294]] # vis_images.append(vis_image) name = "Gradient-based Localization map" cv2.imshow(name, utils.stitch_images(heatmaps)) cv2.waitKey(-1) cv2.destroyWindow(name) # name = "Visualizations » Dense Layers" # cv2.imshow(name, utils.stitch_images(vis_images)) # cv2.waitKey(-1) # cv2.destroyWindow(name)
print(model.summary()) model.load_weights(filepath, by_name=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'fc2_out' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] heatmaps = [] #for path in image_paths: for i in range(5): seed_img = utils.load_img(path, target_size=(32, 32)) print(seed_img.shape) seed_img = x_train[np.random.randint(i * 13, 50000)] print(seed_img.shape) x = np.expand_dims(img_to_array(seed_img), axis=0) x = preprocess_input(x) pred_class = np.argmax(model.predict(x)) # Here we are asking it to show attention such that prob of `pred_class` is maximized. heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) heatmaps.append(heatmap) plt.axis('off') plt.imshow(utils.stitch_images(heatmaps)) plt.title('Activation map') plt.savefig('activation_map_cnn')
from vis.visualization import visualize_activation from keras.models import load_model from keras.layers import Activation from scipy.misc import imsave # Load the fine-tuned plankton model model = load_model('plankton_model.h5') # Change output activation from softmax to linear model.layers[-1].activation = Activation('linear') print('Model loaded.') # The name of the layer we want to visualize layer_name = 'dense_1' layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0] # Get dense layer visualizations for all 12 plankton categories vis_images = [] for idx in range(12): img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=1000 ) img = utils.draw_text(img, str(idx)) vis_images.append(img) stitched = utils.stitch_images(vis_images) imsave('class_%d.png' % (idx), stitched) plt.axis('off') plt.imshow(stitched) plt.title(layer_name) plt.show()
# (see model definition in vggnet.py) layer_name = 'vizblock' layer_idx = 4 #[idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0] # Visualize all filters in this layer. filters = np.arange(get_num_filters(model.layers[layer_idx])) # Generate input image for each filter. Here `text` field is used to overlay `filter_value` on top of the image. vis_images = [] for idx in range(5): img = visualize_class_activation(model, layer_idx, filter_indices=idx) #img = utils.draw_text(img, str(idx)) vis_images.append(img) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=8) plt.axis('off') plt.imshow(stitched) plt.title(layer_name) plt.show() from matplotlib import pyplot as plt from vis.utils import utils from vis.visualization import visualize_class_activation # Build the VGG16 network with ImageNet weights #model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize
thres = 420 see = test_in[NUM].reshape(48, 48) mean = numpy.mean(see) for i in range(48): for j in range(48): if sum(heatmap[i][j]) <= thres: see[i][j] = 0 plt.figure() plt.imshow(see, cmap='gray') plt.colorbar() plt.tight_layout() plt.savefig('tmp.png') see = utils.load_img('tmp.png') mask.append(see) plt.axis('off') plt.imshow(utils.stitch_images(heatmaps)) plt.title('Saliency map') plt.colorbar() plt.tight_layout() plt.savefig('line.png') plt.figure() plt.axis('off') plt.imshow(utils.stitch_images(mask), cmap='gray') plt.tight_layout() plt.savefig('mask.png') os._exit(0)
# Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print(model.summary()) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'block1_conv1' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Visualize all filters in this layer. filters = np.arange(get_num_filters(model.layers[layer_idx])) # Generate input image for each filter. Here `text` field is used to overlay `filter_value` on top of the image. vis_images = [] for idx in filters: img = visualize_activation(model, layer_idx, filter_indices=idx) img = utils.draw_text(img, str(idx)) vis_images.append(img) print(len(vis_images)) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=8) plt.axis('off') plt.imshow(stitched) plt.title(layer_name) plt.savefig('visualization_vgg16_block1_conv1')
#modelName = "/home/daiver/coding/jff/py/keras_shape_reg/checkpoints/2017-05-19 12:47:16.562989_ep_149_train_l_0.00872_test_l_0.02065.h5" modelName = "/home/daiver/coding/jff/py/keras_shape_reg/checkpoints/2017-05-18 16:43:27.041387_ep_9499_train_l_0.00009_test_l_0.00706.h5" model = keras.models.load_model(modelName) print model.layers[0].name#conv1 print model.layers[3].name#conv2 print model.layers[5].name#conv3 print model.layers[8].name#conv4 print model.layers[10].name#conv5 print model.layers[13].name#conv6 print model.layers[15].name#conv7 print model.layers[18].name#conv8 print model.layers[20].name#conv8 print model.layers[25].name#conv9 #exit() layer_idx = 25 num_filters = get_num_filters(model.layers[layer_idx]) print num_filters filters = np.arange(get_num_filters(model.layers[layer_idx]))[:32] vis_images = [] for idx in filters: img = visualize_activation(model, layer_idx, filter_indices=idx) img = utils.draw_text(img, str(idx)) vis_images.append(img) # Generate stitched image palette with 8 cols. stitched = utils.stitch_images(vis_images, cols=8) plt.axis('off') plt.imshow(stitched) plt.show()
#img = vis_utils.draw_text(img, 'Filter {}'.format(idx) ) vis_images.append(img) # In[ ]: print( model.predict( myLCtest ) ) print( model.predict( np.expand_dims( np.expand_dims( np.array( vis_images )[0,:,:,0], 0 ),0 ) ) ) print( model.predict( np.expand_dims( np.expand_dims( np.array( vis_images )[1,:,:,0], 0 ),0 ) ) ) print( model.predict( np.expand_dims( np.expand_dims( np.array( vis_images )[2,:,:,0], 0 ),0 ) ) ) print( model.predict( np.expand_dims( np.expand_dims( np.array( vis_images )[3,:,:,0], 0 ),0 ) ) ) # In[ ]: stitched = vis_utils.stitch_images( vis_images, cols=1 ) plt.figure( figsize=(10,4) ) plt.axis('off') plt.imshow( stitched[:,:,0], cmap = 'seismic') # ## test extract filters # In[ ]: # model.layers # In[ ]: from vis.visualization import get_num_filters
layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Images corresponding to tiger, penguin, dumbbell, speedboat, spider image_paths = [ "http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg", "http://www.slate.com/content/dam/slate/articles/health_and_science/wild_things/2013/10/131025_WILD_AdeliePenguin.jpg.CROP.promo-mediumlarge.jpg", "https://www.kshs.org/cool2/graphics/dumbbell1lg.jpg", "http://tampaspeedboatadventures.com/wp-content/uploads/2010/10/DSC07011.jpg", "http://ichef-1.bbci.co.uk/news/660/cpsprodpb/1C24/production/_85540270_85540265.jpg" ] heatmaps = [] for path in image_paths: # Predict the corresponding class for use in `visualize_saliency`. seed_img = utils.load_img(path, target_size=(224, 224)) pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)]))) # Here we are asking it to show attention such that prob of `pred_class` # is maximized. heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img, text=utils.get_imagenet_label(pred_class)) heatmaps.append(heatmap) cv2.imshow("Saliency map", utils.stitch_images(heatmaps)) cv2.waitKey(0)
vis_images.append(img) new_vis_image = [] for i, idx in enumerate(filters): img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=1000, tv_weight=0.3, lp_norm_weight=0.4, seed_input=vis_images[i], input_modifiers=[Jitter(0.05)]) new_vis_image.append(img) stitched = utils.stitch_images(new_vis_image) plt.axis('off') plt.imshow(stitched) plt.title(layer_name) plt.show() #image_modifiers = [Jitter(16)] ## Generate input image for each filter. #vis_images = [] #for idx in filters: # img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=500, input_modifiers=image_modifiers) # # # Reverse lookup index to imagenet label and overlay it on the image. # #img = utils.draw_text(img, utils.get_imagenet_label(idx)) # vis_images.append(img) #
initial.append(img) tuples.append((name, idx)) i = 0 for name, idx in tuples: img = visualize_activation(model, layer_idx, filter_indices=idx, seed_input=initial[i], max_iter=300, input_modifiers=[Jitter(16)]) img = utils.draw_text(img, name) i += 1 images.append(img) stitched = utils.stitch_images(images, cols=4) plt.figure(figsize=(20, 20)) plt.axis('off') plt.imshow(stitched) plt.show() # # Visualizing Convnet Filters # In[39]: max_filters = 40 selected_indices = [] vis_images = [[], [], [], [], []] i = 0 selected_filters = [[0, 3, 11, 25, 26, 33, 42, 62], [8, 21, 23, 38, 39, 45, 50, 79],