def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True): """Creates an optimizer that minimizes weighted loss function. Args: input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format= channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`. losses: List of ([Loss](vis.losses#Loss), weight) tuples. input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the final optimized input to the given range. (Default value=(0, 255)) wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses` should be minimized with respect to `wrt_tensor`. `wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None which means that loss will simply be minimized with respect to `input_tensor`. norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures a smooth gradient gradient descent process. If you want the actual gradient (for example, visualizing attention), set this to false. """ self.input_tensor = input_tensor self.input_range = input_range self.loss_names = [] self.loss_functions = [] self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor if self.input_tensor is self.wrt_tensor: self.wrt_tensor_is_input_tensor = True self.wrt_tensor = K.identity(self.wrt_tensor) else: self.wrt_tensor_is_input_tensor = False overall_loss = None for loss, weight in losses: # Perf optimization. Don't build loss function with 0 weight. if weight != 0: loss_fn = weight * loss.build_loss() overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn self.loss_names.append(loss.name) self.loss_functions.append(loss_fn) # Compute gradient of overall with respect to `wrt` tensor. if self.wrt_tensor_is_input_tensor: grads = K.gradients(overall_loss, self.input_tensor)[0] else: grads = K.gradients(overall_loss, self.wrt_tensor)[0] if norm_grads: grads = K.l2_normalize(grads) # The main function to compute various quantities in optimization loop. self.compute_fn = K.function( [self.input_tensor, K.learning_phase()], self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def get_gradients(self, model): """ This function outputs a function to calculate the gradients based on the loss function, current weights/biases Input: - model: model that is training. Returns: - func: a function that uses input of features and true labels to calculate loss and hence gradients """ model_weights = model.trainable_weights if self.only_weights: weights = [ weight for weight in model_weights if 'kernel' in weight.name ] else: weights = [weight for weight in model_weights] if self.num_classes > 1: loss = K.mean(categorical_crossentropy(self.y_true, model.output)) else: loss = K.mean(binary_crossentropy(self.y_true, model.output)) func = K.function([model.input, self.y_true], K.gradients(loss, weights)) return func
def compile_saliency_function(model, act_num): input_sig = model.get_input_at(0) layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]]) layer_output = layer_dict[act_num].output max_output = K.max(layer_output, axis=2) saliency = K.gradients(K.sum(max_output), input_sig)[0] return K.function([input_sig, K.learning_phase()], [saliency])
def gradient_penalty_loss(y_true, y_pred, interpolated_samples): gradients = K.gradients(y_pred, interpolated_samples)[0] gradient_l2_norm = K.sqrt( K.sum(K.square(gradients), axis=[range(1, len(gradients.shape))])) gradient_penalty = K.square(1 - gradient_l2_norm) return K.mean(gradient_penalty)
def get_gradients(self, loss, params): """Returns gradients of `loss` with respect to `params`. Arguments: loss: Loss tensor. params: List of variables. Returns: List of gradient tensors. Raises: ValueError: In case any gradient cannot be computed (e.g. if gradient function not implemented). """ grads = K.gradients(loss, params) if None in grads: raise ValueError('An operation has `None` for gradient. ' 'Please make sure that all of your ops have a ' 'gradient defined (i.e. are differentiable). ' 'Common ops without gradient: ' 'K.argmax, K.round, K.eval.') if hasattr(self, 'clipnorm'): grads = [clip_ops.clip_by_norm(g, self.clipnorm) for g in grads] if hasattr(self, 'clipvalue'): grads = [ clip_ops.clip_by_value(g, -self.clipvalue, self.clipvalue) for g in grads ] return grads
def define_deepDream_model_layerBased(model): dream = model.input print('Model loaded.') # Get the symbolic outputs of each "key" layer (we gave them unique names). layer_dict = dict([(layer.name, layer) for layer in model.layers]) # Define the loss. loss = K.variable(0.) for layer_name in settings['features']: # Add the L2 norm of the features of a layer to the loss. if layer_name not in layer_dict: raise ValueError('Layer ' + layer_name + ' not found in model.') coeff = settings['features'][layer_name] x = layer_dict[layer_name].output # We avoid border artifacts by only involving non-border pixels in the loss. scaling = K.prod(K.cast(K.shape(x), 'float32')) if K.image_data_format() == 'channels_first': loss = loss + coeff * K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling else: loss = loss + coeff * K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] fetch_loss_and_grads = K.function([dream], outputs)
def create_iterate(self,input_img, layer_output,filter_index): ''' layer_output[:,:,:,0] is (Nsample, 94, 94) tensor contains: W0^T [f(image)]_{i,j}], i = 1,..., 94, j = 1,..., 94 layer_output[:,:,:,1] contains: W1^T [f(image)]_{i,j}], i = 1,..., 94, j = 1,..., 94 W0 and W1 are different kernel! ''' ## loss is a scalar if len(layer_output.shape) == 4: ## conv layer loss = K.mean(layer_output[:, :, :, filter_index]) elif len(layer_output.shape) ==2: ## fully connected layer loss = K.mean(layer_output[:, filter_index]) # calculate the gradient of the loss evaluated at the provided image grads = K.gradients(loss, input_img)[0] # normalize the gradients grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) # iterate is a function taking (input_img, scalar) and output [loss_value, gradient_value] iterate = K.function([input_img, K.learning_phase()], [loss, grads]) return(iterate)
def __init__(self, model, layer_name): self.model = model self.layer_name = layer_name dream = model.input # Get the symbolic outputs of each "key" layer (we gave them unique names). layers_all = [layer.name for layer in model.layers] if layer_name not in layers_all: raise ValueError('Layer ' + layer_name + ' not found in model.') # Define the loss. loss = K.variable(0.) for layer_local in model.layers: if layer_local.name == layer_name: x = layer_local.output # We avoid border artifacts by only involving non-border pixels in the loss. if K.image_data_format() == 'channels_first': scaling = K.prod(K.cast(K.shape(x), 'float32')) loss = loss + K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling else: scaling = K.prod(K.cast(K.shape(x), 'float32')) loss = loss + K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] self.fetch_loss_and_grads = K.function([dream], outputs)
def grad_cam_batch(input_model, images, classes, layer_name): """GradCAM method for visualizing input saliency. Same as grad_cam but processes multiple images in one run.""" loss = tf.gather_nd(input_model.output, np.dstack([range(images.shape[0]), classes])[0]) layer_output = input_model.get_layer(layer_name).output grads = K.gradients(loss, layer_output)[0] gradient_fn = K.function( [input_model.input, K.learning_phase()], [layer_output, grads]) conv_output, grads_val = gradient_fn([images, 0]) weights = np.mean(grads_val, axis=(1, 2)) cams = np.einsum('ijkl,il->ijk', conv_output, weights) # Process CAMs (H, W) = images.shape[1:-1] new_cams = np.empty((images.shape[0], H, W)) for i in range(new_cams.shape[0]): cam_i = cams[i] - cams[i].mean() cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i, 2) + 1e-10) new_cams[i] = cv2.resize(cam_i, (H, W), cv2.INTER_LINEAR) new_cams[i] = np.maximum(new_cams[i], 0) new_cams[i] = new_cams[i] / new_cams[i].max() return new_cams
def GetGradient(data_x_win,model,perturbation): y_pred = model.output y_true = K.variable(np.array(df_YY_actual.iloc[0])) #ten_x_scale = K.variable(np.array(data_x_win[0])) loss = keras.losses.mean_squared_error(y_true, y_pred) grads = K.gradients(loss,model.input)[0] x_adv = K.sign(grads) sess =K.get_session() init = tf.compat.v1.global_variables_initializer() sess.run(init) x_adv_0 = x_adv[0] adv = [] if len(Y) != len(data_x_win): print("WARNING!!!! Unequal length of X and Y") #len(df_x_scale) for i in range(len(data_x_win)): adv_i = sess.run(x_adv_0, feed_dict={model.input:[data_x_win[i]],y_true:np.array(df_YY_actual.iloc[i])}) if i%1000 == 0: print(i) print(datetime.datetime.now().isoformat()) df_grd_i = pd.DataFrame(adv_i,columns = header) adv.append(np.array(df_grd_i)) return adv
def compile_saliency_function(net, activation_layer): input_img = net.input layer_dict = dict([(layer.name, layer) for layer in net.layers[1:]]) layer_output = layer_dict[activation_layer].output max_output = K.max(layer_output, axis=3) saliency = K.gradients(K.sum(max_output), input_img)[0] return K.function([input_img, K.learning_phase()], [saliency])
def __build_activation_layer_gradients_func(self, dense_layer): shape = dense_layer.output_shape dtype = dense_layer.get_weights()[0].dtype input_data = tf.placeholder(shape=shape, dtype=dtype, name=self.activation_placeholder_name) self.activation_gradient_func = gradients(dense_layer.activation(input_data), input_data)
def example(): image_path = 'D:\\Onepredict_MK\\LG CNS\\cat.jpg' img = image.load_img(image_path, target_size=(224, 224)) img_input = preprocess_input(np.expand_dims(img, 0)) resnet = ResNet50(input_shape=(224, 224, 3), weights='imagenet', include_top=True) probs = resnet.predict(img_input) pred = np.argmax(probs[0]) activation_layer = resnet.layers[-3].name inp = resnet.input # for idx in range(1000): y_c = resnet.output.op.inputs[0][:, pred] A_k = resnet.get_layer(activation_layer).output grads = K.gradients(y_c, A_k)[0] # Model(inputs=[inp], outputs=[A_k, grads, resnet.output]) get_output = K.function(inputs=[inp], outputs=[A_k, grads, resnet.output]) [conv_output, grad_val, model_output] = get_output([img_input]) conv_output = conv_output[0] grad_val = grad_val[0] weights = np.mean(grad_val, axis=(0, 1)) grad_cam = np.zeros(dtype=np.float32, shape=conv_output.shape[0:2]) for k, w in enumerate(weights): grad_cam += w * conv_output[:, :, k] # RELU grad_cam = np.maximum(grad_cam, 0) grad_cam = cv2.resize(grad_cam, (224, 224)) # Guided grad-CAM register_gradient() guided_model, activation_layer = modify_backprop(resnet, 'GuidedBackProp', args.checkpoint_path, args.main_name) saliency_fn = compile_saliency_function(guided_model, activation_layer) saliency = saliency_fn([img_input, 0]) gradcam = saliency[0] * grad_cam[..., np.newaxis] gradcam = deprocess_image(gradcam) # grad_cam = ndimage.zoom(grad_cam, (32, 32), order=1) plt.subplot(1, 2, 1) plt.imshow(img, alpha=0.8) plt.imshow(grad_cam, cmap='jet', alpha=0.5) plt.axis('off') plt.subplot(1, 2, 2) plt.imshow(gradcam, cmap='jet', alpha=0.5) plt.axis('off') plt.show()
def build(self, a_image, ap_image, b_image, output_shape): self.output_shape = output_shape loss = self.build_loss(a_image, ap_image, b_image) # get the gradients of the generated image wrt the loss grads = K.gradients(loss, self.net_input) outputs = [loss] if type(grads) in {list, tuple}: outputs += grads else: outputs.append(grads) self.f_outputs = K.function([self.net_input], outputs)
def __init__(self, model, sess=None): K.set_learning_phase(0) self.model = model with self.model.graph.as_default(): with self.model.session.as_default(): self.class_grads = K.function([self.model.model.input], K.gradients( self.model.model.output, self.model.model.input)) self.out = K.function([self.model.model.input], self.model.model.output)
def build(self, a_image, ap_image, b_image, output_shape): self.output_shape = output_shape loss = self.build_loss(a_image, ap_image, b_image) # get the gradients of the generated image wrt the loss grads = K.gradients(loss, self.net_input) outputs = [loss] if type(grads) in {list, tuple}: outputs += grads else: outputs.append(grads) f_inputs = [self.net_input] for nnf in self.feature_nnfs: f_inputs.append(nnf.placeholder) self.f_outputs = K.function(f_inputs, outputs)
def CAM(self, img): pred = self.net.predict(img).flatten() width = int(pred[0] * util.IMG_WIDTH) height = int(abs(pred[1] * util.IMG_HEIGHT - util.IMG_HEIGHT)) print('PREDICTION: (%s, %s)' % (width, height)) x_out = self.net.output[:, 0] y_out = self.net.output[:, 1] last_conv_layer = self.net.get_layer('conv2d_6') grads = K.gradients(x_out, last_conv_layer.output)[0] pooled_grads = K.mean(grads, axis=(0, 1, 2)) iterate = K.function([self.net.input], [pooled_grads, last_conv_layer.output[0]]) pooled_grads_value, conv_layer_output_value = iterate([img]) for i in range(256): conv_layer_output_value[:, :, i] *= pooled_grads_value[i] heatmap = np.mean(conv_layer_output_value, axis=-1) heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) plt.figure(figsize=(10, 8)) plt.matshow(heatmap) plt.show() img = np.array(img * 255, dtype=np.uint8) img = np.reshape( img, (util.IMG_HEIGHT, util.IMG_WIDTH, util.INPUT_CHANNELS)) gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) print(img.shape) plt.figure(figsize=(10, 8)) plt.imshow(img) plt.show() heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0])) heatmap = np.uint8(255 * heatmap) heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) superimposed_img = heatmap * 0.8 + img superimposed_img = cv2.circle(superimposed_img, (width, height), 6, (255, 0, 0), -1) plt.figure(figsize=(10, 8)) plt.imshow(superimposed_img) plt.show() return
def gradient_penalty_loss(self, y_true, y_pred, averaged_samples): """ Computes gradient penalty based on prediction and weighted real / fake samples """ gradients = K.gradients(y_pred, averaged_samples)[0] # compute the euclidean norm by squaring ... gradients_sqr = K.square(gradients) # ... summing over the rows ... gradients_sqr_sum = K.sum(gradients_sqr, axis=np.arange(1, len(gradients_sqr.shape))) # ... and sqrt gradient_l2_norm = K.sqrt(gradients_sqr_sum) # compute lambda * (1 - ||grad||)^2 still for each single sample gradient_penalty = K.square(1 - gradient_l2_norm) # return the mean as loss over all the batch samples return K.mean(gradient_penalty)
def generate_pattern(layer_name, filter_index, size=150): layer_output = model.get_layer(layer_name).output loss = K.mean(layer_output[:, :, :, filter_index]) grads = K.gradients(loss, model.input)[0] grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) iterate = K.function([model.input], [loss, grads]) input_img_data = np.random.random((1, size, size, 3)) * 20 + 128. step = 1. for i in range(40): loss_value, grads_value = iterate([input_img_data]) input_img_data += grads_value * step img = input_img_data[0] return deprocess_image(img)
def viz_layer_output_cropmodel(img, layer_idx, pred_idx, crop_layer_idx, model, top_crop=30, bottom_crop=10): """Vizualize specific areas of the input image that trigger the network decision to be used for network with crop2D layer img : image np array layer_idx: layer outputs to show pred_idx: label index to show outputs for crop_layer_idx: index of cropping layer in keras model model: keras model top_crop: top pixels to trim off bottom_crop: bottom pixels to trim off""" cropped_img = img[top_crop : -bottom_crop, :, :] # compute prediction for specific image pred = model.predict(img.reshape((1,) + img.shape) / 255 - .5) output = model.output[:, pred_idx] last_conv_layer = model.layers[layer_idx] crop_layer = model.layers[crop_layer_idx] grads = K.gradients(output, last_conv_layer.output)[0] pooled_grads = K.mean(grads, axis=(0, 1, 2)) iterate = K.function([crop_layer.output], [pooled_grads, last_conv_layer.output[0]]) pooled_grads_value, conv_layer_output_value = iterate([croped_img.reshape((1,) + croped_img.shape) / 255 - .5]) n_filters = conv_layer_output_value.shape[-1] for i in range(n_filters): conv_layer_output_value[:, :, i] *= pooled_grads_value[i] heatmap = np.mean(conv_layer_output_value, axis=-1) heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) heatmap = cv2.resize(heatmap, (cropped_img.shape[1], cropped_img.shape[0])) cropped_img = cropped_img / 255 cropped_img = cropped_img.astype(np.float32) gray_img = cv2.cvtColor(cropped_img, cv2.COLOR_RGB2GRAY) added_image = cv2.addWeighted(gray_img, 0.5, heatmap, 0.1, 0) plt.imshow(added_image) plt.title(str(np.round(pred, 3)))
def build_model(self): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" # Define input layers states = layers.Input(shape=(self.state_size, ), name='states') actions = layers.Input(shape=(self.action_size, ), name='actions') # Add hidden layer(s) for state pathway net_states = layers.Dense(units=32, activation='relu')(states) net_states = layers.Dropout(0.8)(net_states) net_states = layers.Dense(units=64, activation='relu')(net_states) net_states = layers.Dropout(0.8)(net_states) # Add hidden layer(s) for action pathway net_actions = layers.Dense(units=32, activation='relu')(actions) net_actions = layers.Dense(units=64, activation='relu')(net_actions) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # Combine state and action pathways net = layers.Add()([net_states, net_actions]) net = layers.Activation('relu')(net) # Add more layers to the combined network if needed # Add final output layer to prduce action values (Q values) Q_values = layers.Dense(units=1, name='q_values')(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam(lr=0.0001) self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
def grad_cam_init(model): # model_output = model.output #[0, cls] model_output = model.output[:, 0] # model_output = model.output[0] # model_output = model.output[0,0] """ All 4 lines above produce same result""" DICT_LAYER_NAMES.clear() DICT_LAYER_OUTPUTS.clear() find_all_layers_recursive(model, 'Conv2D') """output of last convolution layer""" print("Using output of Conv2d Layer = %s" % DICT_LAYER_NAMES.get(max(DICT_LAYER_NAMES))) conv_output = DICT_LAYER_OUTPUTS.get(max(DICT_LAYER_NAMES)) """Chollet book and eqlique produce identical result""" """ Gradient of the positive class w.r.t. the output feature map of last_conv_layer """ grads = K.gradients(model_output, conv_output)[0] """ Vector where each entry is the mean intensity of the gradient over a specific feature-map channel done by np.mean() later """ # grads = K.mean(grads, axis=(0, 1, 2)) # <<<< chollet book; ## Normalize if necessary # grads = normalize(grads) """No difference in result""" """ Lets you access the values of the quantities you just defined: (pooled) grads and the output feature map of last_conv_layer, given a sample image """ # iterate_gradient_function = K.function([model.input], [conv_output[0], grads]) # <<<< chollet book iterate_gradient_function = K.function([model.input], [conv_output, grads]) return iterate_gradient_function
def generate_pattern(layer_name, filter_index, size=150): # Defining loss tensor for filter viz layer_output = mdl.get_layer(layer_name).output loss = K.mean(layer_output[:, :, :, filter_index]) # Obtaining grad of loss w.r.t. input grads = K.gradients(loss, mdl.input)[0] # Grad-normalization trick grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) iterate = K.function([mdl.input], [loss, grads]) # Loss maximization via SGD input_img_data = np.random.random((1, size, size, 3)) * 20 + 128. step = 1 for i in range(40): loss_value, grads_value = iterate([input_img_data]) input_img_data += grads_value * step img = input_img_data[0] return deprocess_image(img)
def grad_cam_modify(model, layer_nm, x, num): """ Args: model: model x: image input category_index: category index layer_name: last convolution layer name """ # outputs = [layer.output for layer in model.layers] # all layer outputs # class_output = model.output[:,num] class_output = model.get_output_at(num) convolution_output = model.get_layer(layer_nm).output # layer output grads = K.gradients(class_output, convolution_output)[0] # get gradients gradient_function = K.function([model.get_input_at(num)], [convolution_output, grads]) # get convolution output and gradients for input output, grads_val = gradient_function([x]) output, grads_val = output[0], grads_val[0] weights = (1/grads_val.shape[0])*np.sum(grads_val, axis=0) cam = np.dot(output, weights) cam = np.maximum(cam, 0) heatmap = cam / np.max(cam) heatmap = heatmap - np.mean(heatmap) heatmap = cv2.resize(heatmap, (x.shape[0], x.shape[1]),cv2.INTER_CUBIC) return heatmap
def define_loss_and_grads(layer_dict, input_tensor, im_size, layer_contributions, continuity, dream_l2): loss = K.variable(0.) for layer_name in layer_contributions: # add the L2 norm of the features of a layer to the loss assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.' coeff = layer_contributions[layer_name] x = layer_dict[layer_name].output shape = layer_dict[layer_name].output_shape # we avoid border artifacts by only involving non-border pixels in the loss loss = loss - coeff*K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:]) # add continuity loss loss = loss + continuity*continuity_loss(input_tensor, im_size[0], im_size[1]) / np.prod(im_size) # add image L2 norm to loss loss = loss + dream_l2*K.sum(K.square(input_tensor)) / np.prod(im_size) grads=K.gradients(loss, input_tensor) output=[loss] if isinstance(grads, (list, tuple)): output += grads else: output.append(grads) f_outputs = K.function([input_tensor], output) def eval_loss_and_grads(x): x = x.reshape((1,) + im_size) outs = f_outputs([x]) loss_value = outs[0] if len(outs[1:]) == 1: grad_values = outs[1].flatten().astype('float64') else: grad_values = np.array(outs[1:]).flatten().astype('float64') return loss_value, grad_values return eval_loss_and_grads
def process_conv_2d_layer(layer, input_img): """Generate images maximizing the activation of conv2d filters""" filter_cnt = layer.get_weights()[0].shape[-1] img_width, img_height, img_chans = input_img.shape[1:4] kept_filters = [] for filter_index in range(filter_cnt): print('{}:, filter {} of {}'.format(layer.name, filter_index, filter_cnt)) # we build a loss function that maximizes the activation # of the nth filter of the layer considered loss = K.mean(layer.output[:, :, :, filter_index]) # we compute the gradient of the input picture wrt this loss grads = K.gradients(loss, input_img)[0] # normalization trick: we normalize the gradient grads = normalize(grads) # this function returns the loss and grads given the input picture iterate = K.function([input_img], [loss, grads]) # we start from a gray image with some random noise input_img_data = np.random.random( (1, img_width, img_height, img_chans)) # run gradient ascent for i in range(GRADIENT_ASCENT_STEPS): loss_value, grads_value = iterate([input_img_data]) input_img_data += grads_value * GRADIENT_ASCENT_STEP_SIZE # decode the resulting input image img = deprocess_image(input_img_data[0]) kept_filters.append((img, loss_value)) return kept_filters
def make_gard_CAM(ckpt_path, main_name): test_images = fault_images # test_images = sorted(glob(args.data_path + '/*.png')) # test_images = normal_images best_model_path = os.path.join(ckpt_path, '201906_Model', main_name + '_model', main_name + '.h5') network = load_model(filepath=best_model_path) activation_layer = network.layers[-9].name inp = network.input A_k = network.get_layer(activation_layer).output register_gradient() guided_model, activation_layer = modify_backprop(network, 'GuidedBackProp', ckpt_path, main_name) saliency_fn = compile_saliency_function(guided_model, activation_layer) for num, img in enumerate(test_images): # if not '10h35m42s' in img.split('\\')[-1]: # continue # image = misc.imread(img) # image = np.expand_dims(image, axis=0) img_pil = image.load_img(img, target_size=(256, 256)) img_input = np.asarray(img_pil, dtype=np.float32) img_input = img_input / 255.0 img_tensor = np.expand_dims(img_input, axis=0) # img_tensor = img_tensor[:, :, :, 0:3] probs = network.predict(img_tensor) pred = np.argmax(probs[0]) "SAVE THE IMAGE" # if os.path.exists(os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred))) != True: # os.makedirs(os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred))) y_c = network.output.op.inputs[0][0, pred] grads = K.gradients(y_c, A_k)[0] get_output = K.function(inputs=[inp], outputs=[A_k, grads, network.output]) [conv_output, grad_val, model_output] = get_output([img_tensor]) conv_output = conv_output[0] grad_val = grad_val[0] weights = np.mean(grad_val, axis=(0, 1)) grad_cam = np.zeros(dtype=np.float32, shape=conv_output.shape[0:2]) for k, w in enumerate(weights): # grad_cam += np.maximum(w * conv_output[:, :, k], 0) grad_cam += w * conv_output[:, :, k] "RELU" grad_cam = np.maximum(grad_cam, 0) "NORMALIZATION" # if grad_cam.max() == 0: # grad_cam += 1e-18 # grad_cam = grad_cam / grad_cam.max() # grad_cam = cv2.resize(grad_cam, (img_input.shape[0], img_input.shape[1])) grad_cam = ndimage.zoom(grad_cam, (32, 32), order=1) f_list, t_list = get_f_t() plt.pcolormesh(f_list[0], t_list[0], img_input) "Guided grad-CAM" saliency = saliency_fn([img_tensor, 0]) gradcam = saliency[0] * grad_cam[..., np.newaxis] guided_back_cam = deprocess_image(gradcam) # guided_back_cam = guided_back_cam/guided_back_cam.max() # plt.imsave(fname=os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred), # img.split('\\')[-1][:-4] + '_guided.png'), arr=guided_back_cam) "PLOT" plt.imshow(img_pil, alpha=0.8) plt.imshow(grad_cam, cmap='jet', alpha=0.5) plt.colorbar() # plt.clim(0, 11) plt.axis('off') save_path = os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred), img.split('\\')[-1]) plt.savefig(save_path) plt.close() print('processing: {}/{}'.format(num + 1, len(test_images))) if args.write_mode == True: Write_mode(index=num, grad_cam=grad_cam)
def __init__(self, model, layer_name, index_feature, kind_of_optim='squared'): """ Initialisation function of the Optimization of the feature map Parameters ---------- model : keras model layer_name : string : the layer you want to use index_feature : string : the index of the feature in the given layer kind_of_optim : string The kind of optimisation maded on the given feature. The default is 'squared'. Use 'pos' for the positive feature maximisation and 'neg' for the negative one Returns ------- None """ self.model = model self.layer_name = layer_name self.index_feature = index_feature self.kind_of_optim = kind_of_optim dream = model.input # Get the symbolic outputs of each "key" layer (we gave them unique names). layers_all = [layer.name for layer in model.layers] if layer_name not in layers_all: raise ValueError('Layer ' + layer_name + ' not found in model.') # Define the loss. loss = K.variable(0.) for layer_local in model.layers: if layer_local.name == layer_name: x = layer_local.output # We avoid border artifacts by only involving non-border pixels in the loss. if K.image_data_format() == 'channels_first': raise (NotImplementedError) x_index_feature = x[:, index_feature, :, :] x_index_feature = K.expand_dims(x_index_feature, axis=1) scaling = K.prod( K.cast(K.shape(x_index_feature), 'float32')) loss = loss + K.sum( K.square(x_index_feature[:, :, 2:-2, 2:-2])) / scaling else: x_index_feature = x[:, :, :, index_feature] x_index_feature = K.expand_dims(x_index_feature, axis=-1) scaling = K.prod( K.cast(K.shape(x_index_feature), 'float32')) if self.kind_of_optim == 'squared': loss = loss + K.sum( K.square(x_index_feature[:, 2:-2, 2:-2, :])) / scaling elif self.kind_of_optim == 'pos': loss = loss + K.sum(x_index_feature[:, 2:-2, 2:-2, :]) / scaling elif self.kind_of_optim == 'neg': loss = loss - K.sum(x_index_feature[:, 2:-2, 2:-2, :]) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] self.fetch_loss_and_grads = K.function([dream], outputs)
content_feat = image_features[layer][CONTENT, :, :, :] target_feat = image_features[layer][TARGET, :, :, :] loss += content_weight * content_loss(content_feat, target_feat) nb_layers = len(style_feature_layers) for i, layer in enumerate(style_feature_layers): style_feat = image_features[layer][STYLE, :, :, :] target_feat = image_features[layer][TARGET, :, :, :] style_masks = mask_features[layer][STYLE, :, :, :] target_masks = mask_features[layer][TARGET, :, :, :] sl = style_loss(style_feat, target_feat, style_masks, target_masks) loss += (style_weight / (2**(nb_layers - (i + 1)))) * sl loss += total_variation_weight * total_variation_loss(target_image) loss_grads = K.gradients(loss, target_image) # Evaluator class for computing efficiency outputs = [loss] if type(loss_grads) in {list, tuple}: outputs += loss_grads else: outputs.append(loss_grads) f_outputs = K.function([target_image], outputs) def eval_loss_and_grads(x): if K.image_dim_ordering() == 'th': x = x.reshape((1, 3, img_nrows, img_ncols)) else:
loss = K.variable(0.) layer_features = outputs_dict[content_layer] target_image_features = layer_features[0, :, :, :] combination_features = layer_features[2, :, :, :] loss += content_weight * content_loss(target_image_features, combination_features) for layer_name in style_layers: layer_features = outputs_dict[layer_name] style_reference_features = layer_features[1, :, :, :] combination_features = layer_features[2, :, :, :] sl = style_loss(style_reference_features, combination_features) loss += (style_weight / len(style_layers)) * sl loss += total_variation_weight * total_variation_loss(combination_image) grads = K.gradients(loss, combination_image)[0] fetch_loss_and_grads = K.function([combination_image], [loss, grads]) evaluator = Evaluator() result_prefix = 'result' iterations = 20 x = preprocess_image(target_image_path) x = x.flatten() for i in range(iterations): print('반복 횟수:', i) start_time = time.time() x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x,