def __init__(self,
input_tensor,
losses,
input_range=(0, 255),
wrt_tensor=None,
norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
if self.input_tensor is self.wrt_tensor:
self.wrt_tensor_is_input_tensor = True
self.wrt_tensor = K.identity(self.wrt_tensor)
else:
self.wrt_tensor_is_input_tensor = False
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
if self.wrt_tensor_is_input_tensor:
grads = K.gradients(overall_loss, self.input_tensor)[0]
else:
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = K.l2_normalize(grads)
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function(
[self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def get_gradients(self, model):
"""
This function outputs a function to calculate the gradients based on the loss function, current weights/biases
Input:
- model: model that is training.
Returns:
- func: a function that uses input of features and true labels to calculate loss and hence gradients
"""
model_weights = model.trainable_weights
if self.only_weights:
weights = [
weight for weight in model_weights if 'kernel' in weight.name
]
else:
weights = [weight for weight in model_weights]
if self.num_classes > 1:
loss = K.mean(categorical_crossentropy(self.y_true, model.output))
else:
loss = K.mean(binary_crossentropy(self.y_true, model.output))
func = K.function([model.input, self.y_true],
K.gradients(loss, weights))
return func
def compile_saliency_function(model, act_num):
input_sig = model.get_input_at(0)
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_output = layer_dict[act_num].output
max_output = K.max(layer_output, axis=2)
saliency = K.gradients(K.sum(max_output), input_sig)[0]
return K.function([input_sig, K.learning_phase()], [saliency])
def gradient_penalty_loss(y_true, y_pred, interpolated_samples):
gradients = K.gradients(y_pred, interpolated_samples)[0]
gradient_l2_norm = K.sqrt(
K.sum(K.square(gradients), axis=[range(1, len(gradients.shape))]))
gradient_penalty = K.square(1 - gradient_l2_norm)
return K.mean(gradient_penalty)
def get_gradients(self, loss, params):
"""Returns gradients of `loss` with respect to `params`.
Arguments:
loss: Loss tensor.
params: List of variables.
Returns:
List of gradient tensors.
Raises:
ValueError: In case any gradient cannot be computed (e.g. if gradient
function not implemented).
"""
grads = K.gradients(loss, params)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
if hasattr(self, 'clipnorm'):
grads = [clip_ops.clip_by_norm(g, self.clipnorm) for g in grads]
if hasattr(self, 'clipvalue'):
grads = [
clip_ops.clip_by_value(g, -self.clipvalue, self.clipvalue)
for g in grads
]
return grads
def define_deepDream_model_layerBased(model):
dream = model.input
print('Model loaded.')
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
if layer_name not in layer_dict:
raise ValueError('Layer ' + layer_name + ' not found in model.')
coeff = settings['features'][layer_name]
x = layer_dict[layer_name].output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss = loss + coeff * K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
loss = loss + coeff * K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
def create_iterate(self,input_img, layer_output,filter_index):
'''
layer_output[:,:,:,0] is (Nsample, 94, 94) tensor contains:
W0^T [f(image)]_{i,j}], i = 1,..., 94, j = 1,..., 94
layer_output[:,:,:,1] contains:
W1^T [f(image)]_{i,j}], i = 1,..., 94, j = 1,..., 94
W0 and W1 are different kernel!
'''
## loss is a scalar
if len(layer_output.shape) == 4:
## conv layer
loss = K.mean(layer_output[:, :, :, filter_index])
elif len(layer_output.shape) ==2:
## fully connected layer
loss = K.mean(layer_output[:, filter_index])
# calculate the gradient of the loss evaluated at the provided image
grads = K.gradients(loss, input_img)[0]
# normalize the gradients
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
# iterate is a function taking (input_img, scalar) and output [loss_value, gradient_value]
iterate = K.function([input_img, K.learning_phase()], [loss, grads])
return(iterate)
def __init__(self, model, layer_name):
self.model = model
self.layer_name = layer_name
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layers_all = [layer.name for layer in model.layers]
if layer_name not in layers_all:
raise ValueError('Layer ' + layer_name + ' not found in model.')
# Define the loss.
loss = K.variable(0.)
for layer_local in model.layers:
if layer_local.name == layer_name:
x = layer_local.output
# We avoid border artifacts by only involving non-border pixels in the loss.
if K.image_data_format() == 'channels_first':
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
self.fetch_loss_and_grads = K.function([dream], outputs)
def get_gradients(self, loss, params):
"""Returns gradients of `loss` with respect to `params`.
Arguments:
loss: Loss tensor.
params: List of variables.
Returns:
List of gradient tensors.
Raises:
ValueError: In case any gradient cannot be computed (e.g. if gradient
function not implemented).
"""
grads = K.gradients(loss, params)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
if hasattr(self, 'clipnorm'):
grads = [clip_ops.clip_by_norm(g, self.clipnorm) for g in grads]
if hasattr(self, 'clipvalue'):
grads = [
clip_ops.clip_by_value(g, -self.clipvalue, self.clipvalue)
for g in grads
]
return grads
def grad_cam_batch(input_model, images, classes, layer_name):
"""GradCAM method for visualizing input saliency.
Same as grad_cam but processes multiple images in one run."""
loss = tf.gather_nd(input_model.output,
np.dstack([range(images.shape[0]), classes])[0])
layer_output = input_model.get_layer(layer_name).output
grads = K.gradients(loss, layer_output)[0]
gradient_fn = K.function(
[input_model.input, K.learning_phase()], [layer_output, grads])
conv_output, grads_val = gradient_fn([images, 0])
weights = np.mean(grads_val, axis=(1, 2))
cams = np.einsum('ijkl,il->ijk', conv_output, weights)
# Process CAMs
(H, W) = images.shape[1:-1]
new_cams = np.empty((images.shape[0], H, W))
for i in range(new_cams.shape[0]):
cam_i = cams[i] - cams[i].mean()
cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i, 2) + 1e-10)
new_cams[i] = cv2.resize(cam_i, (H, W), cv2.INTER_LINEAR)
new_cams[i] = np.maximum(new_cams[i], 0)
new_cams[i] = new_cams[i] / new_cams[i].max()
return new_cams
def GetGradient(data_x_win,model,perturbation):
y_pred = model.output
y_true = K.variable(np.array(df_YY_actual.iloc[0]))
#ten_x_scale = K.variable(np.array(data_x_win[0]))
loss = keras.losses.mean_squared_error(y_true, y_pred)
grads = K.gradients(loss,model.input)[0]
x_adv = K.sign(grads)
sess =K.get_session()
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
x_adv_0 = x_adv[0]
adv = []
if len(Y) != len(data_x_win):
print("WARNING!!!! Unequal length of X and Y")
#len(df_x_scale)
for i in range(len(data_x_win)):
adv_i = sess.run(x_adv_0, feed_dict={model.input:[data_x_win[i]],y_true:np.array(df_YY_actual.iloc[i])})
if i%1000 == 0:
print(i)
print(datetime.datetime.now().isoformat())
df_grd_i = pd.DataFrame(adv_i,columns = header)
adv.append(np.array(df_grd_i))
return adv
def compile_saliency_function(net, activation_layer):
input_img = net.input
layer_dict = dict([(layer.name, layer) for layer in net.layers[1:]])
layer_output = layer_dict[activation_layer].output
max_output = K.max(layer_output, axis=3)
saliency = K.gradients(K.sum(max_output), input_img)[0]
return K.function([input_img, K.learning_phase()], [saliency])
def __build_activation_layer_gradients_func(self, dense_layer):
shape = dense_layer.output_shape
dtype = dense_layer.get_weights()[0].dtype
input_data = tf.placeholder(shape=shape,
dtype=dtype,
name=self.activation_placeholder_name)
self.activation_gradient_func = gradients(dense_layer.activation(input_data), input_data)
def example():
image_path = 'D:\\Onepredict_MK\\LG CNS\\cat.jpg'
img = image.load_img(image_path, target_size=(224, 224))
img_input = preprocess_input(np.expand_dims(img, 0))
resnet = ResNet50(input_shape=(224, 224, 3),
weights='imagenet',
include_top=True)
probs = resnet.predict(img_input)
pred = np.argmax(probs[0])
activation_layer = resnet.layers[-3].name
inp = resnet.input
# for idx in range(1000):
y_c = resnet.output.op.inputs[0][:, pred]
A_k = resnet.get_layer(activation_layer).output
grads = K.gradients(y_c, A_k)[0]
# Model(inputs=[inp], outputs=[A_k, grads, resnet.output])
get_output = K.function(inputs=[inp], outputs=[A_k, grads, resnet.output])
[conv_output, grad_val, model_output] = get_output([img_input])
conv_output = conv_output[0]
grad_val = grad_val[0]
weights = np.mean(grad_val, axis=(0, 1))
grad_cam = np.zeros(dtype=np.float32, shape=conv_output.shape[0:2])
for k, w in enumerate(weights):
grad_cam += w * conv_output[:, :, k]
# RELU
grad_cam = np.maximum(grad_cam, 0)
grad_cam = cv2.resize(grad_cam, (224, 224))
# Guided grad-CAM
register_gradient()
guided_model, activation_layer = modify_backprop(resnet, 'GuidedBackProp',
args.checkpoint_path,
args.main_name)
saliency_fn = compile_saliency_function(guided_model, activation_layer)
saliency = saliency_fn([img_input, 0])
gradcam = saliency[0] * grad_cam[..., np.newaxis]
gradcam = deprocess_image(gradcam)
# grad_cam = ndimage.zoom(grad_cam, (32, 32), order=1)
plt.subplot(1, 2, 1)
plt.imshow(img, alpha=0.8)
plt.imshow(grad_cam, cmap='jet', alpha=0.5)
plt.axis('off')
plt.subplot(1, 2, 2)
plt.imshow(gradcam, cmap='jet', alpha=0.5)
plt.axis('off')
plt.show()
def build(self, a_image, ap_image, b_image, output_shape):
self.output_shape = output_shape
loss = self.build_loss(a_image, ap_image, b_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, self.net_input)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
self.f_outputs = K.function([self.net_input], outputs)
def __init__(self, model, sess=None):
K.set_learning_phase(0)
self.model = model
with self.model.graph.as_default():
with self.model.session.as_default():
self.class_grads = K.function([self.model.model.input],
K.gradients(
self.model.model.output,
self.model.model.input))
self.out = K.function([self.model.model.input],
self.model.model.output)
def build(self, a_image, ap_image, b_image, output_shape):
self.output_shape = output_shape
loss = self.build_loss(a_image, ap_image, b_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, self.net_input)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
f_inputs = [self.net_input]
for nnf in self.feature_nnfs:
f_inputs.append(nnf.placeholder)
self.f_outputs = K.function(f_inputs, outputs)
def CAM(self, img):
pred = self.net.predict(img).flatten()
width = int(pred[0] * util.IMG_WIDTH)
height = int(abs(pred[1] * util.IMG_HEIGHT - util.IMG_HEIGHT))
print('PREDICTION: (%s, %s)' % (width, height))
x_out = self.net.output[:, 0]
y_out = self.net.output[:, 1]
last_conv_layer = self.net.get_layer('conv2d_6')
grads = K.gradients(x_out, last_conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function([self.net.input],
[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([img])
for i in range(256):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.figure(figsize=(10, 8))
plt.matshow(heatmap)
plt.show()
img = np.array(img * 255, dtype=np.uint8)
img = np.reshape(
img, (util.IMG_HEIGHT, util.IMG_WIDTH, util.INPUT_CHANNELS))
gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
print(img.shape)
plt.figure(figsize=(10, 8))
plt.imshow(img)
plt.show()
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = heatmap * 0.8 + img
superimposed_img = cv2.circle(superimposed_img, (width, height), 6,
(255, 0, 0), -1)
plt.figure(figsize=(10, 8))
plt.imshow(superimposed_img)
plt.show()
return
def gradient_penalty_loss(self, y_true, y_pred, averaged_samples):
"""
Computes gradient penalty based on prediction and weighted real / fake samples
"""
gradients = K.gradients(y_pred, averaged_samples)[0]
# compute the euclidean norm by squaring ...
gradients_sqr = K.square(gradients)
# ... summing over the rows ...
gradients_sqr_sum = K.sum(gradients_sqr,
axis=np.arange(1, len(gradients_sqr.shape)))
# ... and sqrt
gradient_l2_norm = K.sqrt(gradients_sqr_sum)
# compute lambda * (1 - ||grad||)^2 still for each single sample
gradient_penalty = K.square(1 - gradient_l2_norm)
# return the mean as loss over all the batch samples
return K.mean(gradient_penalty)
def generate_pattern(layer_name, filter_index, size=150):
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
grads = K.gradients(loss, model.input)[0]
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
iterate = K.function([model.input], [loss, grads])
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.
step = 1.
for i in range(40):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
img = input_img_data[0]
return deprocess_image(img)
def viz_layer_output_cropmodel(img, layer_idx, pred_idx, crop_layer_idx, model, top_crop=30, bottom_crop=10):
"""Vizualize specific areas of the input image that trigger the network decision
to be used for network with crop2D layer
img : image np array
layer_idx: layer outputs to show
pred_idx: label index to show outputs for
crop_layer_idx: index of cropping layer in keras model
model: keras model
top_crop: top pixels to trim off
bottom_crop: bottom pixels to trim off"""
cropped_img = img[top_crop : -bottom_crop, :, :]
# compute prediction for specific image
pred = model.predict(img.reshape((1,) + img.shape) / 255 - .5)
output = model.output[:, pred_idx]
last_conv_layer = model.layers[layer_idx]
crop_layer = model.layers[crop_layer_idx]
grads = K.gradients(output, last_conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function([crop_layer.output], [pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([croped_img.reshape((1,) + croped_img.shape) / 255 - .5])
n_filters = conv_layer_output_value.shape[-1]
for i in range(n_filters):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
heatmap = cv2.resize(heatmap, (cropped_img.shape[1], cropped_img.shape[0]))
cropped_img = cropped_img / 255
cropped_img = cropped_img.astype(np.float32)
gray_img = cv2.cvtColor(cropped_img, cv2.COLOR_RGB2GRAY)
added_image = cv2.addWeighted(gray_img, 0.5, heatmap, 0.1, 0)
plt.imshow(added_image)
plt.title(str(np.round(pred, 3)))
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# Add hidden layer(s) for state pathway
net_states = layers.Dense(units=32, activation='relu')(states)
net_states = layers.Dropout(0.8)(net_states)
net_states = layers.Dense(units=64, activation='relu')(net_states)
net_states = layers.Dropout(0.8)(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(units=32, activation='relu')(actions)
net_actions = layers.Dense(units=64, activation='relu')(net_actions)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(lr=0.0001)
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def grad_cam_init(model):
# model_output = model.output #[0, cls]
model_output = model.output[:, 0]
# model_output = model.output[0]
# model_output = model.output[0,0]
""" All 4 lines above produce same result"""
DICT_LAYER_NAMES.clear()
DICT_LAYER_OUTPUTS.clear()
find_all_layers_recursive(model, 'Conv2D')
"""output of last convolution layer"""
print("Using output of Conv2d Layer = %s" %
DICT_LAYER_NAMES.get(max(DICT_LAYER_NAMES)))
conv_output = DICT_LAYER_OUTPUTS.get(max(DICT_LAYER_NAMES))
"""Chollet book and eqlique produce identical result"""
"""
Gradient of the positive class w.r.t.
the output feature map of last_conv_layer
"""
grads = K.gradients(model_output, conv_output)[0]
"""
Vector where each entry is the mean intensity
of the gradient over a specific feature-map channel
done by np.mean() later
"""
# grads = K.mean(grads, axis=(0, 1, 2)) # <<<< chollet book;
## Normalize if necessary
# grads = normalize(grads)
"""No difference in result"""
"""
Lets you access the values of the quantities
you just defined: (pooled) grads and the
output feature map of last_conv_layer, given
a sample image
"""
# iterate_gradient_function = K.function([model.input], [conv_output[0], grads]) # <<<< chollet book
iterate_gradient_function = K.function([model.input], [conv_output, grads])
return iterate_gradient_function
def generate_pattern(layer_name, filter_index, size=150):
# Defining loss tensor for filter viz
layer_output = mdl.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
# Obtaining grad of loss w.r.t. input
grads = K.gradients(loss, mdl.input)[0]
# Grad-normalization trick
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
iterate = K.function([mdl.input], [loss, grads])
# Loss maximization via SGD
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.
step = 1
for i in range(40):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
img = input_img_data[0]
return deprocess_image(img)
def grad_cam_modify(model, layer_nm, x, num):
"""
Args:
model: model
x: image input
category_index: category index
layer_name: last convolution layer name
"""
# outputs = [layer.output for layer in model.layers] # all layer outputs
# class_output = model.output[:,num]
class_output = model.get_output_at(num)
convolution_output = model.get_layer(layer_nm).output # layer output
grads = K.gradients(class_output, convolution_output)[0] # get gradients
gradient_function = K.function([model.get_input_at(num)], [convolution_output, grads]) # get convolution output and gradients for input
output, grads_val = gradient_function([x])
output, grads_val = output[0], grads_val[0]
weights = (1/grads_val.shape[0])*np.sum(grads_val, axis=0)
cam = np.dot(output, weights)
cam = np.maximum(cam, 0)
heatmap = cam / np.max(cam)
heatmap = heatmap - np.mean(heatmap)
heatmap = cv2.resize(heatmap, (x.shape[0], x.shape[1]),cv2.INTER_CUBIC)
return heatmap
def define_loss_and_grads(layer_dict, input_tensor, im_size, layer_contributions, continuity, dream_l2):
loss = K.variable(0.)
for layer_name in layer_contributions:
# add the L2 norm of the features of a layer to the loss
assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.'
coeff = layer_contributions[layer_name]
x = layer_dict[layer_name].output
shape = layer_dict[layer_name].output_shape
# we avoid border artifacts by only involving non-border pixels in the loss
loss = loss - coeff*K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
# add continuity loss
loss = loss + continuity*continuity_loss(input_tensor, im_size[0], im_size[1]) / np.prod(im_size)
# add image L2 norm to loss
loss = loss + dream_l2*K.sum(K.square(input_tensor)) / np.prod(im_size)
grads=K.gradients(loss, input_tensor)
output=[loss]
if isinstance(grads, (list, tuple)):
output += grads
else:
output.append(grads)
f_outputs = K.function([input_tensor], output)
def eval_loss_and_grads(x):
x = x.reshape((1,) + im_size)
outs = f_outputs([x])
loss_value = outs[0]
if len(outs[1:]) == 1:
grad_values = outs[1].flatten().astype('float64')
else:
grad_values = np.array(outs[1:]).flatten().astype('float64')
return loss_value, grad_values
return eval_loss_and_grads
def process_conv_2d_layer(layer, input_img):
"""Generate images maximizing the activation of conv2d filters"""
filter_cnt = layer.get_weights()[0].shape[-1]
img_width, img_height, img_chans = input_img.shape[1:4]
kept_filters = []
for filter_index in range(filter_cnt):
print('{}:, filter {} of {}'.format(layer.name, filter_index,
filter_cnt))
# we build a loss function that maximizes the activation
# of the nth filter of the layer considered
loss = K.mean(layer.output[:, :, :, filter_index])
# we compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, input_img)[0]
# normalization trick: we normalize the gradient
grads = normalize(grads)
# this function returns the loss and grads given the input picture
iterate = K.function([input_img], [loss, grads])
# we start from a gray image with some random noise
input_img_data = np.random.random(
(1, img_width, img_height, img_chans))
# run gradient ascent
for i in range(GRADIENT_ASCENT_STEPS):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * GRADIENT_ASCENT_STEP_SIZE
# decode the resulting input image
img = deprocess_image(input_img_data[0])
kept_filters.append((img, loss_value))
return kept_filters
def make_gard_CAM(ckpt_path, main_name):
test_images = fault_images
# test_images = sorted(glob(args.data_path + '/*.png'))
# test_images = normal_images
best_model_path = os.path.join(ckpt_path, '201906_Model',
main_name + '_model', main_name + '.h5')
network = load_model(filepath=best_model_path)
activation_layer = network.layers[-9].name
inp = network.input
A_k = network.get_layer(activation_layer).output
register_gradient()
guided_model, activation_layer = modify_backprop(network, 'GuidedBackProp',
ckpt_path, main_name)
saliency_fn = compile_saliency_function(guided_model, activation_layer)
for num, img in enumerate(test_images):
# if not '10h35m42s' in img.split('\\')[-1]:
# continue
# image = misc.imread(img)
# image = np.expand_dims(image, axis=0)
img_pil = image.load_img(img, target_size=(256, 256))
img_input = np.asarray(img_pil, dtype=np.float32)
img_input = img_input / 255.0
img_tensor = np.expand_dims(img_input, axis=0)
# img_tensor = img_tensor[:, :, :, 0:3]
probs = network.predict(img_tensor)
pred = np.argmax(probs[0])
"SAVE THE IMAGE"
# if os.path.exists(os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred))) != True:
# os.makedirs(os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred)))
y_c = network.output.op.inputs[0][0, pred]
grads = K.gradients(y_c, A_k)[0]
get_output = K.function(inputs=[inp],
outputs=[A_k, grads, network.output])
[conv_output, grad_val, model_output] = get_output([img_tensor])
conv_output = conv_output[0]
grad_val = grad_val[0]
weights = np.mean(grad_val, axis=(0, 1))
grad_cam = np.zeros(dtype=np.float32, shape=conv_output.shape[0:2])
for k, w in enumerate(weights):
# grad_cam += np.maximum(w * conv_output[:, :, k], 0)
grad_cam += w * conv_output[:, :, k]
"RELU"
grad_cam = np.maximum(grad_cam, 0)
"NORMALIZATION"
# if grad_cam.max() == 0:
# grad_cam += 1e-18
# grad_cam = grad_cam / grad_cam.max()
# grad_cam = cv2.resize(grad_cam, (img_input.shape[0], img_input.shape[1]))
grad_cam = ndimage.zoom(grad_cam, (32, 32), order=1)
f_list, t_list = get_f_t()
plt.pcolormesh(f_list[0], t_list[0], img_input)
"Guided grad-CAM"
saliency = saliency_fn([img_tensor, 0])
gradcam = saliency[0] * grad_cam[..., np.newaxis]
guided_back_cam = deprocess_image(gradcam)
# guided_back_cam = guided_back_cam/guided_back_cam.max()
# plt.imsave(fname=os.path.join(args.test_result_path, 'Grad_CAM{}'.format(pred),
# img.split('\\')[-1][:-4] + '_guided.png'), arr=guided_back_cam)
"PLOT"
plt.imshow(img_pil, alpha=0.8)
plt.imshow(grad_cam, cmap='jet', alpha=0.5)
plt.colorbar()
# plt.clim(0, 11)
plt.axis('off')
save_path = os.path.join(args.test_result_path,
'Grad_CAM{}'.format(pred),
img.split('\\')[-1])
plt.savefig(save_path)
plt.close()
print('processing: {}/{}'.format(num + 1, len(test_images)))
if args.write_mode == True:
Write_mode(index=num, grad_cam=grad_cam)
def __init__(self,
model,
layer_name,
index_feature,
kind_of_optim='squared'):
"""
Initialisation function of the Optimization of the feature map
Parameters
----------
model : keras model
layer_name : string : the layer you want to use
index_feature : string : the index of the feature in the given layer
kind_of_optim : string
The kind of optimisation maded on the given feature. The default is 'squared'.
Use 'pos' for the positive feature maximisation and 'neg' for the negative one
Returns
-------
None
"""
self.model = model
self.layer_name = layer_name
self.index_feature = index_feature
self.kind_of_optim = kind_of_optim
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layers_all = [layer.name for layer in model.layers]
if layer_name not in layers_all:
raise ValueError('Layer ' + layer_name + ' not found in model.')
# Define the loss.
loss = K.variable(0.)
for layer_local in model.layers:
if layer_local.name == layer_name:
x = layer_local.output
# We avoid border artifacts by only involving non-border pixels in the loss.
if K.image_data_format() == 'channels_first':
raise (NotImplementedError)
x_index_feature = x[:, index_feature, :, :]
x_index_feature = K.expand_dims(x_index_feature, axis=1)
scaling = K.prod(
K.cast(K.shape(x_index_feature), 'float32'))
loss = loss + K.sum(
K.square(x_index_feature[:, :, 2:-2, 2:-2])) / scaling
else:
x_index_feature = x[:, :, :, index_feature]
x_index_feature = K.expand_dims(x_index_feature, axis=-1)
scaling = K.prod(
K.cast(K.shape(x_index_feature), 'float32'))
if self.kind_of_optim == 'squared':
loss = loss + K.sum(
K.square(x_index_feature[:, 2:-2,
2:-2, :])) / scaling
elif self.kind_of_optim == 'pos':
loss = loss + K.sum(x_index_feature[:, 2:-2,
2:-2, :]) / scaling
elif self.kind_of_optim == 'neg':
loss = loss - K.sum(x_index_feature[:, 2:-2,
2:-2, :]) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
self.fetch_loss_and_grads = K.function([dream], outputs)
content_feat = image_features[layer][CONTENT, :, :, :]
target_feat = image_features[layer][TARGET, :, :, :]
loss += content_weight * content_loss(content_feat, target_feat)
nb_layers = len(style_feature_layers)
for i, layer in enumerate(style_feature_layers):
style_feat = image_features[layer][STYLE, :, :, :]
target_feat = image_features[layer][TARGET, :, :, :]
style_masks = mask_features[layer][STYLE, :, :, :]
target_masks = mask_features[layer][TARGET, :, :, :]
sl = style_loss(style_feat, target_feat, style_masks, target_masks)
loss += (style_weight / (2**(nb_layers - (i + 1)))) * sl
loss += total_variation_weight * total_variation_loss(target_image)
loss_grads = K.gradients(loss, target_image)
# Evaluator class for computing efficiency
outputs = [loss]
if type(loss_grads) in {list, tuple}:
outputs += loss_grads
else:
outputs.append(loss_grads)
f_outputs = K.function([target_image], outputs)
def eval_loss_and_grads(x):
if K.image_dim_ordering() == 'th':
x = x.reshape((1, 3, img_nrows, img_ncols))
else:
loss = K.variable(0.)
layer_features = outputs_dict[content_layer]
target_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss += content_weight * content_loss(target_image_features,
combination_features)
for layer_name in style_layers:
layer_features = outputs_dict[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
sl = style_loss(style_reference_features, combination_features)
loss += (style_weight / len(style_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)
grads = K.gradients(loss, combination_image)[0]
fetch_loss_and_grads = K.function([combination_image], [loss, grads])
evaluator = Evaluator()
result_prefix = 'result'
iterations = 20
x = preprocess_image(target_image_path)
x = x.flatten()
for i in range(iterations):
print('반복 횟수:', i)
start_time = time.time()
x, min_val, info = fmin_l_bfgs_b(evaluator.loss,
x,
