def _init_class_gradients(self, label: Optional[Union[int, List[int], np.ndarray]] = None) -> None:
# pylint: disable=E0401
if self.is_tensorflow:
import tensorflow.keras.backend as k
else:
import keras.backend as k
if len(self._output.shape) == 2:
nb_outputs = self._output.shape[1]
else: # pragma: no cover
raise ValueError("Unexpected output shape for classification in Keras model.")
if label is None:
logger.debug("Computing class gradients for all %i classes.", self.nb_classes)
if not hasattr(self, "_class_gradients"):
class_gradients = [k.gradients(self._predictions_op[:, i], self._input)[0] for i in range(nb_outputs)]
self._class_gradients = k.function([self._input], class_gradients)
else:
if isinstance(label, int):
unique_labels = [label]
else:
unique_labels = np.unique(label)
logger.debug("Computing class gradients for classes %s.", str(unique_labels))
if not hasattr(self, "_class_gradients_idx"):
self._class_gradients_idx = [None for _ in range(nb_outputs)]
for current_label in unique_labels:
if self._class_gradients_idx[current_label] is None:
class_gradients = [k.gradients(self._predictions_op[:, current_label], self._input)[0]]
self._class_gradients_idx[current_label] = k.function(
[self._input, k.learning_phase()], class_gradients
)
def _init_class_gradients(self, label=None):
# pylint: disable=E0401
if self.is_tensorflow:
import tensorflow.keras.backend as k
else:
import keras.backend as k
if len(self._output.shape) == 2:
nb_outputs = self._output.shape[1]
else:
raise ValueError('Unexpected output shape for classification in Keras model.')
if label is None:
logger.debug('Computing class gradients for all %i classes.', self.nb_classes())
if not hasattr(self, '_class_gradients'):
class_gradients = [k.gradients(self._predictions_op[:, i], self._input)[0] for i in range(nb_outputs)]
self._class_gradients = k.function([self._input], class_gradients)
else:
if isinstance(label, int):
unique_labels = [label]
else:
unique_labels = np.unique(label)
logger.debug('Computing class gradients for classes %s.', str(unique_labels))
if not hasattr(self, '_class_gradients_idx'):
self._class_gradients_idx = [None for _ in range(nb_outputs)]
for current_label in unique_labels:
if self._class_gradients_idx[current_label] is None:
class_gradients = [k.gradients(self._predictions_op[:, current_label], self._input)[0]]
self._class_gradients_idx[current_label] = k.function([self._input], class_gradients)
def build_model(self):
self.inputs = Input(shape=(
28,
28,
1,
), name='imgs')
self.labels = Input((10, ), name='labels')
self.aa = Conv2D(
32,
kernel_size=(3, 3),
activation='relu',
)(self.inputs)
self.bb = MaxPooling2D(pool_size=(2, 2))(self.aa)
self.cc = Conv2D(64, (3, 3), activation='relu')(self.bb)
self.dd = MaxPooling2D(pool_size=(2, 2))(self.cc)
self.ee = Conv2D(128, (3, 3), activation='relu')(self.dd)
self.a = Conv2D(128, (3, 3), activation='relu')(self.ee)
self.b = MaxPooling2D(pool_size=(2, 2))(self.a)
self.c = Flatten()(self.b)
self.d = Dense(64, activation='relu')(self.c)
self.e = Dropout(0.5)(self.d)
self.before_soft_max = Dense(10)(self.e)
self.predictions = Activation('softmax')(self.before_soft_max)
self.g = Lambda(lambda x: K.gradients(x[0] * x[2], x[1]),
output_shape=list(self.a.shape))(
[self.before_soft_max, self.a, self.labels])
self.cost = Lambda(lambda x: (-1) * K.sum(x[0] * K.log(x[1]), axis=1),
output_shape=list(self.labels.shape))(
[self.labels, self.predictions])
self.gb_grad = Lambda(lambda x: K.gradients(x[0], x[1]),
output_shape=list(
self.inputs.shape))([self.cost, self.inputs])
def get_gradients(self, *args):
grad = K.gradients(loss, model.trainable_weights)
grad_snapshot = K.gradients(loss_snapshot,
snapshot.trainable_weights)
return [
g - gs + bg
for g, gs, bg in zip(grad, grad_snapshot, batch_grad)
]
def __init__(self,
input_tensor,
losses,
input_range=(0, 255),
wrt_tensor=None,
norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
if self.input_tensor is self.wrt_tensor:
self.wrt_tensor_is_input_tensor = True
self.wrt_tensor = K.identity(self.wrt_tensor)
else:
self.wrt_tensor_is_input_tensor = False
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
if self.wrt_tensor_is_input_tensor:
grads = K.gradients(overall_loss, self.input_tensor)[0]
else:
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = K.l2_normalize(grads)
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function(
[self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def gradient_penalty(samples, output, weight):
gradients = K.gradients(output, samples)[0]
gradients_sqr = K.square(gradients)
gradient_penalty = K.sum(gradients_sqr,
axis=np.arange(1, len(gradients_sqr.shape)))
return K.mean(gradient_penalty) * weight
def gradient_penalty_loss(y_true, y_pred, averaged_samples,
gradient_penalty_weight):
"""Calculates the gradient penalty loss for a batch of "averaged" samples.
In Improved WGANs, the 1-Lipschitz constraint is enforced by adding a term to the
loss function that penalizes the network if the gradient norm moves away from 1.
However, it is impossible to evaluate this function at all points in the input
space. The compromise used in the paper is to choose random points on the lines
between real and generated samples, and check the gradients at these points. Note
that it is the gradient w.r.t. the input averaged samples, not the weights of the
discriminator, that we're penalizing!
In order to evaluate the gradients, we must first run samples through the generator
and evaluate the loss. Then we get the gradients of the discriminator w.r.t. the
input averaged samples. The l2 norm and penalty can then be calculated for this
gradient.
Note that this loss function requires the original averaged samples as input, but
Keras only supports passing y_true and y_pred to loss functions. To get around this,
we make a partial() of the function with the averaged_samples argument, and use that
for model training."""
# first get the gradients:
# assuming: - that y_pred has dimensions (batch_size, 1)
# - averaged_samples has dimensions (batch_size, nbr_features)
# gradients afterwards has dimension (batch_size, nbr_features), basically
# a list of nbr_features-dimensional gradient vectors
gradients = K.gradients(y_pred, averaged_samples)[0]
# compute the euclidean norm by squaring ...
gradients_sqr = K.square(gradients)
# ... summing over the rows ...
gradients_sqr_sum = K.sum(gradients_sqr,
axis=np.arange(1, len(gradients_sqr.shape)))
# ... and sqrt
gradient_l2_norm = K.sqrt(gradients_sqr_sum)
# compute lambda * (1 - ||grad||)^2 still for each single sample
gradient_penalty = gradient_penalty_weight * K.square(1 - gradient_l2_norm)
# return the mean as loss over all the batch samples
return K.mean(gradient_penalty)
def _get_gradcam_tensor_function(self):
model_input = self.model.input
y_c = self.model.outputs[0].op.inputs[0][0, self.class_idx]
A_k = self.model.get_layer(self.activation_layer).output
tensor_function = K.function([model_input], [A_k, K.gradients(y_c, A_k)[0]])
return tensor_function
def test_deserialized_gradients(self, ef_multinetwork):
ef_multinetwork._init_model()
s = dumps(ef_multinetwork)
ef_multinetwork = loads(s)
model = ef_multinetwork.model['combined']
grads = K.gradients(model.total_loss, model.trainable_weights)
assert all([g is not None for g in grads])
def train_D(self, real_images):
num_images = K.int_shape(real_images)[0]
noise = make_noise(num_images, self.latent_size)
with tf.GradientTape() as critic_tape:
fake_images = self.generator(noise, training=True)
alpha = tf.random.uniform((num_images, 1, 1, 1)) # [0, 1]
other_samples = alpha * real_images + ((1 - alpha) * fake_images)
fake_logits = self.critic(fake_images, training=True)
real_logits = self.critic(real_images, training=True)
other_logits = self.critic(other_samples, training=True)
critic_loss = K.mean(fake_logits) + (-1.0) * K.mean(real_logits)
# gradient penalty
sample_gradients = K.gradients(other_logits, other_samples)
l2_norm = K.sqrt(K.sum(K.square(sample_gradients), axis=[1, 2, 3]))
gp = K.mean(K.square(l2_norm - 1.0))
# Critic Loss = fake_loss - real_loss + lambda * gp
total_loss = critic_loss + self.gp_coefficient * gp
# Update Critic
critic_gradients = critic_tape.gradient(
total_loss, self.critic.trainable_variables)
self.c_opt.apply_gradients(
zip(critic_gradients, self.critic.trainable_variables))
return total_loss
def generate_pattern(layer_name, filter_index, size=150):
# Build a loss function that maximizes the activation
# of the nth filter of the layer considered.
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
# Compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, model.input)[0]
# Normalization trick: we normalize the gradient
grads /= K.sqrt(K.mean(K.square(grads))) + 1e-5
# This function returns the loss and grads given the input picture
iterate = K.function([model.input], [loss, grads])
# We start from a gray image with some noise
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.0
# Run gradient ascent for 40 steps
step = 1.0
for i in range(40):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
img = input_img_data[0]
return deprocess_image(img)
def gradient_penalty(inputs, preds, weights):
gradients = K.gradients(preds, inputs)[0] #first layer gradients
gradients_sqr = K.square(gradients)
gradients_sqr_norm = K.sum(
gradients_sqr, axis=np.arange(1, len(gradients_sqr.shape))
) #add up gradients of each image (every axis other than 0)
return K.mean(gradients_sqr_norm * weights) #average over batch
def get_gradients(self, loss, params):
grads = super(E2EFS_RMSprop, self).get_gradients(loss, params)
if not (hasattr(self.e2efs_layer, 'regularization_loss')):
return grads
e2efs_grad = grads[0]
e2efs_regularizer_grad = K.gradients(
self.e2efs_layer.regularization_loss, [self.e2efs_layer.kernel])[0]
# norm_e2efs_grad_clipped = K.maximum(0.1, tf.norm(e2efs_grad) + K.epsilon())
# e2efs_regularizer_grad_corrected = e2efs_regularizer_grad / K.max(K.abs(e2efs_regularizer_grad) + K.epsilon())
# e2efs_grad_corrected = e2efs_grad / K.max(K.abs(e2efs_grad) + K.epsilon())
e2efs_regularizer_grad_corrected = e2efs_regularizer_grad / (
K.tf.norm(e2efs_regularizer_grad) + K.epsilon())
e2efs_grad_corrected = e2efs_grad / (K.tf.norm(e2efs_grad) +
K.epsilon())
# e2efs_regularizer_grad_corrected = norm_e2efs_grad_clipped * e2efs_regularizer_grad / (tf.norm(e2efs_regularizer_grad) + K.epsilon())
combined_e2efs_grad = (1. - self.e2efs_layer.moving_factor) * e2efs_grad_corrected + \
self.e2efs_layer.moving_factor * e2efs_regularizer_grad_corrected
# combined_e2efs_grad_norm = tf.norm(combined_e2efs_grad) + K.epsilon()
# combined_e2efs_grad = optimizers.clip_norm(combined_e2efs_grad, self.e2efs_norm_max, combined_e2efs_grad_norm)
# combined_e2efs_grad = K.maximum(combined_e2efs_grad_norm, self.e2efs_norm_min) / combined_e2efs_grad_norm * combined_e2efs_grad
combined_e2efs_grad = K.sign(
self.e2efs_layer.moving_factor) * K.minimum(
self.th, K.max(
K.abs(combined_e2efs_grad))) * combined_e2efs_grad / K.max(
K.abs(combined_e2efs_grad) + K.epsilon())
# combined_e2efs_grad = K.tf.Print(combined_e2efs_grad, [K.max(combined_e2efs_grad), K.min(combined_e2efs_grad)])
grads[0] = combined_e2efs_grad
return grads
def rae_loss(x_true, x_pred, z):
L_rec = utils.recon(x_true, x_pred)
L_rae = K.mean(0.5 * K.sum(K.square(z), axis=-1))
GP = K.sum(K.square(K.gradients(K.square(x_pred), z)), axis=-1)
return L_rec + gamma * L_rae + lamb * GP
def keras_fn():
'''
对梯度值进行了修改，它是怎么做到不超出图片的数值范围的？
:return: 更新后的输入值
'''
conv_base = vgg16.VGG16(include_top=False, weights='imagenet')
im = Image.open('/Users/yongli/Pictures/flower.jpeg')
im = im.resize((224, 224))
im_arr = np.array(im)
im_arr = np.expand_dims(im_arr, axis=0)
im_arr = im_arr.astype('float32')
layer_name = 'block3_conv1'
filter_idx = 0
layer_output = conv_base.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_idx])
grads = K.gradients(loss, conv_base.input)[0]
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
iterate = K.function([conv_base.input], [loss, grads])
step = 1
for i in range(40):
loss_value, grads_value = iterate([im_arr])
im_arr += grads_value
plot.imshow(im_arr[0])
plot.show()
def getwhere(x):
"""Calculate the 'where' mask that contains switches indicating which
index contained the max value when MaxPool2D was applied. Using the
gradient of the sum is a nice trick to keep everything high level.
"""
y_prepool, y_postpool = x
return K.gradients(K.sum(y_postpool), y_prepool)
def get_centralized_gradients(optimizer, loss, params):
"""Compute a list of centralized gradients.
Modified version of tf.keras.optimizers.Optimizer.get_gradients:
https://github.com/keras-team/keras/blob/1931e2186843ad3ca2507a3b16cb09a7a3db5285/keras/optimizers.py#L88-L101
Reference:
https://arxiv.org/pdf/2004.01461.pdf
"""
grads = []
for grad in K.gradients(loss, params):
rank = len(grad.shape)
if rank > 1:
grad -= tf.reduce_mean(grad,
axis=list(range(rank - 1)),
keep_dims=True)
grads.append(grad)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
if hasattr(optimizer, 'clipnorm') and optimizer.clipnorm > 0:
norm = K.sqrt(sum([K.sum(K.square(g)) for g in grads]))
grads = [
tf.keras.optimizers.clip_norm(g, optimizer.clipnorm, norm)
for g in grads
]
if hasattr(optimizer, 'clipvalue') and optimizer.clipvalue > 0:
grads = [
K.clip(g, -optimizer.clipvalue, optimizer.clipvalue) for g in grads
]
return grads
def sensitivityBinaryCrossentropy(self, data, labels, batchSize=128):
"""
Returns the sensitivity of the categorical crossentropy with respect
to the input data.
:param data: Input data.
:param labels: Respective labels.
:param batchSize: The network iterates through the dataset with
batches, whose batch size is given by this parameter.
"""
# Gradient of the binary crossentropy of the model output regarding
# the model input
grads = K.gradients(
K.binary_crossentropy(target=labels,
output=self.network.output,
from_logits=False), self.network.input)[0]
# Define a Keras function to calculate the gradient for a given input
grads_func = K.function([self.network.input], [grads])
# Run the calculation for the gradients
sens = grads_func([data])
return sens
def showimage():
#last_conv_layer = model.get_layer("multiply")
last_conv_layer = model.get_layer("conv2d_5")
grads = K.gradients(model.output, last_conv_layer.output)[0]
print(grads, ' grads')
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function([model.input], [pooled_grads, last_conv_layer.output])
print(iterate)
pooled_grads_value, conv_layer_output_value = iterate([img_data])
print(pooled_grads_value.shape, ' pooled_grads')
print(conv_layer_output_value.shape, ' conv_layer_output_value')
for j in range(128):
conv_layer_output_value[:, :, :, j] *= pooled_grads_value[j]
heatmap = np.mean(conv_layer_output_value, axis=-1)
#print (heatmap.shape)
heatmap = np.squeeze(heatmap)
#print (heatmap.shape,' squeezed')
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
#print(heatmap)
#print (validation_data_dir+'/'+predict_list.filenames[i])
img = cv2.imread(validation_data_dir + '/' + predict_list.filenames[i])
#print(heatmap.dtype)
#print (heatmap.shape,' truoc resize')
heatmap = cv2.resize(np.float32(heatmap), (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
#print(heatmap.dtype)
#print (heatmap.shape)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = cv2.addWeighted(img, 0.6, heatmap, 0.4, 0)
cv2.imshow(predict_list.filenames[i], cv2.resize(img, (512, 512)))
cv2.imshow("GradCam", cv2.resize(superimposed_img, (512, 512)))
cv2.waitKey(0)
cv2.destroyAllWindows()
def CAM(img_path, out_path):
K.clear_session()
tf.compat.v1.disable_eager_execution()
model = VGG16(weights='imagenet')
img = image.load_img(img_path, target_size=(224,224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
preds = model.predict(x)
argmax = np.argmax(preds[0])
output = model.output[:, argmax]
last_conv_layer = model.get_layer('block5_conv3')
grads = K.gradients(output, last_conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0,1,2))
iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([x])
for i in range(512):
conv_layer_output_value[:, :, 1] *= pooled_grads_value[i]
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
img = cv2.imread(img_path)
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
hif = .8
superimposed_img = heatmap * hif + img
cv2.imwrite(out_path, superimposed_img)
def build_multi_reg(self, n_fault, activation=layers.LeakyReLU(),
output_activation=None):
"""
Build multiple reg model from input to y, include build_inf
:param n_fault: int. Amount of faults.
:param activation:
:param output_activation:
:return:
"""
print("build_multi_reg begin: n_fault =", n_fault)
x = self.build_inf(activation)
# x = layers.Dense(64, kernel_initializer=self.W_init)(x)
# x = activation(x)
x = layers.Dense(32, kernel_initializer=self.W_init)(x)
x = activation(x)
self.x_feature['high_out'] = x
self.y = layers.Dense(n_fault, activation=output_activation,
kernel_initializer=self.W_init, name='y')(x)
self.pre_model = Model(self.x, self.y)
self.gradients = []
for i in range(n_fault):
self.gradients.append(K.gradients(self.y[:, i], [self.x]))
self.fault = Input(shape=(n_fault,), dtype='float32', name='fault')
self.y_ = Input(shape=(n_fault,), dtype='float32', name='y_')
loss = MultiRegLossLayer(name='multi_loss_layer')([self.y, self.y_, self.fault])
self.train_model = Model([self.x, self.fault, self.y_], loss)
self.train_model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),
loss=None)
print("build_multi_reg end!")
def compile_saliency_function(model, activation_layer='block5_conv3'):
input_img = model.input
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_output = layer_dict[activation_layer].output
max_output = K.max(layer_output, axis=3)
saliency = K.gradients(K.sum(max_output), input_img)[0]
return K.function([input_img, K.learning_phase()], [saliency])
def __init__(self,
sess,
s_dim,
a_dim,
num_actor_vars,
weights_len,
gamma,
tau,
learning_rate,
scope="critic"):
self.sess = sess
self.s_dim = s_dim
self.a_dim = a_dim
self.num_actor_vars = num_actor_vars
self.weights_len = weights_len
self.gamma = gamma
self.tau = tau
self.learning_rate = learning_rate
self.critic_model = self.create_critic_model()
self.target_critic_model = self.create_critic_model()
self.critic_model.compile(keras.optimizers.Adam(lr=self.learning_rate),
'mse')
self.target_critic_model.compile(
keras.optimizers.Adam(lr=self.learning_rate), 'mse')
# Function to compute Q-value gradients (Actor Optimization)
self.action_grads = K.function(
[self.critic_model.input[0], self.critic_model.input[1]],
K.gradients(self.critic_model.output,
[self.critic_model.input[1]]))
def on_train_begin(self, logs={}):
if not os.path.exists(self.cfg['SAVE_DIR']):
os.makedirs(self.cfg['SAVE_DIR'])
# Indexes of the layers which we keep track of. Basically, this will be any layer
# which has a 'kernel' attribute, which is essentially the "Dense" or "Dense"-like layers
self.layerixs = []
# Functions return activity of each layer
self.layerfuncs = []
# Functions return weights of each layer
self.layerweights = []
for lndx, l in enumerate(self.model.layers):
if hasattr(l, 'kernel'):
self.layerixs.append(lndx)
self.layerfuncs.append(
K.function(self.model.inputs, [
l.output,
]))
self.layerweights.append(l.kernel)
grads = K.gradients(self.model.total_loss,
self.model.trainable_weights)
input_tensors = self.model._feed_inputs + self.model._feed_targets + self.model._feed_sample_weights
self.get_gradients = K.function(input_tensors, grads)
# Get cross-entropy loss
self.get_loss = K.function(inputs=input_tensors,
outputs=[
self.model.total_loss,
])
def custom_loss_gradient(self,
nn_function,
tensors,
input_values,
name="default"):
"""
Returns the gradient of the nn_function with respect to model input
:param nn_function: an intermediate tensor representation of the function to differentiate
:type nn_function: a Keras tensor
:param tensors: the tensors or variables to differentiate with respect to
:type tensors: `list`
:param input_values: the inputs to evaluate the gradient
:type input_values: `list`
:param name: The name of the function. Functions of the same name are cached
:type name: `str`
:return: the gradient of the function w.r.t vars
:rtype: `np.ndarray`
"""
if self.is_tensorflow:
import tensorflow.keras.backend as k
else:
import keras.backend as k
if not hasattr(self, "_custom_loss_func"):
self._custom_loss_func = {}
if name not in self._custom_loss_func:
grads = k.gradients(nn_function, tensors[0])[0]
self._custom_loss_func[name] = k.function(tensors, [grads])
outputs = self._custom_loss_func[name]
return outputs(input_values)
def create_feature_map(model, final_layer_name: str, class_index: int,
processed_image: np.ndarray) -> np.ndarray:
"""Returns last convolution layer multiplied by its importance for a given class.
Args:
model (keras.engine.training.Model): Used ML model.
final_layer_name (str): Name of final convolution layer in ML model.
class_index (int): Index of image class.
processed_image (numpy.ndarray): Processed (convertion to numpy array) loaded figure.
Returns:
numpy.ndarray: Feature map with info about importance of channels for image class.
Usually the tensor shape is (14, 14, 512).
"""
class_output = model.output[:, class_index]
last_conv_layer = model.get_layer(final_layer_name)
filter_count = last_conv_layer.filters
# gradients class_output for layer last_conv_layer
grads = K.gradients(class_output, last_conv_layer.output)[0]
# average of gradients over all axes except the channel one
# i.e. result consists of 512 elemets (number given by ML model layer)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
# function provides access to the above mentioned things
iterate = K.function([model.input],
[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([processed_image])
# for each channel is conv_layer_output_value multiplied by channel importance
# for a given image class
for i in range(filter_count):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
return conv_layer_output_value
def plot_grad_cam(model, layer_name, images):
"""
Plot grad cam for the given layer for given images
"""
for img in images:
class_idx = np.argmax(model.predict(img[np.newaxis,:])) class_out = model[:, class_idx]
conv_layer = model.get_layer(layer_name)
grads = K.gradients(class_out, conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0,1,2))
iterate = K.function([model.input], [pooled_grads, conv_layer.output[0]])
pooled_grads, conv_layer_output = iterate([img[np.newaxis,:]])
for i in range(pooled_grads.shape[0]):
conv_layer_output[:,:,i] *= pooled_grads[i]
heatmap = np.mean(conv_layer_output, axis=-1)
heatmap = np.maximum(heatmap, 0) # Filter out negative values
heatmap /= np.max(heatmap)
heatmap = cv2.resize(heatmap, (img.shape[0], img.shape[1]))
fig, ax = plt.subplots(1,2)
ax[0].imshow(((img+0.5)*255).astype('int'))
ax[0].set_title(f"Predicted class: {class_idx}")
ax[1].set_title(f"GRAD CAM")
ax[1].imshow(((img+0.5)*255).astype('int'),alpha=0.85)
ax[1].imshow(heatmap,alpha=0.6)
def discriminator_with_own_loss(self):
z = tfk.Input(shape=(self.dim, ))
e = K.placeholder(shape=(None, 1, 1, 1))
f_img = self.generator(z)
r_img = tfk.Input(shape=(self.img_shape))
a_img = tfk.Input(shape=(self.img_shape),
tensor=e * r_img + (1 - e) * f_img)
f_out = self.discriminator(f_img)
r_out = self.discriminator(r_img)
a_out = self.discriminator(a_img)
loss_real = K.mean(r_out) / self.batch_size
loss_fake = K.mean(f_out) / self.batch_size
grad_mixed = K.gradients(a_out, [a_img])[0]
norm_grad_mixed = K.sqrt(K.sum(K.square(grad_mixed), axis=[1, 2, 3]))
grad_penalty = K.mean(K.square(norm_grad_mixed - 1))
loss = loss_fake - loss_real + self.gp_weight * grad_penalty
training_updates = tfk.optimizers.Adam(
lr=1e-4, beta_1=0.5,
beta_2=0.9).get_updates(loss, self.discriminator.trainable_weights)
d_train = K.function([r_img, z, e], [loss_real, loss_fake],
training_updates)
return d_train
def get_GCAM(model, inputs, targets, layers=-11):
'''
Grad-CAM
:param model: keras model
:param input: 4-d array, shape is [1, W, H, C]
:param target: 1-d array, shape is [nb_class] (one-hot)
:param layers: int value, visualize which layer
return: 2-d array, shape is [W, H]
'''
class_idx = np.argmax(targets, axis=-1)
class_output = model.output[:, class_idx]
last_conv_layer = model.layers[layers]
class_output = model.output[:, class_idx]
x = inputs.copy()
grads = K.gradients(class_output, last_conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function(model.input,
[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate(x)
for i, grads_value in enumerate(pooled_grads_value):
conv_layer_output_value[:, :, i] *= grads_value
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap[heatmap < 0] = 0 # ReLU
heatmap = cv2.resize(heatmap, (x.shape[2], x.shape[1]))
gcam = (heatmap - heatmap.min()) / (heatmap.max() - heatmap.min())
return gcam
def styleTransfer(cData, sData, tData):
print(" Building transfer model.")
contentTensor = K.variable(cData)
styleTensor = K.variable(sData)
genTensor = K.placeholder((1, CONTENT_IMG_H, CONTENT_IMG_W, 3))
inputTensor = K.concatenate([contentTensor, styleTensor, genTensor], axis=0)
model = vgg19.VGG19(include_top=False, weights="imagenet", input_tensor=inputTensor)
outputDict = dict([(layer.name, layer.output) for layer in model.layers])
print(" VGG19 model loaded.")
styleLayerNames = ["block1_conv1", "block2_conv1", "block3_conv1", "block4_conv1", "block5_conv1"]
contentLayerName = "block5_conv2"
loss = compute_loss(genTensor, outputDict, styleLayerNames, contentLayerName)
# Setup gradients or use K.gradients().
grads = K.gradients(loss, genTensor)
kFunction = K.function([genTensor], [loss] + grads)
evaluator = Evaluator(kFunction)
print(" Beginning transfer.")
x = tData
for i in range(TRANSFER_ROUNDS):
print(" Step %d." % i)
# perform gradient descent using fmin_l_bfgs_b.
x, tLoss, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(), fprime=evaluator.grads, maxiter=20)
print(" Loss: %f." % tLoss)
img = deprocessImage(x.copy())
saveFile = CONTENT_IMG_PATH[:-4] + STYLE_IMG_PATH[:-4] + str(i) + ".jpg"
imageio.imwrite(saveFile, img)
print(" Image saved to \"%s\"." % saveFile)
print(" Transfer complete.")
