def total_variation_loss(x, target_size):
img_height, img_width = target_size
a = K.square(
x[:, :img_height - 1, :img_width - 1, :] - x[:, 1:, :img_width - 1, :])
b = K.square(
x[:, :img_height - 1, :img_width - 1, :] - x[:, :img_height - 1, 1:, :])
return K.sum(K.pow(a + b, 1.25))
def gradient_penalty(y_true, y_pred, interpolate, lamb):
grad = K.gradients(y_pred, interpolate)[0]
norm = K.square(grad)
norm_sum = K.sum(norm,axis=np.arange(1,len(norm.shape)))
l2_norm = K.sqrt(norm_sum)
gp_reg = lamb*K.square(1-l2_norm)
return K.mean(gp_reg)
def contrastive_loss(y_true, y_pred):
'''
Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
@param
y_true : true label 1 for positive pair, 0 for negative pair
y_pred : distance output of the Siamese network
'''
margin = 1
# if positive pair, y_true is 1, penalize for large distance returned by Siamese network
# if negative pair, y_true is 0, penalize for distance smaller than the margin
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def build_loss(self):
r"""Implements the N-dim version of function
$$TV^{\beta}(x) = \sum_{whc} \left ( \left ( x(h, w+1, c) - x(h, w, c) \right )^{2} +
\left ( x(h+1, w, c) - x(h, w, c) \right )^{2} \right )^{\frac{\beta}{2}}$$
to return total variation for all images in the batch.
"""
image_dims = K.ndim(self.img) - 2
# Constructing slice [1:] + [:-1] * (image_dims - 1) and [:-1] * (image_dims)
start_slice = [slice(1, None, None)] + [
slice(None, -1, None) for _ in range(image_dims - 1)
]
end_slice = [slice(None, -1, None) for _ in range(image_dims)]
samples_channels_slice = [
slice(None, None, None),
slice(None, None, None)
]
# Compute pixel diffs by rolling slices to the right per image dim.
tv = None
for i in range(image_dims):
ss = tuple(samples_channels_slice + start_slice)
es = tuple(samples_channels_slice + end_slice)
diff_square = K.square(self.img[utils.slicer[ss]] -
self.img[utils.slicer[es]])
tv = diff_square if tv is None else tv + diff_square
# Roll over to next image dim
start_slice = np.roll(start_slice, 1).tolist()
end_slice = np.roll(end_slice, 1).tolist()
tv = K.sum(K.pow(tv, self.beta / 2.))
return normalize(self.img, tv)
def euclidean_distance(vects):
'''
Auxiliary function to compute the Euclidian distance between two vectors
in a Keras layer.
'''
x, y = vects
return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def style_loss(style, combination):
S = gram_matrix(style)
C = gram_matrix(combination)
channels = 3
size = height * width
return backend.sum(backend.square(S - C)) / (4. * (channels**2) *
(size**2))
def style_loss(style, combination, target_size):
s = gram_matrix(style)
c = gram_matrix(combination)
channels = 3
img_height, img_width = target_size
size = img_height * img_width
return K.sum(K.square(s - c)) / (4.0 * (channels ** 2) * (size ** 2))
def contrastive_loss(y_true, y_pred):
'''
Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
@param
y_true : true label 1 for positive pair, 0 for negative pair
y_pred : distance output of the Siamese network
@return
contrastive_loss between two values
'''
margin = 1
# if positive pair, y_true is 1, penalize for large distance returned by Siamese network
# if negative pair, y_true is 0, penalize for distance smaller than the margin
y_positive_true = 1
y_negative_true = 0
return K.mean(y_true * K.square(y_pred) +
(y_positive_true - y_true) * K.square(K.maximum(margin - y_pred, y_negative_true)))
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = grads / (K.sqrt(K.mean(K.square(grads))) + K.epsilon())
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function([self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def generate_pattern(model, layer_name, filter_index, steps, learning_rate, size=224):
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
# obtain the gradient of the loss with respect to the model's input image
grads_list = K.gradients(loss, model.input)
grads = grads_list[0]
# gradient normalization trick
grads /= (K.sqrt(K.mean(K.square(grads))) + EPSILON)
# fetch loss and normalized-gradients for a given input
iterate = K.function(inputs=[model.input], outputs=[loss, grads])
# loss maximization via stochastic gradient descent
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128 # start from gray image with random noise
for i in range(steps):
loss_value, grads_value = iterate([input_img_data])
print('@{:-4d}: {:.4f}'.format(i, loss_value))
# gradient ascent: adjust the input image in the direction that maximizes the loss
input_img_data += grads_value * learning_rate
img_tensor = input_img_data[0]
return tensor_to_image(img_tensor)
def normalize(x):
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
def content_loss(content, combination):
return K.sum(K.square(combination - content))
def total_variation_loss(x):
a = K.square(x[:, :height-1, :width-1, :] - x[:, 1:, :width-1, :])
b = K.square(x[:, :height-1, :width-1, :] - x[:, :height-1, 1:, :])
return K.sum(K.pow(a + b, 1.25))
def vae_loss(x, x_decoded_mean):
xent_loss = original_dim * keras.losses.categorical_crossentropy(
x, x_decoded_mean)
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def content_loss(content, combination):
return backend.sum(backend.square(combination - content))
def total_variation_loss(x):
a = backend.square(x[:, :height - 1, :width - 1, :] -
x[:, 1:, :width - 1, :])
b = backend.square(x[:, :height - 1, :width - 1, :] -
x[:, :height - 1, 1:, :])
return backend.sum(backend.pow(a + b, 1.25))
def main(_):
# disable all training specific operations
K.set_learning_phase(0)
model = applications.inception_v3.InceptionV3(weights='imagenet',
include_top=False)
layer_contributions = {
'mixed2': 0.2,
'mixed3': 3.0,
'mixed4': 2.0,
'mixed5': 1.5
}
layer_dict = dict([(layer.name, layer) for layer in model.layers])
loss = K.variable(0.,)
for layer_name in layer_contributions:
coeff = layer_contributions[layer_name]
activation = layer_dict[layer_name].output
scaling = K.prod(K.cast(K.shape(activation), 'float32'))
# avoid artifacts by only involving non-boarder pixels
loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling
# start the gradient-ascent process
dream = model.input
grads_list = K.gradients(loss, dream)
grads = grads_list[0]
# trick: normalize gradients
grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
fetch_loss_and_grads = K.function(inputs=[dream],
outputs=[loss, grads])
def gradient_ascent(x, iterations, step_rate, max_loss=None):
for i in range(iterations):
loss_value, grads_value = fetch_loss_and_grads([x])
if max_loss is not None and loss_value > max_loss:
break
print('@{:4d}: {:.4f}'.format(i, loss_value))
x += step_rate * grads_value
return x
img = preprocess_img(FLAGS.img_path)
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, NUM_OCTAVES):
shape = tuple([int(dim / (OCTAVES_SCLAE ** i))
for dim in original_shape])
successive_shapes.append(shape)
# reverse
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
for shape in successive_shapes:
print('Preprocess image with shape: {}'.format(shape))
img = resize_img(img, shape)
img = gradient_ascent(img,
iterations=FLAGS.iterations,
step_rate=FLAGS.step_rate,
max_loss=MAX_LOSS)
same_size_original = resize_img(original_img, shape)
if FLAGS.repair_lost_detail:
upscale_shrunk_original_img = resize_img(shrunk_original_img, shape)
lost_detail = same_size_original - upscale_shrunk_original_img
img += lost_detail
shrunk_original_img = same_size_original
save_img(img, filename='dream_at_scale_{}.png'.format(str(shape)))
save_img(img, filename='dream.png')
def ms_euclidean(y_true, y_pred):
return K.mean(K.sum(K.square(y_true - y_pred), axis=1))
def euclidean_distance(y_true, y_pred):
return K.sqrt(K.sum(K.square(y_true - y_pred), axis=1))
def content_loss(base, combination):
return K.sum(K.square(combination - base))
train_x = []
test_x = []
for row in train_data:
train_x.append(row[0])
for row in test_data:
test_x.append(row[0])
train_x = np.array(train_x).reshape(-1, 128, 128, 3)
test_x = np.array(test_x).reshape(-1, 128, 128, 3)
reconstruction_loss = tf.keras.losses.binary_crossentropy(
K.flatten(input_img), K.flatten(outputs))
reconstruction_loss *= 128 * 128
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
flag = 1
if flag == 0:
vae.fit(train_x,
epochs=10,
batch_size=64,
validation_data=(test_x, None))
vae.save_weights("vae_cnn.h5")
elif flag == 1:
def square(x):
return K.square(x)
