def total_variation_loss(x, target_size): img_height, img_width = target_size a = K.square( x[:, :img_height - 1, :img_width - 1, :] - x[:, 1:, :img_width - 1, :]) b = K.square( x[:, :img_height - 1, :img_width - 1, :] - x[:, :img_height - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def gradient_penalty(y_true, y_pred, interpolate, lamb): grad = K.gradients(y_pred, interpolate)[0] norm = K.square(grad) norm_sum = K.sum(norm,axis=np.arange(1,len(norm.shape))) l2_norm = K.sqrt(norm_sum) gp_reg = lamb*K.square(1-l2_norm) return K.mean(gp_reg)
def contrastive_loss(y_true, y_pred): ''' Contrastive loss from Hadsell-et-al.'06 http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf @param y_true : true label 1 for positive pair, 0 for negative pair y_pred : distance output of the Siamese network ''' margin = 1 # if positive pair, y_true is 1, penalize for large distance returned by Siamese network # if negative pair, y_true is 0, penalize for distance smaller than the margin return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def build_loss(self): r"""Implements the N-dim version of function $$TV^{\beta}(x) = \sum_{whc} \left ( \left ( x(h, w+1, c) - x(h, w, c) \right )^{2} + \left ( x(h+1, w, c) - x(h, w, c) \right )^{2} \right )^{\frac{\beta}{2}}$$ to return total variation for all images in the batch. """ image_dims = K.ndim(self.img) - 2 # Constructing slice [1:] + [:-1] * (image_dims - 1) and [:-1] * (image_dims) start_slice = [slice(1, None, None)] + [ slice(None, -1, None) for _ in range(image_dims - 1) ] end_slice = [slice(None, -1, None) for _ in range(image_dims)] samples_channels_slice = [ slice(None, None, None), slice(None, None, None) ] # Compute pixel diffs by rolling slices to the right per image dim. tv = None for i in range(image_dims): ss = tuple(samples_channels_slice + start_slice) es = tuple(samples_channels_slice + end_slice) diff_square = K.square(self.img[utils.slicer[ss]] - self.img[utils.slicer[es]]) tv = diff_square if tv is None else tv + diff_square # Roll over to next image dim start_slice = np.roll(start_slice, 1).tolist() end_slice = np.roll(end_slice, 1).tolist() tv = K.sum(K.pow(tv, self.beta / 2.)) return normalize(self.img, tv)
def euclidean_distance(vects): ''' Auxiliary function to compute the Euclidian distance between two vectors in a Keras layer. ''' x, y = vects return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def style_loss(style, combination): S = gram_matrix(style) C = gram_matrix(combination) channels = 3 size = height * width return backend.sum(backend.square(S - C)) / (4. * (channels**2) * (size**2))
def style_loss(style, combination, target_size): s = gram_matrix(style) c = gram_matrix(combination) channels = 3 img_height, img_width = target_size size = img_height * img_width return K.sum(K.square(s - c)) / (4.0 * (channels ** 2) * (size ** 2))
def contrastive_loss(y_true, y_pred): ''' Contrastive loss from Hadsell-et-al.'06 http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf @param y_true : true label 1 for positive pair, 0 for negative pair y_pred : distance output of the Siamese network @return contrastive_loss between two values ''' margin = 1 # if positive pair, y_true is 1, penalize for large distance returned by Siamese network # if negative pair, y_true is 0, penalize for distance smaller than the margin y_positive_true = 1 y_negative_true = 0 return K.mean(y_true * K.square(y_pred) + (y_positive_true - y_true) * K.square(K.maximum(margin - y_pred, y_negative_true)))
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True): """Creates an optimizer that minimizes weighted loss function. Args: input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format= channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`. losses: List of ([Loss](vis.losses#Loss), weight) tuples. input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the final optimized input to the given range. (Default value=(0, 255)) wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses` should be minimized with respect to `wrt_tensor`. `wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None which means that loss will simply be minimized with respect to `input_tensor`. norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures a smooth gradient gradient descent process. If you want the actual gradient (for example, visualizing attention), set this to false. """ self.input_tensor = input_tensor self.input_range = input_range self.loss_names = [] self.loss_functions = [] self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor overall_loss = None for loss, weight in losses: # Perf optimization. Don't build loss function with 0 weight. if weight != 0: loss_fn = weight * loss.build_loss() overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn self.loss_names.append(loss.name) self.loss_functions.append(loss_fn) # Compute gradient of overall with respect to `wrt` tensor. grads = K.gradients(overall_loss, self.wrt_tensor)[0] if norm_grads: grads = grads / (K.sqrt(K.mean(K.square(grads))) + K.epsilon()) # The main function to compute various quantities in optimization loop. self.compute_fn = K.function([self.input_tensor, K.learning_phase()], self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def generate_pattern(model, layer_name, filter_index, steps, learning_rate, size=224): layer_output = model.get_layer(layer_name).output loss = K.mean(layer_output[:, :, :, filter_index]) # obtain the gradient of the loss with respect to the model's input image grads_list = K.gradients(loss, model.input) grads = grads_list[0] # gradient normalization trick grads /= (K.sqrt(K.mean(K.square(grads))) + EPSILON) # fetch loss and normalized-gradients for a given input iterate = K.function(inputs=[model.input], outputs=[loss, grads]) # loss maximization via stochastic gradient descent input_img_data = np.random.random((1, size, size, 3)) * 20 + 128 # start from gray image with random noise for i in range(steps): loss_value, grads_value = iterate([input_img_data]) print('@{:-4d}: {:.4f}'.format(i, loss_value)) # gradient ascent: adjust the input image in the direction that maximizes the loss input_img_data += grads_value * learning_rate img_tensor = input_img_data[0] return tensor_to_image(img_tensor)
def normalize(x): return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
def content_loss(content, combination): return K.sum(K.square(combination - content))
def total_variation_loss(x): a = K.square(x[:, :height-1, :width-1, :] - x[:, 1:, :width-1, :]) b = K.square(x[:, :height-1, :width-1, :] - x[:, :height-1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def vae_loss(x, x_decoded_mean): xent_loss = original_dim * keras.losses.categorical_crossentropy( x, x_decoded_mean) kl_loss = -0.5 * K.sum( 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return xent_loss + kl_loss
def content_loss(content, combination): return backend.sum(backend.square(combination - content))
def total_variation_loss(x): a = backend.square(x[:, :height - 1, :width - 1, :] - x[:, 1:, :width - 1, :]) b = backend.square(x[:, :height - 1, :width - 1, :] - x[:, :height - 1, 1:, :]) return backend.sum(backend.pow(a + b, 1.25))
def main(_): # disable all training specific operations K.set_learning_phase(0) model = applications.inception_v3.InceptionV3(weights='imagenet', include_top=False) layer_contributions = { 'mixed2': 0.2, 'mixed3': 3.0, 'mixed4': 2.0, 'mixed5': 1.5 } layer_dict = dict([(layer.name, layer) for layer in model.layers]) loss = K.variable(0.,) for layer_name in layer_contributions: coeff = layer_contributions[layer_name] activation = layer_dict[layer_name].output scaling = K.prod(K.cast(K.shape(activation), 'float32')) # avoid artifacts by only involving non-boarder pixels loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling # start the gradient-ascent process dream = model.input grads_list = K.gradients(loss, dream) grads = grads_list[0] # trick: normalize gradients grads /= K.maximum(K.mean(K.abs(grads)), 1e-7) fetch_loss_and_grads = K.function(inputs=[dream], outputs=[loss, grads]) def gradient_ascent(x, iterations, step_rate, max_loss=None): for i in range(iterations): loss_value, grads_value = fetch_loss_and_grads([x]) if max_loss is not None and loss_value > max_loss: break print('@{:4d}: {:.4f}'.format(i, loss_value)) x += step_rate * grads_value return x img = preprocess_img(FLAGS.img_path) original_shape = img.shape[1:3] successive_shapes = [original_shape] for i in range(1, NUM_OCTAVES): shape = tuple([int(dim / (OCTAVES_SCLAE ** i)) for dim in original_shape]) successive_shapes.append(shape) # reverse successive_shapes = successive_shapes[::-1] original_img = np.copy(img) shrunk_original_img = resize_img(img, successive_shapes[0]) for shape in successive_shapes: print('Preprocess image with shape: {}'.format(shape)) img = resize_img(img, shape) img = gradient_ascent(img, iterations=FLAGS.iterations, step_rate=FLAGS.step_rate, max_loss=MAX_LOSS) same_size_original = resize_img(original_img, shape) if FLAGS.repair_lost_detail: upscale_shrunk_original_img = resize_img(shrunk_original_img, shape) lost_detail = same_size_original - upscale_shrunk_original_img img += lost_detail shrunk_original_img = same_size_original save_img(img, filename='dream_at_scale_{}.png'.format(str(shape))) save_img(img, filename='dream.png')
def ms_euclidean(y_true, y_pred): return K.mean(K.sum(K.square(y_true - y_pred), axis=1))
def euclidean_distance(y_true, y_pred): return K.sqrt(K.sum(K.square(y_true - y_pred), axis=1))
def content_loss(base, combination): return K.sum(K.square(combination - base))
train_x = [] test_x = [] for row in train_data: train_x.append(row[0]) for row in test_data: test_x.append(row[0]) train_x = np.array(train_x).reshape(-1, 128, 128, 3) test_x = np.array(test_x).reshape(-1, 128, 128, 3) reconstruction_loss = tf.keras.losses.binary_crossentropy( K.flatten(input_img), K.flatten(outputs)) reconstruction_loss *= 128 * 128 kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) kl_loss = K.sum(kl_loss, axis=-1) kl_loss *= -0.5 vae_loss = K.mean(reconstruction_loss + kl_loss) vae.add_loss(vae_loss) vae.compile(optimizer='rmsprop') vae.summary() flag = 1 if flag == 0: vae.fit(train_x, epochs=10, batch_size=64, validation_data=(test_x, None)) vae.save_weights("vae_cnn.h5") elif flag == 1:
def square(x): return K.square(x)