def get_gradient(self, o_image_arr):
'''
Calculate the gradient of the loss function with respect to the generated image using K.function
'''
if o_image_arr.shape != (1, self.target_width, self.target_height, 3):
o_image_arr = o_image_arr.reshape((1, self.target_width, self.target_height, 3))
gradient_function = K.function([self.o_model.input], K.gradients(self.get_total_loss(self.o_model.input), [self.o_model.input]))
gradient = gradient_function([o_image_arr])[0].flatten().astype('float64')
return gradient
def get_style_loss(self, ws, Gs, As):
style_loss = K.variable(0.)
for w, G, A in zip(ws, Gs, As):
M_l = K.int_shape(G)[1]
N_l = K.int_shape(G)[0]
G_gram = self.get_gram_matrix(G)
A_gram = self.get_gram_matrix(A)
style_loss += w*0.25*K.sum(K.square(G_gram - A_gram))/ (N_l**2 * M_l**2)
return style_loss
def get_feature_reps(self, x, layer_names, model):
feature_matrices = []
for layer in layer_names:
current_layer = model.get_layer(layer)
feature_raw = current_layer.output
feature_raw_shape = K.shape(feature_raw).eval(session=self.backedn_session)
N_l = feature_raw_shape[-1]
M_l = feature_raw_shape[1]*feature_raw_shape[2]
feature_matrix = K.reshape(feature_raw, (M_l, N_l))
feature_matrix = K.transpose(feature_matrix)
feature_matrices.append(feature_matrix)
return feature_matrices
def calculate_loss(self, o_image_arr):
'''
Calculate total loss using K.function
'''
if o_image_arr.shape != (1, self.target_width, self.target_width, 3):
o_image_arr = o_image_arr.reshape((1, self.target_width, self.target_height, 3))
loss_function = K.function([self.o_model.input], [self.get_total_loss(self.o_model.input)])
return loss_function([o_image_arr])[0].astype('float64')
def process_images(self):
# Image Processing
self.target_height = 512
self.target_width = 512
self.target_size = (self.target_height, self.target_width)
c_image_original = Image.open(self.c_image_path)
self.c_image_original_size = c_image_original.size
c_image = load_img(path=self.c_image_path, target_size=self.target_size)
self.c_image_arr = img_to_array(c_image)
self.c_image_arr = K.variable(preprocess_input(np.expand_dims(self.c_image_arr, axis=0)), dtype='float32')
s_image = load_img(path=self.s_image_path, target_size=self.target_size)
self.s_image_arr = img_to_array(s_image)
self.s_image_arr = K.variable(preprocess_input(np.expand_dims(self.s_image_arr, axis=0)), dtype='float32')
self.o_image_initial = np.random.randint(256, size=(self.target_width, self.target_height, 3)).astype('float64')
self.o_image_initial = preprocess_input(np.expand_dims(self.o_image_initial, axis=0))
self.o_image_placeholder = K.placeholder(shape=(1, self.target_width, self.target_height, 3))
def construct_image(self):
self.backedn_session = K.get_session()
self.c_model = VGG16(include_top=False, weights='imagenet', input_tensor=self.c_image_arr)
self.s_model = VGG16(include_top=False, weights='imagenet', input_tensor=self.s_image_arr)
self.o_model = VGG16(include_top=False, weights='imagenet', input_tensor=self.o_image_placeholder)
self.c_layer_name = 'block4_conv2'
self.s_layer_names = [
'block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
]
self.P = self.get_feature_reps(x=self.c_image_arr, layer_names=[self.c_layer_name], model=self.c_model)[0]
self.As = self.get_feature_reps(x=self.s_image_arr, layer_names=self.s_layer_names, model=self.s_model)
self.ws = np.ones(len(self.s_layer_names)) / float(len(self.s_layer_names))
iterations = 500
x_val = self.o_image_initial.flatten()
start = time.time()
try:
x_output, f_minimum_val, info_dict = fmin_l_bfgs_b(func=calculate_loss, x0=x_val, fprime=get_gradient, maxiter=iterations, disp=True, callback=callback_image_save)
x_output = postprocess_array(x_output)
save_image(x_output, title='final_image')
print(f'Final image saved')
end = time.time()
print(f'Time taken to run whole algorithm {iterations} iterations: {end - start}')
finally:
# Write number of iterations went through to attributes file
with open(self.o_image_directory + 'attributes.txt', 'a') as f:
f.write(f'Number of iterations: {self.current_iteration}')
# Collect images in a gif
images = []
for filename in os.listdir(o_image_directory):
if os.path.splitext(filename)[1] == '.jpg':
images.append(imageio.imread(self.o_image_directory + filename))
imageio.mimsave(self.o_image_directory + 'collected_images.gif', images, duration=0.3)
print('Saved gif of collected images')
def get_gram_matrix(self, F):
'''
Get the gram matrix for style loss function
'''
G = K.dot(F, K.transpose(F))
return G
def get_content_loss(self, F, P):
'''
Calculuates the content loss using mean squared error
'''
content_loss = 0.5*K.sum(K.square(F - P))
return content_loss
from skimage.transform import resize
from skimage.io import imsave
import numpy as np
from keras.models import Model
from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Conv2DTranspose
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint
from keras import backend as K
from keras.callbacks import *
from keras.utils.vis_utils import plot_model
import shutil
#from data import load_train_data, load_test_data
import time
print(K())
K.set_image_data_format('channels_last') # TF dimension ordering in this code
starttime = time.clock()
img_rows = 512
img_cols = 512
smooth = 1.
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)