def main():
# 2
# compare_vgg19_blocks_for_content_layer_reconstruction()
# 3
# compare_vgg19_5ht_block_layer()
# 4
# compare_different_number_layer_style_reconstruction()
# style_transfer('content_images/tubingen.jpg', 'style_images/starry_night.jpg')
content_image = Tools.load_img(Config.content_path, Config.resize_input)
style_image = Tools.load_img(Config.style_path, Config.resize_input)
StyleTransfer2.style_transfer(content_image, style_image)
def upload_content(request):
print("here")
# Handle file upload
if request.method == 'POST':
form = Upload_content_style(request.POST or None, request.FILES
or None)
print(request.FILES)
if form.is_valid():
# extract style path
style_id = form.cleaned_data['Style_num']
style_object = Styles.objects.get(Style_num=style_id)
style_path = style_object.Stylefile.path
# extract content image id
instance = form.save(commit=False)
instance.save()
content_object = Uploadpics.objects.get(id=instance.id)
if content_object.Url_field is None:
content_path = content_object.Contentfile.path
else:
content_path = "nothing"
content_image = Tools.load_img(content_path, True)
style_image = Tools.load_img(style_path, True)
output_image = StyleTransfer2.style_transfer(
content_image, style_image)
random_name = "output{x}.jpg".format(x=np.random.randn())
img_path2 = os.path.join(BASE_DIR, 'media', 'saved_images',
random_name)
output_image.save(img_path2)
img_path2 = '/media/' + 'saved_images/' + random_name
request.session["path_img"] = img_path2
return redirect("style_transfer:saved")
else:
form = Upload_content_style() # A empty, unbound form
context = {
"form": form,
"list": Styles.objects.all(),
}
return render(request, 'style_transfer/upload.html', context)
def reconstruct_style_image(img, conv_layer, file_name, plot_name):
# convert image to array and preprocess for vgg
x = Tools.convert_image_to_array_vgg(img)
# we'll use this throughout the rest of the script
batch_shape = x.shape
shape = x.shape[1:]
vgg = Tools.VGG19_AvgPool(shape)
style_layers_features_outputs, symbolic_conv_outputs, style_features_extractor_model = get_style_image_features(
x, conv_layer, vgg)
# calculate the total style loss
loss = 0
for symbolic, actual in zip(symbolic_conv_outputs,
style_layers_features_outputs):
# gram_matrix() expects a (H, W, C) as input
if conv_layer == 1:
loss += style_loss(symbolic[0], actual)
else:
loss += style_loss(symbolic[0], actual[0])
grads = K.gradients(loss, style_features_extractor_model.input)
# just like theano.function
get_loss_and_grads = K.function(
inputs=[style_features_extractor_model.input], outputs=[loss] + grads)
def get_loss_and_grads_wrapper(x_vec):
l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10,
batch_shape)
# plot loss
plt.plot(losses)
plt.savefig(plot_name)
plt.show()
# save image
final_image = Tools.scale_img(final_image)
plt.imshow(final_image)
plt.imsave(file_name, final_image)
plt.show()
def compare_different_number_layer_style_reconstruction():
mydir = '../media/results/compare_different_number_layer_style_reconstruction'
Tools.create_clear_directory(mydir)
images = []
# select content images
images.append(
Tools.resize_image(Image.open('../media/stylefile/starry_night.jpg'),
512))
images.append(
Tools.resize_image(Image.open('../media/stylefile/picasso.jpg'), 512))
images.append(
Tools.resize_image(Image.open('../media/stylefile/4.jpg'), 512))
images.append(
Tools.resize_image(Image.open('../media/stylefile/frida_kahlo.jpg'),
512))
images.append(
Tools.resize_image(Image.open('../media/stylefile/1.jpg'), 512))
counter = 0
for img in images:
counter += 1
img.save(mydir + "/original_{number}.jpg".format(number=counter))
StyleReconstruction.test_style_reconstruction(img, mydir, counter)
gc.collect()
def compare_vgg19_blocks_for_content_layer_reconstruction():
mydir = '../media/results/compare_vgg19_blocks_for_content_layer_reconstruction'
Tools.create_clear_directory(mydir)
images = []
layers = [1, 4, 9, 14, 19]
# select content images
images.append(
Tools.resize_image(Image.open('../media/contentfile/tubingen.jpg'),
512))
images.append(
Tools.resize_image(
Image.open('../media/contentfile/old-city-jail.jpg'), 512))
images.append(
Tools.resize_image(
Image.open('../media/contentfile/hoovertowernight.jpg'), 512))
images.append(
Tools.resize_image(Image.open('../media/contentfile/golden_gate.jpg'),
512))
images.append(
Tools.resize_image(Image.open('../media/contentfile/5.jpg'), 512))
counter = 0
for img in images:
counter += 1
img.save(mydir + "/original_{number}.jpg".format(number=counter))
ContentReconstruction.test_content_reconstruction(
img, mydir, layers, counter)
gc.collect()
def train_step(image, losses):
with tf.GradientTape() as tape:
outputs = extractor(image)
loss = style_content_loss(outputs)
loss += Config.total_variation_weight * tf.image.total_variation(image)
if losses is not None:
losses.append(loss.numpy()[0])
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(Tools.clip_0_1(image))
def call(self, inputs):
"Expects float input in [0,1]"
inputs = inputs * 255.0
preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)
outputs = self.vgg(preprocessed_input)
style_outputs, content_outputs = (outputs[:self.num_style_layers],
outputs[self.num_style_layers:])
style_outputs = [Tools.gram_matrix(style_output)
for style_output in style_outputs]
content_dict = {content_name: value
for content_name, value
in zip(self.content_layers, content_outputs)}
style_dict = {style_name: value
for style_name, value
in zip(self.style_layers, style_outputs)}
return {'content': content_dict, 'style': style_dict}
def total_variation_loss(image):
x_deltas, y_deltas = Tools.high_pass_x_y(image)
return tf.reduce_sum(tf.abs(x_deltas)) + tf.reduce_sum(tf.abs(y_deltas))
def style_transfer(c_image=None, s_image=None):
if Config.show_log:
# 1: create output directory to save the result
directory = '../media/results/StyleTransfer_V2_Results'
file_name = Tools.get_file_name_to_save_result(directory)
Tools.create_clear_directory(directory)
# 2: load the content and style images
if c_image is not None:
content_image = c_image
else:
content_image = Tools.load_img(Config.content_path, Config.resize_input)
if s_image is not None:
style_image = s_image
else:
style_image = Tools.load_img(Config.style_path, Config.resize_input)
# 3: plot the content and style images
plt.subplot(1, 3, 1)
Tools.imshow(content_image, 'Content Image')
plt.subplot(1, 3, 2)
Tools.imshow(style_image, 'Style Image')
# 4: get fast style transfer output
if Config.get_fast_style_output:
plt.subplot(1, 3, 3)
stylized_fast = get_stylized_image_from_fast_style_transfer(content_image, style_image)
Tools.imshow(stylized_fast, 'Fast Style Transfer')
Tools.tensor_to_image(stylized_fast).save(file_name + '_fast_stylized.png')
# 5: run style transfer algorithm
output_image, losses = do_style_transfer(content_image, style_image)
# 6: pre-processing the output image
output_image = Tools.tensor_to_image(output_image)
if Config.show_log:
output_image.save(file_name + '.png')
plt.imshow(output_image)
# 7: show losses
plt.clf()
x_axis = list(range(100, 100 + Config.epochs * 100, 100))
plt.plot(x_axis, losses)
plt.savefig(file_name + "_losses.png")
plt.xlabel('iterations')
plt.ylabel('total loss')
plt.title('Image Style Transfer Total loss')
plt.grid()
plt.show()
# 8: free memory
global extractor
global opt
global style_targets
global content_targets
del extractor
del opt
del style_targets
del content_targets
return output_image
def do_style_transfer(content_image, style_image):
global num_content_layers
global num_style_layers
global extractor
global opt
global style_targets
global content_targets
num_content_layers = len(Config.content_layers)
num_style_layers = len(Config.style_layers)
# 1: initiate style and content extractor
extractor = StyleContentModel(Config.style_layers, Config.content_layers)
results = extractor(tf.constant(content_image))
# 2: print output details
if Config.show_log:
print('Styles:')
for name, output in sorted(results['style'].items()):
print(" ", name)
print(" shape: ", output.numpy().shape)
print(" min: ", output.numpy().min())
print(" max: ", output.numpy().max())
print(" mean: ", output.numpy().mean())
print()
print("Contents:")
for name, output in sorted(results['content'].items()):
print(" ", name)
print(" shape: ", output.numpy().shape)
print(" min: ", output.numpy().min())
print(" max: ", output.numpy().max())
print(" mean: ", output.numpy().mean())
# 3: get separate output for style and content, the style_targets and content_targets are
# actually the input images
style_targets = extractor(style_image)['style']
content_targets = extractor(content_image)['content']
# 4: define output image
# to make this quick, initialize it with the content image instead of white noise image
output_image = tf.Variable(content_image)
# 5: define adam optimizer
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
# 6: draw the high frequency noises in the inputs
Tools.draw_high_noise(output_image, content_image)
plt.show()
# 7: Start Optimization
start = time.time()
step = 0
losses = []
for n in range(Config.epochs):
temp = []
for m in range(Config.steps_per_epoch):
step += 1
train_step(output_image, temp)
if temp is not None:
losses.extend(temp)
temp = None
print(".", end='')
intermediate_image = Tools.tensor_to_image(output_image)
plt.imshow(intermediate_image)
plt.show()
print("Train step: {}".format(step))
end = time.time()
print("Total time: {:.1f} minutes".format((end - start) / 60))
return output_image, losses
def reconstruct_content_image(img, conv_layer, file_name, plot_name):
""":param
This function reconstructs the input image from the given convolution layer number
in VGG19 architecture
"""
# convert image to array and preprocess for vgg
x = Tools.convert_image_to_array_vgg(img)
# we'll use this throughout the rest of the script
batch_shape = x.shape
shape = x.shape[1:]
#
# # see the image
# plt.imshow(img)
# plt.show()
# make a content model
# try different cutoffs to see the images that result
vgg = Tools.VGG19_AvgPool(shape)
content_features, content_model_extractor = get_content_image_features(
x, conv_layer, vgg)
# define our loss in keras
loss = K.mean(K.square(content_features - content_model_extractor.output))
# gradients which are needed by the optimizer
grads = K.gradients(loss, content_model_extractor.input)
# just like theano.function
get_loss_and_grads = K.function(inputs=[content_model_extractor.input],
outputs=[loss] + grads)
def get_loss_and_grads_wrapper(x_vec):
# scipy's minimizer allows us to pass back
# function value f(x) and its gradient f'(x)
# simultaneously, rather than using the fprime arg
#
# we cannot use get_loss_and_grads() directly
# input to minimizer func must be a 1-D array
# input to get_loss_and_grads must be [batch_of_images]
#
# gradient must also be a 1-D array
# and both loss and gradient must be np.float64
# will get an error otherwise
l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10,
batch_shape)
# plot loss
plt.plot(losses)
plt.savefig(plot_name)
plt.show()
# save image
final_image = Tools.scale_img(final_image)
plt.imshow(final_image)
plt.imsave(file_name, final_image)
plt.show()
def style_transfer(content_image_path, style_image_path):
mydir = './Outputs/style_transfer_details'
# 1: load the content and style images, then rescale the style image to the scale of content image
content_img = Tools.load_img_and_preprocess_resize(content_image_path, resize=512)
h, w = content_img.shape[1:3]
# test_content_reconstruction(content_img[0], mydir, [16, 17, 18, 19], 0)
style_img = Tools.load_img_and_preprocess_shape(style_image_path, (h, w))
# show all blocks output
# test_style_reconstruction(style_img[0], mydir, 1)
batch_shape = content_img.shape
shape = content_img.shape[1:]
vgg = Tools.VGG19_AvgPool(shape)
print(vgg.summary())
# 2: get content and style features + features extractor model
content_features, content_features_extractor_model = ContentReconstruction.get_content_image_features(content_img,
14, vgg)
style_layers_features_outputs, symbolic_conv_outputs, style_features_extractor_model = StyleReconstruction.get_style_image_features(
style_img, 5, vgg)
# we will assume the weight of the content loss is 1
# and only weight the style losses
style_weights = [0.2, 0.4, 0.3, 0.5, 0.2]
# style_weights = [0.4, 0.6, 0.6, 0.7, 0.4]
# create the total loss which is the sum of content + style loss
loss = 1 * K.mean(K.square(content_features_extractor_model.output - content_features))
for w, symbolic, actual in zip(style_weights, symbolic_conv_outputs, style_layers_features_outputs):
# gram_matrix() expects a (H, W, C) as input
loss += w * Tools.style_loss(symbolic[0], actual[0])
# loss += 0.0001 * tf.image.total_variation(vgg.input)
# once again, create the gradients and loss + grads function
# note: it doesn't matter which model's input you use
# they are both pointing to the same keras Input layer in memory
grads = K.gradients(loss, vgg.input)
# just like theano.function
get_loss_and_grads = K.function(
inputs=[vgg.input],
outputs=[loss] + grads
)
def get_loss_and_grads_wrapper(x_vec):
l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10, batch_shape)
# plot loss
plt.plot(losses)
# plt.savefig(plot_name)
plt.show()
# save image
final_image = Tools.scale_img(final_image)
plt.imshow(final_image)
# plt.imsave(file_name, final_image)
plt.show()