def main(): # 2 # compare_vgg19_blocks_for_content_layer_reconstruction() # 3 # compare_vgg19_5ht_block_layer() # 4 # compare_different_number_layer_style_reconstruction() # style_transfer('content_images/tubingen.jpg', 'style_images/starry_night.jpg') content_image = Tools.load_img(Config.content_path, Config.resize_input) style_image = Tools.load_img(Config.style_path, Config.resize_input) StyleTransfer2.style_transfer(content_image, style_image)
def upload_content(request): print("here") # Handle file upload if request.method == 'POST': form = Upload_content_style(request.POST or None, request.FILES or None) print(request.FILES) if form.is_valid(): # extract style path style_id = form.cleaned_data['Style_num'] style_object = Styles.objects.get(Style_num=style_id) style_path = style_object.Stylefile.path # extract content image id instance = form.save(commit=False) instance.save() content_object = Uploadpics.objects.get(id=instance.id) if content_object.Url_field is None: content_path = content_object.Contentfile.path else: content_path = "nothing" content_image = Tools.load_img(content_path, True) style_image = Tools.load_img(style_path, True) output_image = StyleTransfer2.style_transfer( content_image, style_image) random_name = "output{x}.jpg".format(x=np.random.randn()) img_path2 = os.path.join(BASE_DIR, 'media', 'saved_images', random_name) output_image.save(img_path2) img_path2 = '/media/' + 'saved_images/' + random_name request.session["path_img"] = img_path2 return redirect("style_transfer:saved") else: form = Upload_content_style() # A empty, unbound form context = { "form": form, "list": Styles.objects.all(), } return render(request, 'style_transfer/upload.html', context)
def reconstruct_style_image(img, conv_layer, file_name, plot_name): # convert image to array and preprocess for vgg x = Tools.convert_image_to_array_vgg(img) # we'll use this throughout the rest of the script batch_shape = x.shape shape = x.shape[1:] vgg = Tools.VGG19_AvgPool(shape) style_layers_features_outputs, symbolic_conv_outputs, style_features_extractor_model = get_style_image_features( x, conv_layer, vgg) # calculate the total style loss loss = 0 for symbolic, actual in zip(symbolic_conv_outputs, style_layers_features_outputs): # gram_matrix() expects a (H, W, C) as input if conv_layer == 1: loss += style_loss(symbolic[0], actual) else: loss += style_loss(symbolic[0], actual[0]) grads = K.gradients(loss, style_features_extractor_model.input) # just like theano.function get_loss_and_grads = K.function( inputs=[style_features_extractor_model.input], outputs=[loss] + grads) def get_loss_and_grads_wrapper(x_vec): l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)]) return l.astype(np.float64), g.flatten().astype(np.float64) final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10, batch_shape) # plot loss plt.plot(losses) plt.savefig(plot_name) plt.show() # save image final_image = Tools.scale_img(final_image) plt.imshow(final_image) plt.imsave(file_name, final_image) plt.show()
def compare_different_number_layer_style_reconstruction(): mydir = '../media/results/compare_different_number_layer_style_reconstruction' Tools.create_clear_directory(mydir) images = [] # select content images images.append( Tools.resize_image(Image.open('../media/stylefile/starry_night.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/stylefile/picasso.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/stylefile/4.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/stylefile/frida_kahlo.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/stylefile/1.jpg'), 512)) counter = 0 for img in images: counter += 1 img.save(mydir + "/original_{number}.jpg".format(number=counter)) StyleReconstruction.test_style_reconstruction(img, mydir, counter) gc.collect()
def compare_vgg19_blocks_for_content_layer_reconstruction(): mydir = '../media/results/compare_vgg19_blocks_for_content_layer_reconstruction' Tools.create_clear_directory(mydir) images = [] layers = [1, 4, 9, 14, 19] # select content images images.append( Tools.resize_image(Image.open('../media/contentfile/tubingen.jpg'), 512)) images.append( Tools.resize_image( Image.open('../media/contentfile/old-city-jail.jpg'), 512)) images.append( Tools.resize_image( Image.open('../media/contentfile/hoovertowernight.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/contentfile/golden_gate.jpg'), 512)) images.append( Tools.resize_image(Image.open('../media/contentfile/5.jpg'), 512)) counter = 0 for img in images: counter += 1 img.save(mydir + "/original_{number}.jpg".format(number=counter)) ContentReconstruction.test_content_reconstruction( img, mydir, layers, counter) gc.collect()
def train_step(image, losses): with tf.GradientTape() as tape: outputs = extractor(image) loss = style_content_loss(outputs) loss += Config.total_variation_weight * tf.image.total_variation(image) if losses is not None: losses.append(loss.numpy()[0]) grad = tape.gradient(loss, image) opt.apply_gradients([(grad, image)]) image.assign(Tools.clip_0_1(image))
def call(self, inputs): "Expects float input in [0,1]" inputs = inputs * 255.0 preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs) outputs = self.vgg(preprocessed_input) style_outputs, content_outputs = (outputs[:self.num_style_layers], outputs[self.num_style_layers:]) style_outputs = [Tools.gram_matrix(style_output) for style_output in style_outputs] content_dict = {content_name: value for content_name, value in zip(self.content_layers, content_outputs)} style_dict = {style_name: value for style_name, value in zip(self.style_layers, style_outputs)} return {'content': content_dict, 'style': style_dict}
def total_variation_loss(image): x_deltas, y_deltas = Tools.high_pass_x_y(image) return tf.reduce_sum(tf.abs(x_deltas)) + tf.reduce_sum(tf.abs(y_deltas))
def style_transfer(c_image=None, s_image=None): if Config.show_log: # 1: create output directory to save the result directory = '../media/results/StyleTransfer_V2_Results' file_name = Tools.get_file_name_to_save_result(directory) Tools.create_clear_directory(directory) # 2: load the content and style images if c_image is not None: content_image = c_image else: content_image = Tools.load_img(Config.content_path, Config.resize_input) if s_image is not None: style_image = s_image else: style_image = Tools.load_img(Config.style_path, Config.resize_input) # 3: plot the content and style images plt.subplot(1, 3, 1) Tools.imshow(content_image, 'Content Image') plt.subplot(1, 3, 2) Tools.imshow(style_image, 'Style Image') # 4: get fast style transfer output if Config.get_fast_style_output: plt.subplot(1, 3, 3) stylized_fast = get_stylized_image_from_fast_style_transfer(content_image, style_image) Tools.imshow(stylized_fast, 'Fast Style Transfer') Tools.tensor_to_image(stylized_fast).save(file_name + '_fast_stylized.png') # 5: run style transfer algorithm output_image, losses = do_style_transfer(content_image, style_image) # 6: pre-processing the output image output_image = Tools.tensor_to_image(output_image) if Config.show_log: output_image.save(file_name + '.png') plt.imshow(output_image) # 7: show losses plt.clf() x_axis = list(range(100, 100 + Config.epochs * 100, 100)) plt.plot(x_axis, losses) plt.savefig(file_name + "_losses.png") plt.xlabel('iterations') plt.ylabel('total loss') plt.title('Image Style Transfer Total loss') plt.grid() plt.show() # 8: free memory global extractor global opt global style_targets global content_targets del extractor del opt del style_targets del content_targets return output_image
def do_style_transfer(content_image, style_image): global num_content_layers global num_style_layers global extractor global opt global style_targets global content_targets num_content_layers = len(Config.content_layers) num_style_layers = len(Config.style_layers) # 1: initiate style and content extractor extractor = StyleContentModel(Config.style_layers, Config.content_layers) results = extractor(tf.constant(content_image)) # 2: print output details if Config.show_log: print('Styles:') for name, output in sorted(results['style'].items()): print(" ", name) print(" shape: ", output.numpy().shape) print(" min: ", output.numpy().min()) print(" max: ", output.numpy().max()) print(" mean: ", output.numpy().mean()) print() print("Contents:") for name, output in sorted(results['content'].items()): print(" ", name) print(" shape: ", output.numpy().shape) print(" min: ", output.numpy().min()) print(" max: ", output.numpy().max()) print(" mean: ", output.numpy().mean()) # 3: get separate output for style and content, the style_targets and content_targets are # actually the input images style_targets = extractor(style_image)['style'] content_targets = extractor(content_image)['content'] # 4: define output image # to make this quick, initialize it with the content image instead of white noise image output_image = tf.Variable(content_image) # 5: define adam optimizer opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1) # 6: draw the high frequency noises in the inputs Tools.draw_high_noise(output_image, content_image) plt.show() # 7: Start Optimization start = time.time() step = 0 losses = [] for n in range(Config.epochs): temp = [] for m in range(Config.steps_per_epoch): step += 1 train_step(output_image, temp) if temp is not None: losses.extend(temp) temp = None print(".", end='') intermediate_image = Tools.tensor_to_image(output_image) plt.imshow(intermediate_image) plt.show() print("Train step: {}".format(step)) end = time.time() print("Total time: {:.1f} minutes".format((end - start) / 60)) return output_image, losses
def reconstruct_content_image(img, conv_layer, file_name, plot_name): """:param This function reconstructs the input image from the given convolution layer number in VGG19 architecture """ # convert image to array and preprocess for vgg x = Tools.convert_image_to_array_vgg(img) # we'll use this throughout the rest of the script batch_shape = x.shape shape = x.shape[1:] # # # see the image # plt.imshow(img) # plt.show() # make a content model # try different cutoffs to see the images that result vgg = Tools.VGG19_AvgPool(shape) content_features, content_model_extractor = get_content_image_features( x, conv_layer, vgg) # define our loss in keras loss = K.mean(K.square(content_features - content_model_extractor.output)) # gradients which are needed by the optimizer grads = K.gradients(loss, content_model_extractor.input) # just like theano.function get_loss_and_grads = K.function(inputs=[content_model_extractor.input], outputs=[loss] + grads) def get_loss_and_grads_wrapper(x_vec): # scipy's minimizer allows us to pass back # function value f(x) and its gradient f'(x) # simultaneously, rather than using the fprime arg # # we cannot use get_loss_and_grads() directly # input to minimizer func must be a 1-D array # input to get_loss_and_grads must be [batch_of_images] # # gradient must also be a 1-D array # and both loss and gradient must be np.float64 # will get an error otherwise l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)]) return l.astype(np.float64), g.flatten().astype(np.float64) final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10, batch_shape) # plot loss plt.plot(losses) plt.savefig(plot_name) plt.show() # save image final_image = Tools.scale_img(final_image) plt.imshow(final_image) plt.imsave(file_name, final_image) plt.show()
def style_transfer(content_image_path, style_image_path): mydir = './Outputs/style_transfer_details' # 1: load the content and style images, then rescale the style image to the scale of content image content_img = Tools.load_img_and_preprocess_resize(content_image_path, resize=512) h, w = content_img.shape[1:3] # test_content_reconstruction(content_img[0], mydir, [16, 17, 18, 19], 0) style_img = Tools.load_img_and_preprocess_shape(style_image_path, (h, w)) # show all blocks output # test_style_reconstruction(style_img[0], mydir, 1) batch_shape = content_img.shape shape = content_img.shape[1:] vgg = Tools.VGG19_AvgPool(shape) print(vgg.summary()) # 2: get content and style features + features extractor model content_features, content_features_extractor_model = ContentReconstruction.get_content_image_features(content_img, 14, vgg) style_layers_features_outputs, symbolic_conv_outputs, style_features_extractor_model = StyleReconstruction.get_style_image_features( style_img, 5, vgg) # we will assume the weight of the content loss is 1 # and only weight the style losses style_weights = [0.2, 0.4, 0.3, 0.5, 0.2] # style_weights = [0.4, 0.6, 0.6, 0.7, 0.4] # create the total loss which is the sum of content + style loss loss = 1 * K.mean(K.square(content_features_extractor_model.output - content_features)) for w, symbolic, actual in zip(style_weights, symbolic_conv_outputs, style_layers_features_outputs): # gram_matrix() expects a (H, W, C) as input loss += w * Tools.style_loss(symbolic[0], actual[0]) # loss += 0.0001 * tf.image.total_variation(vgg.input) # once again, create the gradients and loss + grads function # note: it doesn't matter which model's input you use # they are both pointing to the same keras Input layer in memory grads = K.gradients(loss, vgg.input) # just like theano.function get_loss_and_grads = K.function( inputs=[vgg.input], outputs=[loss] + grads ) def get_loss_and_grads_wrapper(x_vec): l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)]) return l.astype(np.float64), g.flatten().astype(np.float64) final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10, batch_shape) # plot loss plt.plot(losses) # plt.savefig(plot_name) plt.show() # save image final_image = Tools.scale_img(final_image) plt.imshow(final_image) # plt.imsave(file_name, final_image) plt.show()