def main():
sess = K.get_session()
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
K.set_session(sess)
sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
style = StyleTransfer(stop_layer='block1_conv1')
style.train("content", "style")
class Worker(threading.Thread):
def __init__(self,name,upstream_task_queue,downstream_task_queue):
super(Worker, self).__init__()
self._name = name
self._upstream_task_queue = upstream_task_queue
self._downstream_task_queue = downstream_task_queue
self._stop_flag = False
caffe.set_mode_cpu()
logging.info("Running net on CPU.")
# initialize style transfer
self._style_transfer = StyleTransfer('googlenet')
def _create_art(self,style_image,content_image,params):
# perform style transfer
start = timeit.default_timer()
n_iters = self._style_transfer.transfer_style(style_image, content_image, length=params["length"], init="mixed", ratio=np.float(params["ratio"]), n_iter=params["num_iters"], verbose=params["verbose"])
end = timeit.default_timer()
logging.info("Ran {0} iterations in {1:.0f}s.".format(n_iters, end-start))
img_out = self._style_transfer.get_generated()
img_out = img_as_ubyte(img_out)
return img_out;
def stop(self):
self._stop_flag = True
def run(self):
'''
'''
logging.info("Runing worker ("+self._name+")...")
while self._stop_flag == False:
try:
task = self._upstream_task_queue.get(False)
# validate task
if self._validate_task(task) == False:
self._upstream_task_queue.task_done()
continue
# create art
task.output_image = self._create_art(task.style_image, task.content_image, task.params)
#send event to downstream
self._downstream_task_queue.put(task)
self._upstream_task_queue.task_done()
except Empty:
logging.info('not task request on local queue, retry in 5 seconds later.')
time.sleep(5)
logging.info("Completed worker ("+self._name+").")
def _validate_task(self,task):
return True
def __init__(self,name,upstream_task_queue,downstream_task_queue):
super(Worker, self).__init__()
self._name = name
self._upstream_task_queue = upstream_task_queue
self._downstream_task_queue = downstream_task_queue
self._stop_flag = False
caffe.set_mode_cpu()
logging.info("Running net on CPU.")
# initialize style transfer
self._style_transfer = StyleTransfer('googlenet')
def init(n_workers=1):
"""
Initialize the style transfer backend.
"""
global workers
if n_workers == 0:
n_workers = 1
# assign a lock to each worker
for i in range(n_workers):
worker = StyleTransfer("vgg16", use_pbar=False)
workers.update({Lock(): worker})
parser = argparse.ArgumentParser(
description="Transfer the style.",
usage="demo.py -s <style_image> -c <content_image>")
parser.add_argument("-s",
"--style-img",
type=str,
required=True,
help="the style image")
parser.add_argument("-c",
"--content-img",
type=str,
required=True,
help="the content image")
# use googlenet as defaults
transferer = StyleTransfer("googlenet")
def st_api(img_style, img_content, callback=None):
"""
Style transfer API.
"""
# style transfer arguments
args = {
"length": 512,
"ratio": 2e4,
"n_iter": 16,
"callback": callback,
"init": "content"
}
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 24 16:52:41 2016
@author: zhouc
"""
# library imports
import caffe
import cv2
import numpy as np
# local imports
from style import StyleTransfer
caffe.set_mode_gpu()
style_img_path = 'images/style/the_scream.jpg'
content_img_path = 'images/content/tubingen.jpg'
img_style = caffe.io.load_image(style_img_path)
img_content = caffe.io.load_image(content_img_path)
args = {"length": 600, "ratio": 2e5, "n_iter": 32, "init": "content"}
st = StyleTransfer()
st.transfer_style(img_style, img_content, **args)
img_out = st.get_generated()
# show the image
cv2.imshow("Style", cv2.cvtColor(img_out, cv2.COLOR_RGB2BGR))
cv2.waitKey()
cv2.destroyWindow("Style")
def sampling( self, condition_net, image_encoder, image_net, image_generator, edge_detector,
gen_in_layer, gen_out_layer, start_code, content_layer,
n_iters, lr, lr_end, threshold,
layer, conditions, mask=None, input_image=None, #units=None, xy=0,
epsilon1=1, epsilon2=1, epsilon3=1e-10,
mask_epsilon=1e-6, edge_epsilon=1e-8,
style_epsilon=1e-8, content_epsilon=1e-8,
output_dir=None, reset_every=0, save_every=1):
# Get the input and output sizes
image_shape = condition_net.blobs['data'].data.shape
generator_output_shape = image_generator.blobs[gen_out_layer].data.shape
encoder_input_shape = image_encoder.blobs['data'].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
image_size = util.get_image_size(image_shape)
generator_output_size = util.get_image_size(generator_output_shape)
encoder_input_size = util.get_image_size(encoder_input_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft = self.compute_topleft(image_size, generator_output_size)
topleft_DAE = self.compute_topleft(encoder_input_size, generator_output_size)
src = image_generator.blobs[gen_in_layer] # the input feature layer of the generator
# Make sure the layer size and initial vector size match
assert src.data.shape == start_code.shape
use_style_transfer= style_epsilon != 0 or content_epsilon !=0
# setup style transfer
if input_image is not None and use_style_transfer:
style_transfer = StyleTransfer(image_net, style_weight=style_epsilon, content_weight=content_epsilon)
style_transfer.init_image(input_image)
elif input_image is None:
# TODO setup loading the vector components
raise NotImplementedError('input image must not be None')
elif not use_style_transfer:
print('not using style transfer')
# Variables to store the best sample
last_xx = np.zeros(image_shape) # best image
last_prob = -sys.maxint # highest probability
h = start_code.copy()
condition_idx = 0
list_samples = []
i = 0
while True:
step_size = lr + ((lr_end - lr) * i) / n_iters
condition = conditions[condition_idx] # Select a class
# 1. Compute the epsilon1 term ---
# : compute gradient d log(p(h)) / dh per DAE results in Alain & Bengio 2014
d_prior = self.h_autoencoder_grad(h=h, encoder=image_generator, decoder=image_encoder, gen_out_layer=gen_out_layer, topleft=topleft_DAE, mask=mask, input_image=input_image)
# 2. Compute the epsilon2 term ---
# Push the code through the generator to get an image x
image_generator.blobs["feat"].data[:] = h
generated = image_generator.forward()
x = generated[gen_out_layer].copy() # 256x256
# Crop from 256x256 to 227x227
cropped_x_nomask = x[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]]
if mask is not None:
cropped_x = mask * input_image + (1 - mask) * cropped_x_nomask
else:
cropped_x = cropped_x_nomask
# Forward pass the image x to the condition net up to an unit k at the given layer
# Backprop the gradient through the condition net to the image layer to get a gradient image
d_condition_x, prob, info = self.forward_backward_from_x_to_condition(net=condition_net, end=layer, image=cropped_x, condition=condition)
if mask is not None:
generated_image = (1 - mask) * d_condition_x
else:
generated_image = d_condition_x
d_edge = self.get_edge_gradient(input_image, generated_image, edge_detector)
d_condition_x = epsilon2 * generated_image + edge_epsilon * d_edge
if use_style_transfer:
d_condition_x += style_transfer.get_gradient(generated_image)
if mask is not None:
d_condition_x += mask_epsilon * (mask) * (input_image - cropped_x_nomask)
# Put the gradient back in the 256x256 format
d_condition_x256 = np.zeros_like(x)
d_condition_x256[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]] = d_condition_x.copy()
# Backpropagate the above gradient all the way to h (through generator)
# This gradient 'd_condition' is d log(p(y|h)) / dh (the epsilon2 term in Eq. 11 in the paper)
d_condition = self.backward_from_x_to_h(generator=image_generator, diff=d_condition_x256, start=gen_in_layer, end=gen_out_layer)
# if i % 10 == 0:
# self.print_progress(i, info, condition, prob, d_condition)
# 3. Compute the epsilon3 term ---
noise = np.zeros_like(h)
if epsilon3 > 0:
noise = np.random.normal(0, epsilon3, h.shape) # Gaussian noise
d_h = epsilon1 * d_prior
d_h += d_condition
d_h += noise
h += step_size/np.abs(d_h).mean() * d_h
h = np.clip(h, a_min=0, a_max=30) # Keep the code within a realistic range
# Reset the code every N iters (for diversity when running a long sampling chain)
if reset_every > 0 and i % reset_every == 0 and i > 0:
h = np.random.normal(0, 1, h.shape)
# Experimental: For sample diversity, it's a good idea to randomly pick epsilon1 as well
epsilon1 = np.random.uniform(low=1e-6, high=1e-2)
# Save every sample
last_xx = cropped_x.copy()
last_prob = prob
# Filter samples based on threshold or every N iterations
if save_every > 0 and i % save_every == 0 and prob > threshold:
name = "%s/samples/%05d.jpg" % (output_dir, i)
label = self.get_label(condition)
if mask is not None:
image = last_xx * mask + (1 - mask) * input_image
else:
image = last_xx
# TODO check why this wasn't the case
# list_samples.append( (last_xx.copy(), name, label) )
list_samples.append( (image.copy(), name, label) )
# Stop if grad is 0
if norm(d_h) == 0:
print(" d_h is 0")
break
# Randomly sample a class every N iterations
if i > 0 and i % n_iters == 0:
condition_idx += 1
if condition_idx == len(conditions):
break
i += 1 # Next iter
# returning the last sample
print( "-------------------------")
print("Last sample: prob [%s] " % last_prob)
# if mask is not None:
# return last_xx * mask + (1 - mask) * input_image, list_samples
return last_xx, list_samples
description="Run the optimized style transfer pipeline.",
usage="demo.py -s <style_image> -c <content_image>")
parser.add_argument("-s",
"--style-img",
type=str,
required=True,
help="input style (art) image")
parser.add_argument("-c",
"--content-img",
type=str,
required=True,
help="input content image")
# style transfer
caffe.set_mode_cpu()
st = StyleTransfer("googlenet", use_pbar=True)
def st_api(img_style, img_content):
"""
Style transfer API.
"""
global st
# run iterations
all_args = [{
"length": 360,
"ratio": 4e2,
"n_iter": 80
}, {
"length": 480,
