def optimize(content_targets,
style_target,
content_weight,
style_weight,
tv_weight,
vgg_path,
epochs=2,
print_iterations=1000,
batch_size=4,
save_path='saver/fns.ckpt',
slow=False,
learning_rate=1e-3,
debug=False):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
content_targets = content_targets[:-mod]
print("Train set has been trimmed down to %d" % (len(content_targets)))
style_features = {}
batch_shape = (batch_size, 256, 256, 3)
style_shape = (1, ) + style_target.shape
print("style_shape is", style_shape)
# precompute style features
print("Precomputing style features")
with tf.Graph().as_default(), tf.device('/cpu:0'), tf.Session() as sess:
style_image = tf.placeholder(tf.float32,
shape=style_shape,
name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
print("Computing content features")
with tf.Graph().as_default(), tf.Session() as sess:
X_content = tf.placeholder(tf.float32,
shape=batch_shape,
name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256)
preds_pre = preds
else:
preds = transform.net(X_content / 255.0)
preds_pre = vgg.preprocess(preds)
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(
content_features[CONTENT_LAYER]) * batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(
net[CONTENT_LAYER])
content_loss = content_weight * (
2 * tf.nn.l2_loss(net[CONTENT_LAYER] -
content_features[CONTENT_LAYER]) / content_size)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i: i.value,
layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0, 2, 1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram) /
style_gram.size)
style_loss = style_weight * functools.reduce(tf.add,
style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:, 1:, :, :])
tv_x_size = _tensor_size(preds[:, :, 1:, :])
y_tv = tf.nn.l2_loss(preds[:, 1:, :, :] -
preds[:, :batch_shape[1] - 1, :, :])
x_tv = tf.nn.l2_loss(preds[:, :, 1:, :] -
preds[:, :, :batch_shape[2] - 1, :])
tv_loss = tv_weight * 2 * (x_tv / tv_x_size +
y_tv / tv_y_size) / batch_size
loss = content_loss + style_loss + tv_loss
# overall loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
delta_time = 60
num_examples = len(content_targets)
iterations_per_epoch = num_examples / batch_size
total_iterations = iterations_per_epoch * epochs
iterations_completed = 0
for epoch in range(epochs):
print("Starting epoch %d of %d" % (epoch + 1, epochs))
iterations = 0
while iterations * batch_size < num_examples:
time_remaining = delta_time * (total_iterations -
iterations_completed)
print(
"Epoch %d/%d iteration %d/%d (completed: %d/%d @ %s). %0.2f hours left"
% (epoch + 1, epochs, iterations + 1, iterations_per_epoch,
iterations_completed, total_iterations, delta_time,
time_remaining / (60 * 60) * 1.0))
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p,
(256, 256, 3)).astype(np.float32)
iterations += 1
iterations_completed += 1
assert X_batch.shape[0] == batch_size
feed_dict = {X_content: X_batch}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {X_content: X_batch}
tup = sess.run(to_get, feed_dict=test_feed_dict)
_style_loss, _content_loss, _tv_loss, _loss, _preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess, save_path)
yield (_preds, losses, iterations, epoch)
def stylize(network_path='imagenet-vgg-very0.001p-19.mat', content, styles, iterations=1000,
content_weight=5e0, content_weight_blend=1, style_weight=5e2, style_layer_weight_exp=1, style_blend_weights=None, tv_weight=1e2,
learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08, pooling='avg',
print_iterations=100, checkpoint_iterations=100, checkpoint_path=None, output_path=None):
"""
This is a function to stylelize images,
given the content image, list of style images, path to the network and all the hypter parameters.
Returns
-------
stylized_img : np.ndarray
N x H x W x C image.
"""
# calculate the shape of the network input tensor according to the content image
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network_path)
# scale the importance of each sytle layers according to their depth. (deeper layers are more important if style_layers_weights > 1 (default = 1))
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features of the content image by feeding it into the network
@TODO why put graph on cpu?, what is the high level idea of content_features?
g = tf.Graph()
with g.as_default(), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features of the content image by feeding it into the network
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
# make stylized image using backpropogation
# if the users doesn't specify a input image, start with noise
# @TODO where does the number 0.256 come from?
with tf.Graph().as_default():
initial = tf.random_normal(shape) * 0.256
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss, we can adjust the weight of each CONTENT_LAYERS
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# We can specify different weight for different style images
if style_blend_weights is None:
# default is equal weights
style_blend_weights = [1.0/len(style_images) for _ in style_images]
else:
total_blend_weight = sum(style_blend_weights)
# normalization
style_blend_weights = [weight/total_blend_weight
for weight in style_blend_weights]
# style loss
style_loss = 0
# iterate to calculate style lose with multiple style images
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising, according to the paper
# Mahendran, Aravindh, and Andrea Vedaldi. "Understanding deep image representations by inverting them."
# Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015.
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
# yield (
# (None if last_step else i),
# img_out
# )
output_file = None
if not last_step:
if checkpoint_path:
output_file = checkpoint_path % iteration
else:
output_file = output_path
if output_file:
imsave(output_file, image)
def stylize(network,
content,
styles,
shape,
iterations,
content_weight=5.0,
style_weight=100.0,
tv_weight=100.0,
style_blend_weights=None,
learning_rate=10.0,
initial=None,
use_mrf=False,
use_semantic_masks=False,
mask_resize_as_feature=True,
output_semantic_mask=None,
style_semantic_masks=None,
semantic_masks_weight=1.0,
print_iterations=None,
checkpoint_iterations=None,
semantic_masks_num_layers=4,
content_img_style_weight_mask=None):
# type: (str, Union[None,np.ndarray], List[np.ndarray], Tuple[int,int,int,int], int, float, float, float, Union[None,List[float]], float, Union[None,np.ndarray], bool, bool, bool, Union[None,np.ndarray], Union[None,List[np.ndarray], float, Union[None,int], Union[None,int], Union[None,int], Union[None,np.ndarray], Union[None,int]]) -> Iterable[Tuple[Union[None,int],np.ndarray]]
"""
Stylize images.
:param network: Path to pretrained vgg19 network. It can be downloaded at
http://www.vlfeat.org/matconvnet/models/imagenet-vgg-verydeep-19.mat
:param content: The content image. If left blank, it will enter texture generation mode (style synthesis without
context loss).
:param styles: A list of style images as numpy arrays.
:param shape: The shape of the output image. It should be with format (1, height, width, 3)
:param iterations: The number of iterations to run.
:param content_weight: The weight for content loss. The larger the weight, the more the output will look like
the content image.
:param style_weight: The weight for style loss. The larger the weight, the more the output will have a style that
looks like the style images.
:param tv_weight: The weight for total-variation loss. The larger the weight, the smoother the output will be.
:param style_blend_weights: If inputting multiple style images, this controls the balance between their styles.
If left as None, it will treat all style images as equal.
:param learning_rate: As name suggests.
:param initial: The initial starting point for the output. If left blank, the initial would just be noise.
:param use_mrf: Whether we use markov-random-field loss instead of gramian loss. mrf_util.py contains more info.
:param use_semantic_masks: Whether we use semantic masks as additional semantic information. Please check the paper
"Semantic Style Transfer and Turning Two-Bit Doodles into Fine Artworks" for more information.
:param mask_resize_as_feature: If true, resize the mask and use the resized mask as additional feature besides the
vgg network layers. If false, pass the masks (must have exactly 3 masks) into the vgg network and use the outputted
layers as additional features.
:param output_semantic_mask: The semantic masks you would like to apply to the outputted image.The mask should have
shape (batch_size, height, width, semantic_masks_num_layers) Unlike the neural doodle paper, here I use one
black-and-white image for each semantic mask (the paper had semantic masks represented as rgb images, limiting the
semantic channels to 3).
:param style_semantic_masks: A list of semantic masks you would like to apply to each style image. The mask should
have shape (batch_size, height, width, semantic_masks_num_layers)
:param semantic_masks_weight: How heavily you'd like to weight the semantic masks as compared to other sources of
semantic information obtained through passing the image through vgg network. Default is 1.0.
:param print_iterations: Print loss information every n iterations.
:param checkpoint_iterations: Save a checkpoint as well as the best image so far every n iterations.
:param semantic_masks_num_layers: The number of semantic masks each image have.
:param content_img_style_weight_mask: One black-and-white mask specifying how much we should "stylize" each pixel
in the outputted image. The areas where the mask has higher value would be stylized more than other areas. A
completely white mask would mean that we stylize the output image just as before, while a completely dark mask
would mean that we do not stylize the output image at all, so it should look pretty much the same as content image.
If you do not wish to use this feature, just leave it as None.
:return: a tuple where the first item is either the current iteration or None, indicating it has finished training.
The second item is the image that has the lowest loss so far. The tuples are yielded every 'checkpoint_iterations'
iterations as well as the last iteration.
:rtype: iterator[tuple[int|None,image]]
"""
global STYLE_LAYERS
if content is not None:
STYLE_LAYERS = STYLE_LAYERS_WITH_CONTENT
if use_mrf:
STYLE_LAYERS = STYLE_LAYERS_MRF # Easiest way to be compatible with no-mrf versions.
if use_semantic_masks:
assert semantic_masks_weight is not None
assert output_semantic_mask is not None
assert style_semantic_masks is not None
if content_img_style_weight_mask is not None:
if shape[1] != content_img_style_weight_mask.shape[1] or shape[
2] != content_img_style_weight_mask.shape[2]:
raise AssertionError(
"The shape of style_weight_mask is incorrect. It must have the same height and width "
"as the output image. The output image has shape: %s and the style weight mask has "
"shape: %s" %
(str(shape), str(content_img_style_weight_mask.shape)))
if content_img_style_weight_mask.dtype != np.float32:
raise AssertionError(
'The dtype of style_weight_mask must be float32. it is now %s'
% str(content_img_style_weight_mask.dtype))
if len(styles) == 0:
raise AssertionError("Must feed in at least one style image.")
# Append a (1,) in front of the shapes of the style images. So the style_shapes contains (1, height, width, 3).
# 3 corresponds to rgb.
style_shapes = [(1, ) + style.shape for style in styles]
if style_blend_weights is None:
style_blend_weights = [1.0 / len(styles) for _ in styles]
content_features = {}
style_features = [{} for _ in styles]
output_semantic_mask_features = {}
# The default behavior of tensorflow was to allocate all gpu memory. Here it is set to only use as much gpu memory
# as it needs.
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
vgg_data, mean_pixel = vgg.read_net(network)
# Compute content features in feed-forward mode
content_image = tf.placeholder('float',
shape=shape,
name='content_image')
net = vgg.pre_read_net(vgg_data, content_image)
content_features[CONTENT_LAYER] = net[CONTENT_LAYER]
net_layer_sizes = vgg.get_net_layer_sizes(net)
if content is not None:
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
# Compute style features in feed-forward mode.
if content_img_style_weight_mask is not None:
style_weight_mask_layer_dict = neural_doodle_util.masks_average_pool(
content_img_style_weight_mask)
for i in range(len(styles)):
# Using precompute_image_features, which calculates on cpu and thus allow larger images.
style_features[i] = neural_util.precompute_image_features(
styles[i], STYLE_LAYERS, style_shapes[i], vgg_data, mean_pixel,
use_mrf, use_semantic_masks)
if use_semantic_masks:
output_semantic_mask_features, style_features, content_semantic_mask, style_semantic_masks_images = neural_doodle_util.construct_masks_and_features(
style_semantic_masks,
styles,
style_features,
shape[0],
shape[1],
shape[2],
semantic_masks_num_layers,
STYLE_LAYERS,
net_layer_sizes,
semantic_masks_weight,
vgg_data,
mean_pixel,
mask_resize_as_feature,
use_mrf,
average_pool=False
) # TODO: average pool is not working so well in practice??
if initial is None:
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
_, height, width, number = map(
lambda i: i.value, content_features[CONTENT_LAYER].get_shape())
content_features_size = height * width * number
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features_size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
if content_img_style_weight_mask is not None:
# Apply_style_weight_mask_to_feature_layer, then normalize with average of that style weight mask.
layer = neural_doodle_util.vgg_layer_dot_mask(style_weight_mask_layer_dict[style_layer], layer) \
/ (tf.reduce_mean(style_weight_mask_layer_dict[style_layer]) + 0.000001)
if use_mrf:
if use_semantic_masks:
# TODO: Compare the effect of concatenate masks to vgg layers versus dotting them with vgg
# layers. If you change this to dot, don't forget to also change that in neural_doodle_util.
layer = neural_doodle_util.concatenate_mask_layer_tf(
output_semantic_mask_features[style_layer], layer)
# layer = neural_doodle_util.vgg_layer_dot_mask(output_semantic_mask_features[style_layer], layer)
style_losses.append(
mrf_loss(style_features[i][style_layer],
layer,
name='%d%s' % (i, style_layer)))
else:
if use_semantic_masks:
gram = neural_doodle_util.gramian_with_mask(
layer, output_semantic_mask_features[style_layer])
else:
gram = neural_util.gramian(layer)
style_gram = style_features[i][style_layer]
style_gram_size = get_np_array_num_elements(style_gram)
style_losses.append(
tf.nn.l2_loss(gram - style_gram) / style_gram_size
) # TODO: Check normalization constants. the style loss is way too big compared to the other two.
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_loss = tf.mul(neural_util.total_variation(image), tv_weight)
# overall loss
if content is None: # If we are doing style/texture regeration only.
loss = style_loss + tv_loss
else:
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
def print_progress(i, feed_dict, last=False):
stderr.write('Iteration %d/%d\n' % (i + 1, iterations))
if last or (print_iterations is not None and print_iterations != 0
and i % print_iterations == 0):
if content is not None:
stderr.write(' content loss: %g\n' %
content_loss.eval(feed_dict=feed_dict))
stderr.write(' style loss: %g\n' %
style_loss.eval(feed_dict=feed_dict))
stderr.write(' tv loss: %g\n' %
tv_loss.eval(feed_dict=feed_dict))
stderr.write(' total loss: %g\n' %
loss.eval(feed_dict=feed_dict))
# optimization
best_loss = float('inf')
best = np.zeros(shape=shape)
feed_dict = {}
if content is not None:
feed_dict[content_image] = content_pre
if use_semantic_masks:
feed_dict[content_semantic_mask] = output_semantic_mask
for styles_iter in range(len(styles)):
feed_dict[style_semantic_masks_images[
styles_iter]] = style_semantic_masks[styles_iter]
sess.run(tf.initialize_all_variables(), feed_dict=feed_dict)
for i in range(iterations):
last_step = (i == iterations - 1)
print_progress(i, feed_dict, last=last_step)
train_step.run(feed_dict=feed_dict)
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval(feed_dict=feed_dict)
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
yield ((None if last_step else i),
vgg.unprocess(best.reshape(shape[1:]), mean_pixel))
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations,
content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate, beta1, beta2, epsilon, pooling,
print_iterations=None, checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)#加载vgg19与训练模型
layer_weight = 1.0#layer权重默认为1
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight#style_layer_weight_exp为图像风格的权重默认为1，否则为指数级增长
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/gpu:0'), tf.Session() as sess:#使用gpu训练，cpu训练大约2个小时，gpu5分钟
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/gpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)#池化层默认为max规则
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))#将一维数组根据图像大小转为三维
gram = np.matmul(features.T, features) / features.size#计算gram矩阵
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256#得到一个随机白噪音
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)#将得到的白噪音转为tensorflow对象
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
#网络的高层特征一般是关于输入图像的物体和布局等信息，低层特征一般表达输入图像的像素信息
#最终选择conv4_2
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
#计算content loss
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
#总loos为loss相加
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress():#输出相关信息
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
iteration_times = []
start = time.time()
for i in range(iterations):
iteration_start = time.time()
if i > 0:
elapsed = time.time() - start
# take average of last couple steps to get time per iteration
remaining = np.mean(iteration_times[-10:]) * (iterations - i)
stderr.write('Iteration %4d/%4d (%s elapsed, %s remaining)\n' % (
i + 1,
iterations,
hms(elapsed),
hms(remaining)
))
else:
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. 将风格图像的RGB转为gray
# 2. 将风格图像gray转为ycrcb
# 3. 将事物图像转为ycrcb
# 4. 将图像重组
# 5. 最后转为RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out
)
iteration_end = time.time()
iteration_times.append(iteration_end - iteration_start)
def stylize(network, initial, content, styles, iterations,
content_weight, style_weight, style_blend_weights, tv_weight,
learning_rate, print_iterations=None, checkpoint_iterations=None):
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(styles[i], mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
print 'Initial feature shape: ', features.shape
features = np.reshape(features, (-1, features.shape[3]))
#mask = np.zeros_like(features)
#mask[:49664/2, :] = 1
#print 'Mask shape', mask.shape
print 'Final features shape', features.shape
#features = features*mask
gram = np.matmul(features.T, features) / features.size
print 'Gram matrix shape: ', gram.shape
style_features[i][layer] = gram
#sys.exit()
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
print 'Height, width, number', height, width, number
size = height * width * number
feats = tf.reshape(layer, (-1, number))
#print tf.shape(feats).as_list()
if normal_flag == 0:
mask = np.zeros((height*width, number), dtype=np.float32)
maskt = np.reshape(imread('bottle_mask.jpg').astype(np.float32), (height*width,))
maskt = maskt > 100
for d in xrange(number):
mask[:,d] = maskt
print 'Mask shape', mask.shape
#print sum(sum(mask == 1)) + sum(sum(mask == 0))
#mask[:height*width/2, :] = 1
if i == 0:
mask = tf.constant(mask)
feats = tf.mul(feats,mask)
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
else:
mask2 = mask < 1
feats2 = tf.mul(feats,mask2)
gram2 = tf.matmul(tf.transpose(feats2), feats2) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram2 - style_gram) / style_gram.size)
else:
feats2 = feats
gram2 = tf.matmul(tf.transpose(feats2), feats2) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram2 - style_gram) / style_gram.size)
pass
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
if normal_flag != 0:
print "general mask :"
mask = np.zeros((height*width, number), dtype=np.float32)
maskt = np.reshape(imread('bottle_mask.jpg').astype(np.float32), (height*width,))
maskt = maskt > 100
# for d in xrange(3):
# mask[:,d] = maskt
print 'Mask shape', maskt.shape
maskt = maskt.reshape((height,width))
maskt = np.array([maskt,maskt,maskt])
maskt = maskt.transpose((1,2,0))
mask = tf.constant(maskt, dtype=tf.float32)
# feats = tf.mul(feats,mask)
def capper(a,b,mask):
# (1, 468, 304, 3)
print "orig shape", a
reshaped_in_grad = tf.reshape(a,[-1] )
print "reshaped grad", reshaped_in_grad
print "mask" ,mask
g = tf.mul(a,mask)
# g = tf.reshape(g, (1,height,width,3))
# print a,b
# print g
return g,b
# optimizer setup
# train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# # Create an optimizer.
train_step = tf.train.GradientDescentOptimizer(learning_rate)
# # Compute the gradients for a list of variables.
grads_and_vars = train_step.compute_gradients(loss)
# # grads_and_vars is a list of tuples (gradient, variable). Do whatever you
# # need to the 'gradient' part, for example cap them, etc.
capped_grads_and_vars = [(capper(gv[0], gv[1], mask)) for gv in grads_and_vars]
# # Ask the optimizer to apply the capped gradients.
train_step = train_step.apply_gradients(capped_grads_and_vars)
# opt_op = opt.minimize(cost, var_list=<list of variables>)
def print_progress(i, last=False):
if print_iterations is not None:
if i is not None and i % print_iterations == 0 or last:
print >> stderr, ' content loss: %g' % content_loss.eval()
print >> stderr, ' style loss: %g' % style_loss.eval()
print >> stderr, ' tv loss: %g' % tv_loss.eval()
print >> stderr, ' total loss: %g' % loss.eval()
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
print_progress(i)
print >> stderr, 'Iteration %d/%d' % (i + 1, iterations)
train_step.run()
# print "runningstep: ",i, running_step
if (checkpoint_iterations is not None and
i % checkpoint_iterations == 0) or i == iterations - 1:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
print_progress(None, i == iterations - 1)
if i % 10 == 0 and best is not None:
tmp_img = vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
imsave("iter" + str(i) + ".jpg", tmp_img)
return vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
def stylize(network, initial, content, styles, iterations,
content_weight, style_weight, style_blend_weights, tv_weight,
learning_rate, print_iterations=None, checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(styles[i], mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
def print_progress(i, last=False):
global timenow
stderr.write('Iteration %d/%d, time: %dms\n' % (i + 1, iterations, current_milli_time() - timenow))
timenow = current_milli_time()
if last or (print_iterations and i % print_iterations == 0):
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
last_step = (i == iterations - 1)
print_progress(i, last=last_step)
train_step.run()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
yield (
(None if last_step else i),
vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
)
def model_neural_style(pre_train_vgg_path,
content_image,
style_images,
content_weight=5e0,
content_weight_blend=1.0,
style_weight=5e2,
style_layer_weight_exp=1.0,
pooling='',
initial=None,
initial_noiseblend=1.0,
tv_weight=1e2,
learning_rate=1e1,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
print_iterations=None,
iterations=500,
checkpoint_iterations=50,
preserve_colors=None):
print "++++++++++++++++++++"
# input shape of model
shape = (1, ) + content_image.shape
style_images_shapes = [(1, ) + style_image.shape
for style_image in style_images]
content_features = {}
style_features = [{} for _ in style_images]
# load the weights of pretrained vgg model
vgg_weights, vgg_mean_pixel = vgg.load_weights(pre_train_vgg_path)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_infer(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content_image, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={image: content_pre})
# # for debug
# for layer in CONTENT_LAYERS:
# item = content_features[layer]
# item = item.reshape(item.shape[1], item.shape[2], item.shape[3])
# item_for_plot = []
# for i in range(item.shape[2]):
# item_for_plot.append(item[:, :, i])
#
# tools.show_images(item_for_plot[::8], cols=8)
# compute style features in feedforward mode
# compute styles features in feedforward mode
for i in range(len(style_images)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_images_shapes[i])
net = vgg.net_infer(vgg_weights, image, pooling)
style_pre = np.array(
[vgg.preprocess(style_images[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape,
scale=np.std(content_image) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape,
scale=np.std(content_image) * 0.1)
initial = (initial) * initial_content_noise_coeff + (
tf.random_normal(shape) *
0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_infer(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(
content_layers_weights[content_layer] * content_weight *
(2 * tf.nn.l2_loss(net[content_layer] -
content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(style_images)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 *
tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations
and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]),
vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content_image, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(
styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv,
'YCbCr').convert('RGB'))
yield ((None if last_step else i), img_out)
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations,
content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate, beta1, beta2, epsilon, pooling, exp_sigma, mat_sigma, mat_rho, text_to_print,
print_iterations=None, checkpoint_iterations=None, kernel=3, d=2, gamma_rho=1, gamma=1, rational_rho=1, alpha=1):
tf.logging.set_verbosity(tf.logging.INFO)
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
0 - dot product kernel
1 - exponential kernel
2 - matern kernel
3 - polynomial kernel
"""
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
# sqr = features.T*features.T
# dim = features.shape
if(kernel == 0):
gram2 = np.matmul(features.T, features) / features.size
elif(kernel == 1):
gram2 = gramSquaredExp_np(features, exp_sigma) / features.size # exponential kernal
elif(kernel == 2):
gram2 = gramMatten_np(features, mat_sigma, v, mat_rho) / features.size # Mattern kernal
elif(kernel == 3):
print(d)
gram2 = gramPoly_np(features, C=0, d=d) / features.size
elif(kernel == 4):
gram2 = gramGammaExp_np(features, gamma_rho, gamma) / features.size
elif(kernel == 4):
gram2 = gramRatioanlQuad_np(features, rational_rho, alpha) / features.size
# print(features.shape,"diamention of feature\n")
style_features[i][layer] = gram2
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
g = tf.Graph()
with g.as_default(), g.device('/gpu'):
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
style_gram = style_features[i][style_layer]
dim = feats.get_shape()
# print(dim)
sqr = tf.reduce_sum(tf.transpose(feats) * tf.transpose(feats), axis=1)
if(kernel == 0):
gram = (tf.matmul(tf.transpose(feats), feats)) / size
elif(kernel == 1):
gram = tf.exp(-1 * (tf.transpose(tf.ones([dim[1], dim[1]]) * sqr) + tf.ones([dim[1], dim[1]]) * sqr - 2 *
tf.matmul(tf.transpose(feats), feats)) / 2 / (exp_sigma * exp_sigma)) / size # exponetial kernal
elif(kernel == 2):
# mattern kernal
d2 = tf.nn.relu(tf.transpose(tf.ones([dim[1], dim[1]]) * sqr) + tf.ones([dim[1], dim[1]]) * sqr - 2 * tf.matmul(tf.transpose(feats), feats))
if(v == 0.5):
gram = mat_sigma**2 * tf.exp(-1 * tf.sqrt(d2) / mat_rho) / size
elif(v == 1.5):
gram = mat_sigma**2 * (tf.ones([dim[1], dim[1]]) + tf.sqrt(3.0) * tf.sqrt(d2) / mat_rho) * tf.exp(-1 * tf.sqrt(3.0) * tf.sqrt(d2) / mat_rho) / size
elif(v == 2.5):
gram = mat_sigma**2 * (tf.ones([dim[1], dim[1]]) + tf.sqrt(5.0) * tf.sqrt(d2) / mat_rho + 5 * d2 / 3 / (mat_rho**2)) * tf.exp(-1 * tf.sqrt(5.0) * tf.sqrt(d2) / mat_rho) / size
elif(kernel == 3):
# polynomial kernal
gram = (tf.matmul(tf.transpose(feats), feats))**d / size
elif(kernel == 4):
# gamma exponental kernal
gram = tf.exp(-1 * (tf.sqrt(d2) / gamma_rho)**gamma) / size
elif(kernel == 5):
# gamma exponental kernal
gram = (1 + (d2 / rational_rho**2 / 2 / alpha))**(-1 * alpha) / size
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) /
tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
# train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress(last_loss):
new_loss = loss.eval()
stderr.write('file ===> %s \n' % text_to_print)
stderr.write(' content loss: %1.3e \t' % content_loss.eval())
stderr.write(' style loss: %1.3e \t' % style_loss.eval())
stderr.write(' tv loss: %1.3e \t' % tv_loss.eval())
stderr.write(' total loss: %1.3e \t' % new_loss)
stderr.write(' loss difference: %1.3e \t\n' % (last_loss - new_loss))
return new_loss
def save_progress():
dict = {"content loss": content_loss.eval(), "style loss": style_loss.eval(), "tv loss": tv_loss.eval(), "total loss": loss.eval()}
return dict
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
new_loss = 0
# if (print_iterations and print_iterations != 0):
# print_progress()
for i in range(iterations):
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
new_loss = print_progress(new_loss)
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
dict = save_progress()
this_loss = loss.eval()
print(this_loss, "loss in each check point")
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
try:
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
except:
print("uanlabe to result image due to given parameters")
img_out = "no image"
if preserve_colors and preserve_colors:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out, dict
)
def main():
content_path, style_path, width, style_scale = sys.argv[1:]
width = int(width)
style_scale = float(style_scale)
content_image = imread(content_path)
style_image = imread(style_path)
if width > 0:
new_shape = (int(math.floor(float(content_image.shape[0]) /
content_image.shape[1] * width)), width)
content_image = sm.imresize(content_image, new_shape)
if style_scale > 0:
style_image = sm.imresize(style_image, style_scale)
shape = (1,) + content_image.shape
style_shape = (1,) + style_image.shape
content_features = {}
style_features = {}
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(VGG_PATH, image)
content_pre = np.array([vgg.preprocess(content_image, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shape)
net, _ = vgg.net(VGG_PATH, image)
style_pre = np.array([vgg.preprocess(style_image, mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / (features.size)
style_features[layer] = gram
with tf.Graph().as_default():
noise = np.random.normal(size=shape, scale=np.std(content_image) * 0.1)
init = tf.random_normal(shape) * 256 / 1000
image = tf.Variable(init)
net, _ = vgg.net(VGG_PATH, image)
content_loss = tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER])
style_losses = []
for i in STYLE_LAYERS:
layer = net[i]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / (size)
style_gram = style_features[i]
style_losses.append(tf.nn.l2_loss(gram - style_gram))
style_loss = reduce(tf.add, style_losses) / len(style_losses)
tv_loss = (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) +
tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]))
loss = ALPHA * content_loss + BETA * style_loss + TV_WEIGHT * tv_loss
train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(100000):
print 'i = %d' % i
if i % 10 == 0:
print '\tcontent_loss = %15.0f' % content_loss.eval()
print '\tstyle_loss = %15.0f' % style_loss.eval()
print '\ttv_loss = %15.0f' % tv_loss.eval()
print '\tloss = %15.0f' % loss.eval()
imsave('%05d.jpg' % i, vgg.unprocess(
image.eval().reshape(shape[1:]), mean_pixel))
train_step.run()
def optimize(content_targets, style_target, content_weight, style_weight,
tv_weight, vgg_path, use_IN, epochs=2, print_iterations=1000,
batch_size=4, save_path='checkpoints/fast_style_transfer.ckpt', slow=False,
learning_rate=1e-3, debug=False):
if slow:
batch_size = 1
# content_target is a list of files, 4-D size, so this is about the batch size here.
# If using only one content image, then mod here is 0.
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly...")
content_targets = content_targets[:-mod]
# training image get to be 256 x 256 because of get_img resize,
# it then get into tensorflow graph from Adam optimizer feed_dict.
batch_shape = (batch_size, 256, 256, 3)
style_shape = (1,) + style_target.shape # add 1 in the front for batch size, 4-D.
print(f"batch_shape of the content image is: {batch_shape}")
print(f"style_shape of the style image is: {style_shape}")
### Graph Construction ###
# vgg won't be trained, because in vgg.py the weights are loaded through that matlab file.
# computed vgg style features in gram matrices
# tf.device('/cpu:0')
config = v1.ConfigProto()
config.gpu_options.allow_growth = True
style_features = {}
with tf.Graph().as_default(), v1.Session(config=config) as sess:
style_image = v1.placeholder(tf.float32, shape=style_shape, name='style_image') # 4-D placeholder for feed_dict
vgg_style_net = vgg.net(vgg_path, vgg.preprocess(style_image)) # extract feature volume
np_style_target = np.array([style_target]) # a 3-D numpy array for feed_dict's input
for layer in STYLE_LAYERS:
# vgg_style_net[layer] is a tf.Tensor returned by tf.nn.relu,
# eval at that layer, by running forward to that vgg layer or entire network.
features = vgg_style_net[layer].eval(feed_dict={style_image:np_style_target}) # extract a fVol value
features = np.reshape(features, (-1, features.shape[3])) # (N*H*W, C)
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
# computed vgg content feature map and both losses
with tf.Graph().as_default(), v1.Session(config=config) as sess:
X_content = v1.placeholder(tf.float32, shape=batch_shape, name="X_content") # 4-D
vgg_content_net = vgg.net(vgg_path, vgg.preprocess(X_content)) # run ground truth image through the pre-trained model
# noisy prediction image runs through feed forward conv net, then
# run through vgg to extract feature volume predicitons
if slow:
preds = tf.Variable(
tf.random.normal(X_content.get_shape()) * 0.256
)
preds_pre = preds
else:
preds = transform.net(X_content/255.0, use_IN) # run through the style feed forward network. why need to normalize pixel to 0-1?
net = vgg.net(vgg_path, vgg.preprocess(preds)) # run generated image through the pre-trained model
# _tensor_size is a reduce function only count from [1:],
# so it doesn't have batch_size information.
content_size = _tensor_size(vgg_content_net[CONTENT_LAYER]) * batch_size
vgg_content_net_size = _tensor_size(vgg_content_net[CONTENT_LAYER])
vgg_transform_content_net_size = _tensor_size(net[CONTENT_LAYER])
# print(f"vgg_content_net_size is {vgg_content_net_size}")
# print(vgg_content_net[CONTENT_LAYER])
# print(f"vgg_transform_content_net_size is {vgg_transform_content_net_size}")
# print(net[CONTENT_LAYER])
assert vgg_content_net_size == vgg_transform_content_net_size
# define loss functions
# content loss
content_l2_loss = 2 * tf.nn.l2_loss(net[CONTENT_LAYER] - vgg_content_net[CONTENT_LAYER])
content_loss = content_weight * (content_l2_loss / content_size)
# style loss
style_l2_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
N, H, W, C = map(lambda i : i, layer.get_shape())
feats = tf.reshape(layer, (N, H*W, C)) # N, HW, C
feats_T = tf.transpose(feats, perm=[0, 2, 1]) # N, C, HW
pred_gram = tf.matmul(feats_T, feats) / (H * W * C)
true_gram = style_features[style_layer] # numpy array
style_l2_loss = 2 * tf.nn.l2_loss(pred_gram - true_gram)
style_l2_losses.append(style_l2_loss / true_gram.size)
style_loss = style_weight * functools.reduce(tf.add, style_l2_losses) / batch_size
# total variation denoising regularization loss
# test if not needed in NN conv case and mirror padding
# tv_y_size = _tensor_size(preds[:,1:,:,:])
# tv_x_size = _tensor_size(preds[:,:,1:,:])
# # N, H, W, C
# y_tv = 2 * tf.nn.l2_loss(preds[:, 1:, :, :] - preds[:, :batch_shape[1]-1, :, :]) # H, down - up
# x_tv = 2 * tf.nn.l2_loss(preds[:, :, 1:, :] - preds[:, :, :batch_shape[2]-1, :]) # W, right - left
# tv_loss = tv_weight * (x_tv/tv_x_size + y_tv/tv_y_size) / batch_size
# total loss
# total_loss = content_loss + style_loss + tv_loss
total_loss = content_loss + style_loss
# train the feed forward net, and save weights to a checkpoint.
import random
uid = random.randint(1, 100)
print("This random UID is: %s" % uid)
optimizer = v1.train.AdamOptimizer(learning_rate).minimize(total_loss)
sess.run(v1.global_variables_initializer())
for epoch in range(epochs): # epoch loop
iterations = 0
num_examples = len(content_targets) # COCO train2014 ~20000 images
while iterations * batch_size < num_examples: # batch loop
# start training a batch
start_time = time.time()
X_batch = np.zeros(batch_shape, dtype=np.float32)
start = iterations * batch_size
end = iterations * batch_size + batch_size
for i, img_p in enumerate(content_targets[start:end]): # img_p is a coco images
X_batch[i] = get_img(img_p, (256,256,3)).astype(np.float32) # resize to 256 x 256
optimizer.run(feed_dict={X_content:X_batch})
end_time = time.time()
# end training a batch
# update training information
iterations += 1
is_print_iter = int(iterations) % print_iterations == 0
is_last_train = epoch == epochs - 1 and iterations * batch_size >= num_examples
if slow:
is_print_iter = epoch % print_iterations == 0
if debug:
print("UID: %s, batch training time: %s" % (uid, end_time - start_time))
# monitor the training losses
if is_print_iter or is_last_train:
_style_loss, _content_loss, _total_loss, _preds = \
sess.run([style_loss, content_loss, total_loss, preds],
feed_dict={X_content:X_batch})
losses = (_style_loss, _content_loss, _total_loss)
generated_image = _preds
if slow:
generated_image = vgg.unprocess(generated_image)
else:
res = v1.train.Saver().save(sess, save_path)
print("yield")
yield(generated_image, losses, iterations, epoch)
def stylize(content,
style,
initial,
initial_noiseblend,
content_weight=5e0,
content_layer_num=9,
style_weight=5e2,
style_layer_weight=(0.2, 0.2, 0.2, 0.2, 0.2),
tv_weight=1e2,
learning_rate=1e1,
beta1=0.9,
beta2=0.999,
epsilon=1e-8,
preserve_colors=False,
pooling='max',
iterations=1000,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1, ) + content.shape
content_features = {}
style_features = {}
style_layers_weights = {}
content_layer = CONTENT_LAYERS[content_layer_num]
for i, style_layer in enumerate(STYLE_LAYERS):
style_layers_weights[style_layer] = style_layer_weight[i]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
image = tf.placeholder(tf.float32, shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
style_pre = np.array([vgg.preprocess(style, vgg_mean_pixel)])
# compute content features,style features in feedforward mode
with tf.Session() as sess:
content_features[content_layer] = sess.run(
net[content_layer], feed_dict={image: content_pre})
for layer in STYLE_LAYERS:
features = sess.run(net[layer], feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
# make stylized image using backpropogation
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype(np.float32)
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = initial * (1 - initial_noiseblend) + (
tf.random_normal(shape) * 0.256) * initial_noiseblend
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_loss = content_weight * 2 * tf.nn.l2_loss(
net[content_layer] -
content_features[content_layer]) / content_features[content_layer].size
# style loss
style_loss = 0
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[style_layer]
style_loss += style_weight * style_layers_weights[
style_layer] * 2 * tf.nn.l2_loss(gram -
style_gram) / style_gram.size
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) /
tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
def print_progress():
print(' content loss: %g\n' % content_loss.eval())
print(' style loss: %g\n' % style_loss.eval())
print(' tv loss: %g\n' % tv_loss.eval())
print(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
images = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print('Iteration %4d/%4d\n' % (i + 1, iterations))
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
styled_image = np.clip(
vgg.unprocess(image.eval().reshape(shape[1:]),
vgg_mean_pixel), 0, 255)
if this_loss < best_loss:
best_loss = this_loss
best = styled_image
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
styled_image = np.array(
Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
plt.figure(figsize=(8, 8))
plt.imshow(styled_image.astype(np.uint8))
plt.axis('off')
plt.show()
images.append(styled_image.astype(np.uint8))
return images, best
def inferenceImg(network, initial_img, initial_noiseblend, content, style,
preserve_colors, iterations, content_weight,
content_weight_blend, style_weight, style_layer_weight_exp,
style_blend_weight, tv_weight, learning_rate, beta1, beta2,
epsilon, pooling, print_iterations, checkpoint_iterations):
content_shape = (1, ) + content.shape
style_shape = (1, ) + style.shape
content_features = {}
style_features = {}
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight = layer_weight * style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum = layer_weights_sum + style_layers_weights[
style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = style_layers_weights[
style_layer] / layer_weights_sum
# compute content features in feedforward mode
g1 = tf.Graph()
with g1.as_default(), g1.device('/cpu:0'), tf.Session() as sess:
contentImg = tf.placeholder('float', shape=content_shape)
net = vgg.net_preloaded(vgg_weights, contentImg, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={contentImg: content_pre})
# compute style features in feedforward mode
g2 = tf.Graph()
with g2.as_default(), g2.device('/cpu:0'), tf.Session() as sess:
styleImg = tf.placeholder('float', shape=style_shape)
net = vgg.net_preloaded(vgg_weights, styleImg, pooling)
style_pre = np.array([vgg.preprocess(style, vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={styleImg: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
noise = np.random.normal(size=content_shape,
scale=np.std(content) * 0.1)
initial = tf.random_normal(content_shape) * 0.256
inferenceImg = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, inferenceImg, pooling)
# compute content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(
content_layers_weights[content_layer] * content_weight *
(2 * tf.nn.l2_loss(net[content_layer] -
content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# compute style loss
style_loss = 0
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 *
tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss += style_weight * style_blend_weight * reduce(
tf.add, style_losses)
# skip compute variation denoise, in order to shorten the running time
# total variation denoising
# tv_y_size = _tensor_size(inferenceImg[:, 1:, :, :])
# tv_x_size = _tensor_size(inferenceImg[:, :, 1:, :])
# tv_loss = tv_weight * 2 * (
# (tf.nn.l2_loss(inferenceImg[:, 1:, :, :] - inferenceImg[:, :content_shape[1] - 1, :, :]) /
# tv_y_size) +
# (tf.nn.l2_loss(inferenceImg[:, :, 1:, :] - inferenceImg[:, :, :content_shape[2] - 1, :]) /
# tv_x_size))
tv_loss = 0
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer training
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations
and i % print_iterations == 0):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = inferenceImg.eval()
img_out = vgg.unprocess(best.reshape(content_shape[1:]),
vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(
styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv,
'YCbCr').convert('RGB'))
yield ((None if last_step else i), img_out)
def stylize(self, network, content, styles, iterations, content_weight,
content_weight_blend, style_weight, style_layer_weight_exp,
style_blend_weights, tv_weight, learning_rate, beta1, beta2,
epsilon, pooling):
"""
Nałożenie stylu na obraz
Metoda jest wywoływana iteracyjnie, obliczane są straty i wagi, a potem do rodzica jest przekazywany
tuple z iteratorem i tablicą obrazu oraz, jeśli to ostatnia iteracja, z obliczonymi stratami
:rtype: iterator[tuple[int,image]]
"""
self.style_features = [{} for _ in styles]
self.content_features = {}
self.style_shapes = [(1, ) + style.shape for style in styles]
self.shape = (1, ) + content.shape
self.vgg_weights, vgg_mean_pixel = vgg.load_net(network)
self.layer_weight = 1.0
for style_layer in self.style_layers:
self.style_layers_weights[style_layer] = self.layer_weight
self.layer_weight *= style_layer_weight_exp
self.calculate_sum_weight()
self.calculate_content_feature(pooling, content, vgg_mean_pixel)
self.calculate_style_feature(styles, pooling, vgg_mean_pixel)
# Użycie propagacji wstecznej na stylizowanym obrazie
with tf.Graph().as_default():
initial = tf.random_normal(self.shape) * 0.256
self.image = tf.Variable(initial)
self.net = vgg.net_preloaded(self.vgg_weights, self.image, pooling)
self.calculate_content_loss(content_weight_blend, content_weight)
self.calculate_style_loss(styles, style_weight,
style_blend_weights)
self.denoise_image(tv_weight)
self.calculate_total_loss()
# konfiguracja optymalizatora
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(self.loss)
# optymalizacja
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(iterations):
if i > 0:
print('%4d/%4d' % (i + 1, iterations))
else:
print('%4d/%4d' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step:
loss_vals = self.get_loss_vals(self.loss_store)
else:
loss_vals = None
if last_step:
this_loss = self.loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = self.image.eval()
img_out = vgg.unprocess(best.reshape(self.shape[1:]),
vgg_mean_pixel)
else:
img_out = None
yield i + 1 if last_step else i, img_out, loss_vals
def stylize(network, initial, content, styles, iterations,
content_weight, style_weight, style_blend_weights, tv_weight,
learning_rate, print_iterations=None, checkpoint_iterations=None):
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(styles[i], mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
print 'Initial feature shape: ', features.shape
features = np.reshape(features, (-1, features.shape[3]))
#mask = np.zeros_like(features)
#mask[:49664/2, :] = 1
#print 'Mask shape', mask.shape
print 'Final features shape', features.shape
#features = features*mask
gram = np.matmul(features.T, features) / features.size
print 'Gram matrix shape: ', gram.shape
style_features[i][layer] = gram
#sys.exit()
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
print 'Height, width, number', height, width, number
size = height * width * number
feats = tf.reshape(layer, (-1, number))
#print tf.shape(feats).as_list()
print 'Height', height
print 'Weight', width
print 'Number', number
print 'Style features shape', style_features[i][style_layer].shape
print style_layer
if style_layer == 'relu2_1':
mask = np.zeros((height*width, number), dtype=np.float32)
temp = imread('emma/emma_test_mask.jpg').astype(np.float32)
c = temp.reshape(height,2,width,2)
temp = c.max(axis=1).max(axis=2)
print temp.shape
maskt = np.reshape(temp, (height*width,))
maskt = maskt > 100
for d in xrange(number):
mask[:,d] = maskt
print 'Mask shape', mask.shape
#b = mask.reshape(height*width*2, 2, number/2,2)
#mask = b.max(axis=1).max(axis=2)
#print 'New mask shape', mask.shape
else:
mask = np.zeros((height*width, number), dtype=np.float32)
maskt = np.reshape(imread('emma/emma_test_mask.jpg').astype(np.float32), (height*width,))
maskt = maskt > 100
for d in xrange(number):
mask[:,d] = maskt
print 'Mask shape', mask.shape
if i == 0:
mask = tf.constant(mask)
print 'Mask shape', map(lambda i: i.value, mask.get_shape())
feats = tf.mul(feats,mask)
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
else:
mask2 = mask < 1
feats2 = tf.mul(feats,mask2)
gram2 = tf.matmul(tf.transpose(feats2), feats2) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram2 - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
def print_progress(i, last=False):
if print_iterations is not None:
if i is not None and i % print_iterations == 0 or last:
print >> stderr, ' content loss: %g' % content_loss.eval()
print >> stderr, ' style loss: %g' % style_loss.eval()
print >> stderr, ' tv loss: %g' % tv_loss.eval()
print >> stderr, ' total loss: %g' % loss.eval()
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
print_progress(i)
print >> stderr, 'Iteration %d/%d' % (i + 1, iterations)
train_step.run()
if (checkpoint_iterations is not None and
i % checkpoint_iterations == 0) or i == iterations - 1:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
print_progress(None, i == iterations - 1)
if i % 10 == 0 and best is not None:
tmp_img = vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
imsave("iter" + str(i) + ".jpg", tmp_img)
return vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations,
content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate, beta1, beta2, epsilon, pooling,
print_iterations=None, checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image);
`iteration` is None if this is the final image (the last iteration).
Otherwise tuples are yielded every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
# The shape information in the comment is based on the content image 1-content.jpg with shape (533, 400, 3)
# and 1-style.jpg (316, 400, 3)
# This should be changed with different images.
shape = (1,) + content.shape # (1, 533, 400, 3)
style_shapes = [(1,) + style.shape for style in styles] # (1, 316, 400, 3)
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network) # Load the VGG-19 model.
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight # {'relu1_1': 1.0, 'relu2_1': 1.0, 'relu3_1': 1.0, 'relu4_1': 1.0, 'relu5_1': 1.0}
layer_weight *= style_layer_weight_exp # 1.0
# VGG19 layers:
# 'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',
# 'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',
# 'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4', 'pool3',
# 'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3', 'relu4_3', 'conv4_4', 'relu4_4', 'pool4',
# 'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4'
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS: # ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1')
layer_weights_sum += style_layers_weights[style_layer] # 5.0
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum # {'relu1_1': 0.2, 'relu2_1': 0.2, 'relu3_1': 0.2, 'relu4_1': 0.2, 'relu5_1': 0.2}
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling) # {'conv1_1': Tensor..., relu1_1: Tensor...}
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) # (1, 533, 400, 3) subtract with the mean pixel
for layer in CONTENT_LAYERS: # (relu4_2, relu5_2)
content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # Find the feature values for (relu4_2, relu5_2)
# compute style features in feed forward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i]) # (1, 316, 400, 3)
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS: # # ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1')
features = net[layer].eval(feed_dict={image: style_pre}) # For relu1_1 layer (1, 316, 400, 64)
features = np.reshape(features, (-1, features.shape[3])) # (126400, 64)
gram = np.matmul(features.T, features) / features.size # (64, 64) Gram matrix - measure the dependency of features.
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend # 0
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) # Generate a random image with SD the same as the content image.
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS: # {'relu5_2'}
# Use MSE as content losses
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer] # For relu1_1: (1, 533, 400, 64)
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number)) # (213200, 64)
gram = tf.matmul(tf.transpose(feats), feats) / size # Gram matrix for the features in relu1_1 for the result image.
style_gram = style_features[i][style_layer] # Gram matrix for the style
# Style loss is the MSE for the difference of the 2 Gram matrix
style_losses.append(style_layers_weights[style_layer]
* 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# Total variation denoising: Add cost to penalize neighboring pixel is very different.
# This help to reduce noise.
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out
)
def synthesis(network, initial, initial_noiseblend, content, styles,
iterations, content_weight, content_weight_blend, style_weight,
style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate):
"""
:input
:-styles: a list containing one or multiple images used as style image.(art work)
"""
# calculate the original image (content) shape
image_shape = (1, ) + content.shape
# calculate the art image (style) shape
style_shapes = [(1, ) + style.shape for style in styles]
# style layer weight exponentional increase - weight(layer<n+1>) = weight_exp*weight(layer<n>)
style_layers_weights = style_layer_weight_cal(style_layer_weight_exp)
content_features, style_features, mean_pixel = compute_feature(
network, image_shape, style_shapes, content, styles)
initial_content_coeff = 1.0 - initial_noiseblend
with tf.Graph().as_default():
# overall loss
image, content_loss, style_loss, tv_loss, loss = loss_computaion(
network, initial, image_shape, mean_pixel, initial_content_coeff,
initial_noiseblend, content, content_weight_blend, content_weight,
content_features, styles, style_layers_weights, style_features,
tv_weight, style_weight, style_blend_weights)
# optimizer setup
# The original paper didn't specify which optimization method to use, thus here we choose the classical Adam optimizer
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# optimization
# optimization_process(train_step, image, content_loss, style_loss, tv_loss, loss, vgg_mean_pixel, preserve_colors, content)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step:
print_progress()
if last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
# final step, generate output image
img_out = vgg.unprocess(best.reshape(image_shape[1:]),
mean_pixel)
yield ((None if last_step else i), img_out)
def optimize(content_targets,
style_target,
content_weight,
style_weight,
tv_weight,
vgg_path,
epochs=2,
print_iterations=1000,
batch_size=4,
save_path='saver/fns.ckpt',
slow=False,
learning_rate=1e-3,
device='/cpu:0',
debug=False,
total_iterations=-1,
base_model_path=None):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size, 256, 256, 3)
style_shape = (1, ) + style_target.shape
print(style_shape)
# precompute style features
print("Precomputing style features")
sys.stdout.flush()
with tf.Graph().as_default(), tf.device(device), tf.Session(
config=tf.ConfigProto(allow_soft_placement=True)) as sess:
style_image = tf.placeholder(tf.float32,
shape=style_shape,
name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session() as sess:
X_content = tf.placeholder(tf.float32,
shape=batch_shape,
name="X_content")
X_pre = vgg.preprocess(X_content)
print("Precomputing content features")
sys.stdout.flush()
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256)
preds_pre = preds
else:
preds = transform.net(X_content / 255.0)
preds_pre = vgg.preprocess(preds)
print("Building VGG net")
sys.stdout.flush()
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(
content_features[CONTENT_LAYER]) * batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(
net[CONTENT_LAYER])
content_loss = content_weight * (
2 * tf.nn.l2_loss(net[CONTENT_LAYER] -
content_features[CONTENT_LAYER]) / content_size)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i: i.value,
layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0, 2, 1])
grams = tf.batch_matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram) /
style_gram.size)
style_loss = style_weight * reduce(tf.add, style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:, 1:, :, :])
tv_x_size = _tensor_size(preds[:, :, 1:, :])
y_tv = tf.nn.l2_loss(preds[:, 1:, :, :] -
preds[:, :batch_shape[1] - 1, :, :])
x_tv = tf.nn.l2_loss(preds[:, :, 1:, :] -
preds[:, :, :batch_shape[2] - 1, :])
tv_loss = tv_weight * 2 * (x_tv / tv_x_size +
y_tv / tv_y_size) / batch_size
loss = content_loss + style_loss + tv_loss
# overall loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.initialize_all_variables())
# If base model file is present, load that in to the session
if base_model_path:
saver = tf.train.Saver()
if os.path.isdir(base_model_path):
ckpt = tf.train.get_checkpoint_state(base_model_path)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise Exception("No checkpoint found...")
else:
saver.restore(sess, base_model_path)
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
sys.stdout.flush()
for epoch in range(epochs):
num_examples = len(content_targets)
print("number of examples: %s" % num_examples)
sys.stdout.flush()
iterations = 0
while iterations * batch_size < num_examples:
print("Current iteration : %s" % iterations)
sys.stdout.flush()
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p,
(256, 256, 3)).astype(np.float32)
iterations += 1
assert X_batch.shape[0] == batch_size
feed_dict = {X_content: X_batch}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
if debug:
print("UID: %s, batch time: %s" % (uid, delta_time))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = False
if epoch == epochs - 1 and iterations * batch_size >= num_examples:
is_last = True
if total_iterations > 0 and iterations >= total_iterations:
is_last = True
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {X_content: X_batch}
tup = sess.run(to_get, feed_dict=test_feed_dict)
_style_loss, _content_loss, _tv_loss, _loss, _preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess, save_path)
yield (_preds, losses, iterations, epoch)
if is_last:
break
def optimize(content_targets, style_target, content_weight, style_weight,
tv_weight, vgg_path, epochs=2, print_iterations=1000,
batch_size=4, save_path='saver/fns.ckpt', slow=False,
learning_rate=1e-3, debug=False, device_and_number=False):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size,256,256,3)
style_shape = (1,) + style_target.shape
print(style_shape)
# removed tf.device('/gpu:0'), let system automatically detect available device; this is no longer true
device_type, device_number = device_and_number.strip('/').split(':')
if device_type == 'gpu': # /gpu:0 means use GPU 0; /gpu:1 means use GPU 1; /gpu2: means use GPU2; etc.
os.environ["CUDA_VISIBLE_DEVICES"] = device_number # starts at 0
session_conf = tf.ConfigProto() # session_conf.gpu_options.allow_growth = True # test if growth slows down training
# backprop doubles RAM usage
# for training, takes 2.7 seconds/iter for batch size=20
# for evaluating loss and saving checkpoint, takes 2.6 seconds (depending on size of test image) using 1000x700 px image
else: # /cpu:0 means use all CPUs; /cpu:1 means use 1 CPU; /cpu:2 means use 2 CPUs; etc.
session_conf = tf.ConfigProto(intra_op_parallelism_threads=int(device_number))
with tf.Graph().as_default(), tf.Session(config=session_conf) as sess: # precompute style features
style_image = tf.placeholder(tf.float32, shape=style_shape, name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image:style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session(config=session_conf) as sess:
X_content = tf.placeholder(tf.float32, shape=batch_shape, name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256
)
preds_pre = preds
else:
preds = transform.net(X_content/255.0)
preds_pre = vgg.preprocess(preds)
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(content_features[CONTENT_LAYER])*batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(net[CONTENT_LAYER])
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) / content_size
)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i:i.value,layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0,2,1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram)/style_gram.size)
style_loss = style_weight * functools.reduce(tf.add, style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:,1:,:,:])
tv_x_size = _tensor_size(preds[:,:,1:,:])
y_tv = tf.nn.l2_loss(preds[:,1:,:,:] - preds[:,:batch_shape[1]-1,:,:])
x_tv = tf.nn.l2_loss(preds[:,:,1:,:] - preds[:,:,:batch_shape[2]-1,:])
tv_loss = tv_weight*2*(x_tv/tv_x_size + y_tv/tv_y_size)/batch_size
loss = content_loss + style_loss + tv_loss
# overall loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
checkpoint_number = 0
for epoch in range(epochs):
num_examples = len(content_targets)
iterations = 0
while iterations * batch_size < num_examples:
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p, (256,256,3)).astype(np.float32)
iterations += 1
assert X_batch.shape[0] == batch_size
feed_dict = {
X_content:X_batch
}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
if debug:
print("UID: %s, batch time: %s" % (uid, delta_time))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {
X_content:X_batch
}
tup = sess.run(to_get, feed_dict = test_feed_dict)
_style_loss,_content_loss,_tv_loss,_loss,_preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
parent_dir = os.path.dirname(save_path)
checkpoint_number += 1
actual_dir = os.path.join(parent_dir, "checkpoint_{}".format(checkpoint_number))
if not os.path.exists(actual_dir):
os.mkdir(actual_dir)
filename = os.path.basename(save_path)
actual_path = os.path.join(actual_dir, filename) ### hard coded directory name logic
res = saver.save(sess, actual_path)
if os.path.isfile(actual_path + '.meta'): # delete fns.ckpt.meta file, which takes 160 MBs
os.remove(actual_path + '.meta')
yield _preds, losses, iterations, epoch, checkpoint_number
def stylize(network,
initial,
content,
style,
iterations,
content_weight,
style_weight,
tv_weight,
learning_rate,
print_iter=None):
shape = (1, ) + content.shape
style_shape = (1, ) + style.shape
content_features = {}
style_features = {}
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shape)
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(style, mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / (features.size)
style_features[layer] = gram
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 256 / 1000
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
content_loss = tf.nn.l2_loss(net[CONTENT_LAYER] -
content_features[CONTENT_LAYER])
style_losses = []
for i in STYLE_LAYERS:
layer = net[i]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / (size)
style_gram = style_features[i]
style_losses.append(tf.nn.l2_loss(gram - style_gram))
style_loss = reduce(tf.add, style_losses) / len(style_losses)
tv_loss = (
tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) +
tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]))
loss = content_weight * content_loss + \
style_weight * style_loss + tv_weight * tv_loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
if print_iter is not None and i % print_iter == 0:
print ' content loss: %g' % (content_loss.eval())
print ' style loss: %g' % (style_loss.eval())
print ' tv loss: %g' % (tv_loss.eval())
print ' total loss: %g' % loss.eval()
print 'Iteration %d/%d' % (i + 1, iterations)
train_step.run()
return vgg.unprocess(image.eval().reshape(shape[1:]), mean_pixel)
def optimize(content_targets,style_target,content_weight,style_weight,
tv_weight,vgg_path,epochs=2,print_iterations=1000,batch_size = 4,
save_path='checkpoint/save/model.ckpt',slow=False,learning_rate = 1e-3,debug=False):
if slow:
batch_size=1
# trimming the total training set size
extra_train_img = len(content_targets)%batch_size
if extra_train_img>0:
print("Leaving out {} extra (modulus) train examples ".format(extra_train_img))
content_targets = content_targets[:-extra_train_img]
train_batch_shape = (batch_size,256,256,3)
style_img_gram_features = {}
# appending the batch size for the style image
style_img_shape = (1,)+style_target.shape
# precomputing the style image gram matrix features
with tf.Graph().as_default(),tf.device('/cpu:0'),tf.Session() as sess:
# defining style image placeholder and preprocessing the image
style_image_ph = tf.placeholder(tf.float32,shape=style_img_shape,name='style_image_ph')
style_image_pre = vgg.preprocess(style_image_ph)
# passing the "preproccessed style image" through the VGG19 network
style_net = vgg.net(vgg_path,style_image_pre)
# creating the numpy array of the style image
style_img_feed = np.array([style_target])
for layer in STYLE_LAYERS:
# activations for the style image's different VGG19 layers
activations = style_net[layer].eval(feed_dict = {style_image_ph:style_img_feed})
activations = np.reshape(activations,(-1,activations.shape[3]))
gram = np.matmul(activations.T,activations)/activations.size
style_img_gram_features[layer] = gram
# defining graph for computing the Content cost, Style cost and TV cost
with tf.Graph().as_default(),tf.Session() as sess:
X_content_ph = tf.placeholder(tf.float32,shape =train_batch_shape,name='X_content_ph' )
X_pre = vgg.preprocess(X_content_ph)
# precomputing the content image activation for content loss
content_img_activation = {}
content_net = vgg.net(vgg_path,X_pre)
content_img_activation[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow :
preds = tf.Variable(tf.random_normal(X_content_ph.get_shape())*0.256)
preds_pre = preds
else:
# getting the generated image by transforming the content image
preds = transform_net.net(X_content_ph/255.0)
preds_pre = vgg.preprocess(preds)
# passing the "preproccessed generated image" through the VGG19 network
gen_net = vgg.net(vgg_path,preds_pre)
assert _tensor_size(content_img_activation[CONTENT_LAYER]) == _tensor_size(gen_net[CONTENT_LAYER])
# calculating the content loss
content_img_size = _tensor_size(content_img_activation[CONTENT_LAYER])*batch_size
content_loss = content_weight * ( 2 *
tf.nn.l2_loss(gen_net[CONTENT_LAYER]-content_img_activation[CONTENT_LAYER])/content_img_size
)
# computing the generated image gram matrix features
style_loss = []
for style_layer in STYLE_LAYERS:
activations = gen_net[style_layer]
bs,height,width,filters = map(lambda i:i.value,activations.get_shape())
activation_size = height*width*filters
activations = tf.reshape(activations,(bs,height*width,filters))
activations_T = tf.transpose(activations,perm=[0,2,1])
gram = tf.matmul(activations_T,activations)/activation_size
style_gram = style_img_gram_features[style_layer]
style_loss.append(2 * tf.nn.l2_loss(gram-style_gram)/style_gram.size)
style_loss = style_weight * functools.reduce(tf.add,style_loss)/batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:,1:,:,:])
tv_x_size = _tensor_size(preds[:,:,1:,:])
y_tv = tf.nn.l2_loss(preds[:,1:,:,:] - preds[:,:train_batch_shape[1]-1,:,:])
x_tv = tf.nn.l2_loss(preds[:,:,1:,:] - preds[:,:,:train_batch_shape[2]-1,:])
tv_loss = tv_weight*2*(x_tv/tv_x_size + y_tv/tv_y_size)/batch_size
# Defining total loss
loss = content_loss + style_loss + tv_loss
# Defining optimizer
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# variabel initializing (tensorflow)
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
num_examples = len(content_targets)
print('no of training examples: ',num_examples)
iterations = 0
while iterations*batch_size < num_examples:
start_time = dt.datetime.now()
start_batch = iterations*batch_size
end_batch = start_batch + batch_size
iterations+=1
X_input_feed = np.zeros(train_batch_shape,dtype = np.float32)
# preparing the X_input_feed
for j,input_img in enumerate(content_targets[start_batch:end_batch]):
X_input_feed[j] = get_img(input_img,(256,256,3)).astype(np.float32)
assert X_input_feed.shape[0] == batch_size
# running optimer step
feed_dict = { X_content_ph:X_input_feed}
train_step.run(feed_dict=feed_dict)
end_time = dt.datetime.now()
if debug :
print("iteration : {} , epoch : {} , time for this iteration : {}".format(iterations,epoch,str(end_time-start_time)))
# if iterations >1000 i.e. num_examples is more than 1000*batch_size
print_iter = int(iterations)%print_iterations == 0
if slow:
print_iter = epoch % print_iterations == 0
is_last_iter = epoch == epochs-1 and iterations * batch_size >=num_examples
should_print = print_iter or is_last_iter
if should_print:
calculate_these = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {X_content_ph:X_input_feed}
# calcuating loss and preds i.e. generated image
_style_loss,_content_loss,_tv_loss,_loss,_preds = sess.run(calculate_these,
feed_dict=test_feed_dict)
losses = (_style_loss,_content_loss,_tv_loss,_loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess,save_path)
yield (_preds,losses,iterations,epoch)
def do_shit(content,
style,
iterations=1000,
learning_rate=1e0,
content_weight=5,
style_weight=1e2,
smooth_weight=1e2):
content = imresize(content, [256, 256])
style = imresize(style, [256, 256])
shape = (1, ) + content.shape
content_features = {}
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(NETWORK, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
style_features = {}
style_shape = (1, ) + style.shape
# compute style features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shape)
net, mean_pixel = vgg.net(NETWORK, image)
style_pre = np.array([vgg.preprocess(style, mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
# optimizing the image
with tf.Graph().as_default():
image = tf.Variable(tf.random_normal(shape) * 0.256)
net, channel_avg = vgg.net(NETWORK, image)
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
style_loss = 0
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss = style_weight * reduce(tf.add, style_losses)
y_size = _tensor_size(image[:, 1:, :, :])
x_size = _tensor_size(image[:, :, 1:, :])
smooth_loss = smooth_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ x_size))
loss = content_loss + style_loss + smooth_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
im_out = None
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in xrange(iterations):
print 'Iteration', i
train_step.run()
im = image.eval()
return vgg.unprocess(im.reshape(shape[1:]), channel_avg)
def optimize(content_targets,
style_target,
content_weight,
style_weight,
tv_weight,
vgg_path,
epochs=2,
print_iterations=1000,
batch_size=4,
save_path='saver/fns.ckpt',
slow=False,
learning_rate=1e-3,
debug=False):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size, 256, 256, 3)
style_shape = (1, ) + style_target.shape
print(style_shape)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.333
# precompute style features
with tf.Graph().as_default(), tf.device('/cpu:0'), tf.Session(
config=config) as sess:
style_image = tf.placeholder(tf.float32,
shape=style_shape,
name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session(config=config) as sess:
X_content = tf.placeholder(tf.float32,
shape=batch_shape,
name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
# tensorboar output
train_writer = tf.summary.FileWriter('./logs/1/train', sess.graph)
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256)
preds_pre = preds
else:
preds = transform.net(X_content / 255.0)
preds_pre = vgg.preprocess(preds)
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(
content_features[CONTENT_LAYER]) * batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(
net[CONTENT_LAYER])
content_loss = content_weight * (
2 * tf.nn.l2_loss(net[CONTENT_LAYER] -
content_features[CONTENT_LAYER]) / content_size)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i: i.value,
layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0, 2, 1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram) /
style_gram.size)
style_loss = style_weight * functools.reduce(tf.add,
style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:, 1:, :, :])
tv_x_size = _tensor_size(preds[:, :, 1:, :])
y_tv = tf.nn.l2_loss(preds[:, 1:, :, :] -
preds[:, :batch_shape[1] - 1, :, :])
x_tv = tf.nn.l2_loss(preds[:, :, 1:, :] -
preds[:, :, :batch_shape[2] - 1, :])
tv_loss = tv_weight * 2 * (x_tv / tv_x_size +
y_tv / tv_y_size) / batch_size
loss = content_loss + style_loss + tv_loss
# tensorboard variables
# batch_time = tf.Variable(0)
# tf.summary.scalar("style_loss", style_loss)
# tf.summary.scalar("content_loss", content_loss)
# tf.summary.scalar("tv_loss", tv_loss)
# tf.summary.scalar("loss", loss)
# tf.summary.scalar("batch_time", batch_time)
# overall loss
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
for epoch in range(epochs):
num_examples = len(content_targets)
iterations = 0
while iterations * batch_size < num_examples:
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p,
(256, 256, 3)).astype(np.float32)
iterations += 1
assert X_batch.shape[0] == batch_size
feed_dict = {X_content: X_batch}
sess.run(optimizer, feed_dict=feed_dict)
end_time = time.time()
batch_time = end_time - start_time
# batch_time.load(end_time - start_time, sess)
# merge = tf.summary.merge_all()
# summary = sess.run(merge, feed_dict=feed_dict)
# train_writer.add_summary(summary=summary, global_step=iterations)
if iterations % 200 == 0:
print("batch time: " + str(batch_time) + " iteration: " +
str(iterations))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {X_content: X_batch}
tup = sess.run(to_get, feed_dict=test_feed_dict)
_style_loss, _content_loss, _tv_loss, _loss, _preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
saver.save(sess, save_path)
yield (_preds, losses, iterations, epoch)
def optimize(content_targets, style_target, content_weight, style_weight,
tv_weight, vgg_path, epochs=2, print_iterations=1000,
batch_size=4, save_path='saver/fns.ckpt', slow=False,
learning_rate=1e-3, debug=False):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size,256,256,3)
style_shape = (1,) + style_target.shape
print(style_shape)
# precompute style features
with tf.Graph().as_default(), tf.device('/cpu:0'), tf.Session() as sess:
style_image = tf.placeholder(tf.float32, shape=style_shape, name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image:style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session() as sess:
X_content = tf.placeholder(tf.float32, shape=batch_shape, name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256
)
preds_pre = preds
else:
preds = transform.net(X_content/255.0)
preds_pre = vgg.preprocess(preds)
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(content_features[CONTENT_LAYER])*batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(net[CONTENT_LAYER])
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) / content_size
)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i:i.value,layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0,2,1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram)/style_gram.size)
style_loss = style_weight * functools.reduce(tf.add, style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:,1:,:,:])
tv_x_size = _tensor_size(preds[:,:,1:,:])
y_tv = tf.nn.l2_loss(preds[:,1:,:,:] - preds[:,:batch_shape[1]-1,:,:])
x_tv = tf.nn.l2_loss(preds[:,:,1:,:] - preds[:,:,:batch_shape[2]-1,:])
tv_loss = tv_weight*2*(x_tv/tv_x_size + y_tv/tv_y_size)/batch_size
loss = content_loss + style_loss + tv_loss
# overall loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
for epoch in range(epochs):
num_examples = len(content_targets)
iterations = 0
while iterations * batch_size < num_examples:
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p, (256,256,3)).astype(np.float32)
iterations += 1
assert X_batch.shape[0] == batch_size
feed_dict = {
X_content:X_batch
}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
if debug:
print("UID: %s, batch time: %s" % (uid, delta_time))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {
X_content:X_batch
}
tup = sess.run(to_get, feed_dict = test_feed_dict)
_style_loss,_content_loss,_tv_loss,_loss,_preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess, save_path)
yield(_preds, losses, iterations, epoch)
def stylize_c(network,
initial,
initial_noiseblend,
content,
styles,
preserve_colors,
iterations,
content_weight,
content_weight_blend,
style_weight,
style_layer_weight_exp,
style_blend_weights,
tv_weight,
learning_rate,
beta1,
beta2,
epsilon,
pooling,
prev_style_image,
prev_content_image,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1, ) + content.shape
style_shapes = [(1, ) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
#noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
noise = content
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (
tf.random_normal(shape) *
0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
#content_layers_weights['conv2_2'] = content_weight_blend
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(
content_layers_weights[content_layer] * content_weight *
(2 * tf.nn.l2_loss(net[content_layer] -
content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 *
tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ tv_x_size))
#Continuity Loss
#K = np.array([[1/256,4/256,6/256,4/256,1/256],
#[4/256,16/256,24/256,16/256,4/256],
#[6/256,24/256,36/256,24/256,6/256],
#[4/256,16/256,24/256,16/256,4/256],
#[1/256,4/256,6/256,4/256,1/256]], dtype=np.float32)
#G_filt = np.zeros([5,5,3],dtype=np.float32)
#G_filt[:,:,0] = K
#G_filt[:,:,1] = K
#G_filt[:,:,2] = K
#filterG = tf.convert_to_tensor(G_filt,dtype=tf.float32)
#filterG = tf.reshape(filterG, [5,5,3,1])
#G_filt = tf.reshape(K, [5,5,1,1], name='G_filt')
#G_filt = tf.convert_to_tensor(K, dtype=tf.float32)
#tf.expand_dims(G_filt,0)
#tf.expand_dims(G_filt,0)
tf_org_img = tf.convert_to_tensor(content, dtype=tf.float32)
tf_org_img = tf.reshape(tf_org_img, tf.shape(image))
tf_prev_img = tf.convert_to_tensor(prev_content_image,
dtype=tf.float32)
tf_prev_img = tf.reshape(tf_prev_img, tf.shape(image))
tf_prev_styl = tf.convert_to_tensor(prev_style_image, dtype=tf.float32)
tf_prev_styl = tf.reshape(tf_prev_styl, tf.shape(image))
#smth_org_frame_diff = tf.nn.conv2d(tf_org_img - tf_prev_img,filterG,strides=[1, 1, 1, 1],padding='VALID')
#smth_styl_frame_diff = tf.nn.conv2d(image - tf_prev_styl,filterG,strides=[1, 1, 1, 1],padding='VALID')
#org_frame_diff = tf.norm(smth_org_frame_diff)
#styl_frame_diff = tf.norm(smth_styl_frame_diff)
org_frame_diff = tf.norm(tf_org_img - tf_prev_img)
styl_frame_diff = tf.norm(tf_prev_styl - image)
hyperparam_cl = 10e4
cl_loss = tf.multiply(
hyperparam_cl,
tf.divide(styl_frame_diff, org_frame_diff +
3 * content.shape[0] * content.shape[1]))
# overall loss
loss = content_loss + style_loss + cl_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' Continuity loss: %g\n' % cl_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
iteration_times = []
start = time.time()
for i in range(iterations):
iteration_start = time.time()
if i > 0:
elapsed = time.time() - start
# take average of last couple steps to get time per iteration
remaining = np.mean(
iteration_times[-10:]) * (iterations - i)
stderr.write(
'Iteration %4d/%4d (%s elapsed, %s remaining)\n' %
(i + 1, iterations, hms(elapsed), hms(remaining)))
else:
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations
and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]),
vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(
styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv,
'YCbCr').convert('RGB'))
yield ((None if last_step else i), img_out)
iteration_end = time.time()
iteration_times.append(iteration_end - iteration_start)
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations,
content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate, beta1, beta2, epsilon, pooling,
print_iterations=None, checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
iteration_times = []
start = time.time()
for i in range(iterations):
iteration_start = time.time()
if i > 0:
elapsed = time.time() - start
# take average of last couple steps to get time per iteration
remaining = np.mean(iteration_times[-10:]) * (iterations - i)
stderr.write('Iteration %4d/%4d (%s elapsed, %s remaining)\n' % (
i + 1,
iterations,
hms(elapsed),
hms(remaining)
))
else:
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out
)
iteration_end = time.time()
iteration_times.append(iteration_end - iteration_start)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print('Optimization started...')
for i in range(ITERATIONS):
if i % 100 == 0:
print('Iteration %4d/%4d' % (i + 1, ITERATIONS))
train_step.run()
last_step = (i == ITERATIONS - 1)
if last_step:
print(' content loss: %g' % content_loss.eval())
print(' style loss: %g' % style_loss.eval())
print(' tv loss: %g' % tv_loss.eval())
print(' total loss: %g' % loss.eval())
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
#-------------------------------------------------------------
# img_out은 float형이고 (-)(+)값이 제각각이므로 아래와 같이 데이터를 정제 후 출력해야 한다
img = np.clip(img_out, 0, 255).astype(np.uint8)
plt.imshow(img)
def stylize(network,
initial,
content,
styles,
iterations,
content_weight,
style_weight,
style_blend_weights,
tv_weight,
learning_rate,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1, ) + content.shape
style_shapes = [(1, ) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/gpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/gpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(styles[i], mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
# make stylized image using backpropogation
g = tf.Graph()
with g.as_default(), g.device('/gpu:0'):
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
def print_progress(i, last=False):
stderr.write('Iteration %d/%d\n' % (i + 1, iterations))
if last or (print_iterations and i % print_iterations == 0):
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(iterations):
last_step = (i == iterations - 1)
print_progress(i, last=last_step)
train_step.run()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
yield ((None if last_step else i),
vgg.unprocess(best.reshape(shape[1:]), mean_pixel))
def unprocess(self, image):
return vgg.unprocess(image, self.mean_pixel)[0]
def optimize(
content_targets,
style_target,
content_weight,
style_weight,
tv_weight,
vgg_path,
epochs=2,
print_iterations=1000,
batch_size=4,
save_path='saver/fns.ckpt',
slow=False,
learning_rate=1e-3,
debug=False,
# more cli params
data_format='NHWC',
num_base_channels=32):
#print ("optimize().data_format:{}".format(data_format))
#print ("optimize().num_base_channels:{}".format(num_base_channels))
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size, 256, 256, 3)
#batch_shape = (batch_size,128,128,3) #mcky,smaller size for MX150
style_shape = (1, ) + style_target.shape
#print(style_shape)
# precompute style features
print("precompute style features") #mcky
with tf.Graph().as_default(), tf.device('/cpu:0'), tf.Session() as sess:
style_image = tf.placeholder(tf.float32,
shape=style_shape,
name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session() as sess:
X_content = tf.placeholder(tf.float32,
shape=batch_shape,
name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
print("precompute content features") #mcky
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256)
preds_pre = preds
else:
if data_format == 'NHWC':
#NHWC path
preds = transform.net(X_content / 255.0,
data_format=data_format,
num_base_channels=num_base_channels)
else:
#NCHW path
# use NCHW transformer net. bug vgg net needs NHWC. so transposes needed for input and output.
#nhwc --> nchw --> transform.net --> nhwc
x_content = X_content / 255.0
X_content_nchw = tf.transpose(x_content, [0, 3, 1, 2])
preds_nchw = transform.net(X_content_nchw,
data_format=data_format,
num_base_channels=num_base_channels)
preds = tf.transpose(preds_nchw, [0, 2, 3, 1])
print("preds.shape:{}".format(preds.shape))
preds_pre = vgg.preprocess(preds)
print("preds_pre.shape:{}".format(preds_pre.shape))
net = vgg.net(
vgg_path, preds_pre
) # <-- mcky, this is to feed the output of ITN to VGG, along with content data set.
content_size = _tensor_size(
content_features[CONTENT_LAYER]) * batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(
net[CONTENT_LAYER])
content_loss = content_weight * (
2 * tf.nn.l2_loss(net[CONTENT_LAYER] -
content_features[CONTENT_LAYER]) / content_size)
print("style_losses") #mcky
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i: i.value,
layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0, 2, 1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram) /
style_gram.size)
style_loss = style_weight * functools.reduce(tf.add,
style_losses) / batch_size
# total variation denoising
print("total variation denoising") #mcky
tv_y_size = _tensor_size(preds[:, 1:, :, :])
tv_x_size = _tensor_size(preds[:, :, 1:, :])
y_tv = tf.nn.l2_loss(preds[:, 1:, :, :] -
preds[:, :batch_shape[1] - 1, :, :])
x_tv = tf.nn.l2_loss(preds[:, :, 1:, :] -
preds[:, :, :batch_shape[2] - 1, :])
tv_loss = tv_weight * 2 * (x_tv / tv_x_size +
y_tv / tv_y_size) / batch_size
'''
#mcky, tv for preds in nchw format
tv_y_size = _tensor_size(preds[:,:,1:,:])
tv_x_size = _tensor_size(preds[:,:,:,1:])
y_tv = tf.nn.l2_loss(preds[:,:,1:,:] - preds[:,:,:batch_shape[1]-1,:])
x_tv = tf.nn.l2_loss(preds[:,:,:,1:] - preds[:,:,:,:batch_shape[2]-1])
tv_loss = tv_weight*2*(x_tv/tv_x_size + y_tv/tv_y_size)/batch_size
'''
loss = content_loss + style_loss + tv_loss
# overall loss
#print("overall loss") #mcky
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
#mcky, Variables are printed here
#for v in tf.global_variables():
# print (v)
import random
uid = random.randint(1, 100)
print(
"----------------------------------------------------------------------------------------------------"
) #mcky
print("data_format:{}".format(data_format))
print("num_base_channels:{}".format(num_base_channels))
print("EPOCHS:{}".format(epochs)) #mcky
print("UID: %s" % uid)
for epoch in range(epochs):
num_examples = len(content_targets)
print("num_examples:{}".format(num_examples)) #mcky
iterations = 0
while iterations * batch_size < num_examples:
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p,
(256, 256, 3)).astype(np.float32)
#X_batch[j] = get_img(img_p, (128,128,3)).astype(np.float32) #mcky, smaller size for MX150
iterations += 1
assert X_batch.shape[0] == batch_size
feed_dict = {X_content: X_batch}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
if debug:
print("UID: %s, batch time: %s" % (uid, delta_time))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
#should_print = True#mcky, is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {X_content: X_batch}
tup = sess.run(to_get, feed_dict=test_feed_dict)
_style_loss, _content_loss, _tv_loss, _loss, _preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess, save_path)
yield (_preds, losses, iterations, epoch)
def main():
content_path, style_path, width, style_scale = sys.argv[1:]
width = int(width)
style_scale = float(style_scale)
content_image = imread(content_path)
style_image = imread(style_path)
if width > 0:
new_shape = (int(math.floor(float(content_image.shape[0]) /
content_image.shape[1] * width)), width)
content_image = sm.imresize(content_image, new_shape)
if style_scale > 0:
style_image = sm.imresize(style_image, style_scale)
shape = (1,) + content_image.shape
style_shape = (1,) + style_image.shape
content_features = {}
style_features = {}
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(VGG_PATH, image)
content_pre = np.array([vgg.preprocess(content_image, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shape)
net, _ = vgg.net(VGG_PATH, image)
style_pre = np.array([vgg.preprocess(style_image, mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
grammatrix = np.matmul(features.T, features)
style_features[layer] = grammatrix
g = tf.Graph()
with g.as_default():
global_step = tf.Variable(0, trainable=False)
noise = np.random.normal(size=shape, scale=np.std(content_image) * 0.1)
content_pre = vgg.preprocess(content_image, mean_pixel)
init = content_pre * (1 - NOISE_RATIO) + noise * NOISE_RATIO
init = init.astype('float32')
image = tf.Variable(init)
net, _ = vgg.net(VGG_PATH, image)
content_loss = tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER])
style_losses = []
for i in STYLE_LAYERS:
layer = net[i]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats)
style_gram = style_features[i]
style_losses.append(tf.nn.l2_loss(gram - style_gram) /
(4.0 * number ** 2 * (height * width) ** 2))
style_loss = reduce(tf.add, style_losses) / len(style_losses)
loss = ALPHA * content_loss + BETA * style_loss
learning_rate = tf.train.exponential_decay(LEARNING_RATE_INITIAL,
global_step, LEARNING_DECAY_STEPS, LEARNING_DECAY_BASE,
staircase=True)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss,
global_step=global_step)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(100000):
print 'i = %d' % i
imsave('%05d.jpg' % i, vgg.unprocess(
image.eval().reshape(shape[1:]), mean_pixel))
train_step.run()
def stylize(network, initial, content, styles, iterations,
content_weight, style_weight, style_blend_weights, tv_weight,
learning_rate, print_iterations=None, checkpoint_iterations=None,
print_image_iterations=False):
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(styles[i], mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype('float32')
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
# content loss
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
content_features[CONTENT_LAYER].size)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
def print_progress(i, last=False):
if print_iterations is not None:
if i is not None and i % print_iterations == 0 or last:
print >> stderr, ' content loss: %g' % content_loss.eval()
print >> stderr, ' style loss: %g' % style_loss.eval()
print >> stderr, ' tv loss: %g' % tv_loss.eval()
print >> stderr, ' total loss: %g' % loss.eval()
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
print_progress(i)
print >> stderr, 'Iteration %d/%d' % (i + 1, iterations)
train_step.run()
if (checkpoint_iterations is not None and
i % checkpoint_iterations == 0) or i == iterations - 1:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
print_progress(None, i == iterations - 1)
if (i % 100 == 0) and (print_image_iterations):
temp_image = vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
temp_output = 'iteration_' + str(i) + '.jpg'
imsave(temp_output, temp_image)
return vgg.unprocess(best.reshape(shape[1:]), mean_pixel)
def stylize(network,
initial,
initial_noiseblend,
content,
styles,
preserve_colors,
iterations,
content_weight,
content_weight_blend,
style_weight,
style_layer_weight_exp,
style_blend_weights,
tv_weight,
learning_rate,
beta1,
beta2,
epsilon,
pooling,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image, loss_vals) at every
iteration. However `image` and `loss_vals` are None by default. Each
`checkpoint_iterations`, `image` is not None. Each `print_iterations`,
`loss_vals` is not None.
`loss_vals` is a dict with loss values for the current iteration, e.g.
``{'content': 1.23, 'style': 4.56, 'tv': 7.89, 'total': 13.68}``.
:rtype: iterator[tuple[int,image]]
"""
shape = (1, ) + content.shape
style_shapes = [(1, ) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (
tf.random_normal(shape) *
0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(
content_layers_weights[content_layer] * content_weight *
(2 * tf.nn.l2_loss(net[content_layer] -
content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 *
tf.nn.l2_loss(gram - style_gram) /
style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ tv_x_size))
# total loss
loss = content_loss + style_loss + tv_loss
# We use OrderedDict to make sure we have the same order of loss types
# (content, tv, style, total) as defined by the initial costruction of
# the loss_store dict. This is important for print_progress() and
# saving loss_arrs (column order) in the main script.
#
# Subtle Gotcha (tested with Python 3.5): The syntax
# OrderedDict(key1=val1, key2=val2, ...) does /not/ create the same
# order since, apparently, it first creates a normal dict with random
# order (< Python 3.7) and then wraps that in an OrderedDict. We have
# to pass in a data structure which is already ordered. I'd call this a
# bug, since both constructor syntax variants result in different
# objects. In 3.6, the order is preserved in dict() in CPython, in 3.7
# they finally made it part of the language spec. Thank you!
loss_store = OrderedDict([('content', content_loss),
('style', style_loss), ('tv', tv_loss),
('total', loss)])
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print('Optimization started...')
if (print_iterations and print_iterations != 0):
print_progress(get_loss_vals(loss_store))
iteration_times = []
start = time.time()
for i in range(iterations):
iteration_start = time.time()
if i > 0:
elapsed = time.time() - start
# take average of last couple steps to get time per iteration
remaining = np.mean(
iteration_times[-10:]) * (iterations - i)
print('Iteration %4d/%4d (%s elapsed, %s remaining)' %
(i + 1, iterations, hms(elapsed), hms(remaining)))
else:
print('Iteration %4d/%4d' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations
and i % print_iterations == 0):
loss_vals = get_loss_vals(loss_store)
print_progress(loss_vals)
else:
loss_vals = None
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]),
vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(
styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv,
'YCbCr').convert('RGB'))
else:
img_out = None
yield i + 1 if last_step else i, img_out, loss_vals
iteration_end = time.time()
iteration_times.append(iteration_end - iteration_start)
def stylize(network,
initial,
initial_noiseblend,
content,
content_mask,
styles,
preserve_colors,
iterations,
content_weight,
content_weight_blend,
style_weight,
style_layer_weight_exp,
style_blend_weights,
tv_weight,
learning_rate,
beta1,
beta2,
epsilon,
pooling,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1, ) + content.shape
mask_shape = (1, ) + content.shape[0:2] + (1, )
style_shapes = [(1, ) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
content_mask_features = {}
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
mask = tf.placeholder('float', shape=mask_shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
net_mask = vgg.net_downsample(vgg_weights, mask)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={image: content_pre})
for layer in CONTENT_LAYERS + STYLE_LAYERS:
content_mask_features[layer] = net_mask[layer].eval(feed_dict={
mask:
np.expand_dims(np.expand_dims(content_mask, axis=0), axis=4)
})
# plt.imshow(np.squeeze(content_mask_features[layer]))
# plt.show()
# plt.pause(0.01)
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features_bank = sk_image.extract_patches_2d(
np.squeeze(features), (kernel_s, kernel_s))
style_features[i][layer] = [features_bank, features]
# plt.imshow(np.squeeze(initial).astype(np.uint8))
# plt.show()
# plt.pause(0.01)
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
# noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
# initial_content_noise_coeff = 1.0 - initial_noiseblend
# noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
# initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
# plt.imshow(np.squeeze(initial))
# plt.show()
# plt.pause(0.01)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
for layer in CONTENT_LAYERS:
content_layers_weights[layer] = 1. / len(CONTENT_LAYERS)
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
map_ = (net[content_layer] - content_features[content_layer])
# map_ = (net[content_layer] - content_features[content_layer])*(1.-content_mask_features[content_layer])
loss_ = content_layers_weights[content_layer] * content_weight * (
2 * tf.nn.l2_loss(map_) / content_features[content_layer].size)
content_losses.append(loss_)
content_loss += reduce(tf.add, content_losses)
# plt.imshow(1-np.squeeze(content_mask_features[content_layer]))
# plt.show()
# plt.pause(100)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
# Calculate normalized layer
layer = tf.expand_dims(net[style_layer], axis=4)
paddings = [[0, 0], [pad, pad], [pad, pad], [0, 0], [0, 0]]
layer_depth = layer.get_shape().as_list()[3]
layer_pad = tf.pad(layer, paddings, "CONSTANT")
layer_norm = tf.sqrt(
tf.nn.conv3d(tf.pow(layer_pad, 2),
tf.ones(
(kernel_s, kernel_s, layer_depth, 1, 1),
dtype=tf.float32),
strides=[1, 1, 1, 1, 1],
padding='VALID'))
# Calculate normalized filter bank
style_filters = np.transpose(style_features[i][style_layer][0],
(1, 2, 3, 0))
style_filters = np.expand_dims(style_filters, axis=3)
style_filters_norm = np.sqrt(
np.sum(np.power(style_filters, 2), axis=(0, 1, 2)))
style_filters_normalized = style_filters / style_filters_norm
# Calculate normalized correlations
layer_filtered = tf.nn.conv3d(layer_pad,
style_filters_normalized,
strides=[1, 1, 1, 1, 1],
padding='VALID') / layer_norm
# Find maximum response and index into the filters
max_filter_response_idx = tf.squeeze(
tf.argmax(layer_filtered, axis=4))
max_filter_response_idx = tf.reshape(max_filter_response_idx,
[-1])
max_filter_response_weight = tf.squeeze(
tf.reduce_max(tf.abs(layer_filtered), axis=4))
max_filter_response_weight = tf.reshape(
max_filter_response_weight, [-1])
max_filter_response_weight = max_filter_response_weight / tf.reduce_max(
max_filter_response_weight)
style_filters_tf = tf.transpose(
tf.squeeze(tf.convert_to_tensor(style_filters,
np.float32)), (3, 0, 1, 2))
style_filters_tf_gathered = tf.gather(style_filters_tf,
max_filter_response_idx)
style_filters_tf_gathered = tf.reshape(
style_filters_tf_gathered,
(style_filters_tf_gathered.get_shape().as_list()[0], -1))
layer_patches = tf.extract_image_patches(
tf.squeeze(layer_pad, axis=4), [1, kernel_s, kernel_s, 1],
[1, 1, 1, 1], [1, 1, 1, 1],
padding="VALID")
layer_size = tf.shape(layer_patches)
layer_patches = tf.reshape(layer_patches, (-1, layer_size[3]))
style_weights = np.reshape(content_mask_features[style_layer],
(-1))
# loss_ = tf.reduce_mean(tf.reduce_mean(tf.pow(layer_patches-style_filters_tf_gathered, 2),axis=1)*tf.stop_gradient(max_filter_response_weight))
loss_ = tf.reduce_mean(
tf.reduce_mean(tf.pow(
layer_patches - style_filters_tf_gathered, 2),
axis=1) * style_weights)
style_losses.append(style_layers_weights[style_layer] * 2 *
loss_)
style_loss += style_weight * style_blend_weights[i] * reduce(
tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:, 1:, :, :])
tv_x_size = _tensor_size(image[:, :, 1:, :])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :])
/ tv_y_size) +
(tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :])
/ tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
# print(str(max_filter_response_weight.eval()))
# print(' ')
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations
and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]),
vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(
styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv,
'YCbCr').convert('RGB'))
yield ((None if last_step else i), img_out)
def stylize(style_image,
content_image,
alpha,
beta,
iterations,
vgg_path,
use_avg_pool=False):
# game plan:
# precompute gram matrices for each content and style layer
# make loss function with squared differences
# optimize across
style_shape = (1, ) + style_image.shape
with tf.Graph().as_default(), tf.Session() as sess:
print("precomputing style grams")
style_image_placeholder = vgg.preprocess(
tf.placeholder(tf.float32, shape=style_shape, name='style_image'))
style_net = vgg.net(vgg_path, style_image_placeholder, use_avg_pool)
style_grams = {}
style_pre = np.array([vgg.preprocess(style_image)])
for style_layer in STYLE_LAYERS:
features = style_net[style_layer].eval(
feed_dict={style_image_placeholder: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
style_grams[style_layer] = np.matmul(features.transpose(),
features)
print("precomputing content grams")
content_shape = (1, ) + content_image.shape
content_image_placeholder = tf.placeholder(tf.float32,
shape=content_shape,
name='content_image')
content_net = vgg.net(vgg_path, content_image_placeholder,
use_avg_pool)
content_grams = {}
content_pre = np.array([vgg.preprocess(content_image)])
content_grams[CONTENT_LAYER] = content_net[CONTENT_LAYER].eval(
feed_dict={content_image_placeholder: content_pre})
with tf.Graph().as_default():
# White noise image. 0.256 is taken from online
initial_image = tf.random_normal(content_shape) * 0.256
image = tf.Variable(initial_image)
net = vgg.net(vgg_path, image, use_avg_pool)
# Content Loss
# FROM ONLINE:
# content_weight * (2 * tf.nn.l2_loss(
# net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) /
# content_features[CONTENT_LAYER].size)
# Change this later.
loss_content = tf.nn.l2_loss(
net[CONTENT_LAYER] -
content_grams[CONTENT_LAYER]) / content_grams[CONTENT_LAYER].size
# Style Loss
losses_style = []
style_net = vgg.net(vgg_path, image, use_avg_pool)
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value,
layer.get_shape())
features = tf.reshape(layer, (-1, number))
size = height * width * number
gram = tf.matmul(tf.transpose(features), features) / size
losses_style.append(
tf.nn.l2_loss(gram - style_grams[style_layer]) /
style_grams[style_layer].size)
loss_style = np.sum(losses_style)
loss = alpha * loss_content + beta * loss_style
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
print("starting training")
for i in range(iterations):
last_step = (i == iterations - 1)
train_step.run()
if last_step:
print("finished")
return vgg.unprocess(image.eval().reshape(style_shape[1:]))
def stylize(Ray_render,
ray_steps,
reset_opp,
session,
network,
initial,
initial_noiseblend,
content,
styles,
preserve_colors,
iterations,
content_weight,
content_weight_blend,
style_weight,
style_layer_weight_exp,
style_blend_weights,
tv_weight,
learning_rate,
beta1,
beta2,
epsilon,
pooling,
print_iterations=None,
checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1, ) + content.shape
style_shapes = [(1, ) + style.shape for style in styles]
content_features = {}
style_features = [{} for _ in styles]
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)
]).astype(np.float32)
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(
feed_dict={image: content_pre})
image = initial - tf.cast(tf.reshape(vgg_mean_pixel,
(1, 1, 1, 3)), tf.float32)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = 1 / (1.0 * len(CONTENT_LAYERS))
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(
content_layers_weights * content_weight *
(2 * tf.nn.l2_loss(net[content_layer] -
content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
# overall loss
loss = content_loss + style_loss #+ tv_loss
# optimizer setup
render_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='parameters')
with tf.variable_scope('OPTIMIZATION', reuse=tf.AUTO_REUSE):
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2,
epsilon).minimize(
loss, var_list=render_vars)
session.run(tf.initialize_all_variables())
Ray_render.trace(session, ray_steps, reset_opp, num_steps=50)
# evals_ = session.run(tf.squeeze(initial,axis=0)) # <= returns jpeg data you can write to disk
def print_progress():
stderr.write(' content loss: %g\n' %
content_loss.eval(session=session))
# stderr.write(' style loss: %g\n' % style_loss.eval(session=session))
stderr.write(' total loss: %g\n' % loss.eval(session=session))
print_progress()
# aa= np.squeeze(net[CONTENT_LAYERS[0]].eval(session=session),0)
# bb = np.squeeze(content_features[CONTENT_LAYERS[0]],0)
# pic_aa=np.squeeze(content)
# pic_bb=np.squeeze(initial.eval(session=session),0)
#
# fig = plt.figure(1)
# ax2 = fig.add_subplot(1, 1, 1)
# ax2.imshow(pic_aa)
#
# fig = plt.figure(2)
# ax2 = fig.add_subplot(1, 1, 1)
# ax2.imshow(pic_bb)
# aa.aa=1
# optimization
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run(session=session)
Ray_render.trace(session, ray_steps, reset_opp, num_steps=50)
last_step = (i == iterations - 1)
print_progress()
if (checkpoint_iterations
and i % checkpoint_iterations == 0) or last_step:
image_ = image.eval(session=session)
img_out = vgg.unprocess(image_.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(
Image.fromarray(styled_grayscale_rgb.astype(
np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(
Image.fromarray(original_image.astype(
np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(
Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield ((None if last_step else i), img_out)
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations,
content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
learning_rate, beta1, beta2, epsilon, pooling,
print_iterations=None, checkpoint_iterations=None,
vgg_weights=None, vgg_mean_pixel=None, # Added so that they are no reloaded every time
content_features=None): # Added so that they are not recomputed every time
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1,) + content.shape
style_shapes = [(1,) + style.shape for style in styles]
style_features = [{} for _ in styles]
# Added option to have the net pre-loaded before calling the method
if vgg_weights is None or vgg_mean_pixel is None:
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight
layer_weight *= style_layer_weight_exp
# normalize style layer weights
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum
# Jacob: These content features only need to be computed once, and can be reused for
# each new style image.
# compute content features in feedforward mode
if content_features is None:
content_features = {}
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=shape)
net = vgg.net_preloaded(vgg_weights, image, pooling)
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[i][layer] = gram
initial_content_noise_coeff = 1.0 - initial_noiseblend
# make stylized image using backpropogation
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)])
initial = initial.astype('float32')
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend
content_loss = 0
content_losses = []
for content_layer in CONTENT_LAYERS:
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(
net[content_layer] - content_features[content_layer]) /
content_features[content_layer].size))
content_loss += reduce(tf.add, content_losses)
# style loss
style_loss = 0
for i in range(len(styles)):
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer]
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size)
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
# total variation denoising
tv_y_size = _tensor_size(image[:,1:,:,:])
tv_x_size = _tensor_size(image[:,:,1:,:])
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# overall loss
loss = content_loss + style_loss + tv_loss
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss)
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf')
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0):
print_progress()
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run()
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0):
print_progress()
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval()
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True:
original_image = np.clip(content, 0, 255)
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out
)
def stylize(network, initial, initial_noiseblend, content, styles, matte,
preserve_colors, iterations, content_weight, content_weight_blend,
style_weight, style_layer_weight_exp, style_blend_weights, tv_weight,
matte_weight, learning_rate, beta1, beta2, epsilon, pooling,
output, dest_txt, dest_fig,
print_iterations=None, checkpoint_iterations=None):
"""
Stylize images.
This function yields tuples (iteration, image); `iteration` is None
if this is the final image (the last iteration). Other tuples are yielded
every `checkpoint_iterations` iterations.
:rtype: iterator[tuple[int|None,image]]
"""
shape = (1,) + content.shape #rajoute un 1 en tant que 1ere dimension de content
style_shapes = [(1,) + style.shape for style in styles] #idem sur les images de style
content_features = {} #Création dico
style_features = [{} for _ in styles] #idem pour chaque image de style
vgg_weights, vgg_mean_pixel = vgg.load_net(network)
print('\n',vgg_mean_pixel.shape,'\n')
layer_weight = 1.0
style_layers_weights = {}
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] = layer_weight # => relu1_1 : 1 ; relu2_1 : 1*style_layer_weight_exp ; ... ; relu5_1 : (style_layer_weight_exp)**4
layer_weight *= style_layer_weight_exp # (default : style_layer_weight_exp=1) => seulement des 1
# normalize style layer weights => sum=1
layer_weights_sum = 0
for style_layer in STYLE_LAYERS:
layer_weights_sum += style_layers_weights[style_layer]
for style_layer in STYLE_LAYERS:
style_layers_weights[style_layer] /= layer_weights_sum # => on obtient 1 liste normalisée à 5 élts pour chaque image de style
# compute content features in feedforward mode
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: #Toutes les opérations construites dans ce contexte (indentées) seront placées sur le CPU:0 et dans le graphe g
#"with Session" ferme la session lorsque c'est terminé
image = tf.placeholder('float', shape = shape)
net = vgg.net_preloaded(vgg_weights, image, pooling) #dictionnaire associant à chaque élt de VGG19-LAYERS un tensor , shape.len=4
content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)])
for layer in CONTENT_LAYERS:
content_features[layer] = net[layer].eval(feed_dict={image: content_pre})
# compute style features in feedforward mode
for i in range(len(styles)):
g = tf.Graph()
with g.as_default(), g.device('/cpu:0'), tf.Session() as sess:
image = tf.placeholder('float', shape=style_shapes[i])
net = vgg.net_preloaded(vgg_weights, image, pooling)
style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) #retourne une matrice image_style[i] - vgg_mean_pixel
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
# print("\n")
# print(features)
# print("shape",features.shape, features.size)
# print("\n")
features = np.reshape(features, (-1, features.shape[3]))
# print("\n")
# print(features)
# print("shape",features.shape, features.size)
# print("\n")
gram = np.matmul(features.T, features) / features.size #matmul = matrix multiplication => gram=[features(transposée) x features] / features.size
style_features[i][layer] = gram #style_features = liste de dictionnaires
initial_content_noise_coeff = 1.0 - initial_noiseblend #noiseblend = input (optionnel)
# make stylized image using backpropogation
with tf.Graph().as_default():
#initial = tf.random_normal(shape) * 0 #image de départ = blanche
if initial is None: #initial = image de laquelle on part pour construire l'image suivante
#noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 0.256 #initial non renseignée => aléatoire
else:
initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) # initial - mean_pixel
initial = initial.astype('float32')
#noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff)
#(default : initial_noiseblend=0) => initial = inchangé
image = tf.Variable(initial)
net = vgg.net_preloaded(vgg_weights, image, pooling)
# content loss
content_layers_weights = {}
content_layers_weights['relu4_2'] = content_weight_blend #default : content_weight_blend = 1 ==>...['relu4_2]=1
content_layers_weights['relu5_2'] = 1.0 - content_weight_blend #==>...['relu5_2]=0
content_loss = 0 #initialisation inutile mais on garde le même format pour style loss
content_losses = []
for content_layer in CONTENT_LAYERS: #CONTENT_LAYERS = ('relu4_2', 'relu5_2')
content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( #content_weight = alpha/2
net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) #content_losses = liste de 2 élts
#net[content_layer] = features de l'image générée ; content_features[content_layer] = features de l'image d'origine
content_loss += reduce(tf.add, content_losses) # = somme des élts de content_losses (on calcule l'erreur sur chaque layer, puis on additionne ces erreurs)
#(default : content_layers_weights['relu5_2]=0 => content_loss = content_losses[0])
# style loss
style_loss = 0
for i in range(len(styles)): #nb d'images de style
style_losses = []
for style_layer in STYLE_LAYERS: #STYLE_LAYERS = ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1')
layer = net[style_layer]
_, height, width, number = map(lambda j: j.value, layer.get_shape()) # "_" => discard the first elt of the tuple
#lambda = definit la fonction qui a j associe j.value ; map applique la fonction a tous les élts de layer.get_shape)
size = height * width * number
# print("number ",number)
# print("layer.shape",layer.get_shape())
feats = tf.reshape(layer, (-1, number)) #supprime dim0 (=1), dim0=dim1*dim2, dim1=dim3=number => shape = (dim1*dim2 , number)
# print("feats.shape",feats.get_shape())
gram = tf.matmul(tf.transpose(feats), feats) / size
style_gram = style_features[i][style_layer] #style_features = liste de dictionnaires initialisée dans "compute style featurs in feedforwardmode"
style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) #liste contenant les erreurs de tous les layers de l'image i
#gram = style representation of generated image ; style_gram = style representation of original image
style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses)
#incrémentation de style_loss : reduce=sum(err layers de im[i]) ; style_weight = poids du style par rapp au content
# style_blend_weights[i] = pids de l'im. i par rapp aux autres
# += => on somme les losses de toutes les images
# matting lapacian loss
loader = np.load(matte)
lcoo = csr_matrix((loader['data'], loader['indices'], loader['indptr']),
shape=loader['shape']).tocoo()
lindices = np.mat([lcoo.row, lcoo.col]).transpose()
lvalues = tf.constant(lcoo.data, dtype=tf.float32)
laplacian = tf.SparseTensor(indices=lindices, values=lvalues, dense_shape=lcoo.shape)
matte_loss = 0
matte_losses = []
for i in range(3):
imr = tf.reshape(image[:,:,:,i], [-1, 1])
matte_losses.append(
tf.matmul(tf.transpose(imr),
tf.sparse_tensor_dense_matmul(laplacian, imr))[0][0]
)
matte_loss += matte_weight * reduce(tf.add, matte_losses)
# total variation denoising (pas très important : a remplacer par une autre loss ?)
print("\n total variation denoising") #(possible de désactiver la tv loss avec la commande --tv-weight 0)
tv_y_size = _tensor_size(image[:,1:,:,:])
print(tv_y_size)
tv_x_size = _tensor_size(image[:,:,1:,:])
print(tv_x_size)
print("\n")
tv_loss = tv_weight * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) /
tv_x_size))
# GAN loss
# overall loss
loss = content_loss + style_loss + matte_loss + tv_loss # make alpha etc appear (coeffs)
# optimizer setup
train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) # (operation qui met a jour les variables pour que total loss soit minimise)
# quelles variables ???
def print_progress():
stderr.write(' content loss: %g\n' % content_loss.eval())
stderr.write(' style loss: %g\n' % style_loss.eval())
# stderr.write(' matte loss: %g\n' % matte_loss.eval())
stderr.write(' GAN loss: %g\n' % GAN_loss.eval())
stderr.write(' tv loss: %g\n' % tv_loss.eval())
stderr.write(' total loss: %g\n' % loss.eval())
# optimization
best_loss = float('inf') #???
best = None
with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) #initialise les variables globales
stderr.write('Optimization started...\n')
if (print_iterations and print_iterations != 0): #Si on a rentré un pas pour print_iterations, on affiche avant la 1ere iteration les loss e initial
print_progress()
c_loss = [] #initialisation des listes de valeurs de loss
s_loss = []
t_loss = []
tot_loss = []
x=[i+1 for i in range(iterations)] #initialisation des abscisses des graphes
for i in range(iterations):
stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations))
train_step.run() #on minimise loss a chaque itération
c_loss.append(content_loss.eval()) #incrémentation des listes de valeurs de loss pour chaque itération
s_loss.append(style_loss.eval())
t_loss.append(tv_loss.eval())
tot_loss.append(loss.eval())
last_step = (i == iterations - 1)
if last_step or (print_iterations and i % print_iterations == 0) : #i % print_iterations = reste de la diveucl de i par print_iterations
print_progress() #On affiche les loss instantannées avec une fréquence = print_iterations
if last_step :
if dest_txt is None:
l=len(output)-4 #Création d'un fichier contenant les losses (même nom que l'output mais .txt)
file=output[:l]
F=open(''.join([file,'.txt']),'x') #fusionne file et '.txt'
F.writelines([' content loss: %g\n' % content_loss.eval() , ' style loss: %g\n' % style_loss.eval() ,
' tv loss: %g\n' % tv_loss.eval() , ' total loss: %g\n' % loss.eval()])
F.close
else:
F=open(dest_txt,'x')
F.writelines([' content loss: %g\n' % content_loss.eval() , ' style loss: %g\n' % style_loss.eval() ,
' tv loss: %g\n' % tv_loss.eval() , ' total loss: %g\n' % loss.eval()])
F.close
if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step:
this_loss = loss.eval()
if this_loss < best_loss:
best_loss = this_loss
best = image.eval() #on associe l'image finale à la meilleure loss totale
img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel)
if preserve_colors and preserve_colors == True: #preserve-colors
original_image = np.clip(content, 0, 255) #clip = tous les élts de content >255 -->255, idem <0 -->0
styled_image = np.clip(img_out, 0, 255)
# Luminosity transfer steps:
# 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114)
# 2. Convert stylized grayscale into YUV (YCbCr)
# 3. Convert original image into YUV (YCbCr)
# 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V)
# 5. Convert recombined image from YUV back to RGB
# 1
styled_grayscale = rgb2gray(styled_image)
styled_grayscale_rgb = gray2rgb(styled_grayscale)
# 2
styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr'))
# 3
original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr'))
# 4
w, h, _ = original_image.shape
combined_yuv = np.empty((w, h, 3), dtype=np.uint8)
combined_yuv[..., 0] = styled_grayscale_yuv[..., 0]
combined_yuv[..., 1] = original_yuv[..., 1]
combined_yuv[..., 2] = original_yuv[..., 2]
# 5
img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB'))
yield (
(None if last_step else i),
img_out
)
#Nom de la destination des courbes
if dest_fig is None :
l=len(output)-4
file=output[:l]
dest_fig=''.join([file,'_fig','.jpg'])
print('dest_fig',dest_fig)
#Tracé des graphes
plt.figure(1)
plt.title("Différents types d'erreurs - graphe classique et graphe semi-logarithmique")
plt.subplot(2,1,1)
plt.plot(x, c_loss, label='content_loss')
plt.plot(x, s_loss, label='style_loss')
plt.plot(x, t_loss, label='tv_loss')
plt.plot(x, tot_loss, label='total_loss')
plt.grid('on')
plt.axis('tight')
plt.legend()
plt.ylabel('erreur')
plt.subplot(2,1,2)
plt.semilogy(x, c_loss, label='content_loss')
plt.semilogy(x, s_loss, label='style_loss')
plt.semilogy(x, t_loss, label='tv_loss')
plt.semilogy(x, tot_loss, label='total_loss')
plt.grid('on')
plt.axis('tight')
plt.xlabel("i (Nombre d'itérations)")
plt.ylabel('erreur')
plt.savefig(dest_fig)
def optimize(content_targets, style_target, content_weight, style_weight,
tv_weight, vgg_path, epochs=2, print_iterations=1000,
batch_size=4, save_path='saver/fns.ckpt', slow=False,
learning_rate=1e-3, debug=False):
if slow:
batch_size = 1
mod = len(content_targets) % batch_size
if mod > 0:
print("Train set has been trimmed slightly..")
content_targets = content_targets[:-mod]
style_features = {}
batch_shape = (batch_size,256,256,3)
style_shape = (1,) + style_target.shape
print(style_shape)
# precompute style features
with tf.Graph().as_default(), tf.device('/cpu:0'), tf.Session() as sess:
style_image = tf.placeholder(tf.float32, shape=style_shape, name='style_image')
style_image_pre = vgg.preprocess(style_image)
net = vgg.net(vgg_path, style_image_pre)
style_pre = np.array([style_target])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={style_image:style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / features.size
style_features[layer] = gram
with tf.Graph().as_default(), tf.Session() as sess:
X_content = tf.placeholder(tf.float32, shape=batch_shape, name="X_content")
X_pre = vgg.preprocess(X_content)
# precompute content features
content_features = {}
content_net = vgg.net(vgg_path, X_pre)
content_features[CONTENT_LAYER] = content_net[CONTENT_LAYER]
if slow:
preds = tf.Variable(
tf.random_normal(X_content.get_shape()) * 0.256
)
preds_pre = preds
else:
# Add sin(k1*x+ k2*y+ phi) with 3 channels
# In sess.run, feed_dict should add "phi"
k=tf.get_variable("K",[2,3],tf.float32,tf.random_normal_initializer(stddev=0.02))
phi=tf.placeholder(tf.float32,[3],name="random_phase_offset")
p_noise=transform.periodic_noise(k=k,phi=phi,dp=3,batch_size=4)
X_input=X_content/255.0
X_input=tf.concat((X_input,p_noise),3) # X_input range (0~1), p_noise range (-1~1)
preds = transform.net(X_input)
preds_pre = vgg.preprocess(preds)
net = vgg.net(vgg_path, preds_pre)
content_size = _tensor_size(content_features[CONTENT_LAYER])*batch_size
assert _tensor_size(content_features[CONTENT_LAYER]) == _tensor_size(net[CONTENT_LAYER])
content_loss = content_weight * (2 * tf.nn.l2_loss(
net[CONTENT_LAYER] - content_features[CONTENT_LAYER]) / content_size
)
style_losses = []
for style_layer in STYLE_LAYERS:
layer = net[style_layer]
bs, height, width, filters = map(lambda i:i.value,layer.get_shape())
size = height * width * filters
feats = tf.reshape(layer, (bs, height * width, filters))
feats_T = tf.transpose(feats, perm=[0,2,1])
grams = tf.matmul(feats_T, feats) / size
style_gram = style_features[style_layer]
style_losses.append(2 * tf.nn.l2_loss(grams - style_gram)/style_gram.size)
style_loss = style_weight * functools.reduce(tf.add, style_losses) / batch_size
# total variation denoising
tv_y_size = _tensor_size(preds[:,1:,:,:])
tv_x_size = _tensor_size(preds[:,:,1:,:])
y_tv = tf.nn.l2_loss(preds[:,1:,:,:] - preds[:,:batch_shape[1]-1,:,:])
x_tv = tf.nn.l2_loss(preds[:,:,1:,:] - preds[:,:,:batch_shape[2]-1,:])
tv_loss = tv_weight*2*(x_tv/tv_x_size + y_tv/tv_y_size)/batch_size
loss = content_loss + style_loss + tv_loss
# overall loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
sess.run(tf.global_variables_initializer())
import random
uid = random.randint(1, 100)
print("UID: %s" % uid)
for epoch in range(epochs):
num_examples = len(content_targets)
iterations = 0
while iterations * batch_size < num_examples:
start_time = time.time()
curr = iterations * batch_size
step = curr + batch_size
X_batch = np.zeros(batch_shape, dtype=np.float32)
for j, img_p in enumerate(content_targets[curr:step]):
X_batch[j] = get_img(img_p, (256,256,3)).astype(np.float32)
iterations += 1
assert X_batch.shape[0] == batch_size
batch_phi = np.random.uniform(0, np.pi*2, size = (self.dp))
feed_dict = {
X_content:X_batch
phi:batch_phi
}
train_step.run(feed_dict=feed_dict)
end_time = time.time()
delta_time = end_time - start_time
if debug:
print("UID: %s, batch time: %s" % (uid, delta_time))
is_print_iter = int(iterations) % print_iterations == 0
if slow:
is_print_iter = epoch % print_iterations == 0
is_last = epoch == epochs - 1 and iterations * batch_size >= num_examples
should_print = is_print_iter or is_last
if should_print:
to_get = [style_loss, content_loss, tv_loss, loss, preds]
test_feed_dict = {
X_content:X_batch
phi:batch_phi
}
tup = sess.run(to_get, feed_dict = test_feed_dict)
_style_loss,_content_loss,_tv_loss,_loss,_preds = tup
losses = (_style_loss, _content_loss, _tv_loss, _loss)
if slow:
_preds = vgg.unprocess(_preds)
else:
saver = tf.train.Saver()
res = saver.save(sess, save_path)
yield(_preds, losses, iterations, epoch)
def stylize(
network,
initial,
content,
style,
iterations,
content_weight,
style_weight,
tv_weight,
learning_rate,
print_iter=None,
):
shape = (1,) + content.shape
style_shape = (1,) + style.shape
content_features = {}
style_features = {}
g = tf.Graph()
with g.as_default(), g.device("/cpu:0"), tf.Session() as sess:
image = tf.placeholder("float", shape=shape)
net, mean_pixel = vgg.net(network, image)
content_pre = np.array([vgg.preprocess(content, mean_pixel)])
content_features[CONTENT_LAYER] = net[CONTENT_LAYER].eval(feed_dict={image: content_pre})
g = tf.Graph()
with g.as_default(), g.device("/cpu:0"), tf.Session() as sess:
image = tf.placeholder("float", shape=style_shape)
net, _ = vgg.net(network, image)
style_pre = np.array([vgg.preprocess(style, mean_pixel)])
for layer in STYLE_LAYERS:
features = net[layer].eval(feed_dict={image: style_pre})
features = np.reshape(features, (-1, features.shape[3]))
gram = np.matmul(features.T, features) / (features.size)
style_features[layer] = gram
with tf.Graph().as_default():
if initial is None:
noise = np.random.normal(size=shape, scale=np.std(content) * 0.1)
initial = tf.random_normal(shape) * 256 / 1000
else:
initial = np.array([vgg.preprocess(initial, mean_pixel)])
initial = initial.astype("float32")
image = tf.Variable(initial)
net, _ = vgg.net(network, image)
content_loss = tf.nn.l2_loss(net[CONTENT_LAYER] - content_features[CONTENT_LAYER])
style_losses = []
for i in STYLE_LAYERS:
layer = net[i]
_, height, width, number = map(lambda i: i.value, layer.get_shape())
size = height * width * number
feats = tf.reshape(layer, (-1, number))
gram = tf.matmul(tf.transpose(feats), feats) / (size)
style_gram = style_features[i]
style_losses.append(tf.nn.l2_loss(gram - style_gram))
style_loss = reduce(tf.add, style_losses) / len(style_losses)
tv_loss = tf.nn.l2_loss(image[:, 1:, :, :] - image[:, : shape[1] - 1, :, :]) + tf.nn.l2_loss(
image[:, :, 1:, :] - image[:, :, : shape[2] - 1, :]
)
loss = content_weight * content_loss + style_weight * style_loss + tv_weight * tv_loss
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(iterations):
if print_iter is not None and i % print_iter == 0:
print " content loss: %g" % (content_loss.eval())
print " style loss: %g" % (style_loss.eval())
print " tv loss: %g" % (tv_loss.eval())
print " total loss: %g" % loss.eval()
print "Iteration %d/%d" % (i + 1, iterations)
train_step.run()
return vgg.unprocess(image.eval().reshape(shape[1:]), mean_pixel)
