def style_transfer(content_image,
style_image,
image_size,
style_size,
content_layer,
content_weight,
style_layers,
style_weights,
tv_weight,
init_random=False):
content_img = preprocess_image(load_image(content_image, size=image_size))
feats = model.extract_features(model.image)
content_target = sess.run(feats[content_layer],
{model.image: content_img[None]})
# Extract features from the style image
style_img = preprocess_image(load_image(style_image, size=style_size))
style_feat_vars = [feats[idx] for idx in style_layers]
style_target_vars = []
# Compute list of TensorFlow Gram matrices
for style_feat_var in style_feat_vars:
style_target_vars.append(gram_matrix(style_feat_var))
# Compute list of NumPy Gram matrices by evaluating the TensorFlow graph on the style image
style_targets = sess.run(style_target_vars, {model.image: style_img[None]})
# Initialize generated image to content image
if init_random:
img_var = tf.Variable(tf.random_uniform(content_img[None].shape, 0, 1),
name="image")
else:
img_var = tf.Variable(content_img[None], name="image")
# Extract features on generated image
feats = model.extract_features(img_var)
# Compute loss
c_loss = content_loss(content_weight, feats[content_layer], content_target)
s_loss = style_loss(feats, style_layers, style_targets, style_weights)
t_loss = tv_loss(img_var, tv_weight)
loss = c_loss + s_loss + t_loss
# Set up optimization hyperparameters
initial_lr = 3.0
decayed_lr = 0.1
decay_lr_at = 180
max_iter = 200
# Create and initialize the Adam optimizer
lr_var = tf.Variable(initial_lr, name="lr")
# Create train_op that updates the generated image when run
with tf.variable_scope("optimizer") as opt_scope:
train_op = tf.train.AdamOptimizer(lr_var).minimize(loss,
var_list=[img_var])
# Initialize the generated image and optimization variables
opt_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=opt_scope.name)
sess.run(tf.variables_initializer([lr_var, img_var] + opt_vars))
# Create an op that will clamp the image values when run
clamp_image_op = tf.assign(img_var, tf.clip_by_value(img_var, -1.5, 1.5))
f, axarr = plt.subplots(1, 2)
axarr[0].axis('off')
axarr[1].axis('off')
axarr[0].set_title('Content Source Img.')
axarr[1].set_title('Style Source Img.')
axarr[0].imshow(deprocess_image(content_img))
axarr[1].imshow(deprocess_image(style_img))
plt.show()
plt.figure()
# Hardcoded handcrafted
for t in range(max_iter):
# Take an optimization step to update img_var
sess.run(train_op)
if t < decay_lr_at:
sess.run(clamp_image_op)
if t == decay_lr_at:
sess.run(tf.assign(lr_var, decayed_lr))
if t % 100 == 0:
print('Iteration {}'.format(t))
img = sess.run(img_var)
plt.imshow(deprocess_image(img[0], rescale=True))
plt.axis('off')
plt.show()
print('Iteration {}'.format(t))
img = sess.run(img_var)
plt.imshow(deprocess_image(img[0], rescale=True))
plt.axis('off')
plt.show()
check_scipy()
from cs231n.classifiers.squeezenet import SqueezeNet
import tensorflow as tf
tf.reset_default_graph() # remove all existing variables in the graph
sess = get_session() # start a new Session
# Load pretrained SqueezeNet model
SAVE_PATH = 'cs231n/datasets/squeezenet.ckpt'
# if not os.path.exists(SAVE_PATH):
# raise ValueError("You need to download SqueezeNet!")
model = SqueezeNet(save_path=SAVE_PATH, sess=sess)
# Load data for testing
content_img_test = preprocess_image(load_image('styles/tubingen.jpg',
size=192))[None]
style_img_test = preprocess_image(
load_image('styles/starry_night.jpg', size=192))[None]
answers = np.load('style-transfer-checks-tf.npz')
def content_loss(content_weight, content_current, content_original):
shapes = tf.shape(content_current)
F_l = tf.reshape(content_current, [shapes[1], shapes[2] * shapes[3]])
P_l = tf.reshape(content_original, [shapes[1], shapes[2] * shapes[3]])
loss = content_weight * (tf.reduce_sum((F_l - P_l)**2))
return loss
def content_loss_test(correct):
content_layer = 3
if __name__ == "__main__":
check_scipy()
tf.reset_default_graph() # remove all existing variables in the graph
sess = get_session() # start a new Session
# Load pretrained SqueezeNet model
SAVE_PATH = 'cs231n/datasets/squeezenet.ckpt'
if not os.path.exists(SAVE_PATH + ".index"):
raise ValueError("You need to download SqueezeNet!")
model = SqueezeNet(save_path=SAVE_PATH, sess=sess)
# Load data for testing
content_img_test = preprocess_image(
load_image('styles/tubingen.jpg', size=192))[None]
style_img_test = preprocess_image(
load_image('styles/starry_night.jpg', size=192))[None]
answers = np.load('style-transfer-checks-tf.npz')
# content_loss_test(answers['cl_out'])
# gram_matrix_test(answers['gm_out'])
# style_loss_test(answers['sl_out'])
# tv_loss_test(answers['tv_out'])
# Composition VII + Tubingen
# params1 = {
def style_transfer(content_image,
style_image,
image_size,
style_size,
content_layer,
content_weight,
style_layers,
style_weights,
tv_weight,
init_random=False):
"""Run style transfer!
Inputs:
- content_image: filename of content image
- style_image: filename of style image
- image_size: size of smallest image dimension (used for content loss and generated image)
- style_size: size of smallest style image dimension
- content_layer: layer to use for content loss
- content_weight: weighting on content loss
- style_layers: list of layers to use for style loss
- style_weights: list of weights to use for each layer in style_layers
- tv_weight: weight of total variation regularization term
- init_random: initialize the starting image to uniform random noise
"""
# Extract features from the content image
content_img = preprocess_image(load_image(content_image, size=image_size))
feats = extract_features(content_img[None], model)
content_target = feats[content_layer]
# Extract features from the style image
style_img = preprocess_image(load_image(style_image, size=style_size))
s_feats = extract_features(style_img[None], model)
style_targets = []
# Compute list of TensorFlow Gram matrices
for idx in style_layers:
style_targets.append(gram_matrix(s_feats[idx]))
# Set up optimization hyperparameters
initial_lr = 3.0
decayed_lr = 0.1
decay_lr_at = 180
max_iter = 200
step = tf.Variable(0, trainable=False)
boundaries = [decay_lr_at]
values = [initial_lr, decayed_lr]
learning_rate_fn = tf.keras.optimizers.schedules.PiecewiseConstantDecay(
boundaries, values)
# Later, whenever we perform an optimization step, we pass in the step.
learning_rate = learning_rate_fn(step)
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
# Initialize the generated image and optimization variables
f, axarr = plt.subplots(1, 2)
axarr[0].axis('off')
axarr[1].axis('off')
axarr[0].set_title('Content Source Img.')
axarr[1].set_title('Style Source Img.')
axarr[0].imshow(deprocess_image(content_img))
axarr[1].imshow(deprocess_image(style_img))
plt.show()
plt.figure()
# Initialize generated image to content image
if init_random:
initializer = tf.random_uniform_initializer(0, 1)
img = initializer(shape=content_img[None].shape)
img_var = tf.Variable(img)
print("Intializing randomly.")
else:
img_var = tf.Variable(content_img[None])
print("Initializing with content image.")
for t in range(max_iter):
with tf.GradientTape() as tape:
tape.watch(img_var)
feats = extract_features(img_var, model)
# Compute loss
c_loss = content_loss(content_weight, feats[content_layer],
content_target)
s_loss = style_loss(feats, style_layers, style_targets,
style_weights)
t_loss = tv_loss(img_var, tv_weight)
loss = c_loss + s_loss + t_loss
# Compute gradient
grad = tape.gradient(loss, img_var)
optimizer.apply_gradients([(grad, img_var)])
img_var.assign(tf.clip_by_value(img_var, -1.5, 1.5))
if t % 10 == 0:
print('Iteration {}'.format(t))
#plt.imshow(deprocess_image(img_var[0].numpy(), rescale=True))
#plt.axis('off')
#plt.show()
print('Iteration {}'.format(t))
plt.imshow(deprocess_image(img_var[0].numpy(), rescale=True))
plt.axis('off')
plt.show()
def style_transfer(content_image, style_image, output_image, image_size, style_size, content_layer, content_weight,
style_layers, style_weights, tv_weight, init_random = False, sess=sess, model=model):
"""Run style transfer!
Inputs:
- content_image: filename of content image
- style_image: filename of style image
- output_image: filename to write to
- image_size: size of smallest image dimension (used for content loss and generated image)
- style_size: size of smallest style image dimension
- content_layer: layer to use for content loss
- content_weight: weighting on content loss
- style_layers: list of layers to use for style loss
- style_weights: list of weights to use for each layer in style_layers
- tv_weight: weight of total variation regularization term
- init_random: initialize the starting image to uniform random noise
"""
# Extract features from the content image
content_img = preprocess_image(load_image(content_image, size=image_size))
feats = model.extract_features(model.image)
content_target = sess.run(feats[content_layer],
{model.image: content_img[None]})
# Extract features from the style image
style_img = preprocess_image(load_image(style_image, size=style_size))
style_feat_vars = [feats[idx] for idx in style_layers]
style_target_vars = []
# Compute list of TensorFlow Gram matrices
for style_feat_var in style_feat_vars:
style_target_vars.append(gram_matrix(style_feat_var))
# Compute list of NumPy Gram matrices by evaluating the TensorFlow graph on the style image
style_targets = sess.run(style_target_vars, {model.image: style_img[None]})
# Initialize generated image to content image
if init_random:
img_var = tf.Variable(tf.random_uniform(content_img[None].shape, 0, 1), name="image")
else:
img_var = tf.Variable(content_img[None], name="image")
# Extract features on generated image
feats = model.extract_features(img_var)
# Compute loss
c_loss = content_loss(content_weight, feats[content_layer], content_target)
s_loss = style_loss(feats, style_layers, style_targets, style_weights)
t_loss = tv_loss(img_var, tv_weight)
loss = c_loss + s_loss + t_loss
# Set up optimization hyperparameters
initial_lr = 3.0
decayed_lr = 0.1
decay_lr_at = 180
max_iter = 100
# Create and initialize the Adam optimizer
lr_var = tf.Variable(initial_lr, name="lr")
# Create train_op that updates the generated image when run
with tf.variable_scope("optimizer") as opt_scope:
train_op = tf.train.AdamOptimizer(lr_var).minimize(loss, var_list=[img_var])
# Initialize the generated image and optimization variables
opt_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=opt_scope.name)
sess.run(tf.variables_initializer([lr_var, img_var] + opt_vars))
# Create an op that will clamp the image values when run
clamp_image_op = tf.assign(img_var, tf.clip_by_value(img_var, -1.5, 1.5))
if output_image[-4:] == '.jpg':
output_image = output_image[:-4]
# Hardcoded handcrafted
for t in range(0, max_iter+1):
# Take an optimization step to update img_var
sess.run(train_op)
if t < decay_lr_at:
sess.run(clamp_image_op)
if t == decay_lr_at:
sess.run(tf.assign(lr_var, decayed_lr))
if t % 25 == 0:
print('Iteration {}'.format(t))
img = sess.run(img_var)
cv2.imwrite(output_image + "_iter" + str(t) + ".jpg", deprocess_image(img[0]))
- gram: Tensor of shape (N, C, C) giving the (optionally normalized)
Gram matrices for the input image.
"""
features_shape = tf.shape(features)
features_T = tf.transpose(features, perm=[0, 3, 2, 1])
mult = tf.batch_matmul(features_T, features)
if normalize:
mult = tf.scalar_mul(tf.reciprocal(tf.cast(features_shape[1]*features_shape[2]*features_shape[3], tf.float32)), mult)
return mult
style_layers = [1, 4, 6, 7]
style_weights = [2000000, 800, 12, 1]
style_feats = model.extract_features(model.image)
# TODO: make this a dynamic tensor
style_img = preprocess_image(load_image('./styles/van_gogh.jpg'))
style_feat_vars = [style_feats[idx] for idx in [1, 4, 6, 7]]
style_target_vars = []
# Compute list of TensorFlow Gram matrices
for style_feat_var in style_feat_vars:
style_target_vars.append(gram_matrix(style_feat_var))
# Compute list of NumPy Gram matrices by evaluating the TensorFlow graph on the style image
style_targets = sess.run(style_target_vars, {model.image: style_img[None]})
def gan_style_loss(gan_output_image):
# preprocess the gan image per the constants in cs231n/image_utils
processed_gan_img = tf_preprocess_image(gan_output_image)
gan_img_feats = model.extract_features(processed_gan_img)
loss = tf.constant(0, tf.float32)
for i in range(len(style_layers)):
