def get_feature(model, style_path, content_path):
style = image.pre_process_img(style_path)
content = image.pre_process_img(content_path)
# creating new style matrix for color transfer
style = np.squeeze(style, axis=0)
content = np.squeeze(content, axis=0)
print("Line 70: style's shape", style.shape)
new_style = color_transfer.pixel_transformation('image_analogies', style, content)
new_style = np.expand_dims(new_style, axis=0)
print("Line 74: new_style's shape", new_style.shape)
content = np.expand_dims(content, axis=0)
style_feature_outputs = model(new_style)
content_feature_outputs = model(content)
style_feature_arr, content_feature_arr = [], []
for feature in style_feature_outputs[num_content_layers:]:
style_feature_arr.append(feature[0])
for feature in content_feature_outputs[:num_content_layers]:
content_feature_arr.append(feature[0])
# img = image.pre_process_img(path)
# feature_outputs = model(img)
# feature_arr = []
# if mode == 'style':
# for feature in feature_outputs[num_content_layers:]:
# feature_arr.append(feature[0])
# if mode =='content':
# for feature in feature_outputs[:num_content_layers]:
# feature_arr.append(feature[0])
return (style_feature_arr, content_feature_arr)
def get_feature(model, style_path, content_path):
style = image.pre_process_img(style_path)
content = image.pre_process_img(content_path)
# creating new style matrix for color transfer
style = np.squeeze(style, axis=0)
content = np.squeeze(content, axis=0)
new_style = color_transfer.pixel_transformation('image_analogies', style, content)
new_style = np.expand_dims(new_style, axis=0)
content = np.expand_dims(content, axis=0)
style_feature_outputs = model(new_style)
content_feature_outputs = model(content)
style_feature_arr, content_feature_arr = [], []
for feature in style_feature_outputs[num_content_layers:]:
style_feature_arr.append(feature[0])
for feature in content_feature_outputs[:num_content_layers]:
content_feature_arr.append(feature[0])
# img = image.pre_process_img(path)
# feature_outputs = model(img)
# feature_arr = []
# if mode == 'style':
# for feature in feature_outputs[num_content_layers:]:
# feature_arr.append(feature[0])
# if mode =='content':
# for feature in feature_outputs[:num_content_layers]:
# feature_arr.append(feature[0])
new_content_feature_arr = []
for i in range(num_content_layers):
gain_map = style_feature_arr[i]/np.add(content_feature_arr[i] , 10**(-4))
gain_map = np.clip(gain_map, 0.7, 5)
new_content_feature_arr.append(np.multiply(content_feature_arr[i], gain_map))
return (style_feature_arr, new_content_feature_arr)
def get_feature(model, path, mode):
img = image.pre_process_img(path)
feature_outputs = model(img)
feature_arr = []
if mode == 'style':
for feature in feature_outputs[num_content_layers:]:
feature_arr.append(feature[0])
if mode == 'content':
for feature in feature_outputs[:num_content_layers]:
feature_arr.append(feature[0])
return feature_arr
def get_feature_representation(model, path, mode):
"""
This helper function will load picture and the model,
then produce different layer output according to the need
Arguments:
model : the model we initialized
path : path to the picture
mode : 'content' feature or 'style' feature
Returns:
return the content features or the style features
"""
img = image.pre_process_img(path)
feature_outputs = model(img)
if mode == 'style':
return [
feature[0] for feature in feature_outputs[num_content_layers:]
] # get the last several outputs
if mode == 'content':
return [
feature[0] for feature in feature_outputs[:num_content_layers]
] # get the first several outputs
def get_feature(model, style_paths, content_path):
styles = []
for style_path in style_paths:
style = image.pre_process_img(style_path)
style = np.squeeze(style, axis=0)
styles.append(style)
content = image.pre_process_img(content_path)
# creating new style matrix for color transfer
content = np.squeeze(content, axis=0)
# print("Line 70: style's shape", style.shape)
new_styles = []
for style in styles:
new_style = color_transfer.pixel_transformation(
'image_analogies', style, content)
new_style = np.expand_dims(new_style, axis=0)
new_styles.append(new_style)
# print("Line 74: new_style's shape", new_style.shape)
content = np.expand_dims(content, axis=0)
artist_style_feature_outputs = []
for new_style in new_styles:
style_feature_outputs = model(new_style)
# print(np.array(style_feature_outputs).shape)
artist_style_feature_outputs.append(style_feature_outputs)
content_feature_outputs = model(content)
# print(np.array(content_feature_outputs).shape)
artist_style_feature_arr, artist_content_feature_arr = [], []
for image_style_feature_outputs in artist_style_feature_outputs:
style_feature_arr = []
for feature in image_style_feature_outputs[num_content_layers:]:
style_feature_arr.append(feature[0])
artist_style_feature_arr.append(style_feature_arr)
content_feature_arr = []
for feature in content_feature_outputs[:num_content_layers]:
content_feature_arr.append(feature[0])
for style_feature_arr in artist_style_feature_arr:
new_content_feature_arr = []
for i in range(num_content_layers):
gain_map = style_feature_arr[i] / np.add(content_feature_arr[i], 10
**(-4))
gain_map = np.clip(gain_map, 0.7, 5)
new_content_feature_arr.append(
np.multiply(content_feature_arr[i], gain_map))
artist_content_feature_arr.append(new_content_feature_arr)
# img = image.pre_process_img(path)
# feature_outputs = model(img)
# feature_arr = []
# if mode == 'style':
# for feature in feature_outputs[num_content_layers:]:
# feature_arr.append(feature[0])
# if mode =='content':
# for feature in feature_outputs[:num_content_layers]:
# feature_arr.append(feature[0])
return (artist_style_feature_arr, artist_content_feature_arr)
def run(content_path, style_path, iteration):
content_weight = 1e3
style_weight = 1
model = model_init()
for layer in model.layers:
layer.trainable = False
artist_style_features, artist_content_features = get_feature(
model, style_path, content_path)
init_image = image.pre_process_img(
content_path) # initialize the generated image with content image
init_image = tf.Variable(init_image, dtype=tf.float32)
opt = tf.keras.optimizers.Adam(5, beta_1=0.99, epsilon=1e-1)
loss_weights = (content_weight, style_weight)
cfg = {
'model': model,
'loss_weights': loss_weights,
'init_image': init_image,
'artist_content_features': artist_content_features,
'artist_style_features': artist_style_features
}
#我不太知道这个norm means是怎么来的，image.py里用的也是相同的值，norm means是用来normalize图片的，我看几个github版本用的数值都差不多，但不知道怎么算的
norm_means = np.array([103.939, 116.779, 123.68])
min_vals = -norm_means
max_vals = 255 - norm_means
#store the loss and the img
best_loss, best_img = float('inf'), None
imgs = []
start = datetime.now()
for i in range(iteration):
print(i)
grads, all_loss = compute_grads(cfg)
losss, content_losss, style_losss = all_loss
opt.apply_gradients([(grads, init_image)])
clipped = tf.clip_by_value(init_image, min_vals, max_vals)
init_image.assign(clipped)
# 以下用了一些image.py里的function，用的github上的代码，之后改掉
if losss < best_loss:
# Update best loss and best image from total loss.
best_loss = losss
best_img = image.deprocess_img(init_image.numpy())
if i % 20 == 0:
end = datetime.now()
print('[INFO]Iteration: {}'.format(i))
print('Total loss: {:.4e}, '
'style loss: {:.4e}, '
'content loss: {:.4e}'.format(losss, style_losss,
content_losss))
print(f'20 iters takes {end -start}')
start = datetime.now()
img = init_image.numpy()
img = image.deprocess_img(img)
path = 'output_' + str(i) + '.jpg'
image.saveimg(img, path)
imgs.append(img)
return best_img, best_loss
def run_nst(content_path,
style_path,
iteration=1000,
content_weight=1e3,
style_weight=1):
model = model_init()
for layer in model.layers:
layer.trainable = False
content_features = get_feature_representation(model,
content_path,
mode='content')
style_features = get_feature_representation(model,
style_path,
mode='style')
init_image = image.pre_process_img(
content_path) # initialize the generated image with content image
init_image = tf.Variable(init_image, dtype=tf.float32)
opt = tf.keras.optimizers.Adam(5, beta_1=0.99, epsilon=1e-1)
epoch = 1
loss_weights = (content_weight, style_weight)
cfg = {
'model': model,
'loss_weights': loss_weights,
'init_image': init_image,
'content_features': content_features,
'style_features': style_features
}
norm_means = np.array([103.939, 116.779, 123.68])
min_vals = -norm_means
max_vals = 255 - norm_means
#store the loss and the img
best_loss, best_img = float('inf'), None
imgs = []
start = datetime.now()
for i in range(iteration):
grads, all_loss = compute_grads(cfg)
losss, content_losss, style_losss = all_loss
opt.apply_gradients([(grads, init_image)])
clipped = tf.clip_by_value(init_image, min_vals, max_vals)
init_image.assign(clipped)
if losss < best_loss:
# Update best loss and best image from total loss.
best_loss = losss
best_img = image.deprocess_img(init_image.numpy())
if i % 100 == 0:
end = datetime.now()
print('[INFO]Iteration: {}'.format(i))
print('Total loss: {:.4e}, '
'style loss: {:.4e}, '
'content loss: {:.4e}'.format(losss, style_losss,
content_losss))
print(f'100 iters takes {end -start}')
start = datetime.now()
if i % 500 == 0:
# Use the .numpy() method to get the concrete numpy array
plot_img = init_image.numpy()
plot_img = image.deprocess_img(plot_img)
path = 'output/output_' + str(i) + '.jpg'
image.saveimg(plot_img, path)
imgs.append(plot_img)
return best_img, best_loss