def encoder_decoder(self, inputs):
encoded = self.encoder.encoder(inputs, self.target_layer)
model = Decoder()
decoded, _ = model.decoder(encoded, self.target_layer)
decoded_encoded = self.encoder.encoder(decoded, self.target_layer)
return encoded, decoded, decoded_encoded
def encoder_decoder(self, inputs):
encoded = self.encoder.encoder(inputs, self.target_layer) #特征提取
model = Decoder() #生成图像model
decoded, _ = model.decoder(encoded, self.target_layer) #生成图像
# 再对生成图像，进行特征提取
decoded_encoded = self.encoder.encoder(decoded, self.target_layer)
# 返回值为 a.提取的特征 b.由特征生成的图片 c.由生成图片提取的特征
return encoded, decoded, decoded_encoded
def encoder_decoder(self, inputs):
"""
encoder_decoder function:
Outputs:
encoded: feature map obtained by processing the original image with the encoder
decoded: reconstructed image obtained by processing the original image with both the encoder and decoder
decoded_encoded: feature map obtained by processing the reconstructed image with the encoder
"""
encoded = self.encoder.encoder(inputs, self.target_layer)
model = Decoder()
decoded, _ = model.decoder(encoded, self.target_layer)
decoded_encoded = self.encoder.encoder(decoded, self.target_layer)
return encoded, decoded, decoded_encoded
class WCT_test_single_layer:
def __init__(self, target_layer, content_path, style_path, alpha,
pretrained_vgg, output_path, decoder_weights):
self.target_layer = target_layer
self.content_path = content_path
self.style_path = style_path
self.output_path = output_path
self.alpha = alpha
self.encoder = Vgg19(pretrained_vgg)
self.decoder = Decoder()
self.decoder_weights = decoder_weights
def test(self):
content = tf.placeholder('float', [1, 304, 304, 3])
style = tf.placeholder('float', [1, 304, 304, 3])
content_encode = self.encoder.encoder(content, self.target_layer)
style_encode = self.encoder.encoder(style, self.target_layer)
#blended = wct_tf(content_encode,style_encode,self.alpha)
#blended = Adain(content_encode,style_encode)
#blended = style_swap(content_encode, style_encode, 3, 2)
blended = wct_style_swap(content_encode, style_encode, self.alpha)
stylized = self.decoder.decoder(blended, self.target_layer)
saver = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
saver.restore(sess, self.decoder_weights)
img_c = image.load_img(self.content_path,
target_size=(304, 304, 3))
img_c = image.img_to_array(img_c)
img_c = np.expand_dims(img_c, axis=0)
img_s = image.load_img(self.style_path, target_size=(304, 304, 3))
img_s = image.img_to_array(img_s)
img_s = np.expand_dims(img_s, axis=0)
feed_dict = {content: img_c, style: img_s}
result, e = sess.run([stylized, content_encode],
feed_dict=feed_dict)
result = result[0][0]
result = np.clip(result, 0, 255) / 255.
print(result.shape)
imsave(self.output_path, result)
class WCT_test_all_layer:
# 测试模型初始化，导入命令行参数
def __init__(self, content_path, style_path, alpha_wct, alpha_swap,
pretrained_vgg, output_path):
self.content_path = content_path # 内容图片路径
self.style_path = style_path # 风格图片路径
self.output_path = output_path # 融合后图片输出路径
self.alpha_wct = alpha_wct # wct方法内容与风格图片比重
self.alpha_swap = alpha_swap #wct_swap方法内容与风格图片比重
self.encoder = Vgg19(pretrained_vgg) # 导入VGG19 模型
self.decoder = Decoder() # 导入Decoder 反卷积
# 导入Decoder模型参数
self.decoder_weights = [
'models/decoder_1.ckpt', 'models/decoder_2.ckpt',
'models/decoder_3.ckpt', 'models/decoder_4.ckpt'
]
def test(self):
# 加载content图像,并转成数组,同时进行扩维度（-，h,w,c）
img_c = Image.open(self.content_path)
img_c = np.array(img_c)
img_c = np.expand_dims(img_c, axis=0)
# 加载style图像,并转成数组,同时进行扩维度（-，h,w,c）
img_s = Image.open(self.style_path)
img_s = np.array(img_s)
img_s = np.expand_dims(img_s, axis=0)
# 定义尺寸为 内容图片大小的 占位符变量
content = tf.placeholder('float', shape=img_c.shape)
# 定义尺寸为 风格图片大小的 占位符变量
style = tf.placeholder('float', shape=img_s.shape)
wct_tf_alpha = self.alpha_wct #wct方法比重
wct_swap_alpha = self.alpha_swap #swap方法比重
# 加入relu4_1的style特征，使用wct方法加入
content_encode_4 = self.encoder.encoder(content,
'relu4') # relu4_1的content特征
style_encode_4 = self.encoder.encoder(style,
'relu4') # relu4_1的style特征
blended_4 = wct_tf(content_encode_4, style_encode_4,
wct_tf_alpha) # wct方法生成融合的特征图
stylized_4, var_list4 = self.decoder.decoder(blended_4,
'relu4') # 还原成图片，同时记录变量
#加入relu3_1的style特征，使用wct_swap方法加入
content_encode_3 = self.encoder.encoder(stylized_4,
'relu3') # relu3_1的content特征
style_encode_3 = self.encoder.encoder(style,
'relu3') # relu3_1的style特征
blended_3 = wct_style_swap(content_encode_3, style_encode_3,
wct_swap_alpha) # wct_style_swap方法生成融合的特征图
stylized_3, var_list3 = self.decoder.decoder(blended_3,
'relu3') # 还原成图片，同时记录变量
# 加入relu2_1的style特征，使用wct方法加入
content_encode_2 = self.encoder.encoder(stylized_3,
'relu2') # relu2_1的content特征
style_encode_2 = self.encoder.encoder(style,
'relu2') # relu2_1的style特征
blended_2 = wct_tf(content_encode_2, style_encode_2,
wct_tf_alpha) # wct方法生成融合的特征图
stylized_2, var_list2 = self.decoder.decoder(blended_2,
'relu2') # 还原成图片，同时记录变量
# 加入relu1_1的style特征，使用wct方法加入
content_encode_1 = self.encoder.encoder(stylized_2,
'relu1') # relu2_1的content特征
style_encode_1 = self.encoder.encoder(style,
'relu1') # relu2_1的style特征
blended_1 = wct_tf(content_encode_1, style_encode_1,
wct_tf_alpha) # wct方法生成融合的特征图
stylized_1, var_list1 = self.decoder.decoder(blended_1,
'relu1') # 还原成图片，同时记录变量
# 保存模型
saver1 = tf.train.Saver(var_list1)
saver2 = tf.train.Saver(var_list2)
saver3 = tf.train.Saver(var_list3)
saver4 = tf.train.Saver(var_list4)
# 初始化会话，并开始测试过程
with tf.Session() as sess:
# 变量初始化
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
# 从训练好的模型中加载 参数
saver1.restore(sess, self.decoder_weights[0]) # 加载decoder_1的权重
saver2.restore(sess, self.decoder_weights[1]) # 加载decoder_2的权重
saver3.restore(sess, self.decoder_weights[2]) # 加载decoder_3的权重
saver4.restore(sess, self.decoder_weights[3]) # 加载decoder_4的权重
# 分别喂入内容图片和风格图片
feed_dict = {content: img_c, style: img_s}
# 产生结果
result = sess.run(stylized_1, feed_dict=feed_dict)
# 图片输出前的处理
result = result[0]
# 数据处理使用的 float 变到0-1之间，正确表达图片信息
result = np.clip(result, 0, 255) / 255.
# 存储图片
imsave(self.output_path, result)
class WCT_test_all_layer:
def __init__(self, content_path, style_path, alpha, pretrained_vgg,
output_path):
self.content_path = content_path
self.style_path = style_path
self.output_path = output_path
self.alpha = alpha
self.encoder = Vgg19(pretrained_vgg)
self.decoder = Decoder()
self.decoder_weights = [
'models/decoder_1.ckpt', 'models/decoder_2.ckpt',
'models/decoder_3.ckpt', 'models/decoder_4.ckpt'
]
def test(self):
content = tf.placeholder('float', [1, 304, 304, 3])
style = tf.placeholder('float', [1, 304, 304, 3])
content_encode_4 = self.encoder.encoder(content, 'relu4')
style_encode_4 = self.encoder.encoder(style, 'relu4')
blended_4 = wct_tf(content_encode_4, style_encode_4, self.alpha)
stylized_4, var_list4 = self.decoder.decoder(blended_4, 'relu4')
content_encode_3 = self.encoder.encoder(stylized_4, 'relu3')
style_encode_3 = self.encoder.encoder(style, 'relu3')
blended_3 = wct_tf(content_encode_3, style_encode_3, self.alpha)
stylized_3, var_list3 = self.decoder.decoder(blended_3, 'relu3')
content_encode_2 = self.encoder.encoder(stylized_3, 'relu2')
style_encode_2 = self.encoder.encoder(style, 'relu2')
blended_2 = wct_tf(content_encode_2, style_encode_2, self.alpha)
stylized_2, var_list2 = self.decoder.decoder(blended_2, 'relu2')
content_encode_1 = self.encoder.encoder(stylized_2, 'relu1')
style_encode_1 = self.encoder.encoder(style, 'relu1')
blended_1 = wct_tf(content_encode_1, style_encode_1, self.alpha)
stylized_1, var_list1 = self.decoder.decoder(blended_1, 'relu1')
saver1 = tf.train.Saver(var_list1)
saver2 = tf.train.Saver(var_list2)
saver3 = tf.train.Saver(var_list3)
saver4 = tf.train.Saver(var_list4)
with tf.Session() as sess:
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
saver1.restore(sess, self.decoder_weights[0])
saver2.restore(sess, self.decoder_weights[1])
saver3.restore(sess, self.decoder_weights[2])
saver4.restore(sess, self.decoder_weights[3])
img_c = image.load_img(self.content_path,
target_size=(304, 304, 3))
img_c = image.img_to_array(img_c)
img_c = np.expand_dims(img_c, axis=0)
img_s = image.load_img(self.style_path, target_size=(304, 304, 3))
img_s = image.img_to_array(img_s)
img_s = np.expand_dims(img_s, axis=0)
feed_dict = {content: img_c, style: img_s}
result = sess.run(stylized_1, feed_dict=feed_dict)
result = result[0]
result = np.clip(result, 0, 255) / 255.
imsave(self.output_path, result)
class style_transfer:
"""
Style Transfer Class:
this class is the main algorithm of our project;
generates stylized images given content and style images
"""
def __init__(self, content_path, style_path, alpha, pretrained_vgg,
output_path):
"""
Style Transfer Initialization
Inputs:
content_path: path to content images
style_path: path to style images
alpha: range=[0, 1]; parameter to control ratio of content vs. stylized images
pretrained_vgg: path to pre-trained vgg weights
output_path: path to store result images
"""
self.content_path = content_path
self.style_path = style_path
self.output_path = output_path
self.alpha = alpha
self.encoder = Vgg19(pretrained_vgg)
self.decoder = Decoder()
# load decoder weights from saved models: give names of models to use
self.decoder_weights = [
'models/relu1_1w_2k', 'models/relu2_1w_2k', 'models/relu3_1w_1w',
'models/relu4_8w_1w'
] #,'models/relu5_8w_2w']
def test(self):
"""
Test function: generate stylized images using 5 levels (5 encoder-decoder pairs)
"""
content = tf.placeholder('float', [1, 304, 304, 3])
style = tf.placeholder('float', [1, 304, 304, 3])
# implementation of multi-level stylization: first level is encoder-decoder 5, last level is encoder-decoder 1
# original paper named this "coarse-to-fine stylization"
# content_encode_5 = self.encoder.encoder(content,'relu5')
# style_encode_5 = self.encoder.encoder(style,'relu5')
# blended_5 = wct_tf(content_encode_5,style_encode_5,self.alpha)
# stylized_5 ,var_list5= self.decoder.decoder(blended_5,'relu5')
content_encode_4 = self.encoder.encoder(content, 'relu4')
style_encode_4 = self.encoder.encoder(style, 'relu4')
blended_4 = wct_tf(content_encode_4, style_encode_4, self.alpha)
stylized_4, var_list4 = self.decoder.decoder(blended_4, 'relu4')
content_encode_3 = self.encoder.encoder(stylized_4, 'relu3')
style_encode_3 = self.encoder.encoder(style, 'relu3')
blended_3 = wct_tf(content_encode_3, style_encode_3, self.alpha)
stylized_3, var_list3 = self.decoder.decoder(blended_3, 'relu3')
content_encode_2 = self.encoder.encoder(stylized_3, 'relu2')
style_encode_2 = self.encoder.encoder(style, 'relu2')
blended_2 = wct_tf(content_encode_2, style_encode_2, self.alpha)
stylized_2, var_list2 = self.decoder.decoder(blended_2, 'relu2')
content_encode_1 = self.encoder.encoder(stylized_2, 'relu1')
style_encode_1 = self.encoder.encoder(style, 'relu1')
blended_1 = wct_tf(content_encode_1, style_encode_1, self.alpha)
stylized_1, var_list1 = self.decoder.decoder(blended_1, 'relu1')
saver1 = tf.train.Saver(var_list1)
saver2 = tf.train.Saver(var_list2)
saver3 = tf.train.Saver(var_list3)
saver4 = tf.train.Saver(var_list4)
#saver5 = tf.train.Saver(var_list5)
with tf.Session() as sess:
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
# restore decoder weights from models
saver1.restore(sess, self.decoder_weights[0])
saver2.restore(sess, self.decoder_weights[1])
saver3.restore(sess, self.decoder_weights[2])
saver4.restore(sess, self.decoder_weights[3])
#saver5.restore(sess,self.decoder_weights[4])
# load content image using PIL package: if image is in greyscale, convert it to RGB; then resize image
target_size = (304, 304, 3)
content_img = pil_image.open(self.content_path)
if content_img.mode != 'RGB':
content_img = img.convert('RGB')
hw_tuple = (target_size[1], target_size[0])
if content_img.size != hw_tuple:
content_img = content_img.resize(hw_tuple)
# convert image to numpy array
content_img = np.asarray(content_img)
content_img = np.expand_dims(content_img, axis=0)
# load style image using PIL package: if image is in greyscale, convert it to RGB; then resize image
target_size = (304, 304, 3)
style_img = pil_image.open(self.style_path)
if style_img.mode != 'RGB':
style_img = img.convert('RGB')
hw_tuple = (target_size[1], target_size[0])
if style_img.size != hw_tuple:
style_img = style_img.resize(hw_tuple)
# convert image to numpy array
style_img = np.asarray(style_img)
style_img = np.expand_dims(style_img, axis=0)
feed_dict = {content: content_img, style: style_img}
# generate result images
result = sess.run(stylized_1, feed_dict=feed_dict)
result = result[0]
result = np.clip(result, 0, 255) / 255.
imsave(self.output_path, result)
print('Style Transfer Completed')
