def __init__(self, sess):
self.sess=sess
self.global_step = tf.Variable(0.0, name='global_step',dtype=tf.float32, trainable=False)
#for data input
self.pipline_data_train=cdata.get_pipline_data_train(img_size, batchsize)
self.pipline_data_test=cdata.get_pipline_data_test(img_size, batchsize_test)
#3个placeholder， img和noise,training
self.imgs_pla = tf.placeholder(tf.float32, [batchsize, img_size_h, img_size_w, G_group_img_num*img_channel], name='imgs_in')
self.imgs_pla_eval = tf.placeholder(tf.float32, [batchsize, img_size_h*2, img_size_w*2, G_group_img_num*img_channel], name='imgs_in_eval')
self.training=tf.placeholder(tf.bool, name='training_in') #这里没用上但是为了兼容就保留了
self.timerates_pla=tf.placeholder(tf.float32, [batchsize], name='timerates_in')
self.timerates_expand=tf.expand_dims(self.timerates_pla, -1)
self.timerates_expand=tf.expand_dims(self.timerates_expand, -1)
self.timerates_expand=tf.expand_dims(self.timerates_expand, -1) #12*1*1*1
print ('placeholders:\n','img_placeholder:',self.imgs_pla,self.timerates_pla)
#img_placeholder: Tensor("imgs_in:0", shape=(10, 180, 320, 9), dtype=float32) Tensor("timerates_in:0", shape=(10,), dtype=float32)
self.frame0=self.imgs_pla[:,:,:,:img_channel]
self.frame1=self.imgs_pla[:,:,:,img_channel:img_channel*2]
self.frame2=self.imgs_pla[:,:,:,img_channel*2:]
#这里用来进行evaluate注意这里只用于输出最终图
self.frame0_eval=self.imgs_pla_eval[:,:,:,:img_channel]
self.frame1_eval=self.imgs_pla_eval[:,:,:,img_channel:img_channel*2]
self.frame2_eval=self.imgs_pla_eval[:,:,:,img_channel*2:]
with tf.variable_scope("first_unet", reuse=tf.AUTO_REUSE) as scope:
firstinput=tf.concat([self.frame0, self.frame2], -1)
self.first_opticalflow=my_unet( firstinput, 4,training=self.training , withbias=True, withbn=False) #注意这里是直接作为optical flow
firstinput=tf.concat([self.frame0_eval, self.frame2_eval], -1)
self.first_opticalflow_eval=my_unet( firstinput, 4,training=self.training , withbias=True, withbn=False) #注意这里是直接作为optical flow
#self.first_opticalflow=my_unet_split( firstinput, 4,training=self.training , withbias=True, withbn=True) #注意这里是直接作为optical flow
self.first_opticalflow_0_1=self.first_opticalflow[:, :, :, :2]
self.first_opticalflow_0_1=tf.identity(self.first_opticalflow_0_1, name="first_opticalflow_0_1")
print ('first_opticalflow_0_1:',self.first_opticalflow_0_1)
self.first_opticalflow_1_0=self.first_opticalflow[:, :, :, 2:]
self.first_opticalflow_1_0=tf.identity(self.first_opticalflow_1_0, name="first_opticalflow_1_0")
print ('first_opticalflow_1_0:',self.first_opticalflow_1_0)
#first_opticalflow_0_1: Tensor("first_opticalflow_0_1:0", shape=(10, 180, 320, 2), dtype=float32)
#first_opticalflow_1_0: Tensor("first_opticalflow_1_0:0", shape=(10, 180, 320, 2), dtype=float32)
self.first_opticalflow_0_1_eval=self.first_opticalflow_eval[:, :, :, :2]
self.first_opticalflow_1_0_eval=self.first_opticalflow_eval[:, :, :, 2:]
#输出光流形状
self.flow_size_h=self.first_opticalflow_0_1.get_shape().as_list()[1]
self.flow_size_w=self.first_opticalflow_0_1.get_shape().as_list()[2]
self.flow_channel=self.first_opticalflow_0_1.get_shape().as_list()[-1]
#eval shape
self.flow_size_h_eval=self.first_opticalflow_eval.get_shape().as_list()[1]
self.flow_size_w_eval=self.first_opticalflow_eval.get_shape().as_list()[2]
########################################################
self.step2_flow_channel=4
self.flow_shape=[ self.flow_size_h, self.flow_size_w, self.step2_flow_channel]
#lstm的每个状态（c，h）的形状
self.state_shape=[2, 1, self.flow_size_h, self.flow_size_w, self.step2_flow_channel]
self.state_shape_eval=[2, 1, self.flow_size_h_eval, self.flow_size_w_eval, self.step2_flow_channel]
#获取数据时的一些cpu上的参数，用于扩张数据和判定时序
self.last_flow_init_np=np.zeros(self.state_shape, dtype=np.float32)
self.last_flow_init_np_eval=np.zeros(self.state_shape_eval, dtype=np.float32)
print (self.last_flow_init_np.shape, self.last_flow_init_np_eval.shape) #(2, 1, 180, 320, 5) (2, 1, 360, 640, 5)
##############################################################
self.last_optical_flow=tf.placeholder(tf.float32, self.state_shape, name='second_last_flow')
self.last_optical_flow_eval=tf.placeholder(tf.float32, self.state_shape_eval, name='second_last_flow_eval')
#初始化train和test的初始0状态
self.last_flow_new_train=self.last_flow_init_np
self.last_flow_new_test=self.last_flow_init_np
#反向光流算中间帧
self.first_opticalflow_t_0=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.first_opticalflow_0_1 ,\
self.timerates_expand*self.timerates_expand*self.first_opticalflow_1_0 , name="first_opticalflow_t_0")
self.first_opticalflow_t_2=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.first_opticalflow_0_1 ,\
self.timerates_expand*(self.timerates_expand-1)*self.first_opticalflow_1_0, name="first_opticalflow_t_2")
#反向光流算中间帧
self.first_opticalflow_t_0_eval=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.first_opticalflow_0_1_eval ,\
self.timerates_expand*self.timerates_expand*self.first_opticalflow_1_0_eval , name="first_opticalflow_t_0_eval")
self.first_opticalflow_t_2_eval=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.first_opticalflow_0_1_eval ,\
self.timerates_expand*(self.timerates_expand-1)*self.first_opticalflow_1_0_eval, name="first_opticalflow_t_2_eval")
#2种方法合成t时刻的帧
self.first_img_flow_2_t=self.warp_op(self.frame2, -self.first_opticalflow_t_2) #!!!
self.first_img_flow_0_t=self.warp_op(self.frame0, -self.first_opticalflow_t_0) #!!!
#2种方法合成t时刻的帧
self.first_img_flow_2_t_eval=self.warp_op(self.frame2_eval, -self.first_opticalflow_t_2_eval) #!!!
self.first_img_flow_0_t_eval=self.warp_op(self.frame0_eval, -self.first_opticalflow_t_0_eval) #!!!
#虽然论文里用不到第一步的输出中间帧，但是这里也给他输出看看效果
self.first_output=tf.add( self.timerates_expand*self.first_img_flow_2_t, (1-self.timerates_expand)*self.first_img_flow_0_t , name="first_outputimg")
print ('first output img:',self.first_output)
#first output img: Tensor("first_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
#利用光流前后帧互相合成
self.first_img_flow_2_0=self.warp_op(self.frame2, -self.first_opticalflow_0_1) #frame2->frame0
self.first_img_flow_0_2=self.warp_op(self.frame0, -self.first_opticalflow_1_0) #frame0->frame2
####################################################################################################################3
#第二个unet
with tf.variable_scope("second_unet", reuse=tf.AUTO_REUSE) as scope:
secinput=tf.concat([self.frame0, self.frame2, \
self.first_opticalflow_0_1, self.first_opticalflow_1_0, \
self.first_opticalflow_t_2, self.first_opticalflow_t_0,\
self.first_img_flow_2_t, self.first_img_flow_0_t,\
], -1) #self.last_optical_flow
secinput=tf.expand_dims(secinput, 0)
print ("secinput:",secinput)#secinput: Tensor("second_unet/ExpandDims:0", shape=(1, 10, 180, 320, 20), dtype=float32)
secinput_eval=tf.concat([self.frame0_eval, self.frame2_eval, \
self.first_opticalflow_0_1_eval, self.first_opticalflow_1_0_eval, \
self.first_opticalflow_t_2_eval, self.first_opticalflow_t_0_eval,\
self.first_img_flow_2_t_eval, self.first_img_flow_0_t_eval,\
], -1) #self.last_optical_flow
secinput_eval=tf.expand_dims(secinput_eval, 0)
print ("secinput_eval:",secinput_eval)#secinput_eval: Tensor("second_unet/ExpandDims_1:0", shape=(1, 10, 360, 640, 20), dtype=float32)
lstm_input_channel=secinput.get_shape().as_list()[-1]
self.cell = tf.contrib.rnn.ConvLSTMCell(conv_ndims=2, input_shape=[self.flow_size_h, self.flow_size_w, lstm_input_channel], \
output_channels=self.step2_flow_channel, kernel_shape=[5, 5])
self.cell_eval = tf.contrib.rnn.ConvLSTMCell(conv_ndims=2, input_shape=[self.flow_size_h_eval, self.flow_size_w_eval, lstm_input_channel], \
output_channels=self.step2_flow_channel, kernel_shape=[5, 5])
lstm_outputs, lstm_state_final = tf.nn.dynamic_rnn(self.cell, inputs =secinput , \
initial_state = tf.nn.rnn_cell.LSTMStateTuple(self.last_optical_flow[0], self.last_optical_flow[1]), time_major = False)
lstm_outputs_eval, lstm_state_final_eval = tf.nn.dynamic_rnn(self.cell_eval, inputs =secinput_eval , \
initial_state = tf.nn.rnn_cell.LSTMStateTuple(self.last_optical_flow_eval[0], self.last_optical_flow_eval[1]), time_major = False)
self.second_batch_last_flow=tf.stack([lstm_state_final.c, lstm_state_final.h], 0)
self.second_batch_last_flow=tf.identity(self.second_batch_last_flow, name="second_batch_last_flow")
print ("second_batch_last_flow:",self.second_batch_last_flow)
#second_batch_last_flow: Tensor("second_unet/second_batch_last_flow:0", shape=(2, 1, 180, 320, 5), dtype=float32)
self.second_batch_last_flow_eval=tf.stack([lstm_state_final_eval.c, lstm_state_final_eval.h], 0)
self.second_batch_last_flow_eval=tf.identity(self.second_batch_last_flow_eval, name="second_batch_last_flow_eval")
print ("second_batch_last_flow_eval:",self.second_batch_last_flow_eval)
#Tensor("second_unet/second_batch_last_flow_eval:0", shape=(2, 1, 360, 640, 5), dtype=float32)
self.second_opticalflow=lstm_outputs[0]
print ("self.second_opticalflow:",self.second_opticalflow)
#self.second_opticalflow: Tensor("second_unet/strided_slice_2:0", shape=(10, 180, 320, 5), dtype=float32)
self.second_opticalflow_eval=lstm_outputs_eval[0]
print ("self.second_opticalflow_eval:",self.second_opticalflow_eval)
#Tensor("second_unet/strided_slice_5:0", shape=(10, 360, 640, 5), dtype=float32)
self.second_opticalflow_t_0=tf.add( self.second_opticalflow[:,:,:,:2], self.first_opticalflow_t_0, name="second_opticalflow_t_0")
self.second_opticalflow_t_1=tf.add( self.second_opticalflow[:,:,:,2:4], self.first_opticalflow_t_2, name="second_opticalflow_t_1")
print ('second_opticalflow_t_0:',self.second_opticalflow_t_0)
print ('second_opticalflow_t_1:',self.second_opticalflow_t_1)
#second_opticalflow_t_0: Tensor("second_opticalflow_t_0:0", shape=(10, 180, 320, 2), dtype=float32)
#second_opticalflow_t_1: Tensor("second_opticalflow_t_1:0", shape=(10, 180, 320, 2), dtype=float32)
self.second_opticalflow_t_0_eval=tf.add( self.second_opticalflow_eval[:,:,:,:2], self.first_opticalflow_t_0_eval, name="second_opticalflow_t_0_eval")
self.second_opticalflow_t_1_eval=tf.add( self.second_opticalflow_eval[:,:,:,2:4], self.first_opticalflow_t_2_eval, name="second_opticalflow_t_1_eval")
print ('second_opticalflow_t_0_eval:',self.second_opticalflow_t_0_eval)
print ('second_opticalflow_t_1_eval:',self.second_opticalflow_t_1_eval)
#second_opticalflow_t_0_eval: Tensor("second_opticalflow_t_0_eval:0", shape=(10, 360, 640, 2), dtype=float32)
#second_opticalflow_t_1_eval: Tensor("second_opticalflow_t_1_eval:0", shape=(10, 360, 640, 2), dtype=float32)
'''
self.vmap_t_0=tf.expand_dims( tf.sigmoid(self.second_opticalflow[:,:,:,-1]) , -1)
self.vmap_t_1=1-self.vmap_t_0
self.vmap_t_0_eval=tf.expand_dims( tf.sigmoid(self.second_opticalflow_eval[:,:,:,-1]) , -1)
self.vmap_t_1_eval=1-self.vmap_t_0_eval
'''
#2种方法合成t时刻的帧
self.second_img_flow_1_t=self.warp_op(self.frame2, -self.second_opticalflow_t_1) #!!!
self.second_img_flow_0_t=self.warp_op(self.frame0, -self.second_opticalflow_t_0) #!!!
#2种方法合成t时刻的帧
self.second_img_flow_1_t_eval=self.warp_op(self.frame2_eval, -self.second_opticalflow_t_1_eval) #!!!
self.second_img_flow_0_t_eval=self.warp_op(self.frame0_eval, -self.second_opticalflow_t_0_eval) #!!!
#最终输出的图
print (self.timerates_expand, self.second_img_flow_0_t)
#Tensor("ExpandDims_2:0", shape=(10, 1, 1, 1), dtype=float32) Tensor("ExpandDims_3:0", shape=(10, 180, 320, 1), dtype=float32)
#Tensor("dense_image_warp_5/Reshape_1:0", shape=(10, 180, 320, 3), dtype=float32)
self.second_output=tf.add( (1-self.timerates_expand)*self.second_img_flow_0_t, self.timerates_expand*self.second_img_flow_1_t , name="second_outputimg" )
print ('second output img:',self.second_output)
#second output img: Tensor("second_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
self.second_output_eval=tf.add( (1-self.timerates_expand)*self.second_img_flow_0_t_eval, self.timerates_expand*self.second_img_flow_1_t_eval , name="second_outputimg_eval" )
print ('second output img_eval:',self.second_output_eval)
#second output img_eval: Tensor("second_outputimg_eval:0", shape=(10, 360, 640, 3), dtype=float32)
#判别器的网络构建
self.D_1_net_F, self.D_1_net_F_logit=Discriminator_net(self.second_output, name="D1", training=self.training)
self.D_1_net_T, self.D_1_net_T_logit=Discriminator_net(self.frame1, name="D1", training=self.training)
#D的loss计算
self.D_1_net_loss_sum, _, _=self.D_loss_TandF_logits(self.D_1_net_T_logit, self.D_1_net_F_logit, "D_1_net")
#计算loss
self.second_L1_loss_interframe,self.first_warp_loss,self.second_contex_loss,self.second_local_var_loss_all,self.second_global_var_loss_all,self.second_ssim,self.second_psnr,\
self.first_L1_loss_interframe, self.first_ssim, self.first_psnr, self.second_GAN_loss_mean_D1=self.loss_cal_all()
#训练G的总loss
self.G_loss_all=204 * self.second_L1_loss_interframe + 102 * self.first_warp_loss + 0.005 * self.second_contex_loss \
+self.second_global_var_loss_all
#+ self.second_GAN_loss_mean_D1*0.03
#训练D的总loss
self.D_loss_all=self.D_1_net_loss_sum
#####################################
self.last_label_train='#'
self.last_label_test='#'
self.state_random_row_train=0
self.state_random_col_train=0
self.state_flip_train=False
self.state_random_row_test=0
self.state_random_col_test=0
self.state_flip_test=False
#为了兼容性
self.batchsize_inputimg=batchsize
self.img_size_w=img_size_w
self.img_size_h=img_size_h
t_vars=tf.trainable_variables()
print ("trainable vars cnt:",len(t_vars))
self.first_para=[var for var in t_vars if var.name.startswith('first')]
self.sec_para=[var for var in t_vars if var.name.startswith('second')]
self.vgg_para=[var for var in t_vars if var.name.startswith('VGG')]
self.D_para=[var for var in t_vars if var.name.startswith('D')]
print ("first param len:",len(self.first_para))
print ("second param len:",len(self.sec_para))
print ("VGG param len:",len(self.vgg_para))
print ("D param len:",len(self.D_para))
print (self.vgg_para)
'''
trainable vars cnt: 114
first param len: 46
second param len: 2
VGG param len: 52
D param len: 14
'''
#G训练过程
self.lr_rate = tf.train.exponential_decay(base_lr, global_step=self.global_step, decay_steps=decay_steps, decay_rate=decay_rate)
self.train_op_G = tf.train.AdamOptimizer(self.lr_rate, beta1=beta1, name="superslomo_adam_G").minimize(self.G_loss_all, \
global_step=self.global_step , var_list=self.first_para+self.sec_para )
# weight clipping
self.clip_D = [p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max)) for p in self.D_para]
#D训练过程
self.train_op_D= tf.train.AdamOptimizer(self.lr_rate , beta1=beta1, name="superslomo_adam_D").minimize(self.D_loss_all, var_list=self.D_para)
#最后构建完成后初始化参数
self.sess.run(tf.global_variables_initializer())
def __init__(self, sess):
self.sess = sess
#加载原模型
saver = tf.train.import_meta_graph(op.join(modelpath, meta_name))
saver.restore(self.sess, tf.train.latest_checkpoint(modelpath))
self.graph = tf.get_default_graph()
self.global_step = tf.Variable(0.0,
name='step2_global_step',
dtype=tf.float32,
trainable=False)
#placeholders
self.imgs_pla = self.graph.get_tensor_by_name('imgs_in:0')
self.training = self.graph.get_tensor_by_name("training_in:0")
self.timerates = self.graph.get_tensor_by_name("timerates_in:0")
#self.timerates_pla=tf.placeholder(tf.float32, [batchsize], name='step2_inner_timerates_in')
self.timerates_expand = tf.expand_dims(self.timerates, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand,
-1) #batchsize*1*1*1
#tesorfs
#这里是第一部的输出光流
self.optical_0_1 = self.graph.get_tensor_by_name("G_opticalflow_0_2:0")
self.optical_1_0 = self.graph.get_tensor_by_name("G_opticalflow_2_0:0")
self.outimg = self.graph.get_tensor_by_name("G_net_generate:0")
#第一部中的batchsize，这里可以当作timestep使用
self.batchsize_inputimg = self.imgs_pla.get_shape().as_list()[0]
#输入图像形状
self.img_size_w = self.imgs_pla.get_shape().as_list()[2]
self.img_size_h = self.imgs_pla.get_shape().as_list()[1]
self.img_size = [self.img_size_h, self.img_size_w]
print(self.imgs_pla
) #Tensor("imgs_in:0", shape=(12, 180, 320, 9), dtype=float32)
self.frame0 = self.imgs_pla[:, :, :, :img_channel]
self.frame1 = self.imgs_pla[:, :, :, img_channel:img_channel * 2]
self.frame2 = self.imgs_pla[:, :, :, img_channel * 2:]
self.pipline_data_train = cdata.get_pipline_data_train(
self.img_size, self.batchsize_inputimg)
self.pipline_data_test = cdata.get_pipline_data_test(
self.img_size, self.batchsize_inputimg)
#输出光流形状
self.flow_size_h = self.optical_0_1.get_shape().as_list()[1]
self.flow_size_w = self.optical_0_1.get_shape().as_list()[2]
self.flow_channel = self.optical_0_1.get_shape().as_list()[-1]
self.flow_shape = [
self.flow_size_h, self.flow_size_w, self.flow_channel * 2
]
#last flow placeholder
self.last_optical_flow = tf.placeholder(tf.float32,
self.flow_shape,
name='step2_last_flow')
#这里将batch中的第一组中的前后帧和前后光流拼起来
input_pla = tf.concat([
self.frame0[0], self.frame2[0], self.optical_0_1[0],
self.optical_1_0[0], self.last_optical_flow
], -1) #这里将两个光流拼起来 可以考虑将前后帧也拼起来
print(input_pla
) #Tensor("concat_9:0", shape=(180, 320, 14), dtype=float32)
with tf.variable_scope("STEP2", reuse=tf.AUTO_REUSE) as scopevar:
new_flow = self.step2_network(input_pla, training=self.training)
kep_new_flow = [new_flow]
for ti in range(1, self.batchsize_inputimg):
input_pla = tf.concat([
self.frame0[ti], self.frame2[ti], self.optical_0_1[ti],
self.optical_1_0[ti], new_flow
], -1) #14
new_flow = self.step2_network(input_pla,
training=self.training)
kep_new_flow.append(new_flow)
self.flow_next = new_flow
self.flow_after = tf.stack(kep_new_flow,
axis=0,
name='step2_opticalflow')
print(
'self.flow_after:', self.flow_after
) #Tensor("STEP2/step2_opticalflow:0", shape=(12, 180, 320, 4), dtype=float32)
print(
'self.flow_next:', self.flow_next
) #Tensor("STEP2/strided_slice_79:0", shape=(180, 320, 4), dtype=float32)
self.last_flow_init_np = np.zeros(self.flow_shape, dtype=np.float32)
print(self.last_flow_init_np.shape) #(180, 320, 4)
#初始化train和test的初始0状态
self.last_flow_new_train = self.last_flow_init_np
self.last_flow_new_test = self.last_flow_init_np
#########################################################################
self.opticalflow_0_2 = tf.slice(self.flow_after, [0, 0, 0, 0],
[-1, -1, -1, 2],
name='step2_opticalflow_0_2')
self.opticalflow_2_0 = tf.slice(self.flow_after, [0, 0, 0, 2],
[-1, -1, -1, 2],
name='step2_opticalflow_2_0')
print('original flow:', self.opticalflow_0_2, self.opticalflow_2_0)
#获取数据时的一些cpu上的参数，用于扩张数据和判定时序
self.last_label_train = '#'
self.last_label_test = '#'
self.state_random_row_train = 0
self.state_random_col_train = 0
self.state_flip_train = False
self.state_random_row_test = 0
self.state_random_col_test = 0
self.state_flip_test = False
t_vars = tf.trainable_variables()
print("trainable vars cnt:", len(t_vars))
self.G_para = [var for var in t_vars if var.name.startswith('G')]
self.D_para = [var for var in t_vars if var.name.startswith('D')]
self.STEP2_para = [
var for var in t_vars if var.name.startswith('STEP2')
]
print("G param len:", len(self.G_para))
print("D param len:", len(self.D_para))
print("STEP2 param len:", len(self.STEP2_para))
print(self.STEP2_para)
'''
trainable vars cnt: 184
G param len: 60
D param len: 16
STEP2 param len: 56
剩下的52个是VGG
相比于前面不加第二部的128个，这里注意将VGG与step1中的VGG共享参数，否则会白白多用内存
'''
# weight clipping
self.clip_D = [
p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max))
for p in self.D_para
]
self.step1_L1_loss_all,self.step1_contex_loss,self.step1_local_var_loss_all,self.step1_global_var_loss_all,self.step1_G_loss_all,\
self.step1_ssim,self.step1_psnr,self.step1_G_net=self.loss_cal(self.optical_0_1, self.optical_1_0, LR_step1, self.G_para, scopevar.name+"_step1_losscal")
self.step2_L1_loss_all,self.step2_contex_loss,self.step2_local_var_loss_all, self.step2_global_var_loss_all, self.step2_G_loss_all,\
self.step2_ssim,self.step2_psnr,self.step2_G_net=self.loss_cal(self.opticalflow_0_2, self.opticalflow_2_0, LR, self.STEP2_para, scopevar.name+"_step2_losscal")
Loss_merge=10*self.step1_L1_loss_all + 5*self.step1_contex_loss +\
102*self.step2_L1_loss_all + 10*self.step2_contex_loss + self.step2_global_var_loss_all*0.1
#训练过程
self.lr_rate = tf.train.exponential_decay(LR,
global_step=self.global_step,
decay_steps=decay_steps,
decay_rate=decay_rate)
self.train_op = tf.train.AdamOptimizer(
self.lr_rate, name="step2_v2_adam").minimize(
Loss_merge,
global_step=self.global_step,
var_list=self.STEP2_para + self.G_para)
#最后构建完成后初始化参数
self.sess.run(tf.global_variables_initializer())
def __init__(self, sess):
self.sess = sess
self.global_step = tf.Variable(0.0,
name='global_step',
dtype=tf.float32,
trainable=False)
#for data input
self.pipline_data_train = cdata.get_pipline_data_train(
img_size, batchsize)
self.pipline_data_test = cdata.get_pipline_data_test(
img_size, batchsize_test)
#3个placeholder， img和noise,training
self.imgs_pla = tf.placeholder(
tf.float32,
[batchsize, img_size_h, img_size_w, G_group_img_num * img_channel],
name='imgs_in')
self.training = tf.placeholder(tf.bool,
name='training_in') #这里没用上但是为了兼容就保留了
self.timerates_pla = tf.placeholder(tf.float32, [batchsize],
name='timerates_in')
self.timerates_expand = tf.expand_dims(self.timerates_pla, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand,
-1) #12*1*1*1
print('placeholders:\n', 'img_placeholder:', self.imgs_pla,
self.timerates_pla)
#img_placeholder: Tensor("imgs_in:0", shape=(10, 180, 320, 9), dtype=float32) Tensor("timerates_in:0", shape=(10,), dtype=float32)
self.frame0 = self.imgs_pla[:, :, :, :img_channel]
self.frame1 = self.imgs_pla[:, :, :, img_channel:img_channel * 2]
self.frame2 = self.imgs_pla[:, :, :, img_channel * 2:]
with tf.variable_scope("first_unet", reuse=tf.AUTO_REUSE) as scope:
firstinput = tf.concat([self.frame0, self.frame2], -1)
self.first_opticalflow = my_unet(
firstinput, 4, withbias=True) #注意这里是直接作为optical flow
self.first_opticalflow_0_1 = self.first_opticalflow[:, :, :, :2]
self.first_opticalflow_0_1 = tf.identity(self.first_opticalflow_0_1,
name="first_opticalflow_0_1")
print('first_opticalflow_0_1:', self.first_opticalflow_0_1)
self.first_opticalflow_1_0 = self.first_opticalflow[:, :, :, 2:]
self.first_opticalflow_1_0 = tf.identity(self.first_opticalflow_1_0,
name="first_opticalflow_1_0")
print('first_opticalflow_1_0:', self.first_opticalflow_1_0)
#first_opticalflow_0_1: Tensor("first_opticalflow_0_1:0", shape=(10, 180, 320, 2), dtype=float32)
#first_opticalflow_1_0: Tensor("first_opticalflow_1_0:0", shape=(10, 180, 320, 2), dtype=float32)
#输出光流形状
self.flow_size_h = self.first_opticalflow_0_1.get_shape().as_list()[1]
self.flow_size_w = self.first_opticalflow_0_1.get_shape().as_list()[2]
self.flow_channel = self.first_opticalflow_0_1.get_shape().as_list(
)[-1]
self.flow_shape = [
self.flow_size_h, self.flow_size_w, self.flow_channel * 2
]
#反向光流算中间帧
self.first_opticalflow_t_0=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.first_opticalflow_0_1 ,\
self.timerates_expand*self.timerates_expand*self.first_opticalflow_1_0 , name="first_opticalflow_t_0")
self.first_opticalflow_t_2=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.first_opticalflow_0_1 ,\
self.timerates_expand*(self.timerates_expand-1)*self.first_opticalflow_1_0, name="first_opticalflow_t_2")
#2种方法合成t时刻的帧
self.first_img_flow_2_t = self.warp_op(
self.frame2, -self.first_opticalflow_t_2) #!!!
self.first_img_flow_0_t = self.warp_op(
self.frame0, -self.first_opticalflow_t_0) #!!!
#虽然论文里用不到第一步的输出中间帧，但是这里也给他输出看看效果
self.first_output = tf.add(
self.timerates_expand * self.first_img_flow_2_t,
(1 - self.timerates_expand) * self.first_img_flow_0_t,
name="first_outputimg")
print('first output img:', self.first_output)
#first output img: Tensor("first_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
#利用光流前后帧互相合成
self.first_img_flow_2_0 = self.warp_op(
self.frame2, -self.first_opticalflow_0_1) #frame2->frame0
self.first_img_flow_0_2 = self.warp_op(
self.frame0, -self.first_opticalflow_1_0) #frame0->frame2
####################################################################################################################3
#第二个unet
with tf.variable_scope("second_unet", reuse=tf.AUTO_REUSE) as scope:
secinput=tf.concat([self.frame0, self.frame2, \
self.first_opticalflow_0_1, self.first_opticalflow_1_0, \
self.first_opticalflow_t_2, self.first_opticalflow_t_0,\
self.first_img_flow_2_t, self.first_img_flow_0_t], -1)
print(secinput)
self.second_opticalflow = my_unet(
secinput, 5, withbias=True) #注意这里是直接作为optical flow
self.second_opticalflow_t_0 = tf.add(
self.second_opticalflow[:, :, :, :2],
self.first_opticalflow_t_0,
name="second_opticalflow_t_0")
self.second_opticalflow_t_1 = tf.add(self.second_opticalflow[:, :, :,
2:4],
self.first_opticalflow_t_2,
name="second_opticalflow_t_1")
print('second_opticalflow_t_0:', self.second_opticalflow_t_0)
print('second_opticalflow_t_1:', self.second_opticalflow_t_1)
#second_opticalflow_t_0: Tensor("second_opticalflow_t_0:0", shape=(10, 180, 320, 2), dtype=float32)
#second_opticalflow_t_1: Tensor("second_opticalflow_t_1:0", shape=(10, 180, 320, 2), dtype=float32)
self.vmap_t_0 = tf.expand_dims(
tf.sigmoid(self.second_opticalflow[:, :, :, -1]), -1)
self.vmap_t_1 = 1 - self.vmap_t_0
#2种方法合成t时刻的帧
self.second_img_flow_1_t = self.warp_op(
self.frame2, -self.second_opticalflow_t_1) #!!!
self.second_img_flow_0_t = self.warp_op(
self.frame0, -self.second_opticalflow_t_0) #!!!
#最终输出的图
print(self.timerates_expand, self.vmap_t_0, self.second_img_flow_0_t)
#Tensor("ExpandDims_2:0", shape=(6, 1, 1, 1), dtype=float32) Tensor("Sigmoid:0", shape=(6, 180, 320, 1), dtype=float32)
#Tensor("dense_image_warp_5/Reshape_1:0", shape=(6, 180, 320, 3), dtype=float32)
self.second_output=tf.div( ( (1-self.timerates_expand)*self.vmap_t_0*self.second_img_flow_0_t+self.timerates_expand*self.vmap_t_1*self.second_img_flow_1_t), \
((1-self.timerates_expand)*self.vmap_t_0+self.timerates_expand*self.vmap_t_1) , name="second_outputimg" )
print('second output img:', self.second_output)
#second output img: Tensor("second_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
#计算loss
self.second_L1_loss_interframe,self.first_warp_loss,self.second_contex_loss,self.second_local_var_loss_all,self.second_global_var_loss_all,self.second_ssim,self.second_psnr,\
self.first_L1_loss_interframe, self.first_ssim, self.first_psnr=self.loss_cal_all()
self.G_loss_all = 204 * self.second_L1_loss_interframe + 102 * self.first_warp_loss + 0.005 * self.second_contex_loss + self.second_global_var_loss_all
#获取数据时的一些cpu上的参数，用于扩张数据和判定时序
self.last_flow_init_np = np.zeros(self.flow_shape, dtype=np.float32)
print(self.last_flow_init_np.shape) #(180, 320, 4)
#初始化train和test的初始0状态
self.last_flow_new_train = self.last_flow_init_np
self.last_flow_new_test = self.last_flow_init_np
self.last_label_train = '#'
self.last_label_test = '#'
self.state_random_row_train = 0
self.state_random_col_train = 0
self.state_flip_train = False
self.state_random_row_test = 0
self.state_random_col_test = 0
self.state_flip_test = False
#为了兼容性
self.batchsize_inputimg = batchsize
self.img_size_w = img_size_w
self.img_size_h = img_size_h
t_vars = tf.trainable_variables()
print("trainable vars cnt:", len(t_vars))
self.first_para = [
var for var in t_vars if var.name.startswith('first')
]
self.sec_para = [
var for var in t_vars if var.name.startswith('second')
]
self.vgg_para = [var for var in t_vars if var.name.startswith('VGG')]
print("first param len:", len(self.first_para))
print("second param len:", len(self.sec_para))
print("VGG param len:", len(self.vgg_para))
print(self.vgg_para)
'''
trainable vars cnt: 144
first param len: 46
second param len: 46
VGG param len: 52
'''
#训练过程
self.lr_rate = tf.train.exponential_decay(base_lr,
global_step=self.global_step,
decay_steps=decay_steps,
decay_rate=decay_rate)
self.train_op = tf.train.AdamOptimizer(self.lr_rate, name="superslomo_adam").minimize(self.G_loss_all, \
global_step=self.global_step , var_list=self.first_para+self.sec_para )
# weight clipping
#self.clip_D = [p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max)) for p in self.D_para]
#最后构建完成后初始化参数
self.sess.run(tf.global_variables_initializer())
def __init__(self, sess):
self.sess = sess
#加载原模型
saver = tf.train.import_meta_graph(op.join(modelpath, meta_name) )
saver.restore(self.sess, tf.train.latest_checkpoint(modelpath))
self.graph = tf.get_default_graph()
self.global_step = tf.Variable(0.0, name='step2_global_step',dtype=tf.float32, trainable=False)
#placeholders
self.imgs_pla= self.graph.get_tensor_by_name('imgs_in:0')
self.training= self.graph.get_tensor_by_name("training_in:0")
self.timerates= self.graph.get_tensor_by_name("timerates_in:0")
#tesorfs
self.optical_0_1=self.graph.get_tensor_by_name("G_opticalflow_0_2:0")
self.optical_1_0=self.graph.get_tensor_by_name("G_opticalflow_2_0:0")
self.outimg = self.graph.get_tensor_by_name("G_net_generate:0")
# 注意这里构建lstm的输入是根据第一部的输出直接来的
#self.input_pla = tf.placeholder(tf.float32, [batchsize, flow_size_h, flow_size_w, input_channel], name='step2_opticalflow_in')
self.input_pla=tf.concat([self.imgs_pla[:,:,:,:img_channel], self.optical_0_1, self.optical_1_0, self.imgs_pla[:,:,:, img_channel*2:]], -1) #这里将两个光流拼起来 可以考虑将前后帧也拼起来
self.lstm_input_channel=self.input_pla.get_shape().as_list()[-1]
print (self.input_pla) #Tensor("concat_9:0", shape=(12, 180, 320, ?), dtype=float32)
#第一部中的batchsize，这里可以当作timestep使用
self.batchsize_inputimg=self.imgs_pla.get_shape().as_list()[0]
#输入图像形状
self.img_size_w=self.imgs_pla.get_shape().as_list()[2]
self.img_size_h=self.imgs_pla.get_shape().as_list()[1]
self.img_size=[self.img_size_h, self.img_size_w]
print (self.imgs_pla) #Tensor("imgs_in:0", shape=(12, 180, 320, 9), dtype=float32)
self.pipline_data_train=cdata.get_pipline_data_train(self.img_size, self.batchsize_inputimg)
self.pipline_data_test=cdata.get_pipline_data_test(self.img_size, self.batchsize_inputimg)
#输出光流形状
self.flow_size_h=self.optical_0_1.get_shape().as_list()[1]
self.flow_size_w=self.optical_0_1.get_shape().as_list()[2]
self.flow_channel=self.optical_0_1.get_shape().as_list()[-1]
#lstm的每个状态（c，h）的形状
self.state_shape=[batchsize, self.flow_size_h, self.flow_size_w, output_channel]
#state placeholder
self.state_pla_c = tf.placeholder(tf.float32, self.state_shape, name='step2_state_in_c')
self.state_pla_h = tf.placeholder(tf.float32, self.state_shape, name='step2_state_in_h')
#获取数据时的一些cpu上的参数，用于扩张数据和判定时序
self.last_label_train='#'
self.last_label_test='#'
self.state_random_row_train=0
self.state_random_col_train=0
self.state_flip_train=False
self.state_random_row_test=0
self.state_random_col_test=0
self.state_flip_test=False
#self.imgs_pla = tf.placeholder(tf.float32, [batchsize, img_size_h, img_size_w, group_img_num*img_channel], name='step2_oriimgs_in')
self.frame0=self.imgs_pla[:,:,:,:img_channel]
self.frame1=self.imgs_pla[:,:,:,img_channel:img_channel*2]
self.frame2=self.imgs_pla[:,:,:,img_channel*2:]
#self.timerates_pla=tf.placeholder(tf.float32, [batchsize], name='step2_inner_timerates_in')
self.timerates_expand=tf.expand_dims(self.timerates, -1)
self.timerates_expand=tf.expand_dims(self.timerates_expand, -1)
self.timerates_expand=tf.expand_dims(self.timerates_expand, -1) #batchsize*1*1*1
with tf.variable_scope("STEP2", reuse=tf.AUTO_REUSE) as scopevar:
self.cell = tf.contrib.rnn.ConvLSTMCell(conv_ndims=2, input_shape=[self.flow_size_h, self.flow_size_w, self.lstm_input_channel], \
output_channels=output_channel, kernel_shape=[kernel_len, kernel_len])
self.state_init = self.cell.zero_state(batch_size=batchsize, dtype=tf.float32)
self.state_init_np=( np.zeros(self.state_shape, dtype=np.float32), np.zeros(self.state_shape, dtype=np.float32) )
print (self.state_init) #LSTMStateTuple(c=<tf.Tensor 'ConvLSTMCellZeroState/zeros:0' shape=(2, 180, 320, 12) dtype=float32>, h=<tf.Tensor 'ConvLSTMCellZeroState/zeros_1:0' shape=(2, 180, 320, 12) dtype=float32>)
#初始化train和test的初始0状态
self.state_new_train=self.state_init_np
self.state_new_test=self.state_init_np
#这里开始搞lstm了
self.input_dynamic_lstm=tf.expand_dims(self.input_pla, 0) #这里默认lstm的输入batchsize=1，注意，设置里batchsize必须为1
print (self.input_dynamic_lstm) #Tensor("ExpandDims_6:0", shape=(1, 12, 180, 320, 4), dtype=float32)
self.outputs, self.state_final = tf.nn.dynamic_rnn(self.cell, inputs =self.input_dynamic_lstm , initial_state = tf.nn.rnn_cell.LSTMStateTuple(self.state_pla_c, self.state_pla_h), time_major = False)
#self.output,self.state_final=self.cell.call(inputs=self.input_pla,state=(self.state_pla_c, self.state_pla_h) )
print (self.outputs,self.state_final)
#Tensor("rnn/transpose_1:0", shape=(1, 12, 180, 320, 12), dtype=float32)
#LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(1, 180, 320, 12) dtype=float32>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(1, 180, 320, 12) dtype=float32>)
self.flow_after=self.final_convlayer(self.outputs[0])
print (self.flow_after) #Tensor("final_convlayer/BiasAdd:0", shape=(12, 180, 320, 4), dtype=float32)
self.opticalflow_0_2=tf.slice(self.flow_after, [0, 0, 0, 0], [-1, -1, -1, 2], name='step2_opticalflow_0_2')
self.opticalflow_2_0=tf.slice(self.flow_after, [0, 0, 0, 2], [-1, -1, -1, 2], name='step2_opticalflow_2_0')
print ('original flow:',self.opticalflow_0_2, self.opticalflow_2_0)
#original flow: Tensor("step2_opticalflow_0_2:0", shape=(12, 180, 320, 2), dtype=float32) Tensor("step2_opticalflow_2_0:0", shape=(12, 180, 320, 2), dtype=float32)
# #反向光流算中间帧
# self.opticalflow_t_0=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.opticalflow_0_2 ,\
# self.timerates_expand*self.timerates_expand*self.opticalflow_2_0 , name="step2_opticalflow_t_0")
# self.opticalflow_t_2=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.opticalflow_0_2 ,\
# self.timerates_expand*(self.timerates_expand-1)*self.opticalflow_2_0, name="step2_opticalflow_t_2")
#
# print ('two optical flow:',self.opticalflow_t_0, self.opticalflow_t_2)
# #two optical flow: Tensor("step2_opticalflow_t_0:0", shape=(12, 180, 320, 2), dtype=float32) Tensor("step2_opticalflow_t_2:0", shape=(12, 180, 320, 2), dtype=float32)
#
# #2种方法合成t时刻的帧
# self.img_flow_2_t=self.warp_op(self.frame2, -self.opticalflow_t_2) #!!!
# self.img_flow_0_t=self.warp_op(self.frame0, -self.opticalflow_t_0) #!!!
#
# self.G_net=tf.add(self.timerates_expand*self.img_flow_2_t , (1-self.timerates_expand)*self.img_flow_0_t, name="step2_net_generate" )
#
# #利用光流前后帧互相合成
# self.img_flow_2_0=self.warp_op(self.frame2, self.opticalflow_2_0) #frame2->frame0
# self.img_flow_0_2=self.warp_op(self.frame0, self.opticalflow_0_2) #frame0->frame2
#
#
# #1、contex loss
# print ("forming conx loss：")
# tep_G_shape=self.G_net.get_shape().as_list()[1:]
#
# self.contex_Genera =tf.keras.applications.VGG16(include_top=False, input_tensor=self.G_net, input_shape=tep_G_shape).get_layer("block4_conv3").output
# self.contex_frame1 =tf.keras.applications.VGG16(include_top=False, input_tensor=self.frame1, input_shape=tep_G_shape).get_layer("block4_conv3").output
#
# self.contex_loss= tf.reduce_mean(tf.squared_difference( self.contex_frame1, self.contex_Genera), name='step2_Contex_loss')
# print ('step2_loss_mean_contex form finished..')
#
#
# #2、L1 loss
# print ("forming L1 loss:生成帧与GT、frame2->frame0与frame0、frame0->frame2与frame2")
# self.L1_loss_interframe =tf.reduce_mean(tf.abs( self.G_net-self.frame1 ))
# self.L1_loss_all =tf.reduce_mean(tf.abs( self.G_net-self.frame1 ) + \
# tf.abs(self.img_flow_2_0-self.frame0) + \
# tf.abs(self.img_flow_0_2-self.frame2), name='step2_G_clear_l1_loss')
# #self.G_loss_mean_Square= self.contex_loss*1 + self.L1_loss_all
# print ('step2_loss_mean_l1 form finished..')
#
# #4 local var loss
# self.local_var_loss_0_2=self.local_var_loss(self.opticalflow_0_2)
# self.local_var_loss_2_0=self.local_var_loss(self.opticalflow_2_0)
# #print ("local _var loss:",self.local_var_loss_0_2, self.G_loss_mean_D1)
# #local _var loss: Tensor("mean_local_var:0", shape=(), dtype=float32) Tensor("Mean_3:0", shape=(), dtype=float32)
# self.local_var_loss_all=tf.add(self.local_var_loss_0_2, self.local_var_loss_2_0, name="step2_local_var_add")
#
# #5 global var loss
# self.global_var_loss_0_2=self.global_var_loss(self.opticalflow_0_2)
# self.global_var_loss_2_0=self.global_var_loss(self.opticalflow_2_0)
# self.global_var_loss_all=tf.add(self.global_var_loss_0_2, self.global_var_loss_2_0, name="step2_global_var_add")
#
# #6 SSIM
# self.ssim = tf.image.ssim(self.G_net, self.frame1, max_val=2.0)
# print ("ssim:",self.ssim) #ssim: Tensor("Mean_10:0", shape=(12,), dtype=float32)
#
# #7 PSNR
# self.psnr = tf.image.psnr(self.G_net, self.frame1, max_val=2.0, name="step2_frame1_psnr")
# print ("psnr:", self.psnr) #psnr: Tensor("G_frame1_psnr/Identity_3:0", shape=(12,), dtype=float32)
#
#
# self.G_loss_all=self.contex_loss + self.L1_loss_all + self.local_var_loss_all*0.06
#
# self.train_op = tf.train.AdamOptimizer(LR, name="step2_adam").minimize(self.G_loss_all)
t_vars=tf.trainable_variables()
print ("trainable vars cnt:",len(t_vars))
self.G_para=[var for var in t_vars if var.name.startswith('G')]
self.D_para=[var for var in t_vars if var.name.startswith('D')]
self.STEP2_para=[var for var in t_vars if var.name.startswith('STEP2')]
print ("G param len:",len(self.G_para))
print ("D param len:",len(self.D_para))
print ("STEP2 param len:",len(self.STEP2_para))
print (self.STEP2_para)
'''
trainable vars cnt: 186
G param len: 60
D param len: 16
STEP2 param len: 6
相比于前面不加第二部的128个，这里注意将VGG与step1中的VGG共享参数，否则会白白多用内存
'''
# weight clipping
self.clip_D = [p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max)) for p in self.D_para]
self.step1_train_op,\
self.step1_L1_loss_all,self.step1_contex_loss,self.step1_local_var_loss_all,self.step1_G_loss_all,\
self.step1_ssim,self.step1_psnr,self.step1_G_net=self.loss_cal(self.optical_0_1, self.optical_1_0, LR_step1, self.G_para, scopevar.name)
self.step2_train_op,\
self.step2_L1_loss_all,self.step2_contex_loss,self.step2_local_var_loss_all,self.step2_G_loss_all,\
self.step2_ssim,self.step2_psnr,self.step2_G_net=self.loss_cal(self.opticalflow_0_2, self.opticalflow_2_0, LR, self.STEP2_para, scopevar.name)
#最后构建完成后初始化参数
self.sess.run(tf.global_variables_initializer())
def __init__(self, sess):
self.sess = sess
self.global_step = tf.Variable(0.0,
name='global_step',
dtype=tf.float32,
trainable=False)
#for data input
self.pipline_data_train = cdata.get_pipline_data_train(
img_size, batchsize)
self.pipline_data_test = cdata.get_pipline_data_test(
img_size, batchsize_test)
#3个placeholder， img和noise,training
self.imgs_pla = tf.placeholder(
tf.float32,
[batchsize, img_size_h, img_size_w, G_group_img_num * img_channel],
name='imgs_in')
self.training = tf.placeholder(tf.bool,
name='training_in') #这里没用上但是为了兼容就保留了
self.timerates_pla = tf.placeholder(tf.float32, [batchsize],
name='timerates_in')
self.timerates_expand = tf.expand_dims(self.timerates_pla, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand, -1)
self.timerates_expand = tf.expand_dims(self.timerates_expand,
-1) #12*1*1*1
print('placeholders:\n', 'img_placeholder:', self.imgs_pla,
self.timerates_pla)
#img_placeholder: Tensor("imgs_in:0", shape=(10, 180, 320, 9), dtype=float32) Tensor("timerates_in:0", shape=(10,), dtype=float32)
self.frame0 = self.imgs_pla[:, :, :, :img_channel]
self.frame1 = self.imgs_pla[:, :, :, img_channel:img_channel * 2]
self.frame2 = self.imgs_pla[:, :, :, img_channel * 2:]
with tf.variable_scope("first_unet", reuse=tf.AUTO_REUSE) as scope:
firstinput = tf.concat([self.frame0, self.frame2], -1)
#self.first_opticalflow=my_unet( firstinput, 4,training=self.training , withbias=True, withbn=False) #注意这里是直接作为optical flow
self.first_opticalflow = my_unet_split(
firstinput,
4,
training=self.training,
withbias=True,
withbn=True) #注意这里是直接作为optical flow
self.first_opticalflow_0_1 = self.first_opticalflow[:, :, :, :2]
self.first_opticalflow_0_1 = tf.identity(self.first_opticalflow_0_1,
name="first_opticalflow_0_1")
print('first_opticalflow_0_1:', self.first_opticalflow_0_1)
self.first_opticalflow_1_0 = self.first_opticalflow[:, :, :, 2:]
self.first_opticalflow_1_0 = tf.identity(self.first_opticalflow_1_0,
name="first_opticalflow_1_0")
print('first_opticalflow_1_0:', self.first_opticalflow_1_0)
#first_opticalflow_0_1: Tensor("first_opticalflow_0_1:0", shape=(10, 180, 320, 2), dtype=float32)
#first_opticalflow_1_0: Tensor("first_opticalflow_1_0:0", shape=(10, 180, 320, 2), dtype=float32)
#输出光流形状
self.flow_size_h = self.first_opticalflow_0_1.get_shape().as_list()[1]
self.flow_size_w = self.first_opticalflow_0_1.get_shape().as_list()[2]
self.flow_channel = self.first_opticalflow_0_1.get_shape().as_list(
)[-1]
########################################################
self.step2_flow_channel = 5
self.flow_shape = [
self.flow_size_h, self.flow_size_w, self.step2_flow_channel
]
#获取数据时的一些cpu上的参数，用于扩张数据和判定时序
self.last_flow_init_np = np.zeros(self.flow_shape, dtype=np.float32)
print(self.last_flow_init_np.shape) #(180, 320, 5)
##############################################################
self.last_optical_flow = tf.placeholder(tf.float32,
self.flow_shape,
name='second_last_flow')
#初始化train和test的初始0状态
self.last_flow_new_train = self.last_flow_init_np
self.last_flow_new_test = self.last_flow_init_np
#反向光流算中间帧
self.first_opticalflow_t_0=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.first_opticalflow_0_1 ,\
self.timerates_expand*self.timerates_expand*self.first_opticalflow_1_0 , name="first_opticalflow_t_0")
self.first_opticalflow_t_2=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.first_opticalflow_0_1 ,\
self.timerates_expand*(self.timerates_expand-1)*self.first_opticalflow_1_0, name="first_opticalflow_t_2")
#2种方法合成t时刻的帧
self.first_img_flow_2_t = self.warp_op(
self.frame2, -self.first_opticalflow_t_2) #!!!
self.first_img_flow_0_t = self.warp_op(
self.frame0, -self.first_opticalflow_t_0) #!!!
#虽然论文里用不到第一步的输出中间帧，但是这里也给他输出看看效果
self.first_output = tf.add(
self.timerates_expand * self.first_img_flow_2_t,
(1 - self.timerates_expand) * self.first_img_flow_0_t,
name="first_outputimg")
print('first output img:', self.first_output)
#first output img: Tensor("first_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
#利用光流前后帧互相合成
self.first_img_flow_2_0 = self.warp_op(
self.frame2, -self.first_opticalflow_0_1) #frame2->frame0
self.first_img_flow_0_2 = self.warp_op(
self.frame0, -self.first_opticalflow_1_0) #frame0->frame2
####################################################################################################################3
#第二个unet
with tf.variable_scope("second_unet", reuse=tf.AUTO_REUSE) as scope:
secinput=tf.concat([self.frame0[0], self.frame2[0], \
self.first_opticalflow_0_1[0], self.first_opticalflow_1_0[0], \
self.first_opticalflow_t_2[0], self.first_opticalflow_t_0[0],\
self.first_img_flow_2_t[0], self.first_img_flow_0_t[0],\
self.last_optical_flow], -1)
secinput = tf.expand_dims(secinput, 0)
print(
"secinput:", secinput
) #secinput: Tensor("second_unet/ExpandDims:0", shape=(1, 180, 320, 25), dtype=float32)
step2_withbn = False
new_step2_flow = my_unet(
secinput,
self.step2_flow_channel,
training=self.training,
withbias=True,
withbn=step2_withbn) #注意这里是直接作为optical flow
kep_step2_flow = [new_step2_flow]
print("new_step2_flow:", new_step2_flow)
#new_step2_flow: Tensor("second_unet/unet_end0_relu/LeakyRelu:0", shape=(1, 180, 320, 5), dtype=float32)
for ti in range(1, batchsize):
secinput=tf.concat([self.frame0[ti], self.frame2[ti], \
self.first_opticalflow_0_1[ti], self.first_opticalflow_1_0[ti], \
self.first_opticalflow_t_2[ti], self.first_opticalflow_t_0[ti],\
self.first_img_flow_2_t[ti], self.first_img_flow_0_t[ti],\
new_step2_flow[0] ], -1)
secinput = tf.expand_dims(secinput, 0)
new_step2_flow = my_unet(secinput,
self.step2_flow_channel,
withbias=True,
withbn=step2_withbn)
kep_step2_flow.append(new_step2_flow)
self.second_batch_last_flow = new_step2_flow[0]
#self.second_batch_last_flow=tf.identity(self.second_batch_last_flow, name="second_batch_last_flow")
print(
"second_batch_last_flow:", self.second_batch_last_flow
) #Tensor("second_unet/strided_slice_89:0", shape=(180, 320, 5), dtype=float32)
self.second_opticalflow = tf.concat(kep_step2_flow, 0)
print(
"self.second_opticalflow:", self.second_opticalflow
) #self.second_opticalflow: Tensor("second_unet/concat_60:0", shape=(10, 180, 320, 5), dtype=float32)
self.second_opticalflow_t_0 = tf.add(
self.second_opticalflow[:, :, :, :2],
self.first_opticalflow_t_0,
name="second_opticalflow_t_0")
self.second_opticalflow_t_1 = tf.add(self.second_opticalflow[:, :, :,
2:4],
self.first_opticalflow_t_2,
name="second_opticalflow_t_1")
print('second_opticalflow_t_0:', self.second_opticalflow_t_0)
print('second_opticalflow_t_1:', self.second_opticalflow_t_1)
#second_opticalflow_t_0: Tensor("second_opticalflow_t_0:0", shape=(10, 180, 320, 2), dtype=float32)
#second_opticalflow_t_1: Tensor("second_opticalflow_t_1:0", shape=(10, 180, 320, 2), dtype=float32)
self.vmap_t_0 = tf.expand_dims(
tf.sigmoid(self.second_opticalflow[:, :, :, -1]), -1)
self.vmap_t_1 = 1 - self.vmap_t_0
#2种方法合成t时刻的帧
self.second_img_flow_1_t = self.warp_op(
self.frame2, -self.second_opticalflow_t_1) #!!!
self.second_img_flow_0_t = self.warp_op(
self.frame0, -self.second_opticalflow_t_0) #!!!
#最终输出的图
print(self.timerates_expand, self.vmap_t_0, self.second_img_flow_0_t)
#Tensor("ExpandDims_2:0", shape=(6, 1, 1, 1), dtype=float32) Tensor("Sigmoid:0", shape=(6, 180, 320, 1), dtype=float32)
#Tensor("dense_image_warp_5/Reshape_1:0", shape=(6, 180, 320, 3), dtype=float32)
self.second_output=tf.div( ( (1-self.timerates_expand)*self.vmap_t_0*self.second_img_flow_0_t+self.timerates_expand*self.vmap_t_1*self.second_img_flow_1_t), \
((1-self.timerates_expand)*self.vmap_t_0+self.timerates_expand*self.vmap_t_1) , name="second_outputimg" )
print('second output img:', self.second_output)
#second output img: Tensor("second_outputimg:0", shape=(10, 180, 320, 3), dtype=float32)
#判别器的网络构建
self.D_1_net_F, self.D_1_net_F_logit = Discriminator_net(
self.second_output, name="D1", training=self.training)
self.D_1_net_T, self.D_1_net_T_logit = Discriminator_net(
self.frame1, name="D1", training=self.training)
#D的loss计算
self.D_1_net_loss_sum, _, _ = self.D_loss_TandF_logits(
self.D_1_net_T_logit, self.D_1_net_F_logit, "D_1_net")
#计算loss
self.second_L1_loss_interframe,self.first_warp_loss,self.second_contex_loss,self.second_local_var_loss_all,self.second_global_var_loss_all,self.second_ssim,self.second_psnr,\
self.first_L1_loss_interframe, self.first_ssim, self.first_psnr, self.second_GAN_loss_mean_D1=self.loss_cal_all()
#训练G的总loss
self.G_loss_all = 100 * self.second_L1_loss_interframe + 30 * (
self.first_L1_loss_interframe +
self.first_warp_loss) + 0.05 * self.second_contex_loss
#self.second_global_var_loss_all
#+ self.second_GAN_loss_mean_D1*0.03
#训练D的总loss
self.D_loss_all = self.D_1_net_loss_sum
#####################################
self.last_label_train = '#'
self.last_label_test = '#'
self.state_random_row_train = 0
self.state_random_col_train = 0
self.state_flip_train = False
self.state_random_row_test = 0
self.state_random_col_test = 0
self.state_flip_test = False
#为了兼容性
self.batchsize_inputimg = batchsize
self.img_size_w = img_size_w
self.img_size_h = img_size_h
t_vars = tf.trainable_variables()
print("trainable vars cnt:", len(t_vars))
self.first_para = [
var for var in t_vars if var.name.startswith('first')
]
self.sec_para = [
var for var in t_vars if var.name.startswith('second')
]
self.vgg_para = [var for var in t_vars if var.name.startswith('VGG')]
self.D_para = [var for var in t_vars if var.name.startswith('D')]
print("first param len:", len(self.first_para))
print("second param len:", len(self.sec_para))
print("VGG param len:", len(self.vgg_para))
print("D param len:", len(self.D_para))
print(self.vgg_para)
'''
trainable vars cnt: 144
first param len: 46
second param len: 46
VGG param len: 52
'''
#G训练过程
self.lr_rate = tf.train.exponential_decay(base_lr,
global_step=self.global_step,
decay_steps=decay_steps,
decay_rate=decay_rate)
self.train_op_G = tf.train.AdamOptimizer(self.lr_rate, beta1=beta1, name="superslomo_adam_G").minimize(self.G_loss_all, \
global_step=self.global_step , var_list=self.first_para+self.sec_para )
# weight clipping
self.clip_D = [
p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max))
for p in self.D_para
]
#D训练过程
self.train_op_D = tf.train.AdamOptimizer(
self.lr_rate, beta1=beta1,
name="superslomo_adam_D").minimize(self.D_loss_all,
var_list=self.D_para)
#最后构建完成后初始化参数
self.sess.run(tf.global_variables_initializer())