def train_c3d(self,max_iteration = 100000 , restore_tags = True, trainable_whole=True, trainable_mid = True, ):
my_multi_train_datasets = multi_train_datasets(batch_size = self.batch_size, video_num = self.video_imgs_num, frame_interval = 2, is_frame = True, is_Optical = True,crop_size=4, img_size=self.img_size_h)
gpu_options = tf.GPUOptions(allow_growth=True)
summaries_dir = self.summaries_dir + 'SINGLE_GPU%d.CPTK' % time.time()
train_writer = tf.summary.FileWriter(summaries_dir)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
sess.run(tf.global_variables_initializer())
if restore_tags :
self.restore_model_weghts(sess)
start_time = time.time()
for idx in range(max_iteration):
try:
batch_data = my_multi_train_datasets.get_batches()
if trainable_whole:
sess.run([ self.c3d_apply ], feed_dict={self.train_in_ph : batch_data,self.phase : True})
elif trainable_mid:
sess.run([ self.mid_stage_apply ], feed_dict={self.train_in_ph : batch_data,self.phase : True})
else:
sess.run([ self.optical_apply, self.gray_apply ], feed_dict={self.train_in_ph : batch_data, self.phase : True})
except Exception:
print('training failed')
if (idx+1)%200 == 0:
try:
batch_data = my_multi_train_datasets.get_batches()
train_loss = self.fetch_net_loss(sess, batch_data)
print('Epoches Idx%d' %(idx + 1))
t_elp = time.time() - start_time
h,m,s = time_hms(t_elp)
print('Time Elapsed : %d hours %d minutes %d seconds'%(h, m, s) )
t_eta = t_elp/(idx+1)*(max_iteration - idx -1)
h,m,s = time_hms(t_eta)
print('Time ETA : %d hours %d minutes %d seconds'%(h, m, s) )
print ('loss %.8f, extraloss %.8f,gray_loss %.8f, optical_loss %.8f, mid_stage_loss %.8f'%(train_loss['total'],train_loss['extraloss'],train_loss['gray'] ,train_loss['optical_flow'] ,train_loss['midstage'] ))
train_writer.add_summary(train_loss['summary'], (idx+1))
train_writer.flush()
except Exception:
print('fet net loss failed')
if (idx+1)%500 == 0 and (idx+1) >=2000:
# if (idx+1)%50 == 0:
self.save_model_weghts(sess)
return
def train_c3d(
self,
max_iteration=100000,
restore_tags=True,
trainable_whole=True,
trainable_mid=True,
):
my_multi_train_datasets = multi_train_datasets(batch_size=8,
video_num=4,
frame_interval=2,
is_frame=True,
is_Optical=True,
crop_size=4,
img_size=256)
gpu_options = tf.GPUOptions(allow_growth=True)
with tf.Session(config=tf.ConfigProto(
gpu_options=gpu_options)) as sess:
sess.run(tf.global_variables_initializer())
if restore_tags:
self.restore_model_weghts(sess)
start_time = time.time()
for idx in range(max_iteration):
batch_data = my_multi_train_datasets.get_batches()
if trainable_whole:
sess.run([self.c3d_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
elif trainable_mid:
sess.run([self.mid_stage_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
else:
sess.run([self.optical_apply, self.gray_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
if (idx + 1) % 100 == 0:
batch_data = my_multi_train_datasets.get_batches()
train_loss = self.fetch_net_loss(sess, batch_data)
print('Epoches Idx%d' % (idx + 1))
t_elp = time.time() - start_time
h, m, s = time_hms(t_elp)
print('Time Elapsed : %d hours %d minutes %d seconds' %
(h, m, s))
t_eta = t_elp / (idx + 1) * (max_iteration - idx - 1)
h, m, s = time_hms(t_eta)
print('Time ETA : %d hours %d minutes %d seconds' %
(h, m, s))
print(
'loss %.8f, gray_loss %.8f, optical_loss %.8f, mid_stage_loss %.8f'
% (train_loss['total'], train_loss['gray'],
train_loss['optical_flow'], train_loss['midstage']))
if (idx + 1) % 500 == 0 and (idx + 1) >= 2000:
self.save_model_weghts(sess)
return
def train(self, max_iteration, GPU_IN_USE):
my_multi_train_datasets = multi_train_datasets(
batch_size=self.batch_size,
video_num=self.video_imgs_num,
frame_interval=2,
is_frame=True,
is_Optical=False,
crop_size=4,
img_size=self.img_size_h)
loss_f = nn.MSELoss()
summaries_dir = self.summaries_dir + 'SINGLE_GPU%d.CPTK' % time.time()
writer = SummaryWriter(summaries_dir)
for idx in range(max_iteration):
self.optimizer.zero_grad()
batch_data = my_multi_train_datasets.get_batches()
# batch_data = np.transpose(batch_data, (0,4,2,3,1))
batch_data = np.squeeze(batch_data, 4)
# print(batch_data.shape)
if GPU_IN_USE:
# temp1 = Variable(ToTensor()(temp1)).view(1,-1,29,29)
batch_data = Variable(
torch.tensor(batch_data, dtype=torch.float)).cuda()
else:
batch_data = Variable(torch.tensor(batch_data),
requires_grad=True).cpu()
batch_data1 = Variable(batch_data[:, 0:4:2, :, :],
requires_grad=True)
# batch_data1 = np.transpose(batch_data1,(0,3,2,1))
batch_data2 = Variable(batch_data[:, 1:4:2, :, :],
requires_grad=False)
# batch_data2 = np.transpose(batch_data2,(0,3,2,1))
output1 = self.model(batch_data1)
output2 = self.model(batch_data2)
output2 = Variable(output2, requires_grad=False)
loss = -1.0 * torch.mean((output1 - output2)**2)
# loss = loss * -1.0
loss.backward()
# print('%d\n'%(idx+1), loss.item())
self.optimizer.step()
if (idx + 1) % 200 == 0:
writer.add_scalar('loss', loss.item(), idx + 1)
writer.add_image('input1_image', batch_data1[0, 0:1, :, :],
idx + 1)
writer.add_image('input2_image', batch_data1[0, 1:2, :, :],
idx + 1)
writer.add_image('input3_image', batch_data2[0, 0:1, :, :],
idx + 1)
writer.add_image('input4_image', batch_data2[0, 1:2, :, :],
idx + 1)
writer.add_image('out1_image', output1[0, :, :, :], idx + 1)
writer.add_image('out2_image', output2[0, :, :, :], idx + 1)
writer.add_image('input1_diff_image',
(batch_data1[0, 0:1, :, :] -
batch_data2[0, 0:1, :, :])**2, idx + 1)
writer.add_image('input2_diff_image',
(batch_data1[0, 1:2, :, :] -
batch_data2[0, 1:2, :, :])**2, idx + 1)
writer.add_image('out_difference', (output1[0, :, :, :] -
output2[0, :, :, :])**2,
idx + 1)
# writer.add_image('out2_image', output2[0,:,:,:], idx+1)
print('%d\n' % (idx + 1), loss.item())
# writer.export_scalars_to_json("./all_scalars.json")
writer.close()
def train_c3d(self,
max_iteration=100000,
restore_tags=True,
mse_tag=True,
gan_tag=True,
g_iteration=5,
d_iteration=1):
self.g_iteration = g_iteration
self.d_iteration = d_iteration
my_multi_train_datasets = multi_train_datasets(
batch_size=self.batch_size,
video_num=self.video_imgs_num,
frame_interval=2,
is_frame=True,
is_Optical=False,
crop_size=4,
img_size=self.img_size_h)
gpu_options = tf.GPUOptions(allow_growth=True)
summaries_dir = self.summaries_dir + 'SINGLE_GPU%d.CPTK' % time.time()
train_writer = tf.summary.FileWriter(summaries_dir)
with tf.Session(config=tf.ConfigProto(
gpu_options=gpu_options)) as sess:
sess.run(tf.global_variables_initializer())
if restore_tags:
self.restore_model_weghts(sess)
start_time = time.time()
for idx in range(max_iteration):
batch_data = my_multi_train_datasets.get_batches()
if mse_tag:
sess.run([self.gray_mse_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
if gan_tag:
for g_iter in range(self.g_iteration):
sess.run([self.gray_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
for d_iter in range(self.d_iteration):
sess.run([self.discriminator_apply],
feed_dict={
self.train_in_ph: batch_data,
self.phase: True
})
if (idx + 1) % 100 == 0:
batch_data = my_multi_train_datasets.get_batches()
train_loss = self.fetch_net_loss(sess, batch_data)
print('Epoches Idx%d' % (idx + 1))
t_elp = time.time() - start_time
h, m, s = time_hms(t_elp)
print('Time Elapsed : %d hours %d minutes %d seconds' %
(h, m, s))
t_eta = t_elp / (idx + 1) * (max_iteration - idx - 1)
h, m, s = time_hms(t_eta)
print('Time ETA : %d hours %d minutes %d seconds' %
(h, m, s))
print(
'generator %.8f, discriminator %.8f ' %
(train_loss['generator'], train_loss['discriminator']))
print('gray_mse_loss', train_loss['gray_mse_loss'])
print('disc_sequence_loss',
train_loss['disc_sequence_loss'])
train_writer.add_summary(train_loss['summary'], (idx + 1))
train_writer.flush()
if (idx + 1) % 500 == 0 and (idx + 1) >= 2000:
# if (idx+1)%50 == 0:
self.save_model_weghts(sess)
return