import tensorflow as tf
import os
tf.app.flags.DEFINE_string("configfile", "config/config_mpii.cfg",
"config file name")
tf.app.flags.DEFINE_string("loadmodel", None,
"model name used to continue training")
FLAGS = tf.app.flags.FLAGS
if __name__ == '__main__':
print('--Parsing Config File')
params = process_config(FLAGS.configfile)
os.system('mkdir -p {}'.format(params['saver_directory']))
os.system('cp {0} {1}'.format(FLAGS.configfile, params['saver_directory']))
print('--Creating Dataset')
dataset = DataGenerator(params['joint_list'], params['img_directory'],
params['training_txt_file'], params['img_size'])
dataset._create_train_table()
dataset._randomize()
dataset._create_sets()
model = HourglassModel(params=params, dataset=dataset, training=True)
model.create_model()
model.do_train(nEpochs=params['nepochs'],
epochSize=params['epoch_size'],
saveStep=params['saver_step'],
dataset=None,
load=FLAGS.loadmodel)
remove_joints=params['remove_joints'],
img_dir_test=params['img_directory_test2'],
test_data_file=params['test_txt_file2'])
dataset3 = DataGenerator(params['joint_list'],
params['img_directory3'],
params['training_txt_file3'],
remove_joints=params['remove_joints'],
img_dir_test=params['img_directory_test3'],
test_data_file=params['test_txt_file3'])
dataset4 = DataGenerator(params['joint_list'],
params['img_directory4'],
params['training_txt_file4'],
remove_joints=params['remove_joints'],
img_dir_test=params['img_directory_test4'],
test_data_file=params['test_txt_file4'])
dataset1._create_train_table()
dataset1._randomize()
dataset1._create_sets()
dataset2._create_train_table()
dataset2._randomize()
dataset2._create_sets()
dataset3._create_train_table()
dataset3._randomize()
dataset3._create_sets()
dataset4._create_train_table()
dataset4._randomize()
dataset4._create_sets()
dataset = [dataset1, dataset2, dataset3, dataset4]
camera1 = scipy.io.loadmat(params['camera1'])
camera2 = scipy.io.loadmat(params['camera2'])
def train(resnet_model, is_training, F, H, F_curr, H_curr,
input_images_blur, input_images_boundary, next_boundary_gt, labels,
data_dir, data_dir_valid, img_list, img_list_valid,
dropout_ratio):
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
val_step = tf.get_variable('val_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
# define the losses.
lambda_ = 1e-5
loss_1 = resnet_model.l2_loss_(resnet_model.logits, labels)
loss_2 = resnet_model.l2_loss_(resnet_model.next_frame,next_boundary_gt)
loss_3 = resnet_model.l2_loss_(input_images_blur[:,:,:,-3:],resnet_model.video_deblur_output)
loss_ = loss_1+loss_2+loss_3+tf.reduce_sum(tf.square(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))*lambda_
ema = tf.train.ExponentialMovingAverage(resnet_model.MOVING_AVERAGE_DECAY, global_step)
tf.add_to_collection(resnet_model.UPDATE_OPS_COLLECTION, ema.apply([loss_]))
tf.summary.scalar('loss_avg', ema.average(loss_))
ema = tf.train.ExponentialMovingAverage(0.9, val_step)
val_op = tf.group(val_step.assign_add(1), ema.apply([loss_]))
tf.summary.scalar('loss_valid', ema.average(loss_))
tf.summary.scalar('learning_rate', FLAGS.learning_rate)
# define the optimizer and back propagate.
opt = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads = opt.compute_gradients(loss_)
for grad, var in grads:
if grad is not None and not FLAGS.minimal_summaries:
tf.summary.histogram(var.op.name + '/gradients', grad)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
batchnorm_updates = tf.get_collection(resnet_model.UPDATE_OPS_COLLECTION)
batchnorm_updates_op = tf.group(*batchnorm_updates)
train_op = tf.group(apply_gradient_op, batchnorm_updates_op)
saver_all = tf.train.Saver(tf.all_variables())
summary_op = tf.summary.merge_all()
# initialize all variables
init = tf.initialize_all_variables()
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
sess.run(init)
summary_writer = tf.summary.FileWriter(FLAGS.end_2_end_train_dir, sess.graph)
val_summary_writer = tf.summary.FileWriter(FLAGS.end_2_end_valid_dir)
val_save_root = os.path.join(FLAGS.end_2_end_valid_dir,'visualization')
compare_save_root = os.path.join(FLAGS.end_2_end_valid_dir,'deblur_compare')
# resume weights
resume(sess, FLAGS.resume_structure_predictor, FLAGS.structure_predictor_train_dir, 'voxel_flow_model_')
resume(sess, FLAGS.resume_video_deblur, FLAGS.video_deblur_train_dir, 'video_deblur_model_')
resume(sess, FLAGS.resume_resnet, FLAGS.resnet_train_dir, 'resnet_model_')
resume(sess, FLAGS.resume_all, FLAGS.end_2_end_train_dir, '')
# create data generator
if FLAGS.training_period == 'pretrain':
dataset = DataGenerator(data_dir, img_list, data_dir_valid, img_list_valid)
dataset._create_train_sets_for_300W()
dataset._create_valid_sets_for_300W()
elif FLAGS.training_period == 'train':
dataset = DataGenerator(data_dir,img_list)
dataset._create_train_table()
dataset._create_sets_for_300VW()
else:
raise NameError("No such training_period!")
train_gen = dataset._aux_generator(batch_size = FLAGS.batch_size,
num_input_imgs = num_input_imgs,
NUM_CLASSES = POINTS_NUM*2,
sample_set='train')
valid_gen = dataset._aux_generator(batch_size = FLAGS.batch_size,
num_input_imgs = num_input_imgs,
NUM_CLASSES = POINTS_NUM*2,
sample_set='valid')
# main training process.
for x in xrange(FLAGS.max_steps + 1):
start_time = time.time()
step = sess.run(global_step)
i = [train_op, loss_]
write_summary = step > 1 and not (step % 100)
if write_summary:
i.append(summary_op)
i.append(resnet_model.logits)
i.append(F_curr)
i.append(H_curr)
train_line_num, frame_name, input_boundaries, boundary_gt_train, input_images_blur_generated, landmark_gt_train = next(train_gen)
if (frame_name == '2.jpg'):
input_images_boundary_init = copy.deepcopy(input_boundaries)
F_init = np.zeros([FLAGS.batch_size,
IMAGE_SIZE//2,
IMAGE_SIZE//2,
structure_predictor_net_channel//2], dtype=np.float32)
H_init = np.zeros([1,
FLAGS.batch_size,
IMAGE_SIZE//2,
IMAGE_SIZE//2,
structure_predictor_net_channel], dtype=np.float32)
feed_dict={
input_images_boundary:input_images_boundary_init,
input_images_blur:input_images_blur_generated,
F:F_init,
H:H_init,
labels:landmark_gt_train,
next_boundary_gt:boundary_gt_train,
dropout_ratio:0.5
}
else:
output_points = o[-3]
output_points = np.reshape(output_points,(POINTS_NUM,2))
boundary_from_points = points_to_heatmap_rectangle_68pt(output_points)
boundary_from_points = np.expand_dims(boundary_from_points,axis=0)
boundary_from_points = np.expand_dims(boundary_from_points,axis=3)
input_images_boundary_init = np.concatenate([input_images_boundary_init[:,:,:,1:2],
boundary_from_points], axis=3)
feed_dict={
input_images_boundary:input_images_boundary_init,
input_images_blur:input_images_blur_generated,
F:o[-2],
H:o[-1],
labels:landmark_gt_train,
next_boundary_gt:boundary_gt_train,
dropout_ratio:0.5
}
o = sess.run(i,feed_dict=feed_dict)
loss_value = o[1]
duration = time.time() - start_time
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
if step > 1 and step % 300 == 0:
examples_per_sec = FLAGS.batch_size / float(duration)
format_str = ('step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str % (step, loss_value, examples_per_sec, duration))
if write_summary:
summary_str = o[2]
summary_writer.add_summary(summary_str, step)
if step > 1 and step % 300 == 0:
checkpoint_path = os.path.join(FLAGS.end_2_end_train_dir, 'model.ckpt')
ensure_dir(checkpoint_path)
saver_all.save(sess, checkpoint_path, global_step=global_step)
# Run validation periodically
if step > 1 and step % 300 == 0:
valid_line_num, frame_name, input_boundaries, boundary_gt_valid, input_images_blur_generated, landmark_gt_valid = next(valid_gen)
if (frame_name == '2.jpg') or valid_line_num <= 3:
input_images_boundary_init = copy.deepcopy(input_boundaries)
F_init = np.zeros([FLAGS.batch_size,
IMAGE_SIZE//2,
IMAGE_SIZE//2,
structure_predictor_net_channel//2], dtype=np.float32)
H_init = np.zeros([1, FLAGS.batch_size,
IMAGE_SIZE//2,
IMAGE_SIZE//2,
structure_predictor_net_channel], dtype=np.float32)
feed_dict={input_images_boundary:input_images_boundary_init,
input_images_blur:input_images_blur_generated,
F:F_init,
H:H_init,
labels:landmark_gt_valid,
next_boundary_gt:boundary_gt_valid,
dropout_ratio:1.0
}
else:
output_points = o_valid[-3]
output_points = np.reshape(output_points,(POINTS_NUM,2))
boundary_from_points = points_to_heatmap_rectangle_68pt(output_points)
boundary_from_points = np.expand_dims(boundary_from_points,axis=0)
boundary_from_points = np.expand_dims(boundary_from_points,axis=3)
input_images_boundary_init = np.concatenate([input_images_boundary_init[:,:,:,1:2],
boundary_from_points], axis=3)
feed_dict={
input_images_boundary:input_images_boundary_init,
input_images_blur:input_images_blur_generated,
F:o_valid[-2],
H:o_valid[-1],
labels:landmark_gt_valid,
next_boundary_gt:boundary_gt_valid,
dropout_ratio:1.0
}
i_valid = [loss_,resnet_model.logits,F_curr,H_curr]
o_valid = sess.run(i_valid,feed_dict=feed_dict)
print('Validation top1 error %.2f' % o_valid[0])
if write_summary:
val_summary_writer.add_summary(summary_str, step)
img_video_deblur_output = sess.run(resnet_model.video_deblur_output,feed_dict=feed_dict)[0]*255
img = input_images_blur_generated[0,:,:,0:3]*255
compare_img = np.concatenate([img,img_video_deblur_output],axis=1)
points = o_valid[1][0]*255
for point_num in range(int(points.shape[0]/2)):
cv2.circle(img,(int(round(points[point_num*2])),int(round(points[point_num*2+1]))),1,(55,225,155),2)
val_save_path = os.path.join(val_save_root,str(step)+'.jpg')
compare_save_path = os.path.join(compare_save_root,str(step)+'.jpg')
ensure_dir(val_save_path)
ensure_dir(compare_save_path)
cv2.imwrite(val_save_path,img)
cv2.imwrite(compare_save_path,compare_img)
def test(resnet_model, is_training, F, H, F_curr, H_curr, input_images_blur,
input_images_boundary, next_boundary_gt, labels, data_dir, img_list,
dropout_ratio):
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
init = tf.initialize_all_variables()
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
sess.run(init)
val_save_root = os.path.join(FLAGS.end_2_end_test_dir, 'visualization')
################################ resume part #################################
# resume weights
resume(sess, FLAGS.resume_structure_predictor,
FLAGS.structure_predictor_train_dir, 'voxel_flow_model_')
resume(sess, FLAGS.resume_video_deblur, FLAGS.video_deblur_train_dir,
'video_deblur_model_')
resume(sess, FLAGS.resume_resnet, FLAGS.resnet_train_dir, 'resnet_model_')
resume(sess, FLAGS.resume_all, FLAGS.end_2_end_train_dir, '')
##############################################################################
gt_file_path = os.path.join(FLAGS.end_2_end_test_dir, 'gt.txt')
pre_file_path = os.path.join(FLAGS.end_2_end_test_dir, 'pre.txt')
ensure_dir(gt_file_path)
ensure_dir(pre_file_path)
gt_file = open(gt_file_path, 'w')
pre_file = open(pre_file_path, 'w')
dataset = DataGenerator(data_dir, img_list)
dataset._create_train_table()
dataset._create_sets_for_300VW()
test_gen = dataset._aux_generator(batch_size=FLAGS.batch_size,
num_input_imgs=num_input_imgs,
NUM_CLASSES=POINTS_NUM * 2,
sample_set='test')
test_break_flag = False
for x in xrange(len(dataset.train_table) - 2):
step = sess.run(global_step)
if not test_break_flag:
test_line_num, frame_name, input_boundaries, boundary_gt_test, input_images_blur_generated, landmark_gt_test, names, test_break_flag = next(
test_gen)
if (frame_name == '2.jpg') or test_line_num <= 3:
input_images_boundary_init = copy.deepcopy(input_boundaries)
F_init = np.zeros([
FLAGS.batch_size, IMAGE_SIZE // 2, IMAGE_SIZE // 2,
structure_predictor_net_channel // 2
],
dtype=np.float32)
H_init = np.zeros([
1, FLAGS.batch_size, IMAGE_SIZE // 2, IMAGE_SIZE // 2,
structure_predictor_net_channel
],
dtype=np.float32)
feed_dict = {
input_images_boundary: input_images_boundary_init,
input_images_blur: input_images_blur_generated,
F: F_init,
H: H_init,
labels: landmark_gt_test,
next_boundary_gt: boundary_gt_test,
dropout_ratio: 1.0
}
else:
output_points = o[0]
output_points = np.reshape(output_points, (POINTS_NUM, 2))
boundary_from_points = points_to_heatmap_rectangle_68pt(
output_points)
boundary_from_points = np.expand_dims(boundary_from_points, axis=0)
boundary_from_points = np.expand_dims(boundary_from_points, axis=3)
input_images_boundary_init = np.concatenate([
input_images_boundary_init[:, :, :, 1:2], boundary_from_points
],
axis=3)
feed_dict = {
input_images_boundary: input_images_boundary_init,
input_images_blur: input_images_blur_generated,
F: o[-2],
H: o[-1],
labels: landmark_gt_test,
next_boundary_gt: boundary_gt_test,
dropout_ratio: 1.0
}
i = [resnet_model.logits, F_curr, H_curr]
o = sess.run(i, feed_dict=feed_dict)
pres = o[0]
for batch_num, pre in enumerate(pres):
for v in pre:
pre_file.write(str(v * 255.0) + ' ')
if len(names) > 1:
pre_file.write(names[-1])
else:
pre_file.write(names[batch_num])
pre_file.write('\n')
for batch_num, g in enumerate(landmark_gt_test):
for v in g:
gt_file.write(str(v * 255.0) + ' ')
if len(names) > 1:
gt_file.write(names[-1])
else:
gt_file.write(names[batch_num])
gt_file.write('\n')
img = input_images_blur_generated[0, :, :, 0:3] * 255
points = o[0][0] * 255
for point_num in range(int(points.shape[0] / 2)):
cv2.circle(img, (int(round(points[point_num * 2])),
int(round(points[point_num * 2 + 1]))), 1,
(55, 225, 155), 2)
val_save_path = os.path.join(val_save_root, str(step) + '.jpg')
ensure_dir(val_save_path)
cv2.imwrite(val_save_path, img)
global_step = global_step + 1
print('Test done!')
print('--Parsing Config File')
params = process_config('config.cfg')
print('--Creating Dataset')
dataset = DataGenerator(params['img_directory'],
params['training_txt_file'],
params['num_joints'],
params['val_directory'],
params['val_txt_file'],
resolutions=params['resolutions'],
headed=params['headed'],
head_train=params['head_train'],
head_test=params['head_test'],
head_val=params['head_val'])
dataset._create_train_table(
) #creates the lists with dicts of the coord. of boxes, joints and the corresp. weights
dataset._randomize() # shuffles the previous lists
dataset._create_sets() # validation and training lists
model = HourglassModel(nFeat=params['nfeats'],
nStack=params['nstacks'],
nModules=params['nmodules'],
nLow=params['nlow'],
outputDim=params['num_joints'],
batch_size=params['batch_size'],
attention=params['mcam'],
training=True,
drop_rate=params['dropout_rate'],
lear_rate=params['learning_rate'],
decay=params['learning_rate_decay'],
decay_step=params['decay_step'],
params[option] = eval(config.get(section, option))
if section == 'Network':
for option in config.options(section):
params[option] = eval(config.get(section, option))
if section == 'Train':
for option in config.options(section):
params[option] = eval(config.get(section, option))
if section == 'Validation':
for option in config.options(section):
params[option] = eval(config.get(section, option))
if section == 'Saver':
for option in config.options(section):
params[option] = eval(config.get(section, option))
return params
if __name__ == '__main__':
print('--Parsing Config File')
params = process_config('config.cfg')
print('--Creating Dataset')
dataset = DataGenerator(params['joint_list'], params['img_directory'], params['training_txt_file'], remove_joints=params['remove_joints'])
dataset._create_train_table()
dataset._randomize()
dataset._create_sets()
model = HourglassModel(nFeat=params['nfeats'], nStack=params['nstacks'], nModules=params['nmodules'], nLow=params['nlow'], outputDim=params['num_joints'], batch_size=params['batch_size'], attention = params['mcam'],training=True, drop_rate= params['dropout_rate'], lear_rate=params['learning_rate'], decay=params['learning_rate_decay'], decay_step=params['decay_step'], dataset=dataset, name=params['name'], logdir_train=params['log_dir_train'], logdir_test=params['log_dir_test'], tiny= params['tiny'], w_loss=params['weighted_loss'] , joints= params['joint_list'],modif=False)
model.generate_model()
model.training_init(nEpochs=params['nepochs'], epochSize=params['epoch_size'], saveStep=params['saver_step'], dataset = None)
