def training(args): # DIRECTORY FOR CKPTS and META FILES # ROOT_DIR = '/neuhaus/movie/dataset/tf_records' ROOT_DIR = '/media/data/movie/dataset/tf_records' TRAIN_REC_PATH = os.path.join(ROOT_DIR, args.experiment_name, 'train.tfrecords') VAL_REC_PATH = os.path.join(ROOT_DIR, args.experiment_name, 'val.tfrecords') CKPT_PATH = os.path.join(ROOT_DIR, args.experiment_name, args.ckpt_folder_name + '/') # SCOPING BEGINS HERE with tf.Session().as_default() as sess: train_queue = tf.train.string_input_producer([TRAIN_REC_PATH], num_epochs=None) train_fFrames, train_lFrames, train_iFrames, train_mfn =\ read_and_decode( filename_queue=train_queue, is_training=True, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) val_queue = tf.train.string_input_producer([VAL_REC_PATH], num_epochs=None) val_fFrames, val_lFrames, val_iFrames, val_mfn = \ read_and_decode( filename_queue=val_queue, is_training=False, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) with tf.variable_scope('slomo'): print('TRAIN FRAMES (first):') train_output = slomo.SloMo_model(train_fFrames, train_lFrames, first_kernel=7, second_kernel=5, reuse=False, t_steps=args.n_IF, verbose=False) train_rec_iFrames = train_output[0] train_flow_01 = train_output[1] train_flow_10 = train_output[2] train_weighted_ft0 = train_output[3] train_weighted_ft1 = train_output[4] with tf.variable_scope('slomo', reuse=tf.AUTO_REUSE): print('VAL FRAMES (first):') val_output = slomo.SloMo_model(val_fFrames, val_lFrames, first_kernel=7, second_kernel=5, reuse=False, t_steps=args.n_IF, verbose=False) val_rec_iFrames = val_output[0] val_flow_01 = val_output[1] val_flow_10 = val_output[2] val_weighted_ft0 = val_output[3] val_weighted_ft1 = val_output[4] # Weights should be kept locally ~ 500 MB space with tf.variable_scope('vgg16'): train_iFrames_features = vgg16.build_vgg16( train_iFrames, end_point='pool5').features with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE): train_rec_iFrames_features = vgg16.build_vgg16( train_rec_iFrames, end_point='pool5').features print('Global parameters:{}'.format( count_parameters(tf.global_variables()))) print('Learnable model parameters:{}'.format( count_parameters(tf.trainable_variables()))) train_l2_loss = slomo.l2_loss(train_iFrames, train_rec_iFrames) percep_loss = slomo.l2_loss(train_iFrames_features, train_rec_iFrames_features) wrap_loss = slomo.wrapping_loss(train_fFrames, train_lFrames, train_iFrames, train_flow_01, train_flow_10, train_weighted_ft0, train_weighted_ft1) smooth_loss = slomo.smoothness_loss(train_flow_01, train_flow_10) # DEFINE METRICS val_loss = slomo.l2_loss(val_iFrames, val_rec_iFrames) total_train_loss = 0.1*train_l2_loss+1.0*percep_loss+\ 1.0*wrap_loss+50.0*smooth_loss # SUMMARIES tf.summary.scalar('train_l2_loss', train_l2_loss) tf.summary.scalar('wrap_loss', wrap_loss) tf.summary.scalar('smooth_loss', smooth_loss) tf.summary.scalar('percep_loss', percep_loss) tf.summary.scalar('total_val_l2_loss', val_loss) tf.summary.scalar('total_train_loss', total_train_loss) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(CKPT_PATH + 'train', sess.graph) with tf.variable_scope("global_step_and_learning_rate"): global_step = tf.contrib.framework.get_or_create_global_step() learning_rate = tf.train.exponential_decay( args.learning_rate, global_step, 100000, #FLAGS.decay_step 0.1, #FLAGS.decay_rate staircase=True) #FLAGS.stair incr_global_step = tf.assign(global_step, global_step + 1) with tf.variable_scope("optimizer"): with tf.control_dependencies( tf.get_collection(tf.GraphKeys.UPDATE_OPS)): tvars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='slomo') optimizer = tf.train.AdamOptimizer(learning_rate, beta1=0.9) grads_and_vars = optimizer.compute_gradients( total_train_loss, tvars) train_op = optimizer.apply_gradients(grads_and_vars) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # START TRAINING HERE for iteration in range(args.train_iters): _, t_summ, t_loss = sess.run([train_op, merged, total_train_loss]) train_writer.add_summary(t_summ, iteration) print('Iter:{}/{}, Train Loss:{}'.format(iteration, args.train_iters, t_loss)) if iteration % args.val_every == 0: v_loss = sess.run(val_loss) print('Iter:{}, Val Loss:{}'.format(iteration, v_loss)) if iteration % args.save_every == 0: saver.save( sess, CKPT_PATH + 'iter:{}_val:{}'.format( str(iteration), str(round(v_loss, 3)))) if iteration % args.plot_every == 0: start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [train_fFrames, train_lFrames,\ train_iFrames,\ train_rec_iFrames]) visualize_frames(start_frames, end_frames, mid_frames, rec_mid_frames, training=True, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'train_plots/')) start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [val_fFrames, val_lFrames,\ val_iFrames, val_rec_iFrames]) visualize_frames(start_frames, end_frames, mid_frames, rec_mid_frames, training=False, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'validation_plots/')) print('Training complete.....')
def training(args): # DIRECTORY FOR CKPTS and META FILES # ROOT_DIR = '/neuhaus/movie/dataset/tf_records' ROOT_DIR = '/media/data/movie/dataset/tf_records' TRAIN_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'train.tfrecords') VAL_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'val.tfrecords') CKPT_PATH = os.path.join( ROOT_DIR, args.experiment_name, args.ckpt_folder_name, '/') # SCOPING BEGINS HERE with tf.Session().as_default() as sess: global_step = tf.train.get_global_step() train_queue = tf.train.string_input_producer( [TRAIN_REC_PATH], num_epochs=None) train_fFrames, train_lFrames, train_iFrames, train_mfn =\ read_and_decode( filename_queue=train_queue, is_training=True, batch_size=args.batch_size) val_queue = tf.train.string_input_producer( [VAL_REC_PATH], num_epochs=None) val_fFrames, val_lFrames, val_iFrames, val_mfn = \ read_and_decode( filename_queue=val_queue, is_training=False, batch_size=args.batch_size) with tf.variable_scope('bipn'): print('TRAIN FRAMES (first):') train_rec_iFrames = bipn.build_bipn( train_fFrames, train_lFrames, use_batch_norm=True, is_training=True) with tf.variable_scope('bipn', reuse=tf.AUTO_REUSE): print('VAL FRAMES (first):') val_rec_iFrames = bipn.build_bipn( val_fFrames, val_lFrames, use_batch_norm=True, is_training=False) print('Model parameters:{}'.format( count_parameters())) # Weights should be kept locally ~ 500 MB space with tf.variable_scope('vgg16'): train_iFrames_features = vgg16.build_vgg16( train_iFrames, end_point='conv4_3').features with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE): train_rec_iFrames_features = vgg16.build_vgg16( train_rec_iFrames, end_point='conv4_3').features if args.perceptual_loss_weight: # Weights should be kept locally ~ 500 MB space with tf.variable_scope('vgg16'): train_iFrames_features = vgg16( train_iFrames, end_point='conv4_3') with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE): train_rec_iFrames_features = vgg16( train_rec_iFrames, end_point='conv4_3') # DEFINE METRICS if args.loss_id == 0: train_loss = huber_loss( train_iFrames, train_rec_iFrames, delta=1.) val_loss = huber_loss( val_iFrames, val_rec_iFrames, delta=1.) elif args.loss_id == 1: train_loss = l2_loss( train_iFrames, train_rec_iFrames) val_loss = l2_loss( val_iFrames, val_rec_iFrames) total_train_loss = train_loss tf.summary.scalar('train_l2_loss', train_loss) tf.summary.scalar('total_val_l2_loss', val_loss) if args.perceptual_loss_weight: train_perceptual_loss = perceptual_loss( train_iFrames_features, train_rec_iFrames_features) tf.summary.scalar('train_perceptual_loss',\ train_perceptual_loss) total_train_loss += train_perceptual_loss\ * args.perceptual_loss_weight # SUMMARIES tf.summary.scalar('total_train_loss',\ total_train_loss) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( CKPT_PATH + 'train', sess.graph) # DEFINE OPTIMIZER optimizer = get_optimizer( train_loss, optim_id=args.optim_id, learning_rate=args.learning_rate, use_batch_norm=True) init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners( coord=coord) # START TRAINING HERE try: for iteration in range(args.train_iters): _, t_summ, t_loss = sess.run( [optimizer, merged, total_train_loss]) train_writer.add_summary(t_summ, iteration) print('Iter:{}/{}, Train Loss:{}'.format( iteration, args.train_iters, t_loss)) if iteration % args.val_every == 0: v_loss = sess.run(val_loss) print('Iter:{}, Val Loss:{}'.format( iteration, v_loss)) if iteration % args.save_every == 0: saver.save( sess, CKPT_PATH + 'iter:{}_val:{}'.format( str(iteration), str(round(v_loss, 3)))) if iteration % args.plot_every == 0: start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [train_fFrames, train_lFrames,\ train_iFrames,\ train_rec_iFrames]) visualize_frames( start_frames, end_frames, mid_frames, rec_mid_frames, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'plots/')) except Exception as e: coord.request_stop(e)
def training(args): # DIRECTORY FOR CKPTS and META FILES ROOT_DIR = '/neuhaus/movie/dataset/tf_records' TRAIN_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'train.tfrecords') VAL_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'val.tfrecords') CKPT_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'skip_merge_conv_summary_image_tanh_separate_bipn_wd_l2_adam_1e-3/') # SCOPING BEGINS HERE with tf.Session().as_default() as sess: global_step = tf.train.get_global_step() train_queue = tf.train.string_input_producer( [TRAIN_REC_PATH], num_epochs=None) train_fFrames, train_lFrames, train_iFrames, train_mfn =\ read_and_decode( filename_queue=train_queue, is_training=True, batch_size=args.batch_size) val_queue = tf.train.string_input_producer( [VAL_REC_PATH], num_epochs=None) val_fFrames, val_lFrames, val_iFrames, val_mfn = \ read_and_decode( filename_queue=val_queue, is_training=False, batch_size=args.batch_size) with tf.variable_scope('separate_bipn'): print('TRAIN FRAMES (first):') train_rec_iFrames = skip_merge_conv_separate_encoder_bipn.build_bipn( train_fFrames, train_lFrames, use_batch_norm=True, is_training=True) with tf.variable_scope('separate_bipn', reuse=tf.AUTO_REUSE): print('VAL FRAMES (first):') val_rec_iFrames = skip_merge_conv_separate_encoder_bipn.build_bipn( val_fFrames, val_lFrames, use_batch_norm=True, is_training=False) print('Model parameters:{}'.format( count_parameters())) # DEFINE METRICS if args.loss_id == 0: train_loss = huber_loss( train_iFrames, train_rec_iFrames, delta=1.) val_loss = huber_loss( val_iFrames, val_rec_iFrames, delta=1.) elif args.loss_id == 1: train_loss = l2_loss( train_iFrames, train_rec_iFrames) val_loss = l2_loss( val_iFrames, val_rec_iFrames) if args.weight_decay: decay_loss = ridge_weight_decay( tf.trainable_variables()) train_loss += args.weight_decay * decay_loss # SUMMARIES tf.summary.scalar('train_loss', train_loss) tf.summary.scalar('val_loss', val_loss) with tf.contrib.summary.\ record_summaries_every_n_global_steps( n=args.summary_image_every): summary_true, summary_fake = visualize_tensorboard( train_fFrames, train_lFrames, train_iFrames, train_rec_iFrames, num_plots=3) tf.summary.image('true frames', summary_true) tf.summary.image('fake frames', summary_fake) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( CKPT_PATH + 'train', sess.graph) # DEFINE OPTIMIZER optimizer = get_optimizer( train_loss, optim_id=args.optim_id, learning_rate=args.learning_rate, use_batch_norm=True) init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners( coord=coord) # START TRAINING HERE try: for iteration in range(args.train_iters): _, t_summ, t_loss = sess.run( [optimizer, merged, train_loss]) train_writer.add_summary(t_summ, iteration) print('Iter:{}/{}, Train Loss:{}'.format( iteration, args.train_iters, t_loss)) if iteration % args.val_every == 0: v_loss = sess.run(val_loss) print('Iter:{}, Val Loss:{}'.format( iteration, v_loss)) if iteration % args.save_every == 0: saver.save( sess, CKPT_PATH + 'iter:{}_val:{}'.format( str(iteration), str(round(v_loss, 3)))) ''' if iteration % args.plot_every == 0: start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [train_fFrames, train_lFrames,\ train_iFrames,\ train_rec_iFrames]) visualize_frames( start_frames, end_frames, mid_frames, rec_mid_frames, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'plots/')) ''' coord.join(threads) except Exception as e: coord.request_stop(e)
def training(args): # DIRECTORY FOR CKPTS and META FILES # ROOT_DIR = '/neuhaus/movie/dataset/tf_records' ROOT_DIR = '/media/data/movie/dataset/tf_records' TRAIN_REC_PATH = os.path.join(ROOT_DIR, args.experiment_name, 'train.tfrecords') VAL_REC_PATH = os.path.join(ROOT_DIR, args.experiment_name, 'val.tfrecords') CKPT_PATH = os.path.join(ROOT_DIR, args.experiment_name, args.ckpt_folder_name + '/') # SCOPING BEGINS HERE with tf.Session().as_default() as sess: global_step = tf.train.get_global_step() train_queue = tf.train.string_input_producer([TRAIN_REC_PATH], num_epochs=None) train_fFrames, train_lFrames, train_iFrames, train_mfn =\ read_and_decode( filename_queue=train_queue, is_training=True, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) val_queue = tf.train.string_input_producer([VAL_REC_PATH], num_epochs=None) val_fFrames, val_lFrames, val_iFrames, val_mfn = \ read_and_decode( filename_queue=val_queue, is_training=False, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) # Apply gaussian blurring manually ''' train_fFrames = gaussian_filter(train_fFrames, std=std_dev) train_lFrames = gaussian_filter(train_lFrames, std=std_dev) train_iFrames = gaussian_filter(train_iFrames, std=std_dev) val_fFrames = gaussian_filter(val_fFrames, std=std_dev) val_lFrames = gaussian_filter(val_lFrames, std=std_dev) val_iFrames = gaussian_filter(val_iFrames, std=std_dev) ''' # TRAINABLE print('---------------------------------------------') print('----------------- GENERATOR -----------------') print('---------------------------------------------') with tf.variable_scope('generator'): train_rec_iFrames = generator_unet.build_generator( train_fFrames, train_lFrames, use_batch_norm=True, is_training=True, n_IF=args.n_IF, starting_out_channels=args.starting_out_channels, use_attention=args.use_attention, spatial_attention=args.spatial_attention, is_verbose=True) print('---------------------------------------------') print('-------------- DISCRIMINATOR ----------------') print('---------------------------------------------') # discriminator for classifying real images with tf.variable_scope('discriminator'): train_real_output_discriminator = discriminator.build_discriminator( train_iFrames, use_batch_norm=True, is_training=True, starting_out_channels=args.discri_starting_out_channels, is_verbose=True) # discriminator for classifying fake images with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE): train_fake_output_discriminator = discriminator.build_discriminator( train_rec_iFrames, use_batch_norm=True, is_training=True, starting_out_channels=args.discri_starting_out_channels, is_verbose=False) # VALIDATION with tf.variable_scope('generator', reuse=tf.AUTO_REUSE): val_rec_iFrames = generator_unet.build_generator( val_fFrames, val_lFrames, use_batch_norm=True, n_IF=args.n_IF, is_training=False, starting_out_channels=args.starting_out_channels, use_attention=args.use_attention, spatial_attention=args.spatial_attention, is_verbose=False) with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE): val_real_output_discriminator = discriminator.build_discriminator( val_iFrames, use_batch_norm=True, is_training=False, starting_out_channels=args.discri_starting_out_channels, is_verbose=False) with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE): val_fake_output_discriminator = discriminator.build_discriminator( val_rec_iFrames, use_batch_norm=True, is_training=False, starting_out_channels=args.discri_starting_out_channels, is_verbose=False) if args.perceptual_loss_weight: # Weights should be kept locally ~ 500 MB space with tf.variable_scope('vgg16'): train_iFrames_features = vgg16.build_vgg16( train_iFrames, end_point=args.perceptual_loss_endpoint).features with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE): train_rec_iFrames_features = vgg16.build_vgg16( train_rec_iFrames, end_point=args.perceptual_loss_endpoint).features print('Global parameters:{}'.format( count_parameters(tf.global_variables()))) print('Learnable model parameters:{}'.format( count_parameters(tf.trainable_variables()))) # DEFINE GAN losses: train_discri_real_loss = tf.reduce_sum( tf.square(train_real_output_discriminator - 1)) / (2 * args.batch_size) train_discri_fake_loss = tf.reduce_sum( tf.square(train_fake_output_discriminator)) / (2 * args.batch_size) train_discriminator_loss = train_discri_real_loss + train_discri_fake_loss train_generator_fake_loss = tf.reduce_sum( tf.square(train_fake_output_discriminator - 1)) / args.batch_size train_reconstruction_loss = l2_loss( train_rec_iFrames, train_iFrames) * args.reconstruction_loss_weight train_generator_loss = train_generator_fake_loss + train_reconstruction_loss val_discri_real_loss = tf.reduce_sum( tf.square(val_real_output_discriminator - 1)) / (2 * args.batch_size) val_discri_fake_loss = tf.reduce_sum( tf.square(val_fake_output_discriminator)) / (2 * args.batch_size) val_discriminator_loss = val_discri_real_loss + val_discri_fake_loss val_generator_fake_loss = tf.reduce_sum( tf.square(val_fake_output_discriminator - 1)) / args.batch_size val_reconstruction_loss = l2_loss( val_rec_iFrames, val_iFrames) * args.reconstruction_loss_weight val_generator_loss = val_generator_fake_loss + val_reconstruction_loss if args.perceptual_loss_weight: train_percp_loss = perceptual_loss(train_rec_iFrames_features, train_iFrames_features) train_generator_loss += args.perceptual_loss_weight * train_percp_loss # SUMMARIES tf.summary.scalar('train_discri_real_loss', train_discri_real_loss) tf.summary.scalar('train_discri_fake_loss', train_discri_fake_loss) tf.summary.scalar('train_discriminator_loss', train_discriminator_loss) tf.summary.scalar('train_generator_fake_loss', train_generator_fake_loss) tf.summary.scalar('train_reconstruction_loss', train_reconstruction_loss) tf.summary.scalar('train_generator_loss', train_generator_loss) tf.summary.scalar('val_discri_real_loss', val_discri_real_loss) tf.summary.scalar('val_discri_fake_loss', val_discri_fake_loss) tf.summary.scalar('val_discriminator_loss', val_discriminator_loss) tf.summary.scalar('val_generator_fake_loss', val_generator_fake_loss) tf.summary.scalar('val_reconstruction_loss', val_reconstruction_loss) tf.summary.scalar('val_generator_loss', val_generator_loss) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(CKPT_PATH + 'train', sess.graph) # get variables responsible for generator and discriminator trainable_vars = tf.trainable_variables() generator_vars = [ var for var in trainable_vars if 'generator' in var.name ] discriminator_vars = [ var for var in trainable_vars if 'discriminator' in var.name ] # DEFINE OPTIMIZERS generator_optimizer = get_optimizer(train_generator_loss, optim_id=args.optim_id, learning_rate=args.learning_rate, use_batch_norm=True, var_list=generator_vars) discriminator_optimizer = get_optimizer( train_discriminator_loss, optim_id=args.optim_id, learning_rate=args.learning_rate * 2., use_batch_norm=True, var_list=discriminator_vars) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # START TRAINING HERE for iteration in range(args.train_iters): for d_iteration in range(args.disc_train_iters): disc_, td_loss = sess.run( [discriminator_optimizer, train_discriminator_loss]) gene_, tgf_loss, tr_loss, t_summ = sess.run( [generator_optimizer, train_generator_fake_loss,\ train_reconstruction_loss, merged]) train_writer.add_summary(t_summ, iteration) print( 'Iter:{}/{}, Disc. Loss:{}, Gen. Loss:{}, Rec. Loss:{}'.format( iteration, args.train_iters, str(round(td_loss, 6)), str(round(tgf_loss, 6)), str(round(tr_loss, 6)))) if iteration % args.val_every == 0: vd_loss, vgf_loss, vr_loss = sess.run( [val_discriminator_loss, val_generator_fake_loss,\ val_reconstruction_loss]) print('Iter:{}, VAL Disc. Loss:{}, Gen. Loss:{}, Rec. Loss:{}'. format(iteration, str(round(vd_loss, 6)), str(round(vgf_loss, 6)), str(round(vr_loss, 6)))) if iteration % args.save_every == 0: saver.save( sess, CKPT_PATH + 'iter:{}_valDisc:{}_valGen:{}_valRec:{}'.format( str(iteration), str(round(vd_loss, 6)), str(round(vgf_loss, 6)), str(round(vr_loss, 6)))) if iteration % args.plot_every == 0: start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [train_fFrames, train_lFrames,\ train_iFrames,\ train_rec_iFrames]) visualize_frames(start_frames, end_frames, mid_frames, rec_mid_frames, training=True, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'train_plots/')) start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [val_fFrames, val_lFrames,\ val_iFrames, val_rec_iFrames]) visualize_frames(start_frames, end_frames, mid_frames, rec_mid_frames, training=False, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'validation_plots/')) print('Training complete.....')
def training(args): # DIRECTORY FOR CKPTS and META FILES # ROOT_DIR = '/neuhaus/movie/dataset/tf_records' ROOT_DIR = '/media/data/movie/dataset/tf_records' TRAIN_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'train.tfrecords') VAL_REC_PATH = os.path.join( ROOT_DIR, args.experiment_name, 'val.tfrecords') CKPT_PATH = os.path.join( ROOT_DIR, args.experiment_name, args.ckpt_folder_name + '/') # SCOPING BEGINS HERE with tf.Session().as_default() as sess: global_step = tf.train.get_global_step() train_queue = tf.train.string_input_producer( [TRAIN_REC_PATH], num_epochs=None) train_fFrames, train_lFrames, train_iFrames, train_mfn =\ read_and_decode( filename_queue=train_queue, is_training=True, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) val_queue = tf.train.string_input_producer( [VAL_REC_PATH], num_epochs=None) val_fFrames, val_lFrames, val_iFrames, val_mfn = \ read_and_decode( filename_queue=val_queue, is_training=False, batch_size=args.batch_size, n_intermediate_frames=args.n_IF) # Apply gaussian blurring manually ''' train_fFrames = gaussian_filter(train_fFrames, std=std_dev) train_lFrames = gaussian_filter(train_lFrames, std=std_dev) train_iFrames = gaussian_filter(train_iFrames, std=std_dev) val_fFrames = gaussian_filter(val_fFrames, std=std_dev) val_lFrames = gaussian_filter(val_lFrames, std=std_dev) val_iFrames = gaussian_filter(val_iFrames, std=std_dev) ''' with tf.variable_scope('separate_bipn'): print('TRAIN FRAMES (first):') train_rec_iFrames = skip_separate_encoder_bipn.build_bipn( train_fFrames, train_lFrames, use_batch_norm=True, is_training=True, n_IF=args.n_IF, starting_out_channels=args.starting_out_channels, use_attention=args.use_attention, spatial_attention=args.spatial_attention, is_verbose=True) with tf.variable_scope('separate_bipn', reuse=tf.AUTO_REUSE): print('VAL FRAMES (first):') val_rec_iFrames = skip_separate_encoder_bipn.build_bipn( val_fFrames, val_lFrames, use_batch_norm=True, n_IF=args.n_IF, is_training=False, starting_out_channels=args.starting_out_channels, use_attention=args.use_attention, spatial_attention=args.spatial_attention, is_verbose=False) if args.perceptual_loss_weight: # Weights should be kept locally ~ 500 MB space with tf.variable_scope('vgg16'): train_iFrames_features = vgg16.build_vgg16( train_iFrames, end_point=args.perceptual_loss_endpoint).features with tf.variable_scope('vgg16', reuse=tf.AUTO_REUSE): train_rec_iFrames_features = vgg16.build_vgg16( train_rec_iFrames, end_point=args.perceptual_loss_endpoint).features print('Global parameters:{}'.format( count_parameters(tf.global_variables()))) print('Learnable model parameters:{}'.format( count_parameters(tf.trainable_variables()))) # DEFINE METRICS if args.loss_id == 0: train_loss = huber_loss( train_iFrames, train_rec_iFrames, delta=1.) val_loss = huber_loss( val_iFrames, val_rec_iFrames, delta=1.) elif args.loss_id == 1: train_loss = tf_l2_loss( train_iFrames, train_rec_iFrames) val_loss = tf_l2_loss( val_iFrames, val_rec_iFrames) elif args.loss_id == 2: train_loss = l1_loss( train_iFrames, train_rec_iFrames) val_loss = l1_loss( val_iFrames, val_rec_iFrames) elif args.loss_id == 3: train_loss = ssim_loss( train_rec_iFrames, train_iFrames) val_loss = ssim_loss( val_rec_iFrames, val_iFrames) total_train_loss = train_loss tf.summary.scalar('train_main_loss', train_loss) tf.summary.scalar('total_val_loss', val_loss) if args.perceptual_loss_weight: train_perceptual_loss = tf_perceptual_loss( train_iFrames_features, train_rec_iFrames_features) tf.summary.scalar('train_perceptual_loss',\ train_perceptual_loss) total_train_loss += train_perceptual_loss\ * args.perceptual_loss_weight if args.weight_decay: decay_loss = ridge_weight_decay( tf.trainable_variables()) tf.summary.scalar('ridge_l2_weight_decay',\ decay_loss) total_train_loss += decay_loss\ * args.weight_decay # SUMMARIES tf.summary.scalar('total_train_loss',\ total_train_loss) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( CKPT_PATH + 'train', sess.graph) # DEFINE OPTIMIZER optimizer = get_optimizer( train_loss, optim_id=args.optim_id, learning_rate=args.learning_rate, use_batch_norm=True) init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners( coord=coord) # START TRAINING HERE for iteration in range(args.train_iters + 1): _, t_summ, t_loss = sess.run( [optimizer, merged, total_train_loss]) train_writer.add_summary(t_summ, iteration) print('Iter:{}/{}, Train Loss:{}'.format( iteration, args.train_iters, t_loss)) if iteration % args.val_every == 0: v_loss = sess.run(val_loss) print('Iter:{}, Val Loss:{}'.format( iteration, v_loss)) if iteration % args.save_every == 0: saver.save( sess, CKPT_PATH + 'iter:{}_val:{}'.format( str(iteration), str(round(v_loss, 3)))) if iteration % args.plot_every == 0: start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [train_fFrames, train_lFrames,\ train_iFrames,\ train_rec_iFrames]) visualize_frames( start_frames, end_frames, mid_frames, rec_mid_frames, training=True, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'train_plots/')) start_frames, end_frames, mid_frames,\ rec_mid_frames = sess.run( [val_fFrames, val_lFrames,\ val_iFrames, val_rec_iFrames]) visualize_frames( start_frames, end_frames, mid_frames, rec_mid_frames, training=False, iteration=iteration, save_path=os.path.join( CKPT_PATH, 'validation_plots/')) print('Training complete.....')
def testing(info): # Get the best checkpoint path weight_path = os.listdir(info['model_path']) weight_paths = [i for i in weight_path if 'meta' in i] di_weight = {} for path in weight_paths: di_weight[path] = int(path.split(':')[-1].split('.')[0]) di_weight = { k: v for k, v in sorted(di_weight.items(), key=lambda item: item[1]) } weight_path = [*di_weight][0] weight_path = weight_path[:-5] weight_path = os.path.join(info['model_path'], weight_path) n_IF = info['n_IF'] batch_size = info['batch_size'] # get #test_samples based on experiment if n_IF == 3: test_samples = 18300 elif n_IF == 4: test_samples = 18296 elif n_IF == 5: test_samples = 18265 elif n_IF == 6: test_samples = 18235 elif n_IF == 7: test_samples = 18204 test_iters = test_samples // batch_size test_samples = test_samples - (test_samples % batch_size) # get attention if info['attention']: use_attention = 1 if info['use_spatial_attention']: spatial_attention = 1 else: spatial_attention = 0 else: use_attention = 0 spatial_attention = 0 # SCOPING BEGINS HERE tf.reset_default_graph() with tf.Session() as sess: global_step = tf.train.get_global_step() test_queue = tf.train.string_input_producer([info['TEST_REC_PATH']], num_epochs=1) test_fFrames, test_lFrames, test_iFrames, test_mfn =\ read_and_decode( filename_queue=test_queue, is_training=False, batch_size=batch_size, n_intermediate_frames=n_IF, allow_smaller_final_batch=False) if info['model_name'] in ['skip', 'wnet']: with tf.variable_scope('separate_bipn'): print('TEST FRAMES (first):') if info['model_name'] == 'skip': test_rec_iFrames = skip_separate_encoder_bipn.build_bipn( test_fFrames, test_lFrames, use_batch_norm=True, is_training=False, n_IF=n_IF, starting_out_channels=info['out_channels'], use_attention=use_attention, spatial_attention=spatial_attention, is_verbose=False) elif info['model_name'] == 'wnet': test_rec_iFrames = wnet.build_wnet( test_fFrames, test_lFrames, use_batch_norm=True, is_training=False, n_IF=n_IF, starting_out_channels=info['out_channels'], use_attention=use_attention, spatial_attention=spatial_attention, is_verbose=False) elif info['model_name'] == 'slomo': with tf.variable_scope('slomo'): test_output = slomo.SloMo_model(test_fFrames, test_lFrames, first_kernel=7, second_kernel=5, reuse=False, t_steps=n_IF, verbose=False) test_rec_iFrames = test_output[0] elif info['model_name'] == 'bipn': with tf.variable_scope('bipn'): test_rec_iFrames = BiPN.build_bipn(test_fFrames, test_lFrames, n_IF=n_IF, use_batch_norm=True, is_training=False) print('Global parameters:{}'.format( count_parameters(tf.global_variables()))) print('Learnable model parameters:{}'.format( count_parameters(tf.trainable_variables()))) # DEFINE LOSS if info['loss'] == 'l2': test_loss = l2_loss(test_iFrames, test_rec_iFrames) elif info['loss'] == 'l1': test_loss = l1_loss(test_iFrames, test_rec_iFrames) # DEFINE METRICS repeat_fFrame = metric_repeat_fframe(test_fFrames, test_iFrames) repeat_lFrame = metric_repeat_lframe(test_lFrames, test_iFrames) weighted_frame = metric_weighted_frame(test_fFrames, test_iFrames, test_lFrames) inter_frame = metric_interpolated_frame(test_iFrames, test_rec_iFrames) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() sess.run(init_op) # Load checkpoints saver.restore(sess, weight_path) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) metrics = {} metrics['learnable_parameters'] = count_parameters( tf.trainable_variables()) metrics['repeat_first'] = [] metrics['repeat_last'] = [] metrics['weighted_frames'] = [] metrics['inter_frames'] = [] metrics['repeat_first_psnr'] = [] metrics['repeat_last_psnr'] = [] metrics['weighted_frames_psnr'] = [] metrics['inter_frames_psnr'] = [] print('EVALUATING:{}--------------------------->'.format( info['model_path'])) # START TRAINING HERE for iteration in range(test_iters): # get frames and metrics start_frames, end_frames, mid_frames, rec_mid_frames,\ repeat_first, repeat_last, weighted, true_metric = sess.run( [test_fFrames, test_lFrames, test_iFrames, test_rec_iFrames,\ repeat_fFrame, repeat_lFrame, weighted_frame,\ inter_frame]) samples = start_frames.shape[0] metrics['repeat_first'].append(repeat_first[0] * samples) metrics['repeat_last'].append(repeat_last[0] * samples) metrics['weighted_frames'].append(weighted[0] * samples) metrics['inter_frames'].append(true_metric[0] * samples) metrics['repeat_first_psnr'].append(repeat_first[1] * samples) metrics['repeat_last_psnr'].append(repeat_last[1] * samples) metrics['weighted_frames_psnr'].append(weighted[1] * samples) metrics['inter_frames_psnr'].append(true_metric[1] * samples) visualize_frames(start_frames, end_frames, mid_frames, rec_mid_frames, training=False, iteration=iteration, save_path=os.path.join(info['model_path'], 'test_plots' + '/')) if iteration % 50 == 0: print('{}/{} iters complete'.format(iteration, test_iters)) print('Testing complete.....') # Calculate metrics: mean_rf = sum(metrics['repeat_first']) / test_samples mean_rl = sum(metrics['repeat_last']) / test_samples mean_wf = sum(metrics['weighted_frames']) / test_samples mean_if = sum(metrics['inter_frames']) / test_samples metrics['mean_repeat_first'] = mean_rf metrics['mean_repeat_last'] = mean_rl metrics['mean_weighted_frames'] = mean_wf metrics['mean_inter_frames'] = mean_if mean_rf_psnr = sum(metrics['repeat_first_psnr']) / test_samples mean_rl_psnr = sum(metrics['repeat_last_psnr']) / test_samples mean_wf_psnr = sum(metrics['weighted_frames_psnr']) / test_samples mean_if_psnr = sum(metrics['inter_frames_psnr']) / test_samples metrics['mean_psnr_repeat_first'] = mean_rf_psnr metrics['mean_psnr_repeat_last'] = mean_rl_psnr metrics['mean_psnr_weighted_frames'] = mean_wf_psnr metrics['mean_psnr_inter_frames'] = mean_if_psnr with open(info['model_path'] + '/evaluation.pkl', 'wb') as handle: pickle.dump(metrics, handle) print('Pickle file dumped.....')