def testCallModelFnWithPlaceholders(self): with _reset_for_test() as session: config = configuration.ModelConfig() model = show_and_tell_model.ShowAndTellModel(config, mode='train') def model_fn(images, input_seq, target_seq, input_mask): model.build_model_for_tpu(images, input_seq, target_seq, input_mask) return model.total_loss images = tf.placeholder(tf.float32, shape=(1, 224, 224, 3)) input_seq = tf.placeholder(tf.int32, shape=(1, 128)) target_seq = tf.placeholder(tf.int32, shape=(1, 128)) input_mask = tf.placeholder(tf.int32, shape=(1, 128)) tpu_model_fn = tpu.rewrite(model_fn, [images, input_seq, target_seq, input_mask]) caption = np.random.randint(low=0, high=1000, size=128).reshape((1, 128)) session.run(tpu.initialize_system()) session.run(tf.global_variables_initializer()) inputs = { images: np.random.randn(1, 224, 224, 3), input_seq: caption, target_seq: caption, input_mask: np.random.random_integers(0, 1, size=128).reshape(1, 128), } session.run(tpu_model_fn, inputs) session.run(tpu.shutdown_system())
def run(): eval_dir=FLAGS.eval_dir if not tf.gfile.IsDirectory(eval_dir): tf.logging.info("Creating eval directory: %s",eval_dir) tf.gfile.MakeDirs(eval_dir) g=tf.Graph() with g.as_default(): model_config=configuration.ModelConfig() model_config.input_file_pattern=FLAGS.input_file_pattern model=show_and_tell_model.ShowAndTellModel(model_config,mode="eval") model.build() saver=tf.train.Saver() summary_op=tf.summary.merge_all() summary_writer=tf.summary.FileWriter(eval_dir) g.finalize() i=-1 while True: start=time.time() tf.logging.info("Starting evaluation at "+time.strftime("%Y-%m-%d-%H:%M:%S",time.localtime())) current_filenames=os.listdir(FLAGS.checkpoint_dir) nums=[] for x in current_filenames: if x[-5:]=='index': nums.append(int(re.findall(r'\d+',x)[0])) nums.sort() for x in nums: if x>i: run_once(model,saver,os.path.join(FLAGS.checkpoint_dir,'model.ckpt-'+str(x)),summary_writer,summary_op) i=x break time_to_next_eval=start+FLAGS.eval_interval_secs-time.time() if time_to_next_eval>0: time.sleep(time_to_next_eval)
def run(): """Runs evaluation in a loop, and logs summaries to TensorBoard.""" # Create the evaluation directory if it doesn't exist. eval_dir = FLAGS.eval_dir if not tf.gfile.IsDirectory(eval_dir): tf.logging.info("Creating eval directory: %s", eval_dir) tf.gfile.MakeDirs(eval_dir) g = tf.Graph() with g.as_default(): # Build the model for evaluation. model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model = show_and_tell_model.ShowAndTellModel(model_config, mode="eval") model.build() # Create the Saver to restore model Variables. saver = tf.train.Saver() # Create the summary operation and the summary writer. summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(eval_dir) g.finalize() # Run a new evaluation run every eval_interval_secs. while True: start = time.time() tf.logging.info("Starting evaluation at " + time.strftime( "%Y-%m-%d-%H:%M:%S", time.localtime())) run_once(model, saver, summary_writer, summary_op) time_to_next_eval = start + FLAGS.eval_interval_secs - time.time() if time_to_next_eval > 0: time.sleep(time_to_next_eval)
def main(unused_argv): assert FLAGS.input_file_pattern,"--input_file_pattern is required" assert FLAGS.train_dir,"--train_dir is required" model_config=configuration.ModelConfig() model_config.input_file_pattern=FLAGS.input_file_pattern model_config.inception_checkpoint_file=FLAGS.inception_checkpoint_file training_config=configuration.TrainingConfig() train_dir=FLAGS.train_dir if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s",train_dir) tf.gfile.MakeDirs(train_dir) g=tf.Graph() with g.as_default(): model=show_and_tell_model.ShowAndTellModel(model_config,mode="train",train_inception=FLAGS.train_inception) model.build() learning_rate_decay_fn=None if FLAGS.train_inception: learning_rate=tf.constant(training_config.train_inception_learning_rate) else: learning_rate=tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor>0: num_batches_per_epoch=(training_config.num_examples_per_epoch/model_config.batch_size) decay_steps=int(num_batches_per_epoch*training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate,global_step): return tf.train.exponential_decay(learning_rate,global_step,decay_steps=decay_steps,decay_rate=training_config.learning_rate_decay_factor,staircase=True) learning_rate_decay_fn=_learning_rate_decay_fn train_op=tf.contrib.layers.optimize_loss(loss=model.total_loss,global_step=model.global_step,learning_rate=learning_rate,optimizer=training_config.optimizer,clip_gradients=training_config.clip_gradients,learning_rate_decay_fn=learning_rate_decay_fn) saver=tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep) tf.contrib.slim.learning.train(train_op,train_dir,log_every_n_steps=FLAGS.log_every_n_steps,graph=g,global_step=model.global_step,number_of_steps=FLAGS.number_of_steps,init_fn=model.init_fn,saver=saver)
def model_fn(features, labels, mode, params): im_mode = MODEKEY_TO_MODE[mode] model_config = configuration.ModelConfig() training_config = configuration.TrainingConfig() model = show_and_tell_model.ShowAndTellModel( model_config, mode=im_mode, train_inception=FLAGS.train_inception) model.build_model_for_tpu(images=features["images"], input_seqs=features["input_seqs"], target_seqs=features["target_seqs"], input_mask=features["input_mask"]) optimizer = tf.train.GradientDescentOptimizer( learning_rate=training_config.initial_learning_rate) optimizer = tf.contrib.estimator.clip_gradients_by_norm( optimizer, training_config.clip_gradients) if FLAGS.use_tpu: optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) train_op = optimizer.minimize( model.total_loss, global_step=tf.train.get_or_create_global_step()) def scaffold_fn(): """Load pretrained Inception checkpoint at initialization time.""" return tf.train.Scaffold(init_fn=model.init_fn) return tf.contrib.tpu.TPUEstimatorSpec(mode=mode, loss=model.total_loss, train_op=train_op, scaffold_fn=scaffold_fn)
def input_fn(params): model_config = configuration.ModelConfig() model_config.input_file_pattern = params["input_file_pattern"] model_config.batch_size = params["batch_size"] model_config.mode = params["mode"] model = show_and_tell_model.ShowAndTellModel(model_config, mode="train") model.build_inputs() return { "images": model.images, "input_seqs": model.input_seqs, "target_seqs": model.target_seqs, "input_mask": model.input_mask }
def main(_): model = show_and_tell_model.ShowAndTellModel( FLAGS.model_path ) vocabi = vocab.Vocabulary(FLAGS.vocab_file) filenames = _load_filenames() generator = caption_generator.CaptionGenerator (model, vocabi) for filename in filenames: with tf.gfile.GFile(filename, "rb") as f: image = f.read() captions = generator.beam_search(image) print("Captions for image %s:" % os.path.basename(filename)) for i, caption in enumerate(captions): # Ignore begin and end tokens <S> and </S>. sentence = [ vocab.Vocabulary.id_to_token(vocabi,w) for w in caption.sentence[1:-1]] sentence = " ".join(sentence) print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
def main(unused_argv): assert FLAGS.input_file_pattern, "--input_file_pattern is required" assert FLAGS.train_dir, "--train_dir is required" model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern training_config = configuration.TrainingConfig() # Create training directory. train_dir = FLAGS.train_dir if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s", train_dir) tf.gfile.MakeDirs(train_dir) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model. model = show_and_tell_model.ShowAndTellModel(model_config, mode="train") model.build() # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model.total_loss, global_step=model.global_step, learning_rate=None, optimizer=tf.train.AdamOptimizer()) # Set up the Saver for saving and restoring model checkpoints. saver = tf.train.Saver( max_to_keep=training_config.max_checkpoints_to_keep) # Run training. tf.contrib.slim.learning.train(train_op, train_dir, log_every_n_steps=FLAGS.log_every_n_steps, graph=g, global_step=model.global_step, number_of_steps=FLAGS.number_of_steps, init_fn=model.init_fn, saver=saver, save_interval_secs=300)
def run(): inference_dir = FLAGS.inference_dir if not tf.gfile.IsDirectory(inference_dir): tf.logging.info("Creating inference directory: %s", inference_dir) tf.gfile.MakeDirs(inference_dir) g = tf.Graph() with g.as_default(): # Build the model for evaluation. model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model = show_and_tell_model.ShowAndTellModel(model_config, mode="inference") model.build() saver = tf.train.Saver() summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(inference_dir) g.finalize() tf.logging.info("Starting inference at " + time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())) run_once(model, saver, summary_writer, summary_op) tf.logging.info("Competed inference")
model_config = configuration.ModelConfig() training_config = configuration.TrainingConfig() conf = configuration.MyConfig() iter = read_data.DataIterator(encoded_image_path=conf.feat_path, caption_vector_path=conf.original_train_vec_path, image_size=conf.label_image_size) # gpu_config = tf.ConfigProto(device_count = {'GPU': 1}) # sess = tf.InteractiveSession(config=gpu_config) os.environ["CUDA_VISIBLE_DEVICES"] = "1" sess = tf.InteractiveSession() model = show_and_tell_model.ShowAndTellModel(model_config, mode="train") model.build() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(max_to_keep=1) loss_stored = [] # step = epoch * train_data_set / batch_size # with tf.device('/gpu:1'): for i in range(conf.original_train_steps): images, in_seqs, tar_seqs, masks = iter.next_batch(model_config.batch_size) loss = model.run_batch(sess, images, in_seqs, tar_seqs, masks) #every 100 steps print loss value
def main(unused_argv): assert FLAGS.input_file_pattern, "--input_file_pattern is required" assert FLAGS.train_dir, "--train_dir is required" model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file training_config = configuration.TrainingConfig() # Create training directory. train_dir = FLAGS.train_dir if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s", train_dir) tf.gfile.MakeDirs(train_dir) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model. model = show_and_tell_model.ShowAndTellModel( model_config, mode="train", train_inception=FLAGS.train_inception) model.build() # model.images: [batch_size, 299, 299, 3] image scaled to [-1, 1] # model.input_seqs: [batch_size, 20] numpy array of int64, padding to 20 with 0s # Call visualize_input(model) to visualize input for debug purposes # Set up the learning rate. learning_rate_decay_fn = None if FLAGS.train_inception: learning_rate = tf.constant( training_config.train_inception_learning_rate) else: learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = ( training_config.num_examples_per_epoch / model_config.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model.total_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn) # Set up the Saver for saving and restoring model checkpoints. saver = tf.train.Saver( max_to_keep=training_config.max_checkpoints_to_keep) # Run training. tf.contrib.slim.learning.train(train_op, train_dir, log_every_n_steps=FLAGS.log_every_n_steps, graph=g, global_step=model.global_step, number_of_steps=FLAGS.number_of_steps, init_fn=model.init_fn, saver=saver)
def main(unused_argv): model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file model_config.vgg19_checkpoint_file = FLAGS.vgg19_checkpoint_file model_config.word_embedding_file = FLAGS.word_embedding_file training_config = configuration.TrainingConfig() if cnn_model == 'InceptionV3': trained_models_dir = inception_trained_models_dir elif cnn_model == 'VGG19': trained_models_dir = vgg_trained_models_dir else: print('Unknown cnn model {0}'.format(cnn_model)) exit(0) if not tf.gfile.IsDirectory(trained_models_dir): tf.logging.info("Creating training directory: %s", trained_models_dir) tf.gfile.MakeDirs(trained_models_dir) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model, train from scratch model = show_and_tell_model.ShowAndTellModel( model_config, mode="train", cnn_model = FLAGS.cnn_model, train_cnn_model=FLAGS.train_cnn_model, custom_word_embedding=FLAGS.custom_word_embedding) model.build() # Set up the learning rate. learning_rate_decay_fn = None if FLAGS.train_cnn_model: learning_rate = tf.constant( training_config.train_inception_learning_rate) else: learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = (training_config.num_examples_per_epoch / model_config.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model.total_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn) # Set up the Saver for saving and restoring model checkpoints. saver = tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep) # Run training. tf.contrib.slim.learning.train( train_op, trained_models_dir, log_every_n_steps=FLAGS.log_every_n_steps, graph=g, global_step=model.global_step, number_of_steps=FLAGS.number_of_steps, init_fn=model.init_fn, saver=saver)
import tensorflow as tf import configuration import show_and_tell_model import read_data model_config = configuration.ModelConfig() training_config = configuration.TrainingConfig() iter = read_data.DataIterator( encoded_image_path="data/image_vgg19_fc1_feature.h5", caption_vector_path="data/train_vector.txt") sess = tf.InteractiveSession() model = show_and_tell_model.ShowAndTellModel(model_config, mode="train", train_inception=False) model.build() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() loss_stored = [] # step = epoch * train_data_set / batch_size for i in range(1000000): images, in_seqs, tar_seqs, masks = iter.next_batch(model_config.batch_size) loss = model.run_batch(sess, images, in_seqs, tar_seqs, masks) #every 100 steps print loss value if (i + 1) % 100 == 0:
def build_model(self, model_config): model = show_and_tell_model.ShowAndTellModel( model_config, mode="inference") model.build() return model
def main(unused_argv): assert FLAGS.input_file_pattern, "--input_file_pattern is required" assert FLAGS.train_dir, "--train_dir is required" # Create training directory. train_dir = FLAGS.train_dir filename_saved_model = os.path.join(FLAGS.train_dir, 'im2txt') if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s", train_dir) tf.gfile.MakeDirs(train_dir) save_flags(os.path.join(FLAGS.train_dir, 'flags.txt')) model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file training_config = configuration.TrainingConfig() vocab = vocabulary.Vocabulary(FLAGS.vocab_file) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model (teacher-forcing mode). model = show_and_tell_model.ShowAndTellModel( model_config, mode="train", train_inception=FLAGS.train_inception) model.build() # Build the model (free-running mode). model_free = show_and_tell_model.ShowAndTellModel( model_config, mode="free", train_inception=FLAGS.train_inception, vocab=vocab, reuse=True) model_free.build([ model.images, model.input_seqs, model.target_seqs, model.input_mask ]) # Build the model for validation with variable sharing model_valid = show_and_tell_model.ShowAndTellModel(model_config, mode="inference", reuse=True) model_valid.build() # get teacher behavior teacher_outputs, [teacher_state_c, teacher_state_h] = model.behavior teacher_state_c = tf.expand_dims(teacher_state_c, axis=1) teacher_state_h = tf.expand_dims(teacher_state_h, axis=1) # get free behavior free_outputs, [free_state_c, free_state_h] = model_free.behavior free_state_c = tf.expand_dims(free_state_c, axis=1) free_state_h = tf.expand_dims(free_state_h, axis=1) # get free sentence free_sentence = model_free.free_sentence # prepare behavior to be LSTM's input teacher_behavior = tf.concat( [teacher_outputs, teacher_state_c, teacher_state_h], axis=1) free_behavior = tf.concat([free_outputs, free_state_c, free_state_h], axis=1) d_lstm_cell = tf.contrib.rnn.BasicLSTMCell(model_config.num_lstm_units) d_lstm_cell = tf.contrib.rnn.DropoutWrapper( d_lstm_cell, input_keep_prob=model_config.lstm_dropout_keep_prob, output_keep_prob=model_config.lstm_dropout_keep_prob) with tf.variable_scope("discriminator") as scope_disc: teacher_lengths = tf.reduce_sum(model.input_mask, 1) + 2 free_lengths = tf.ones_like(teacher_lengths) * (30 + 2) # teacher-behavior into Discriminator-LSTM d_outputs_teacher, _ = tf.nn.dynamic_rnn( cell=d_lstm_cell, inputs=teacher_behavior, sequence_length=teacher_lengths, dtype=tf.float32, scope=scope_disc) # gather last outputs (deals with variable length of captions) teacher_lengths = tf.expand_dims(teacher_lengths, 1) batch_range = tf.expand_dims( tf.constant(np.array(range(model_config.batch_size)), dtype=tf.int32), 1) gather_idx = tf.concat([batch_range, teacher_lengths - 1], axis=1) d_last_output_teacher = tf.gather_nd(d_outputs_teacher, gather_idx) # concat inception feature d_teacher_concat = tf.concat( [d_last_output_teacher, model.image_embeddings], 1) # FC to get T/F logits # d_logits_teacher = tf.contrib.layers.fully_connected( inputs = d_last_output_teacher, #d_teacher_concat, d_logits_teacher = tf.contrib.layers.fully_connected( inputs=d_teacher_concat, num_outputs=2, activation_fn=None, weights_initializer=model.initializer, scope=scope_disc) scope_disc.reuse_variables() # teacher with wrong image wrong_image_embeddings = tf.concat( [model.image_embeddings[1:], model.image_embeddings[0:1]], 0) d_teacher_wrong_concat = tf.concat( [d_last_output_teacher, wrong_image_embeddings], 1) d_logits_teacher_wrong = tf.contrib.layers.fully_connected( inputs=d_teacher_wrong_concat, num_outputs=2, activation_fn=None, weights_initializer=model.initializer, scope=scope_disc) # free-behavior into Discriminator-LSTM d_outputs_free, _ = tf.nn.dynamic_rnn(cell=d_lstm_cell, inputs=free_behavior, sequence_length=free_lengths, dtype=tf.float32, scope=scope_disc) d_last_output_free = d_outputs_free[:, -1, :] d_free_concat = tf.concat( [d_last_output_free, model.image_embeddings], 1) # d_logits_free = tf.contrib.layers.fully_connected( inputs = d_last_output_free, #d_free_concat, d_logits_free = tf.contrib.layers.fully_connected( inputs=d_free_concat, num_outputs=2, activation_fn=None, weights_initializer=model.initializer, scope=scope_disc) d_accuracy_teacher = tf.reduce_mean( tf.cast(tf.argmax(d_logits_teacher, axis=1), tf.float32)) d_accuracy_teacher_wrong = tf.reduce_mean( tf.cast(1 - tf.argmax(d_logits_teacher_wrong, axis=1), tf.float32)) d_accuracy_free = tf.reduce_mean( tf.cast(1 - tf.argmax(d_logits_free, axis=1), tf.float32)) d_accuracy = (d_accuracy_teacher + d_accuracy_teacher_wrong + d_accuracy_free) / 3 NLL_loss = model.total_loss d_loss_teacher = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( name='d_loss_teacher', logits=d_logits_teacher, labels=tf.ones_like(d_logits_teacher))) d_loss_teacher_wrong = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( name='d_loss_teacher_wrong', logits=d_logits_teacher_wrong, labels=tf.zeros_like(d_logits_teacher_wrong))) d_loss_free = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( name='d_loss_free', logits=d_logits_free, labels=tf.zeros_like(d_logits_free))) d_loss = d_loss_teacher + d_loss_teacher_wrong + d_loss_free g_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( name='g_loss', logits=d_logits_free, labels=tf.ones_like(d_logits_free))) g_and_NLL_loss = g_loss + NLL_loss # tf.summary summary_NLL_loss = tf.summary.scalar('NLL_loss', NLL_loss) summary_temp_list = [ tf.summary.scalar('d_loss', d_loss), tf.summary.scalar('d_loss_teacher', d_loss_teacher), tf.summary.scalar('d_loss_teacher_wrong', d_loss_teacher_wrong), tf.summary.scalar('d_loss_free', d_loss_free), tf.summary.scalar('d_accuracy', d_accuracy), tf.summary.scalar('d_accuracy_teacher', d_accuracy_teacher), tf.summary.scalar('d_accuracy_teacher_wrong', d_accuracy_teacher_wrong), tf.summary.scalar('d_accuracy_free', d_accuracy_free) ] summary_disc = tf.summary.merge(summary_temp_list) summary_temp_list = [ tf.summary.scalar('g_and_NLL_loss', g_and_NLL_loss), summary_NLL_loss, tf.summary.scalar('g_loss', g_loss), tf.summary.scalar('d_accuracy_free', d_accuracy_free) ] summary_gen = tf.summary.merge(summary_temp_list) # Set up the learning rate for training ops learning_rate_decay_fn = None if FLAGS.train_inception: learning_rate = tf.constant( training_config.train_inception_learning_rate) else: learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = ( training_config.num_examples_per_epoch // model_config.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Collect trainable variables vars_all = [ v for v in tf.trainable_variables() if v not in model.inception_variables ] d_vars = [v for v in vars_all if 'discr' in v.name] g_vars = [v for v in vars_all if 'discr' not in v.name] # Set up the training ops. train_op_NLL = tf.contrib.layers.optimize_loss( loss=NLL_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn, variables=g_vars, name='optimize_NLL_loss') train_op_disc = tf.contrib.layers.optimize_loss( loss=d_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn, variables=d_vars, name='optimize_disc_loss') train_op_gen = tf.contrib.layers.optimize_loss( loss=NLL_loss + g_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn, variables=g_vars, name='optimize_gen_loss') # Set up the Saver for saving and restoring model checkpoints. saver = tf.train.Saver( max_to_keep=training_config.max_checkpoints_to_keep) with tf.Session() as sess: # initialize all variables tf.global_variables_initializer().run() # load inception variables model.init_fn(sess) # Set up the training ops nBatches = num_batches_per_epoch summaryWriter = tf.summary.FileWriter(train_dir, sess.graph) # start input enqueue threads coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) counter = 0 start_time = time.time() could_load, checkpoint_counter = load(sess, saver, train_dir) if could_load: counter = checkpoint_counter generator = caption_generator.CaptionGenerator(model_valid, vocab, beam_size=1) try: # for validation f_valid_text = open(os.path.join(train_dir, 'valid.txt'), 'a') filenames = os.listdir('testimgs') filenames.sort() valid_images = [] print('validation image filenames') for filename in filenames: with tf.gfile.GFile(os.path.join('testimgs', filename), 'r') as f: valid_images.append(f.read()) print(filename) # # run inference for not-trained model # for i, valid_image in enumerate(valid_images): # captions = generator.beam_search( sess, valid_image ) # f_valid_text.write( 'initial caption (beam) {}\n'.format( str(datetime.datetime.now().time())[:-7] ) ) # for j, caption in enumerate(captions): # sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]] # sentence = " ".join(sentence) # sentence = " {}-{}) {} (p={:.8f})".format(i+1,j+1, sentence, math.exp(caption.logprob)) # print( sentence ) # f_valid_text.write( sentence +'\n' ) # f_valid_text.flush() # run training loop lossnames_to_print = [ 'NLL_loss', 'g_loss', 'd_loss', 'd_acc', 'g_acc' ] val_NLL_loss = float('Inf') val_g_loss = float('Inf') val_d_loss = float('Inf') val_d_acc = 0 val_g_acc = 0 for epoch in range(FLAGS.number_of_steps): for batch_idx in range(nBatches): counter += 1 is_disc_trained = False is_gen_trained = False # train NLL loss only (for im2txt sanity check) # _, val_NLL_loss, smry, val_free_sentence, val_teacher_sentence = \ # sess.run( # [train_op_NLL, NLL_loss, summary_NLL_loss,free_sentence, model.input_seqs] ) # summaryWriter.add_summary(smry, counter) if val_NLL_loss > 3: _, val_NLL_loss, smry, val_free_sentence, val_teacher_sentence = sess.run( [ train_op_NLL, NLL_loss, summary_NLL_loss, free_sentence, model.input_seqs ]) summaryWriter.add_summary(smry, counter) else: # train discriminator _, val_d_loss, val_d_acc, smry = sess.run([ train_op_disc, d_loss, d_accuracy, summary_disc ]) summaryWriter.add_summary(smry, counter) # train generator twice _, val_g_loss, val_NLL_loss, val_g_acc, smry, val_free_sentence, val_teacher_sentence = sess.run( [ train_op_gen, g_loss, NLL_loss, d_accuracy, summary_gen, free_sentence, model.input_seqs ]) summaryWriter.add_summary(smry, counter) _, val_g_loss, val_NLL_loss, val_g_acc, smry, val_free_sentence, val_teacher_sentence = sess.run( [ train_op_gen, g_loss, NLL_loss, d_accuracy, summary_gen, free_sentence, model.input_seqs ]) summaryWriter.add_summary(smry, counter) if counter % FLAGS.log_every_n_steps == 0: elapsed = time.time() - start_time log(epoch, batch_idx, nBatches, lossnames_to_print, [ val_NLL_loss, val_g_loss, val_d_loss, val_d_acc, val_g_acc ], elapsed, counter) if counter % 500 == 1 or \ (epoch==FLAGS.number_of_steps-1 and batch_idx==nBatches-1) : saver.save(sess, filename_saved_model, global_step=counter) # if True: if (batch_idx + 1) % (nBatches // 10) == 0 or batch_idx == nBatches - 1: # run test after every epoch f_valid_text.write( 'count {} epoch {} batch {}/{} ({})\n'.format( \ counter, epoch, batch_idx, nBatches, str(datetime.datetime.now().time())[:-7] ) ) for i, valid_image in enumerate(valid_images): captions = generator.beam_search( sess, valid_image) for j, caption in enumerate(captions): sentence = [ vocab.id_to_word(w) for w in caption.sentence[1:-1] ] sentence = " ".join(sentence) sentence = " {}-{}) {} (p={:.8f})".format( i + 1, j + 1, sentence, math.exp(caption.logprob)) print(sentence) f_valid_text.write(sentence + '\n') # free sentence check for i, caption in enumerate(val_free_sentence): if i > 9: break sentence = [ vocab.id_to_word(w) for w in caption ] sentence = " ".join(sentence) sentence = " free %d) %s" % (i + 1, sentence) print(sentence) f_valid_text.write(sentence + '\n') sentence = [ vocab.id_to_word(w) for w in val_teacher_sentence[i, 1:] ] sentence = " ".join(sentence) sentence = " teacher %d) %s\n" % (i + 1, sentence) print(sentence) f_valid_text.write(sentence + '\n') f_valid_text.flush() except tf.errors.OutOfRangeError: print('Finished training: epoch limit reached') finally: coord.request_stop() coord.join(threads)
def main(unused_argv): assert FLAGS.input_file_pattern, "--input_file_pattern is required" assert FLAGS.train_dir, "--train_dir is required" model_config = configuration.ModelConfig() model_config.input_file_pattern = FLAGS.input_file_pattern model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file training_config = configuration.TrainingConfig() # Create training directory. train_dir = FLAGS.train_dir if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s", train_dir) tf.gfile.MakeDirs(train_dir) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model. model = show_and_tell_model.ShowAndTellModel( model_config, mode="train", train_inception=FLAGS.train_inception) model.build() # Set up the learning rate. learning_rate_decay_fn = None if FLAGS.train_inception: learning_rate = tf.constant( training_config.train_inception_learning_rate) else: learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = ( training_config.num_examples_per_epoch / model_config.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model.total_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn) # Set up the Saver for saving and restoring model checkpoints. saver = tf.train.Saver( max_to_keep=training_config.max_checkpoints_to_keep) # Run training.automatic initialize the threads? summary_op = tf.summary.merge_all() save_summaries_secs = 10 summary_writer = tf.summary.FileWriter('./log_train') tf.contrib.slim.learning.train(train_op, train_dir, log_every_n_steps=FLAGS.log_every_n_steps, graph=g, global_step=model.global_step, number_of_steps=FLAGS.number_of_steps, init_fn=model.init_fn, saver=saver, summary_op=summary_op, save_summaries_secs=save_summaries_secs, summary_writer=summary_writer)
def main(unused_argv): '''assert FLAGS.input_file_pattern, "--input_file_pattern is required" assert FLAGS.train_dir, "--train_dir is required"''' model_config = configuration.ModelConfig() model_config.input_file_pattern = base_folder + "/mscoco/train-?????-of-00256" #FLAGS.input_file_pattern model_config.inception_checkpoint_file = base_folder + "/model/inception_v3.ckpt" #FLAGS.inception_checkpoint_file training_config = configuration.TrainingConfig() # Create training directory. train_dir = base_folder + "/model/3m_wa_n_all/train" #FLAGS.train_dir if not tf.gfile.IsDirectory(train_dir): tf.logging.info("Creating training directory: %s", train_dir) tf.gfile.MakeDirs(train_dir) # Build the TensorFlow graph. g = tf.Graph() with g.as_default(): # Build the model. model = show_and_tell_model.ShowAndTellModel( model_config, mode="train", train_inception=FLAGS.train_inception) model.build() # Set up the learning rate. learning_rate_decay_fn = None if FLAGS.train_inception: learning_rate = tf.constant( training_config.train_inception_learning_rate) else: learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = ( training_config.num_examples_per_epoch / model_config.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model.total_loss, global_step=model.global_step, learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn) # Set up the Saver for saving and restoring model checkpoints. var_list = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES) #, scope = the_scope) print("VL ", var_list) saver = tf.train.Saver( var_list=var_list, max_to_keep=training_config.max_checkpoints_to_keep, save_relative_paths=True) var_list2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='(?!attention_var)') print("test", var_list2) saver2 = tf.train.Saver(var_list=var_list2) attVar = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="attention_var") local_init_op = tf.variables_initializer(attVar) def init_fn_2(sess): tf.logging.info("Restoring the original weight %s", model_config.inception_checkpoint_file) checkpoint_path2 = model_config.base_folder + "/model/3m/train/model.ckpt-3003677" saver2.restore(sess, checkpoint_path2) sess.run(local_init_op) config = tf.ConfigProto() config.gpu_options.visible_device_list = "0" gpu_fract = .75 config.gpu_options.per_process_gpu_memory_fraction = gpu_fract # Run training. tf.contrib.slim.learning.train( train_op, train_dir, log_every_n_steps=FLAGS.log_every_n_steps, graph=g, global_step=model.global_step, number_of_steps=FLAGS.number_of_steps, init_fn=init_fn_2, saver=saver, #local_init_op = local_init_op, session_config=config)