def train(main_config, args): main_cfg = MainConfig(main_config, args) main_cfg.max_sequence_len = args.max_seq_length main_cfg.syn_weight = args.syn_weight main_cfg.vocab_size = args.vocab_size batch_size = tf.placeholder(tf.int64) vocab = tf.contrib.lookup.index_table_from_file( vocabulary_file=os.path.join(main_cfg.data_dir, main_cfg.vocab_file), num_oov_buckets=0, default_value=1) with tf.device('/cpu:0'): train_iter, dev_iter, test_iter, dev_run_iter = get_quora_datasets( main_cfg, batch_size, vocab) train_handle, test_handle, dev_handle, dev_run_handle = train_iter.string_handle(), \ test_iter.string_handle(), \ dev_iter.string_handle(), dev_run_iter.string_handle() model_name = 'bilstm_{}'.format(main_config['PARAMS']['embedding_size']) # Switcher handle placeholder, iterator handle = tf.placeholder(tf.string, shape=[], name='handle') quora_iter = tf.data.Iterator.from_string_handle(handle, train_iter.output_types, train_iter.output_shapes) quora_example = quora_iter.get_next() # obtaining finished step = tf.train.get_or_create_global_step() main_config['DATA']['emb_path'] = os.path.join( args.data_dir, main_config['DATA']['embeddings']) model = BiLSTMSiamese(quora_example, args, main_config) best_loss = float("inf") best_ckpt_saver = BestCheckpointSaver( save_dir=args.model_dir, num_to_keep=main_cfg.checkpoints_to_keep, maximize=True) best_accuracy = best_ckpt_saver.get_best_accuracy() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) saver = tf.train.Saver() with tf.Session(config=config) as session: session.run(tf.tables_initializer()) saver.restore( session, get_best_checkpoint(args.model_dir, select_maximum_value=True)) model.set_session(session) model_evaluator = EvalHelper(model, session, logger) session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) dev_handle = session.run(dev_handle) all_dev_acc, all_dev_loss = model_evaluator.evaluate_dev( dev_handle, 1, logger) test_handle = session.run(test_handle) session.run(test_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) test_acc, test_loss = model_evaluator.evaluate_test( test_handle, logger) model_evaluator.save_test_evaluation(args.model_dir)
def train(main_config, args): main_cfg = MainConfig(main_config, args) main_cfg.max_sequence_len = args.max_seq_length main_cfg.syn_weight = args.syn_weight main_cfg.vocab_size = args.vocab_size batch_size = tf.placeholder(tf.int64) vocab = tf.contrib.lookup.index_table_from_file(vocabulary_file=os.path.join(main_cfg.data_dir, main_cfg.vocab_file), num_oov_buckets=0, default_value=1) with tf.device('/cpu:0'): train_iter, dev_iter, test_iter, dev_run_iter = get_quora_datasets(main_cfg, batch_size, vocab) train_handle, test_handle, dev_handle, dev_run_handle = train_iter.string_handle(), \ test_iter.string_handle(), \ dev_iter.string_handle(), dev_run_iter.string_handle() model_name = 'bilstm_{}'.format(main_config['PARAMS']['embedding_size']) # Switcher handle placeholder, iterator handle = tf.placeholder(tf.string, shape=[], name='handle') quora_iter = tf.data.Iterator.from_string_handle(handle, train_iter.output_types, train_iter.output_shapes) quora_example = quora_iter.get_next() # obtaining finished step = tf.train.get_or_create_global_step() main_config['DATA']['emb_path'] = os.path.join(args.data_dir, main_config['DATA']['embeddings']) model = BiLSTMSiamese(quora_example, args, main_config) best_loss = float("inf") best_ckpt_saver = BestCheckpointSaver( save_dir=args.model_dir, num_to_keep=main_cfg.checkpoints_to_keep, maximize=True ) best_accuracy = best_ckpt_saver.get_best_accuracy() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.train.MonitoredTrainingSession(checkpoint_dir=main_cfg.model_dir, save_checkpoint_steps=main_cfg.save_every, config=config, save_summaries_steps=0, save_summaries_secs=None, log_step_count_steps=0, hooks=[ SaveAtEnd(args.model_dir, model_name, main_cfg.checkpoints_to_keep)]) as session: model.set_session(session) log_saver = LogSaver(args.model_dir, 'summaries', session.graph) model_evaluator = EvalHelper(model, session, logger) start_time = time.time() patience_left = main_cfg.patience train_handle, dev_handle, dev_run_handle = session.run([train_handle, dev_handle, dev_run_handle]) session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size}) session.run(dev_run_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) logger.info('Starting training...') while patience_left: try: # run train batch loss, _, current_quora_ids, _ = model.run_train_batch(train_handle) global_step = session.run(step) # Use paraphrases to help model on difficult examples if global_step % main_cfg.eval_every == 0: model.evaluation_stats(train_handle, dev_run_handle, global_step, logger, log_saver) except tf.errors.OutOfRangeError: session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) all_dev_acc, all_dev_loss = model_evaluator.evaluate_dev(dev_handle, global_step, logger) session.run(model.inc_gstep) # decay LR if all_dev_acc > best_accuracy: best_accuracy = all_dev_acc best_ckpt_saver.handle(all_dev_acc, session, step) patience_left = main_cfg.patience else: patience_left -= 1 if best_loss > all_dev_loss: best_loss = all_dev_loss session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size}) logger.info('No improvement observed over {} epochs. Initiating early stopping'.format(main_cfg.patience)) end_time = time.time() total_time = timer(start_time, end_time) logger.info('Training took {}, best accuracy is {}'.format(total_time, best_accuracy)) # model_evaluator.save_dev_evaluation(args.model_dir, total_time) with open(os.path.join(args.model_dir, 'run_config.ini'), 'w') as configfile: # save main_config.write(configfile) saver = tf.train.Saver() with tf.Session(config=config) as session: session.run(tf.tables_initializer()) saver.restore(session, get_best_checkpoint(args.model_dir, select_maximum_value=True)) model.set_session(session) model_evaluator = EvalHelper(model, session, logger) test_handle = session.run(test_handle) session.run(test_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size}) test_acc, test_loss = model_evaluator.evaluate_test(test_handle, logger) model_evaluator.save_test_evaluation(args.model_dir)