def train(main_config, args):
main_cfg = MainConfig(main_config, args)
main_cfg.max_sequence_len = args.max_seq_length
main_cfg.syn_weight = args.syn_weight
main_cfg.vocab_size = args.vocab_size
batch_size = tf.placeholder(tf.int64)
vocab = tf.contrib.lookup.index_table_from_file(
vocabulary_file=os.path.join(main_cfg.data_dir, main_cfg.vocab_file),
num_oov_buckets=0,
default_value=1)
with tf.device('/cpu:0'):
train_iter, dev_iter, test_iter, dev_run_iter = get_quora_datasets(
main_cfg, batch_size, vocab)
train_handle, test_handle, dev_handle, dev_run_handle = train_iter.string_handle(), \
test_iter.string_handle(), \
dev_iter.string_handle(), dev_run_iter.string_handle()
model_name = 'bilstm_{}'.format(main_config['PARAMS']['embedding_size'])
# Switcher handle placeholder, iterator
handle = tf.placeholder(tf.string, shape=[], name='handle')
quora_iter = tf.data.Iterator.from_string_handle(handle,
train_iter.output_types,
train_iter.output_shapes)
quora_example = quora_iter.get_next()
# obtaining finished
step = tf.train.get_or_create_global_step()
main_config['DATA']['emb_path'] = os.path.join(
args.data_dir, main_config['DATA']['embeddings'])
model = BiLSTMSiamese(quora_example, args, main_config)
best_loss = float("inf")
best_ckpt_saver = BestCheckpointSaver(
save_dir=args.model_dir,
num_to_keep=main_cfg.checkpoints_to_keep,
maximize=True)
best_accuracy = best_ckpt_saver.get_best_accuracy()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
saver = tf.train.Saver()
with tf.Session(config=config) as session:
session.run(tf.tables_initializer())
saver.restore(
session,
get_best_checkpoint(args.model_dir, select_maximum_value=True))
model.set_session(session)
model_evaluator = EvalHelper(model, session, logger)
session.run(dev_iter.initializer,
feed_dict={batch_size: main_cfg.eval_batch_size})
dev_handle = session.run(dev_handle)
all_dev_acc, all_dev_loss = model_evaluator.evaluate_dev(
dev_handle, 1, logger)
test_handle = session.run(test_handle)
session.run(test_iter.initializer,
feed_dict={batch_size: main_cfg.eval_batch_size})
test_acc, test_loss = model_evaluator.evaluate_test(
test_handle, logger)
model_evaluator.save_test_evaluation(args.model_dir)
def train(main_config, args):
main_cfg = MainConfig(main_config, args)
main_cfg.max_sequence_len = args.max_seq_length
main_cfg.syn_weight = args.syn_weight
main_cfg.vocab_size = args.vocab_size
batch_size = tf.placeholder(tf.int64)
vocab = tf.contrib.lookup.index_table_from_file(vocabulary_file=os.path.join(main_cfg.data_dir,
main_cfg.vocab_file),
num_oov_buckets=0,
default_value=1)
with tf.device('/cpu:0'):
train_iter, dev_iter, test_iter, dev_run_iter = get_quora_datasets(main_cfg, batch_size, vocab)
train_handle, test_handle, dev_handle, dev_run_handle = train_iter.string_handle(), \
test_iter.string_handle(), \
dev_iter.string_handle(), dev_run_iter.string_handle()
model_name = 'bilstm_{}'.format(main_config['PARAMS']['embedding_size'])
# Switcher handle placeholder, iterator
handle = tf.placeholder(tf.string, shape=[], name='handle')
quora_iter = tf.data.Iterator.from_string_handle(handle,
train_iter.output_types,
train_iter.output_shapes)
quora_example = quora_iter.get_next()
# obtaining finished
step = tf.train.get_or_create_global_step()
main_config['DATA']['emb_path'] = os.path.join(args.data_dir, main_config['DATA']['embeddings'])
model = BiLSTMSiamese(quora_example, args, main_config)
best_loss = float("inf")
best_ckpt_saver = BestCheckpointSaver(
save_dir=args.model_dir,
num_to_keep=main_cfg.checkpoints_to_keep,
maximize=True
)
best_accuracy = best_ckpt_saver.get_best_accuracy()
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
with tf.train.MonitoredTrainingSession(checkpoint_dir=main_cfg.model_dir,
save_checkpoint_steps=main_cfg.save_every,
config=config,
save_summaries_steps=0, save_summaries_secs=None,
log_step_count_steps=0,
hooks=[
SaveAtEnd(args.model_dir, model_name,
main_cfg.checkpoints_to_keep)]) as session:
model.set_session(session)
log_saver = LogSaver(args.model_dir, 'summaries', session.graph)
model_evaluator = EvalHelper(model, session, logger)
start_time = time.time()
patience_left = main_cfg.patience
train_handle, dev_handle, dev_run_handle = session.run([train_handle, dev_handle, dev_run_handle])
session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size})
session.run(dev_run_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
logger.info('Starting training...')
while patience_left:
try:
# run train batch
loss, _, current_quora_ids, _ = model.run_train_batch(train_handle)
global_step = session.run(step)
# Use paraphrases to help model on difficult examples
if global_step % main_cfg.eval_every == 0:
model.evaluation_stats(train_handle, dev_run_handle, global_step, logger, log_saver)
except tf.errors.OutOfRangeError:
session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
all_dev_acc, all_dev_loss = model_evaluator.evaluate_dev(dev_handle, global_step, logger)
session.run(model.inc_gstep) # decay LR
if all_dev_acc > best_accuracy:
best_accuracy = all_dev_acc
best_ckpt_saver.handle(all_dev_acc, session, step)
patience_left = main_cfg.patience
else:
patience_left -= 1
if best_loss > all_dev_loss:
best_loss = all_dev_loss
session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size})
logger.info('No improvement observed over {} epochs. Initiating early stopping'.format(main_cfg.patience))
end_time = time.time()
total_time = timer(start_time, end_time)
logger.info('Training took {}, best accuracy is {}'.format(total_time, best_accuracy))
# model_evaluator.save_dev_evaluation(args.model_dir, total_time)
with open(os.path.join(args.model_dir, 'run_config.ini'), 'w') as configfile: # save
main_config.write(configfile)
saver = tf.train.Saver()
with tf.Session(config=config) as session:
session.run(tf.tables_initializer())
saver.restore(session, get_best_checkpoint(args.model_dir, select_maximum_value=True))
model.set_session(session)
model_evaluator = EvalHelper(model, session, logger)
test_handle = session.run(test_handle)
session.run(test_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
test_acc, test_loss = model_evaluator.evaluate_test(test_handle, logger)
model_evaluator.save_test_evaluation(args.model_dir)