def dev_step(sess, model, target_loss_weight): results = defaultdict(list) num_test = 0 num_correct = 0.0 test_batches = data_helpers.batch_iter(valid_dataset, FLAGS.batch_size, 1, target_loss_weight, FLAGS.max_sequence_length, charVocab, FLAGS.max_word_length, shuffle=True) for test_batch in test_batches: x_question, x_answer, x_question_len, x_answer_len, x_target, x_target_weight, id_pairs, x_q_char, x_q_len, x_a_char, x_a_len = test_batch feed_dict = { model.question: x_question, model.answer: x_answer, model.question_len: x_question_len, model.answer_len: x_answer_len, model.target: x_target, model.target_loss_weight: x_target_weight, model.dropout_keep_prob: 1.0, model.q_charVec: x_q_char, model.q_charLen: x_q_len, model.a_charVec: x_a_char, model.a_charLen: x_a_len } batch_accuracy, predicted_prob = sess.run( [model.accuracy, model.probs], feed_dict) num_test += len(predicted_prob) if num_test % 1000 == 0: print(num_test) num_correct += len(predicted_prob) * batch_accuracy for i, prob_score in enumerate(predicted_prob): question_id, answer_id, label = id_pairs[i] results[question_id].append((answer_id, label, prob_score)) #calculate top-1 precision print('num_test_samples: {} test_accuracy: {}'.format( num_test, num_correct / num_test)) accu, precision, recall, f1, loss = metrics.classification_metrics(results) print('Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'.format( accu, precision, recall, f1, loss)) mrr = metrics.mean_reciprocal_rank(results) top_1_precision = metrics.top_k_precision(results, k=1) top_2_precision = metrics.top_k_precision(results, k=2) top_5_precision = metrics.top_k_precision(results, k=5) top_10_precision = metrics.top_k_precision(results, k=10) total_valid_query = metrics.get_num_valid_query(results) print( 'MRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}\n'. format(mrr, top_1_precision, total_valid_query)) print('Top-2 precision: {}'.format(top_2_precision)) print('Top-5 precision: {}'.format(top_5_precision)) print('Top-10 precision: {}'.format(top_10_precision)) return mrr
def dev_step(): results = defaultdict(list) num_test = 0 num_correct = 0.0 test_batches = data_helpers.batch_iter(test_dataset, FLAGS.batch_size, 1, target_loss_weight, idf, SEQ_LEN, charVocab, FLAGS.max_word_length, shuffle=True) for test_batch in test_batches: x_question, x_answer, x_question_len, x_answer_len, x_target, x_target_weight, id_pairs, extra_feature, q_feature, a_feature, x_q_char, x_q_len, x_a_char, x_a_len = test_batch feed_dict = { esim.question: x_question, esim.answer: x_answer, esim.question_len: x_question_len, esim.answer_len: x_answer_len, esim.target: x_target, esim.target_loss_weight: x_target_weight, esim.dropout_keep_prob: 1.0, esim.extra_feature: extra_feature, esim.q_word_feature: q_feature, esim.a_word_feature: a_feature, esim.q_charVec: x_q_char, esim.q_charLen: x_q_len, esim.a_charVec: x_a_char, esim.a_charLen: x_a_len } batch_accuracy, predicted_prob = sess.run( [esim.accuracy, esim.probs], feed_dict) num_test += len(predicted_prob) if num_test % 1000 == 0: print(num_test) num_correct += len(predicted_prob) * batch_accuracy for i, prob_score in enumerate(predicted_prob): question_id, answer_id, label = id_pairs[i] results[question_id].append( (answer_id, label, prob_score)) #calculate top-1 precision print('num_test_samples: {} test_accuracy: {}'.format( num_test, num_correct / num_test)) accu, precision, recall, f1, loss = metrics.classification_metrics( results) print( 'Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'. format(accu, precision, recall, f1, loss)) mvp = metrics.mean_average_precision(results) mrr = metrics.mean_reciprocal_rank(results) top_1_precision = metrics.top_1_precision(results) total_valid_query = metrics.get_num_valid_query(results) print( 'MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}' .format(mvp, mrr, top_1_precision, total_valid_query)) return mrr
mvp = metrics.mean_average_precision(results) mrr = metrics.mean_reciprocal_rank(results) top_1_precision = metrics.top_1_precision(results) total_valid_query = metrics.get_num_valid_query(results) print( 'MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}' .format(mvp, mrr, top_1_precision, total_valid_query)) return mrr best_mrr = 0.0 batches = data_helpers.batch_iter(train_dataset, FLAGS.batch_size, FLAGS.num_epochs, target_loss_weight, idf, SEQ_LEN, charVocab, FLAGS.max_word_length, shuffle=True) for batch in batches: x_question, x_answer, x_question_len, x_answer_len, x_target, x_target_weight, id_pairs, extra_feature, q_feature, a_feature, x_q_char, x_q_len, x_a_char, x_a_len = batch train_step(x_question, x_answer, x_question_len, x_answer_len, x_target, x_target_weight, id_pairs, extra_feature, q_feature, a_feature, x_q_char, x_q_len, x_a_char, x_a_len) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") valid_mrr = dev_step() if valid_mrr > best_mrr:
q_char_len = graph.get_operation_by_name( "question_char_len").outputs[0] a_char_feature = graph.get_operation_by_name("answer_char").outputs[0] a_char_len = graph.get_operation_by_name("answer_char_len").outputs[0] # Tensors we want to evaluate prob = graph.get_operation_by_name("convolution-1/prob").outputs[0] results = defaultdict(list) num_test = 0 test_batches = data_helpers.batch_iter(test_dataset, FLAGS.batch_size, 1, target_loss_weight, idf, SEQ_LEN, charVocab, FLAGS.max_word_length, shuffle=False) for test_batch in test_batches: batch_question, batch_answer, batch_question_len, batch_answer_len, batch_target, batch_target_weight, batch_id_pairs, extra_feature, q_feature, a_feature, x_q_char, x_q_len, x_a_char, x_a_len = test_batch feed_dict = { question_x: batch_question, answer_x: batch_answer, question_len: batch_question_len, answer_len: batch_answer_len, dropout_keep_prob: 1.0, model_extra_feature: extra_feature, question_word_feature: q_feature, answer_word_feature: a_feature,
def train(): graph = tf.Graph() with graph.as_default(): with tf.device("/gpu:0"): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): model = DualEncoder(sequence_length=FLAGS.max_sequence_length, vocab_size=len(vocab), embedding_size=FLAGS.embedding_dim, vocab=vocab, rnn_size=FLAGS.rnn_size, maxWordLength=FLAGS.max_word_length, charVocab=charVocab, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) starter_learning_rate = 0.001 target_loss_weight = [1.0, 1.0] learning_rate = tf.train.exponential_decay( starter_learning_rate, global_step, 5000, 0.96, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads_and_vars = optimizer.compute_gradients(model.mean_loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(out_dir)) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables()) # Initialize all variables sess.run(tf.global_variables_initializer()) best_mrr = 0.0 batches = data_helpers.batch_iter(train_dataset, FLAGS.batch_size, FLAGS.num_epochs, target_loss_weight, FLAGS.max_sequence_length, charVocab, FLAGS.max_word_length, shuffle=True) for batch in batches: x_question, x_answer, x_question_len, x_answer_len, x_target, x_target_weight, id_pairs, x_q_char, x_q_len, x_a_char, x_a_len = batch feed_dict = { model.question: x_question, model.answer: x_answer, model.question_len: x_question_len, model.answer_len: x_answer_len, model.target: x_target, model.target_loss_weight: x_target_weight, model.dropout_keep_prob: FLAGS.dropout_keep_prob, model.q_charVec: x_q_char, model.q_charLen: x_q_len, model.a_charVec: x_a_char, model.a_charLen: x_a_len } train_step(sess, train_op, global_step, model, feed_dict) current_step = tf.train.global_step(sess, global_step) if current_step > 10 and current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:") valid_mrr = dev_step(sess, model, target_loss_weight) if valid_mrr > best_mrr: best_mrr = valid_mrr path = saver.save(sess, checkpoint_prefix, global_step=current_step) print( "Saved model checkpoint to {}\n".format(path))