def generate_qas(questions_list): qas_list = [] for ques in questions_list: rand_id = ''.join(random.choice('0123456789abcde') for i in range(24)) qas_list.append({ 'answers': generate_answers(), 'id': rand_id, 'question': ques }) return qas_list
def getAnswers(): data = json.loads(request.data) print data qn_uuid_data, context_token_data, qn_token_data = get_json(data) print "qn_uuid_data" print qn_uuid_data print "#" * 100 print "context_token_data" print context_token_data print "#" * 100 print "qn_token_data" print qn_token_data answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) return jsonify(answers_dict)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) print "qn_uuid_data" print qn_uuid_data print "#" * 100 print "context_token_data" print context_token_data print "#" * 100 print "qn_token_data" print qn_token_data with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or \ os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings timer.start("glove_getter") emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) id2idf = get_idf(os.path.abspath(FLAGS.idf_path), word2id) logger.warn("Get glove embedding of size {} takes {:.2f} s".format( FLAGS.embedding_size, timer.stop("glove_getter"))) # Print out Tensorflow version # print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ ensumble = FLAGS.ensumble print(ensumble) if not ensumble and not FLAGS.attn_layer and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --attn_layer or --train_dir") # Define train_dir if not FLAGS.experiment_name: FLAGS.experiment_name = "A_{}_E_{}_D_{}".format( FLAGS.attn_layer, FLAGS.embedding_size, FLAGS.dropout) checkptr_name = FLAGS.experiment_name + "/glove{}".format( FLAGS.embedding_size) FLAGS.train_dir = FLAGS.train_dir or\ os.path.join(EXPERIMENTS_DIR, checkptr_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True is_training = (FLAGS.mode == "train") if not ensumble: # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2idf, is_training) else: ensumbler = Ensumbler(ensumble, config, id2word, word2id, emb_matrix, id2idf) # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model qa_model.initialize_from_checkpoint(sess, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores f1, em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) logger.info("Dev: F1 = {0:.3}, EM = {0:.3}".format(f1, em)) elif FLAGS.mode == "eval": if ensumble: # train train_f1, train_em = ensumbler.check_f1_em(train_context_path, train_qn_path, train_ans_path, "train", FLAGS.n_eval) # dev dev_f1, dev_em = ensumbler.check_f1_em(dev_context_path, dev_qn_path, dev_ans_path, "dev", FLAGS.n_eval) else: with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) logger.info("Model initialzed from checkpoint") # train train_f1, train_em = qa_model.check_f1_em( sess, train_context_path, train_qn_path, train_ans_path, "train", num_samples=10, print_to_screen=False) # dev dev_f1, dev_em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=False) logger.error("Train: F1 = {:.3}, EM = {:.3}".format( train_f1, train_em)) logger.error("Dev: F1 = {:.3}, EM = {:.3}".format(dev_f1, dev_em)) elif FLAGS.mode == "official_eval": if not ensumble: if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path" ) if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir" ) # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) if ensumble: answers_dict = ensumbler.generate_answers(qn_uuid_data, context_token_data, qn_token_data) else: with tf.Session(config=config) as sess: # Load model from ckpt_load_dir qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, id2idf, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Initialize bestmodel directory bestmodel_dir = os.path.join( "F:\Sid\Learnings\Data Scientist\Text Analysis\Machine Comprehension\data\ModelLogs\best_weights", "best_checkpoint") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception("You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config=tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception("For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception("For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
dev_loss = sum(loss_per_batch) / float(total_num_examples) return dev_loss if __name__ == "__main__": mode = sys.argv[1] processor = Processor() if mode == "train": model_file_path = None if len(sys.argv) > 2: model_file_path = sys.argv[2] processor.train(model_file_path) elif mode == "show_examples": model_file_path = sys.argv[2] model = processor.get_model(model_file_path) processor.check_f1_em(model, num_samples=10, print_to_screen=True) elif mode == "official_eval": model_file_path = sys.argv[2] json_in_path = "" json_out_path = "" qn_uuid_data, context_token_data, qn_token_data = get_json_data(json_in_path) model = processor.get_model(model_file_path) answers_dict = generate_answers(config, model, processor, qn_uuid_data, context_token_data, qn_token_data) print ("Writing predictions to %s..." % json_out_path) with open(json_out_path, 'w') as f: f.write(json.dumps(answers_dict, ensure_ascii=False)) print ("Wrote predictions to %s" % json_out_path)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # if FLAGS.mode != 'loadProbs': # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir # with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: # json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=20, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == "official_eval_with_bidaf": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict, bidaf_dict, self_dict1, self_dict2, out_dict = generate_answers_with_bidaf( sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path print "Writing sims to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-bidaf', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(bidaf_dict, ensure_ascii=False))) print "Wrote sims to %s" % FLAGS.json_out_path print "Writing self sims1 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self1', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict1, ensure_ascii=False))) print "Wrote self sims1 to %s" % FLAGS.json_out_path print "Writing self sims2 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self2', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict2, ensure_ascii=False))) print "Wrote self sims2 to %s" % FLAGS.json_out_path print "Writing preds to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-preds', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(out_dict, ensure_ascii=False))) print "Wrote preds to %s" % FLAGS.json_out_path elif FLAGS.mode == 'saveProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = save_answer_probs(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'wb') as f: pickle.dump(answers_dict, f, protocol=2) # f.write(unicode(pickle.dumps(answers_dict, ensure_ascii=False))) # f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == 'loadProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) # word2id = pickle.load(open('word2id', 'rb')) # pickle.dump(word2id, open('word2id', 'wb')) print 'Loaded data' dictLists = [] for file in os.listdir('./pickles'): f = os.path.join('./pickles', file) print 'Loading predictions from ', f prob_dict = pickle.load(open(f, 'rb')) dictLists += [prob_dict] # mainDict = {} # stdiDict = {} # for probs in dictLists: # for k in dictLists[0].keys(): # stdi = 1.0 / (np.std(np.array(probs[k][0])) + np.std(np.array(probs[k][1])) + 1e-2) # stdiDict[k] = stdi # try: # mainDict[k] = (mainDict[k][0] + stdi * np.array(probs[k][0]), mainDict[k][1] + stdi* np.array(probs[k][1])) # except KeyError: # mainDict[k] = (stdi* np.array(probs[k][0]), stdi*np.array(probs[k][1])) uuid2ans = {} # maps uuid to string containing predicted answer detokenizer = MosesDetokenizer() # for k in mainDict.keys(): # start_dist = mainDict[k][0] / stdiDict[k] # end_dist = mainDict[k][1] / stdiDict[k] # # Take argmax to get start_pos and end_post, both shape (batch_size) # end_dp = np.zeros(end_dist.shape) # # start_pos = np.argmax(start_dist) # # end_pos = np.argmax(end_dist) # end_dp[-1]=end_dist[-1] # for i in range(len(end_dist)-2,-1,-1): # end_dp[i]=np.amax([end_dist[i],end_dp[i+1]]) # start_pos=np.argmax(start_dist*end_dp) # end_pos = start_pos + np.argmax(end_dist[start_pos:]) # uuid2ans[k] = (start_pos, end_pos) for k in dictLists[0].keys(): spanDict = {} for probs in dictLists: start_dist = np.array(probs[k][0]) end_dist = np.array(probs[k][1]) # Take argmax to get start_pos and end_post, both shape (batch_size) end_dp = np.zeros(end_dist.shape) end_dp[-1] = end_dist[-1] for i in range(len(end_dist) - 2, -1, -1): end_dp[i] = np.amax([end_dist[i], end_dp[i + 1]]) start_pos = np.argmax(start_dist * end_dp) end_pos = start_pos + np.argmax(end_dist[start_pos:]) try: spanDict[(start_pos, end_pos)] += [ start_dist[start_pos] * end_dist[end_pos] ] except KeyError: spanDict[(start_pos, end_pos)] = [ start_dist[start_pos] * end_dist[end_pos] ] best_span = (0, 0) best_span_votes = 0 best_span_prob = 0 for span in spanDict.keys(): if len(spanDict[span]) > best_span_votes: best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) elif len( spanDict[span] ) == best_span_votes and best_span_prob < max(spanDict[span]): best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) uuid2ans[k] = (best_span[0], best_span[1]) result = {} data_size = len(qn_uuid_data) num_batches = ((data_size - 1) / FLAGS.batch_size) + 1 batch_num = 0 print "Generating answers..." for batch in get_batch_generator(word2id, qn_uuid_data, context_token_data, qn_token_data, FLAGS.batch_size, FLAGS.context_len, FLAGS.question_len): # For each example in the batch: for ex_idx in range(FLAGS.batch_size): # Detokenize and add to dict try: uuid = batch.uuids[ex_idx] pred_start, pred_end = uuid2ans[uuid] # Original context tokens (no UNKs or padding) for this example context_tokens = batch.context_tokens[ ex_idx] # list of strings # Check the predicted span is in range assert pred_start in range(len(context_tokens)) assert pred_end in range(len(context_tokens)) # Predicted answer tokens pred_ans_tokens = context_tokens[pred_start:pred_end + 1] # list of strings result[uuid] = detokenizer.detokenize(pred_ans_tokens, return_str=True) except IndexError: pass batch_num += 1 if batch_num % 10 == 0: print "Generated answers for %i/%i batches = %.2f%%" % ( batch_num, num_batches, batch_num * 100.0 / num_batches) print "Finished generating answers for dataset." answers_dict = result # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Check for ensemble model param setting if FLAGS.enable_ensemble_model and (FLAGS.mode != "official_eval" or not FLAGS.ensemble_model_names): raise Exception( "ERROR: model ensemble is only supported in official_eval mode, you must specify ensemble_model_names" ) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if (not FLAGS.enable_ensemble_model and not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval" ) or (FLAGS.enable_ensemble_model and not FLAGS.ensemble_model_names): raise Exception( "You need to specify either --experiment_name or --train_dir, or ensemble_model_names if ensemble is enabled." ) FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Build character level vocab mappings char2id, id2char = get_char_mapping() # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") if not FLAGS.enable_ensemble_model: # Initialize model only when ensemble model is disabled. qa_model_name = FLAGS.model_name + '_model' QAModel = importlib.import_module(qa_model_name).QAModel print('model loaded from: %s' % qa_model_name) qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, char2id, id2char) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if not FLAGS.enable_ensemble_model and FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir or use ensemble_model_names" ) # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) if FLAGS.enable_ensemble_model: print('FLAGS.ensemble_model_names: %s' % FLAGS.ensemble_model_names) print('FLAGS.sum_weights: %s' % FLAGS.sum_weights) # KV is 'label': ('model_file', 'exp_name', 'codalab_bundle_name', 'has_cnn', weight), ensemble_label_to_model_meta = { 'binco_legacy': [ 'binco_legacy_model', 'binco_30b15_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'binco_30b15', False, 0.6692 ], # 0.6900 (74.0, 63.5) 'chsebinco_real': [ 'chsebinco_model', 'chsebinco_real_1c999_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chsebinco_real_1c999', True, 0.6733 ], # 0.6958, (74.7, 64.0) 'chsebinco_legacy': [ 'chsebinco_legacy_model', 'chsebinco_4a81a_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chsebinco_4a81a', True, 0.6507 ], # 0.6954, (?, ?) 'chgasebinco': [ 'chgasebinco_model', 'chgasebinco_1c999_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chgasebinco_1c999', True, 0.7045 ], # 0.7101 (76.6, 66.4) 'chgasebinco_91ca1': [ 'chgasebinco_model', 'chgasebinco_91ca1_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chgasebinco_91ca1', True, 0.69 ], # 0.68 (? ?) 'chgasebinco_888ca': [ 'chgasebinco_model', 'chgasebinco_888ca_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chgasebinco_888ca', True, 0.69 ], # 0.67 (? ?) 'chgasebinco_888ca_run2': [ 'chgasebinco_model', 'chgasebinco_888ca_run2_hidden=100_lr=0.001_batch=100_context=400_qn=27', 'chgasebinco_888ca_run2', True, 0.69 ], # 0.6911 (? ?) } model_labels = FLAGS.ensemble_model_names.split(';') if len(model_labels) == 1 and model_labels[0].lower() == 'all': model_labels = ensemble_label_to_model_meta.keys() else: for label in model_labels: assert label in ensemble_label_to_model_meta # A list to store the output of all predictions # each entry is a map, storing the start and end dist for that batch. # len(ensemble_model_pred) is len(model_labels) # len(ensemble_model_pred[0]) is number of batches # len(ensemble_model_pred[0]['start']) is batch_size # len(ensemble_model_pred[0]['end']) is batch_size ensemble_model_pred = [] sum_weights_list = [] for label in model_labels: tf.reset_default_graph() model_name, model_exp_name, cl_bundle_name, has_cnn, weight = ensemble_label_to_model_meta[ label] sum_weights_list += str(weight), print "Loading model: %s" % model_name # TODO(binbinx): change this to appropriate models QAModel = importlib.import_module(model_name).QAModel qa_model = (QAModel(FLAGS, id2word, word2id, emb_matrix, char2id, id2char) if has_cnn else QAModel( FLAGS, id2word, word2id, emb_matrix)) with tf.Session(config=config) as sess: # Initialize bestmodel directory ckpt_load_dir = (os.path.join( EXPERIMENTS_DIR, model_exp_name, "best_checkpoint") if not FLAGS.is_codalab_eval else cl_bundle_name) # Load model from ckpt_load_dir initialize_model(sess, qa_model, ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer # WE MUST USE A DEEPCOPY HERE!! qn_uuid_data_ = copy.deepcopy(qn_uuid_data) context_token_data_ = copy.deepcopy(context_token_data) qn_token_data_ = copy.deepcopy(qn_token_data) answers_dict = generate_answers(sess, qa_model, word2id, char2id, qn_uuid_data_, context_token_data_, qn_token_data_, ensemble_model_pred) sum_weights = ';'.join( sum_weights_list) if FLAGS.sum_weights.lower( ) == 'default' else FLAGS.sum_weights pred_start_batches, pred_end_batches = resolve_ensemble_model_preds( ensemble_model_pred, sum_weights=sum_weights) final_ans_dict = generate_answers_with_start_end( FLAGS, word2id, char2id, qn_uuid_data, context_token_data, qn_token_data, pred_start_batches, pred_end_batches) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(final_ans_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, char2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write( unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and \ FLAGS.mode != "official_eval" and FLAGS.mode!= "ensemble_write" and FLAGS.mode!= "ensemble_predict": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") small_context_path = os.path.join(FLAGS.data_dir, "small.context") small_qn_path = os.path.join(FLAGS.data_dir, "small.question") small_ans_path = os.path.join(FLAGS.data_dir, "small.span") qa_model = None # Initialize model if FLAGS.model_name == "baseline": print("Using baseline model") qa_model = QABaselineModel(FLAGS, id2word, word2id, emb_matrix) elif FLAGS.model_name == "bidaf": qa_model = QABidafModel(FLAGS, id2word, word2id, emb_matrix) elif FLAGS.model_name == "selfattn": print("Using Self Attention") qa_model = QASelfAttnModel(FLAGS, id2word, word2id, emb_matrix) elif FLAGS.model_name == "stack": print("Using stack BIDAF/SA") qa_model = QAStackModel(FLAGS, id2word, word2id, emb_matrix) elif FLAGS.model_name == "pointer": print("Using pointer model") qa_model = QAPointerModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "test": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, small_context_path, small_qn_path, small_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "visualize": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Get distribution of begin and end spans. begin_total, end_total, f1_em_scores = qa_model.get_spans( sess, dev_context_path, dev_qn_path, dev_ans_path, "dev") np.save(os.path.join(FLAGS.train_dir, "begin_span"), begin_total) np.save(os.path.join(FLAGS.train_dir, "end_span"), end_total) np.save(os.path.join(FLAGS.train_dir, "f1_em"), f1_em_scores) # Visualize distribution of Context to Question attention c2q_attn = qa_model.get_c2q_attention(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=0) np.save(os.path.join(FLAGS.train_dir, "c2q_attn"), c2q_attn) q2c_attn = qa_model.get_q2c_attention(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=0) if len(q2c_attn > 0): np.save(os.path.join(FLAGS.train_dir, "q2c_attn"), q2c_attn) else: print 'This model doesn\'t have question to context attention' self_attn = qa_model.get_self_attention(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=20) if len(self_attn > 0): np.save(os.path.join(FLAGS.train_dir, "self_attn"), self_attn) else: print 'This model doesn\'t have self attention' elif FLAGS.mode == "ensemble_write": if FLAGS.json_in_path == "": raise Exception( "For ensembling mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For ensembling mode, you need to specify --ckpt_load_dir") if FLAGS.ensemble_name == "": raise Exception( "For ensembling mode, you need to specify --ensemble_name") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) distributions = generate_distributions(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # np uuid -> [start_dist, end_dist] # Write the uuid->answer mapping a to json file in root dir save_path = os.path.join( FLAGS.ensemble_dir, "distribution_" + FLAGS.ensemble_name + '.json') print "Writing distributions to %s..." % save_path with io.open(save_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(distributions, ensure_ascii=False))) print "Wrote distributions to %s" % save_path elif FLAGS.mode == "ensemble_predict": if FLAGS.json_in_path == "": raise Exception( "For ensembling mode, you need to specify --json_in_path") models = ['stack', 'pointer'] distributions = [ os.path.join(FLAGS.ensemble_dir, "distribution_" + m + ".json") for m in models ] total_dict = {} for d in distributions: with open(d) as prediction_file: print d predictions = json.load(prediction_file) for (key, item) in predictions.items(): if total_dict.get(key, None) is None: total_dict[key] = np.asarray(item) else: total_dict[key] += np.asarray(item) for (key, item) in total_dict.items(): total_dict[key][0] /= len(models) total_dict[key][1] /= len(models) # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) answers_dict = generate_answers_from_dist(None, qa_model, total_dict, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)