def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or \ os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings timer.start("glove_getter") emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) id2idf = get_idf(os.path.abspath(FLAGS.idf_path), word2id) logger.warn("Get glove embedding of size {} takes {:.2f} s".format( FLAGS.embedding_size, timer.stop("glove_getter"))) # Print out Tensorflow version # print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ ensumble = FLAGS.ensumble print(ensumble) if not ensumble and not FLAGS.attn_layer and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --attn_layer or --train_dir") # Define train_dir if not FLAGS.experiment_name: FLAGS.experiment_name = "A_{}_E_{}_D_{}".format( FLAGS.attn_layer, FLAGS.embedding_size, FLAGS.dropout) checkptr_name = FLAGS.experiment_name + "/glove{}".format( FLAGS.embedding_size) FLAGS.train_dir = FLAGS.train_dir or\ os.path.join(EXPERIMENTS_DIR, checkptr_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True is_training = (FLAGS.mode == "train") if not ensumble: # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2idf, is_training) else: ensumbler = Ensumbler(ensumble, config, id2word, word2id, emb_matrix, id2idf) # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model qa_model.initialize_from_checkpoint(sess, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores f1, em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) logger.info("Dev: F1 = {0:.3}, EM = {0:.3}".format(f1, em)) elif FLAGS.mode == "eval": if ensumble: # train train_f1, train_em = ensumbler.check_f1_em(train_context_path, train_qn_path, train_ans_path, "train", FLAGS.n_eval) # dev dev_f1, dev_em = ensumbler.check_f1_em(dev_context_path, dev_qn_path, dev_ans_path, "dev", FLAGS.n_eval) else: with tf.Session(config=config) as sess: # Load best model qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) logger.info("Model initialzed from checkpoint") # train train_f1, train_em = qa_model.check_f1_em( sess, train_context_path, train_qn_path, train_ans_path, "train", num_samples=10, print_to_screen=False) # dev dev_f1, dev_em = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=False) logger.error("Train: F1 = {:.3}, EM = {:.3}".format( train_f1, train_em)) logger.error("Dev: F1 = {:.3}, EM = {:.3}".format(dev_f1, dev_em)) elif FLAGS.mode == "official_eval": if not ensumble: if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path" ) if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir" ) # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) if ensumble: answers_dict = ensumbler.generate_answers(qn_uuid_data, context_token_data, qn_token_data) else: with tf.Session(config=config) as sess: # Load model from ckpt_load_dir qa_model.initialize_from_checkpoint(sess, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, id2idf, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) print "qn_uuid_data" print qn_uuid_data print "#" * 100 print "context_token_data" print context_token_data print "#" * 100 print "qn_token_data" print qn_token_data with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): #First check the FLAGS enter correctly (format), python version and tensorflow version #Then check if train_dir or experiment_dir defined #set bestmodel path which named best_checkpoint #set context, question. ans path #read glove #Initialise the model architecture #gpu setting #mode choice if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) if sys.version_info[0] != 3: raise Exception( "ERROR: You must use Python 3 but you are running Python %i" % sys.version_info[0]) print( "This code was developed and tested on TensorFlow 1.8.0. Your TensorFlow version: %s" % tf.__version__) if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) #glove path emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) char2id, id2char = get_char_embed() #path for context, question, ans_span train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") #Initialise the model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2char, char2id) # Some GPU settings config = tf.ConfigProto() #set configuration for sess.run config.gpu_options.allow_growth = True #make gpu storage usage based for condition #different modes if FLAGS.mode == "train": #setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump( FLAGS.flag_values_dict(), fout ) #NoteL changed from FLAGS.__flags to FLAGS.flag_values_dict() after tensorflow 1.5 # Make bestmodel dir if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: #Added tfdbg # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) #Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) """ elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception("For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception("For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print ("Writing predictions to %s..." % FLAGS.json_out_path) with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print ("Wrote predictions to %s" % FLAGS.json_out_path) """ else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Initialize bestmodel directory bestmodel_dir = os.path.join( "F:\Sid\Learnings\Data Scientist\Text Analysis\Machine Comprehension\data\ModelLogs\best_weights", "best_checkpoint") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception("You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config=tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception("For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception("For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.graph") train_qn_path = os.path.join(FLAGS.data_dir, "train.instruction") train_ans_path = os.path.join(FLAGS.data_dir, "train.answer") dev_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") dev_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") dev_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") # Create vocabularies of the appropriate sizes for output answer. context_vocab_path = os.path.join( FLAGS.data_dir, "vocab%d.context" % FLAGS.context_vocabulary_size) # ans_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d." % FLAGS.ans_vocabulary_size) # initialize the vocabulary. context_vocab, rev_context_vocab = create_vocabulary( context_vocab_path, train_context_path, FLAGS.context_vocabulary_size) # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, context_vocab, rev_context_vocab, context_vocab) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir # with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: # json.dump(FLAGS(sys.argv), fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": """ To show a few examples without attention map. """ with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") _, _, _, _ = qa_model.check_f1_em( sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=FLAGS.print_num, print_to_screen=True) #, summary_writer=summary_writer) # summary_writer.close() elif FLAGS.mode == "show_attention": """ To show a few examples of attention map. """ with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") qa_model.demo(sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=FLAGS.print_num, print_to_screen=True, shuffle=False) # , summary_writer=summary_writer) elif FLAGS.mode == "official_eval": with tf.Session(config=config) as sess: if FLAGS.ckpt_load_dir: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) else: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) eval_context_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".graph") eval_qn_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".instruction") eval_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer") f1, em, edit_dist, rem = qa_model.check_f1_em( sess, eval_context_path, eval_qn_path, eval_ans_path, FLAGS.file_in_path, num_samples=0, print_to_screen=False, write_out=FLAGS.write_out, file_out=FLAGS.file_out_path, shuffle=False) logging.info( "F1 score: %f, EM score: %f, edit distance: %f, rough EM score: %f" % (f1, em, edit_dist, rem)) else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv): # Print an error message if you've entered flags incorrectly if len(unused_argv) != 1: raise Exception("There is a problem with how you entered flags: %s" % unused_argv) # Check for Python 2 if sys.version_info[0] != 2: raise Exception( "ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0]) # Print out Tensorflow version print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__ # Define train_dir if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval": raise Exception( "You need to specify either --experiment_name or --train_dir") FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name) # Initialize bestmodel directory bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint") # Define path for glove vecs FLAGS.glove_path = FLAGS.glove_path or os.path.join( DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size)) # if FLAGS.mode != 'loadProbs': # Load embedding matrix and vocab mappings emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size) # Get filepaths to train/dev datafiles for tokenized queries, contexts and answers train_context_path = os.path.join(FLAGS.data_dir, "train.context") train_qn_path = os.path.join(FLAGS.data_dir, "train.question") train_ans_path = os.path.join(FLAGS.data_dir, "train.span") dev_context_path = os.path.join(FLAGS.data_dir, "dev.context") dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question") dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span") # Initialize model qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix) # Some GPU settings config = tf.ConfigProto() config.gpu_options.allow_growth = True # Split by mode if FLAGS.mode == "train": # Setup train dir and logfile if not os.path.exists(FLAGS.train_dir): os.makedirs(FLAGS.train_dir) file_handler = logging.FileHandler( os.path.join(FLAGS.train_dir, "log.txt")) logging.getLogger().addHandler(file_handler) # Save a record of flags as a .json file in train_dir # with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout: # json.dump(FLAGS.__flags, fout) # Make bestmodel dir if necessary if not os.path.exists(bestmodel_dir): os.makedirs(bestmodel_dir) with tf.Session(config=config) as sess: # Load most recent model initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False) # Train qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path) elif FLAGS.mode == "show_examples": with tf.Session(config=config) as sess: # Load best model initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True) # Show examples with F1/EM scores _, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=20, print_to_screen=True) elif FLAGS.mode == "official_eval": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == "official_eval_with_bidaf": if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict, bidaf_dict, self_dict1, self_dict2, out_dict = generate_answers_with_bidaf( sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path print "Writing sims to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-bidaf', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(bidaf_dict, ensure_ascii=False))) print "Wrote sims to %s" % FLAGS.json_out_path print "Writing self sims1 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self1', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict1, ensure_ascii=False))) print "Wrote self sims1 to %s" % FLAGS.json_out_path print "Writing self sims2 to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-self2', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(self_dict2, ensure_ascii=False))) print "Wrote self sims2 to %s" % FLAGS.json_out_path print "Writing preds to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path + '-preds', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(out_dict, ensure_ascii=False))) print "Wrote preds to %s" % FLAGS.json_out_path elif FLAGS.mode == 'saveProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) with tf.Session(config=config) as sess: # Load model from ckpt_load_dir initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True) # Get a predicted answer for each example in the data # Return a mapping answers_dict from uuid to answer answers_dict = save_answer_probs(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data) # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'wb') as f: pickle.dump(answers_dict, f, protocol=2) # f.write(unicode(pickle.dumps(answers_dict, ensure_ascii=False))) # f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path elif FLAGS.mode == 'loadProbs': if FLAGS.json_in_path == "": raise Exception( "For official_eval mode, you need to specify --json_in_path") if FLAGS.ckpt_load_dir == "": raise Exception( "For official_eval mode, you need to specify --ckpt_load_dir") # Read the JSON data from file qn_uuid_data, context_token_data, qn_token_data = get_json_data( FLAGS.json_in_path) # word2id = pickle.load(open('word2id', 'rb')) # pickle.dump(word2id, open('word2id', 'wb')) print 'Loaded data' dictLists = [] for file in os.listdir('./pickles'): f = os.path.join('./pickles', file) print 'Loading predictions from ', f prob_dict = pickle.load(open(f, 'rb')) dictLists += [prob_dict] # mainDict = {} # stdiDict = {} # for probs in dictLists: # for k in dictLists[0].keys(): # stdi = 1.0 / (np.std(np.array(probs[k][0])) + np.std(np.array(probs[k][1])) + 1e-2) # stdiDict[k] = stdi # try: # mainDict[k] = (mainDict[k][0] + stdi * np.array(probs[k][0]), mainDict[k][1] + stdi* np.array(probs[k][1])) # except KeyError: # mainDict[k] = (stdi* np.array(probs[k][0]), stdi*np.array(probs[k][1])) uuid2ans = {} # maps uuid to string containing predicted answer detokenizer = MosesDetokenizer() # for k in mainDict.keys(): # start_dist = mainDict[k][0] / stdiDict[k] # end_dist = mainDict[k][1] / stdiDict[k] # # Take argmax to get start_pos and end_post, both shape (batch_size) # end_dp = np.zeros(end_dist.shape) # # start_pos = np.argmax(start_dist) # # end_pos = np.argmax(end_dist) # end_dp[-1]=end_dist[-1] # for i in range(len(end_dist)-2,-1,-1): # end_dp[i]=np.amax([end_dist[i],end_dp[i+1]]) # start_pos=np.argmax(start_dist*end_dp) # end_pos = start_pos + np.argmax(end_dist[start_pos:]) # uuid2ans[k] = (start_pos, end_pos) for k in dictLists[0].keys(): spanDict = {} for probs in dictLists: start_dist = np.array(probs[k][0]) end_dist = np.array(probs[k][1]) # Take argmax to get start_pos and end_post, both shape (batch_size) end_dp = np.zeros(end_dist.shape) end_dp[-1] = end_dist[-1] for i in range(len(end_dist) - 2, -1, -1): end_dp[i] = np.amax([end_dist[i], end_dp[i + 1]]) start_pos = np.argmax(start_dist * end_dp) end_pos = start_pos + np.argmax(end_dist[start_pos:]) try: spanDict[(start_pos, end_pos)] += [ start_dist[start_pos] * end_dist[end_pos] ] except KeyError: spanDict[(start_pos, end_pos)] = [ start_dist[start_pos] * end_dist[end_pos] ] best_span = (0, 0) best_span_votes = 0 best_span_prob = 0 for span in spanDict.keys(): if len(spanDict[span]) > best_span_votes: best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) elif len( spanDict[span] ) == best_span_votes and best_span_prob < max(spanDict[span]): best_span = span best_span_votes = len(spanDict[span]) best_span_prob = max(spanDict[span]) uuid2ans[k] = (best_span[0], best_span[1]) result = {} data_size = len(qn_uuid_data) num_batches = ((data_size - 1) / FLAGS.batch_size) + 1 batch_num = 0 print "Generating answers..." for batch in get_batch_generator(word2id, qn_uuid_data, context_token_data, qn_token_data, FLAGS.batch_size, FLAGS.context_len, FLAGS.question_len): # For each example in the batch: for ex_idx in range(FLAGS.batch_size): # Detokenize and add to dict try: uuid = batch.uuids[ex_idx] pred_start, pred_end = uuid2ans[uuid] # Original context tokens (no UNKs or padding) for this example context_tokens = batch.context_tokens[ ex_idx] # list of strings # Check the predicted span is in range assert pred_start in range(len(context_tokens)) assert pred_end in range(len(context_tokens)) # Predicted answer tokens pred_ans_tokens = context_tokens[pred_start:pred_end + 1] # list of strings result[uuid] = detokenizer.detokenize(pred_ans_tokens, return_str=True) except IndexError: pass batch_num += 1 if batch_num % 10 == 0: print "Generated answers for %i/%i batches = %.2f%%" % ( batch_num, num_batches, batch_num * 100.0 / num_batches) print "Finished generating answers for dataset." answers_dict = result # Write the uuid->answer mapping a to json file in root dir print "Writing predictions to %s..." % FLAGS.json_out_path with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(answers_dict, ensure_ascii=False))) print "Wrote predictions to %s" % FLAGS.json_out_path else: raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)