def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or \
os.path.join(DEFAULT_DATA_DIR,
"glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
timer.start("glove_getter")
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
id2idf = get_idf(os.path.abspath(FLAGS.idf_path), word2id)
logger.warn("Get glove embedding of size {} takes {:.2f} s".format(
FLAGS.embedding_size, timer.stop("glove_getter")))
# Print out Tensorflow version
# print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
ensumble = FLAGS.ensumble
print(ensumble)
if not ensumble and not FLAGS.attn_layer and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --attn_layer or --train_dir")
# Define train_dir
if not FLAGS.experiment_name:
FLAGS.experiment_name = "A_{}_E_{}_D_{}".format(
FLAGS.attn_layer, FLAGS.embedding_size, FLAGS.dropout)
checkptr_name = FLAGS.experiment_name + "/glove{}".format(
FLAGS.embedding_size)
FLAGS.train_dir = FLAGS.train_dir or\
os.path.join(EXPERIMENTS_DIR, checkptr_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
is_training = (FLAGS.mode == "train")
if not ensumble:
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2idf,
is_training)
else:
ensumbler = Ensumbler(ensumble, config, id2word, word2id, emb_matrix,
id2idf)
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
qa_model.initialize_from_checkpoint(sess,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
qa_model.initialize_from_checkpoint(sess,
bestmodel_dir,
expect_exists=True)
# Show examples with F1/EM scores
f1, em = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
logger.info("Dev: F1 = {0:.3}, EM = {0:.3}".format(f1, em))
elif FLAGS.mode == "eval":
if ensumble:
# train
train_f1, train_em = ensumbler.check_f1_em(train_context_path,
train_qn_path,
train_ans_path, "train",
FLAGS.n_eval)
# dev
dev_f1, dev_em = ensumbler.check_f1_em(dev_context_path,
dev_qn_path, dev_ans_path,
"dev", FLAGS.n_eval)
else:
with tf.Session(config=config) as sess:
# Load best model
qa_model.initialize_from_checkpoint(sess,
FLAGS.ckpt_load_dir,
expect_exists=True)
logger.info("Model initialzed from checkpoint")
# train
train_f1, train_em = qa_model.check_f1_em(
sess,
train_context_path,
train_qn_path,
train_ans_path,
"train",
num_samples=10,
print_to_screen=False)
# dev
dev_f1, dev_em = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=False)
logger.error("Train: F1 = {:.3}, EM = {:.3}".format(
train_f1, train_em))
logger.error("Dev: F1 = {:.3}, EM = {:.3}".format(dev_f1, dev_em))
elif FLAGS.mode == "official_eval":
if not ensumble:
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path"
)
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir"
)
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
if ensumble:
answers_dict = ensumbler.generate_answers(qn_uuid_data,
context_token_data,
qn_token_data)
else:
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
qa_model.initialize_from_checkpoint(sess,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
id2idf, qn_uuid_data,
context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
print "qn_uuid_data"
print qn_uuid_data
print "#" * 100
print "context_token_data"
print context_token_data
print "#" * 100
print "qn_token_data"
print qn_token_data
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
#First check the FLAGS enter correctly (format), python version and tensorflow version
#Then check if train_dir or experiment_dir defined
#set bestmodel path which named best_checkpoint
#set context, question. ans path
#read glove
#Initialise the model architecture
#gpu setting
#mode choice
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
if sys.version_info[0] != 3:
raise Exception(
"ERROR: You must use Python 3 but you are running Python %i" %
sys.version_info[0])
print(
"This code was developed and tested on TensorFlow 1.8.0. Your TensorFlow version: %s"
% tf.__version__)
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
#glove path
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
char2id, id2char = get_char_embed()
#path for context, question, ans_span
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
#Initialise the model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2char, char2id)
# Some GPU settings
config = tf.ConfigProto() #set configuration for sess.run
config.gpu_options.allow_growth = True #make gpu storage usage based for condition
#different modes
if FLAGS.mode == "train":
#setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(
FLAGS.flag_values_dict(), fout
) #NoteL changed from FLAGS.__flags to FLAGS.flag_values_dict() after tensorflow 1.5
# Make bestmodel dir
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
#Added tfdbg
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
#Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
"""
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception("For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception("For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print ("Writing predictions to %s..." % FLAGS.json_out_path)
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print ("Wrote predictions to %s" % FLAGS.json_out_path)
"""
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Initialize bestmodel directory
bestmodel_dir = os.path.join(
"F:\Sid\Learnings\Data Scientist\Text Analysis\Machine Comprehension\data\ModelLogs\best_weights",
"best_checkpoint")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" % unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception("You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config=tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception("For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception("For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.graph")
train_qn_path = os.path.join(FLAGS.data_dir, "train.instruction")
train_ans_path = os.path.join(FLAGS.data_dir, "train.answer")
dev_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
dev_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
dev_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer")
# Create vocabularies of the appropriate sizes for output answer.
context_vocab_path = os.path.join(
FLAGS.data_dir, "vocab%d.context" % FLAGS.context_vocabulary_size)
# ans_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d." % FLAGS.ans_vocabulary_size)
# initialize the vocabulary.
context_vocab, rev_context_vocab = create_vocabulary(
context_vocab_path, train_context_path, FLAGS.context_vocabulary_size)
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, context_vocab,
rev_context_vocab, context_vocab)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
# with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
# json.dump(FLAGS(sys.argv), fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
""" To show a few examples without attention map.
"""
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
_, _, _, _ = qa_model.check_f1_em(
sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=FLAGS.print_num,
print_to_screen=True) #, summary_writer=summary_writer)
# summary_writer.close()
elif FLAGS.mode == "show_attention":
""" To show a few examples of attention map.
"""
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
qa_model.demo(sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=FLAGS.print_num,
print_to_screen=True,
shuffle=False) # , summary_writer=summary_writer)
elif FLAGS.mode == "official_eval":
with tf.Session(config=config) as sess:
if FLAGS.ckpt_load_dir:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
else:
# Load best model
initialize_model(sess,
qa_model,
bestmodel_dir,
expect_exists=True)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
f1, em, edit_dist, rem = qa_model.check_f1_em(
sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=0,
print_to_screen=False,
write_out=FLAGS.write_out,
file_out=FLAGS.file_out_path,
shuffle=False)
logging.info(
"F1 score: %f, EM score: %f, edit distance: %f, rough EM score: %f"
% (f1, em, edit_dist, rem))
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# if FLAGS.mode != 'loadProbs':
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
# with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
# json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=20,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
elif FLAGS.mode == "official_eval_with_bidaf":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict, bidaf_dict, self_dict1, self_dict2, out_dict = generate_answers_with_bidaf(
sess, qa_model, word2id, qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
print "Writing sims to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-bidaf', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(bidaf_dict, ensure_ascii=False)))
print "Wrote sims to %s" % FLAGS.json_out_path
print "Writing self sims1 to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-self1', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(self_dict1, ensure_ascii=False)))
print "Wrote self sims1 to %s" % FLAGS.json_out_path
print "Writing self sims2 to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-self2', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(self_dict2, ensure_ascii=False)))
print "Wrote self sims2 to %s" % FLAGS.json_out_path
print "Writing preds to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-preds', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(out_dict, ensure_ascii=False)))
print "Wrote preds to %s" % FLAGS.json_out_path
elif FLAGS.mode == 'saveProbs':
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = save_answer_probs(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'wb') as f:
pickle.dump(answers_dict, f, protocol=2)
# f.write(unicode(pickle.dumps(answers_dict, ensure_ascii=False)))
# f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
elif FLAGS.mode == 'loadProbs':
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
# word2id = pickle.load(open('word2id', 'rb'))
# pickle.dump(word2id, open('word2id', 'wb'))
print 'Loaded data'
dictLists = []
for file in os.listdir('./pickles'):
f = os.path.join('./pickles', file)
print 'Loading predictions from ', f
prob_dict = pickle.load(open(f, 'rb'))
dictLists += [prob_dict]
# mainDict = {}
# stdiDict = {}
# for probs in dictLists:
# for k in dictLists[0].keys():
# stdi = 1.0 / (np.std(np.array(probs[k][0])) + np.std(np.array(probs[k][1])) + 1e-2)
# stdiDict[k] = stdi
# try:
# mainDict[k] = (mainDict[k][0] + stdi * np.array(probs[k][0]), mainDict[k][1] + stdi* np.array(probs[k][1]))
# except KeyError:
# mainDict[k] = (stdi* np.array(probs[k][0]), stdi*np.array(probs[k][1]))
uuid2ans = {} # maps uuid to string containing predicted answer
detokenizer = MosesDetokenizer()
# for k in mainDict.keys():
# start_dist = mainDict[k][0] / stdiDict[k]
# end_dist = mainDict[k][1] / stdiDict[k]
# # Take argmax to get start_pos and end_post, both shape (batch_size)
# end_dp = np.zeros(end_dist.shape)
# # start_pos = np.argmax(start_dist)
# # end_pos = np.argmax(end_dist)
# end_dp[-1]=end_dist[-1]
# for i in range(len(end_dist)-2,-1,-1):
# end_dp[i]=np.amax([end_dist[i],end_dp[i+1]])
# start_pos=np.argmax(start_dist*end_dp)
# end_pos = start_pos + np.argmax(end_dist[start_pos:])
# uuid2ans[k] = (start_pos, end_pos)
for k in dictLists[0].keys():
spanDict = {}
for probs in dictLists:
start_dist = np.array(probs[k][0])
end_dist = np.array(probs[k][1])
# Take argmax to get start_pos and end_post, both shape (batch_size)
end_dp = np.zeros(end_dist.shape)
end_dp[-1] = end_dist[-1]
for i in range(len(end_dist) - 2, -1, -1):
end_dp[i] = np.amax([end_dist[i], end_dp[i + 1]])
start_pos = np.argmax(start_dist * end_dp)
end_pos = start_pos + np.argmax(end_dist[start_pos:])
try:
spanDict[(start_pos, end_pos)] += [
start_dist[start_pos] * end_dist[end_pos]
]
except KeyError:
spanDict[(start_pos, end_pos)] = [
start_dist[start_pos] * end_dist[end_pos]
]
best_span = (0, 0)
best_span_votes = 0
best_span_prob = 0
for span in spanDict.keys():
if len(spanDict[span]) > best_span_votes:
best_span = span
best_span_votes = len(spanDict[span])
best_span_prob = max(spanDict[span])
elif len(
spanDict[span]
) == best_span_votes and best_span_prob < max(spanDict[span]):
best_span = span
best_span_votes = len(spanDict[span])
best_span_prob = max(spanDict[span])
uuid2ans[k] = (best_span[0], best_span[1])
result = {}
data_size = len(qn_uuid_data)
num_batches = ((data_size - 1) / FLAGS.batch_size) + 1
batch_num = 0
print "Generating answers..."
for batch in get_batch_generator(word2id, qn_uuid_data,
context_token_data, qn_token_data,
FLAGS.batch_size, FLAGS.context_len,
FLAGS.question_len):
# For each example in the batch:
for ex_idx in range(FLAGS.batch_size):
# Detokenize and add to dict
try:
uuid = batch.uuids[ex_idx]
pred_start, pred_end = uuid2ans[uuid]
# Original context tokens (no UNKs or padding) for this example
context_tokens = batch.context_tokens[
ex_idx] # list of strings
# Check the predicted span is in range
assert pred_start in range(len(context_tokens))
assert pred_end in range(len(context_tokens))
# Predicted answer tokens
pred_ans_tokens = context_tokens[pred_start:pred_end +
1] # list of strings
result[uuid] = detokenizer.detokenize(pred_ans_tokens,
return_str=True)
except IndexError:
pass
batch_num += 1
if batch_num % 10 == 0:
print "Generated answers for %i/%i batches = %.2f%%" % (
batch_num, num_batches, batch_num * 100.0 / num_batches)
print "Finished generating answers for dataset."
answers_dict = result
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)