def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception(
"There is a problem with how you entered flags: {}".format(
unused_argv))
# check for Python 3
if sys.version_info[0] != 3:
raise Exception(
"Error: You must use python 3 but you are running Python {}".
format(sys.version_info[0]))
# Print out Tensorflow version
print(
"This code war developed and tested an Tensorflow 1.4.1. Your Tensorflow version : {}"
.format(tf.__version__))
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove directory
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir),
"log.txt")
logging.getLogger().addHandler(file_handler)
# Save a record of flags as .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_context_path, dev_qn_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print("Writing predictions to %s..." % FLAGS.json_out_path)
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print("Wrote predictions to %s" % FLAGS.json_out_path)
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" % unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception("You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config=tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception("For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception("For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and \
FLAGS.mode != "official_eval" and FLAGS.mode!= "ensemble_write" and FLAGS.mode!= "ensemble_predict":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
small_context_path = os.path.join(FLAGS.data_dir, "small.context")
small_qn_path = os.path.join(FLAGS.data_dir, "small.question")
small_ans_path = os.path.join(FLAGS.data_dir, "small.span")
qa_model = None
# Initialize model
if FLAGS.model_name == "baseline":
print("Using baseline model")
qa_model = QABaselineModel(FLAGS, id2word, word2id, emb_matrix)
elif FLAGS.model_name == "bidaf":
qa_model = QABidafModel(FLAGS, id2word, word2id, emb_matrix)
elif FLAGS.model_name == "selfattn":
print("Using Self Attention")
qa_model = QASelfAttnModel(FLAGS, id2word, word2id, emb_matrix)
elif FLAGS.model_name == "stack":
print("Using stack BIDAF/SA")
qa_model = QAStackModel(FLAGS, id2word, word2id, emb_matrix)
elif FLAGS.model_name == "pointer":
print("Using pointer model")
qa_model = QAPointerModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "test":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, small_context_path, small_qn_path,
small_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "visualize":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Get distribution of begin and end spans.
begin_total, end_total, f1_em_scores = qa_model.get_spans(
sess, dev_context_path, dev_qn_path, dev_ans_path, "dev")
np.save(os.path.join(FLAGS.train_dir, "begin_span"), begin_total)
np.save(os.path.join(FLAGS.train_dir, "end_span"), end_total)
np.save(os.path.join(FLAGS.train_dir, "f1_em"), f1_em_scores)
# Visualize distribution of Context to Question attention
c2q_attn = qa_model.get_c2q_attention(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=0)
np.save(os.path.join(FLAGS.train_dir, "c2q_attn"), c2q_attn)
q2c_attn = qa_model.get_q2c_attention(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=0)
if len(q2c_attn > 0):
np.save(os.path.join(FLAGS.train_dir, "q2c_attn"), q2c_attn)
else:
print 'This model doesn\'t have question to context attention'
self_attn = qa_model.get_self_attention(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=20)
if len(self_attn > 0):
np.save(os.path.join(FLAGS.train_dir, "self_attn"), self_attn)
else:
print 'This model doesn\'t have self attention'
elif FLAGS.mode == "ensemble_write":
if FLAGS.json_in_path == "":
raise Exception(
"For ensembling mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For ensembling mode, you need to specify --ckpt_load_dir")
if FLAGS.ensemble_name == "":
raise Exception(
"For ensembling mode, you need to specify --ensemble_name")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
distributions = generate_distributions(sess, qa_model, word2id,
qn_uuid_data,
context_token_data,
qn_token_data)
# np uuid -> [start_dist, end_dist]
# Write the uuid->answer mapping a to json file in root dir
save_path = os.path.join(
FLAGS.ensemble_dir,
"distribution_" + FLAGS.ensemble_name + '.json')
print "Writing distributions to %s..." % save_path
with io.open(save_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(distributions, ensure_ascii=False)))
print "Wrote distributions to %s" % save_path
elif FLAGS.mode == "ensemble_predict":
if FLAGS.json_in_path == "":
raise Exception(
"For ensembling mode, you need to specify --json_in_path")
models = ['stack', 'pointer']
distributions = [
os.path.join(FLAGS.ensemble_dir, "distribution_" + m + ".json")
for m in models
]
total_dict = {}
for d in distributions:
with open(d) as prediction_file:
print d
predictions = json.load(prediction_file)
for (key, item) in predictions.items():
if total_dict.get(key, None) is None:
total_dict[key] = np.asarray(item)
else:
total_dict[key] += np.asarray(item)
for (key, item) in total_dict.items():
total_dict[key][0] /= len(models)
total_dict[key][1] /= len(models)
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
answers_dict = generate_answers_from_dist(None, qa_model, total_dict,
word2id, qn_uuid_data,
context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)