def __init__(self, FLAGS, id2word, word2id, emb_matrix):
"""
Initializes the QA model.
Inputs:
FLAGS: the flags passed in from main.py
id2word: dictionary mapping word idx (int) to word (string)
word2id: dictionary mapping word (string) to word idx (int)
emb_matrix: numpy array shape (400002, embedding_size) containing pre-traing GloVe embeddings
"""
QAModel.__init__(self, FLAGS, id2word, word2id, emb_matrix)
def get_predictions(self, batch):
"""
Return:
"""
starts, ends = [], []
for ckpt, FLAGS in zip(self.ckpts, self.flags):
qa_model = QAModel(FLAGS,
self.id2word,
self.word2id,
self.emb_matrix,
self.id2idf,
is_training=False)
with tf.Session(config=self.tf_config) as session:
qa_model.initialize_from_checkpoint(session, ckpt, True)
pred_start_pos, pred_end_pos = qa_model.get_start_end_pos(
session, batch)
starts.append(pred_start_pos)
ends.append(pred_end_pos)
del qa_model
tf.reset_default_graph()
starts, _ = stats.mode(np.array(starts))
ends, _ = stats.mode(np.array(ends))
return (starts[0].astype(np.int), ends[0].astype(np.int))
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
print "qn_uuid_data"
print qn_uuid_data
print "#" * 100
print "context_token_data"
print context_token_data
print "#" * 100
print "qn_token_data"
print qn_token_data
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or \
os.path.join(DEFAULT_DATA_DIR,
"glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
timer.start("glove_getter")
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
id2idf = get_idf(os.path.abspath(FLAGS.idf_path), word2id)
logger.warn("Get glove embedding of size {} takes {:.2f} s".format(
FLAGS.embedding_size, timer.stop("glove_getter")))
# Print out Tensorflow version
# print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
ensumble = FLAGS.ensumble
print(ensumble)
if not ensumble and not FLAGS.attn_layer and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --attn_layer or --train_dir")
# Define train_dir
if not FLAGS.experiment_name:
FLAGS.experiment_name = "A_{}_E_{}_D_{}".format(
FLAGS.attn_layer, FLAGS.embedding_size, FLAGS.dropout)
checkptr_name = FLAGS.experiment_name + "/glove{}".format(
FLAGS.embedding_size)
FLAGS.train_dir = FLAGS.train_dir or\
os.path.join(EXPERIMENTS_DIR, checkptr_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
is_training = (FLAGS.mode == "train")
if not ensumble:
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2idf,
is_training)
else:
ensumbler = Ensumbler(ensumble, config, id2word, word2id, emb_matrix,
id2idf)
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
qa_model.initialize_from_checkpoint(sess,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
qa_model.initialize_from_checkpoint(sess,
bestmodel_dir,
expect_exists=True)
# Show examples with F1/EM scores
f1, em = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
logger.info("Dev: F1 = {0:.3}, EM = {0:.3}".format(f1, em))
elif FLAGS.mode == "eval":
if ensumble:
# train
train_f1, train_em = ensumbler.check_f1_em(train_context_path,
train_qn_path,
train_ans_path, "train",
FLAGS.n_eval)
# dev
dev_f1, dev_em = ensumbler.check_f1_em(dev_context_path,
dev_qn_path, dev_ans_path,
"dev", FLAGS.n_eval)
else:
with tf.Session(config=config) as sess:
# Load best model
qa_model.initialize_from_checkpoint(sess,
FLAGS.ckpt_load_dir,
expect_exists=True)
logger.info("Model initialzed from checkpoint")
# train
train_f1, train_em = qa_model.check_f1_em(
sess,
train_context_path,
train_qn_path,
train_ans_path,
"train",
num_samples=10,
print_to_screen=False)
# dev
dev_f1, dev_em = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=False)
logger.error("Train: F1 = {:.3}, EM = {:.3}".format(
train_f1, train_em))
logger.error("Dev: F1 = {:.3}, EM = {:.3}".format(dev_f1, dev_em))
elif FLAGS.mode == "official_eval":
if not ensumble:
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path"
)
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir"
)
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
if ensumble:
answers_dict = ensumbler.generate_answers(qn_uuid_data,
context_token_data,
qn_token_data)
else:
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
qa_model.initialize_from_checkpoint(sess,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
id2idf, qn_uuid_data,
context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
# coding: utf-8
import telebot
from qa_model import QAModel
from rubert import RuBertWrapper
CONFIG_PATH = 'models/rubert_cased_L-12_H-768_A-12_pt'
rubert = RuBertWrapper(CONFIG_PATH)
QA_PATH = 'questions_answers/qa_small.txt'
qai = QAModel(QA_PATH, rubert)
TOKEN = '1409080754:AAHRHF35y-BwFPTF6Mb5REuxILvAg0e0gM4'
bot = telebot.TeleBot(TOKEN)
START_MSG = 'Предлагаем Вам ответить на несколько вопросов по системному анализу.'
@bot.message_handler(commands=['start'])
def handle_start(message):
qai.idx = 0
qai.session_active = True
cid = message.chat.id
bot.send_message(cid, START_MSG)
first_question = qai.questions[qai.idx]
bot.send_message(cid, first_question)
@bot.message_handler(func=lambda message: True, content_types=['text'])
def process_answer(m):
if not qai.session_active:
def main():
print("Your TensorFlow version: %s" % tf.__version__)
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR + "/glove.6B/", "glove.6B.{}d.txt".format(
FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
global_step = 1
epoch = 0
print("Beginning training loop...")
# Initialize model
model = QAModel(FLAGS, id2word, word2id, emb_matrix)
optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate)
while FLAGS.num_epochs == 0 or epoch < FLAGS.num_epochs:
epoch += 1
epoch_tic = time.time()
for batch in get_batch_generator( \
word2id, train_context_path, train_qn_path, \
train_ans_path, FLAGS.batch_size, context_len=FLAGS.context_len, \
question_len=FLAGS.question_len, discard_long=True):
# print(batch.ans_span)
with tf.GradientTape() as tape:
prob_start, prob_end = model([
batch.context_ids, batch.context_mask, batch.qn_ids,
batch.qn_mask
])
# prob_start, prob_end = model(batch.context_ids, batch.context_mask, batch.qn_ids, batch.qn_mask)
loss_start = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prob_start, labels=batch.ans_span[:, 0])
loss_start = tf.reduce_mean(loss_start)
loss_end = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prob_end, labels=batch.ans_span[:, 1])
loss_end = tf.reduce_mean(loss_end)
loss = loss_start + loss_end
# print("loss %f" % (loss.numpy()))
grads = tape.gradient(loss, model.variables)
optimizer.apply_gradients(
grads_and_vars=zip(grads, model.variables))
if global_step % FLAGS.eval_every == 0:
print("==== start evaluating ==== ")
dev_f1, dev_em = evaluate(model, word2id, FLAGS,
dev_context_path, dev_qn_path,
dev_ans_path)
print("Epoch %d, Iter %d, Dev F1 score: %f, Dev EM score: %f" %
(epoch, global_step, dev_f1, dev_em))
print("==========================")
global_step += 1
epoch_toc = time.time()
print("End of epoch %i. Time for epoch: %f" %
(epoch, epoch_toc - epoch_tic))
sys.stdout.flush()
os.path.abspath(__file__)))) # relative path of the main directory
DEFAULT_DATA_DIR = os.path.join(MAIN_DIR, "data") # relative path of data dir
EXPERIMENTS_DIR = os.path.join(
MAIN_DIR, "experiments") # relative path of experiments dir
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
print "glove_path"
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
print FLAGS.glove_path
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
def initialize_model(session, model, train_dir, expect_exists):
"""
Initializes model from train_dir.
Inputs:
session: TensorFlow session
model: QAModel
train_dir: path to directory where we'll look for checkpoint
expect_exists: If True, throw an error if no checkpoint is found.
If False, initialize fresh model if no checkpoint is found.
"""
print "Looking for model at %s..." % train_dir
ckpt = tf.train.get_checkpoint_state(train_dir)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
global bestmodel_dir
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
global emb_matrix, word2id, id2word
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Initialize model
global qa_model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
global config
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
global sess
sess = tf.Session(config=config)
global global_context
global_context = 'INSOFE has awarded over Rs 3.2 Crores in merit scholarships in the last 2 years alone. INSOFE recognizes top performers and rewards them for demonstrating outstanding achievement at every phase of the program based on their performance and eligibility criteria. At each phase of the program, top performers are awarded rankings based on which scholarship winners are announced. Top performers can potentially win scholarships ranging from Rs 25,000 to entire program fee and this can be attained on the successful completion of the program.'
global global_context_list
global_context_list = [
'INSOFE has awarded over Rs 3.2 Crores in merit scholarships in the last 2 years alone. INSOFE recognizes top performers and rewards them for demonstrating outstanding achievement at every phase of the program based on their performance and eligibility criteria. At each phase of the program, top performers are awarded rankings based on which scholarship winners are announced. Top performers can potentially win scholarships ranging from Rs 25,000 to entire program fee and this can be attained on the successful completion of the program.',
'INSOFE is working on developing a video surveillance tool with enhanced smart capabilities. The tool identifies the violation and sends out instant automated response without requiring any manual interference. Since the current process involves manually going through the footage and checking for violations, it is not only a time-consuming process but also requires manual hours and effort. The tool makes the entire process automated with an Embedded Machine Learning chip Question',
'Dr Dakshinamurthy, is the Founder and President of INSOFE. He did his PhD in Materials Science and Engineering from Carnegie Mellon University. He is known for simplifying complex ideas and communicating them clearly and excitingly. Dr Sridhar Pappu is the Executive VP - Academics of INSOFE. He leads the academic administration of the institute and ensures the highest standards in learning for the students. He teaches statistics. He loves data, soo much that he wears two fitness trackers.'
]
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# app.run(host='0.0.0.0', port=443, ssl_context=('/home/gem/.ssh/certificate.pem', '/home/gem/.ssh/private-key.pem'))
app.run(
host='0.0.0.0',
port=443,
ssl_context=
('/etc/letsencrypt/live/gem.eastus2.cloudapp.azure.com/fullchain.pem',
'/etc/letsencrypt/live/gem.eastus2.cloudapp.azure.com/privkey.pem'))
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_competition_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
if not FLAGS.attention_type or not FLAGS.reduction_type:
raise Exception(
"You have to specify both --attention_type (dot_product, bidaf, self_attention) and --reduction_type (max, mean) to proceed."
)
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
bestmodel_dir_dev_loss = os.path.join(FLAGS.train_dir,
"best_checkpoint_dev_loss")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_headline_path = os.path.join(FLAGS.data_dir, "train.headline")
train_body_path = os.path.join(FLAGS.data_dir, "train.body")
train_ans_path = os.path.join(FLAGS.data_dir, "train.stance")
dev_headline_path = os.path.join(FLAGS.data_dir, "dev.headline")
dev_body_path = os.path.join(FLAGS.data_dir, "dev.body")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.stance")
test_headline_path = os.path.join(FLAGS.data_dir, "test.headline")
test_body_path = os.path.join(FLAGS.data_dir, "test.body")
test_ans_path = os.path.join(FLAGS.data_dir, "test.stance")
# Initialize model
qa_model = QAModel(
FLAGS, id2word, word2id, emb_matrix
) # create entire computation graph, add loss, opimizer etc...
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
attention_type = FLAGS.attention_type
reduction_type = FLAGS.reduction_type
if attention_type == 'dot_product' and reduction_type == 'max':
FLAGS.max_gradient_norm = 10.0
if attention_type == 'bidaf':
FLAGS.hidden_size = 120
if attention_type == 'self_attention':
FLAGS.self_attn_zsize = 60
FLAGS.hidden_size = 70
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
if not os.path.exists(bestmodel_dir_dev_loss):
os.makedirs(bestmodel_dir_dev_loss)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.custom_train(sess, train_body_path, train_headline_path,
train_ans_path, dev_headline_path,
dev_body_path, dev_ans_path)
elif FLAGS.mode == "check_eval":
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For check_eval mode, you need to specify --ckpt_load_dir")
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
dev_score = qa_model.check_score_cm(sess,
dev_body_path,
dev_headline_path,
dev_ans_path,
"dev",
num_samples=0)
print("Dev score:=>", dev_score)
test_score = qa_model.check_score_cm(sess,
test_body_path,
test_headline_path,
test_ans_path,
"test",
num_samples=0)
print("Test score:=>", test_score)
elif FLAGS.mode == "official_competition_eval":
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_competition_eval mode, you need to specify --ckpt_load_dir"
)
competition_dataset = DataSet(
name="competition_test",
path=FLAGS.data_dir) # Dataset competition read from csv.
#Retreive list of body/ article ids for competition dataset
comp_body_ids = list(
competition_dataset.articles.keys()) # get a list of article ids
#Retrieve stance for all body ids
comp_stances_list = get_stances(competition_dataset, comp_body_ids)
# get body and headline tokens
body_token_data_list, headline_token_data_list, input_body_id_list, headline_list = get_preprocessed_data(
competition_dataset, comp_stances_list, 'competition')
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
#As text not number
pred_label_answer_list = get_answers(sess, qa_model, word2id,
body_token_data_list,
headline_token_data_list)
#stance_df = pd.DataFrame()
#stance_df['Stance'] = pred_label_answer_list
#stance_df.to_csv(os.path.join(FLAGS.result_output_path,"stance.csv"), index=False)
np.savetxt(os.path.join(FLAGS.result_output_path, "stance.csv"),
pred_label_answer_list,
delimiter="\n",
fmt='%s')
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
#First check the FLAGS enter correctly (format), python version and tensorflow version
#Then check if train_dir or experiment_dir defined
#set bestmodel path which named best_checkpoint
#set context, question. ans path
#read glove
#Initialise the model architecture
#gpu setting
#mode choice
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
if sys.version_info[0] != 3:
raise Exception(
"ERROR: You must use Python 3 but you are running Python %i" %
sys.version_info[0])
print(
"This code was developed and tested on TensorFlow 1.8.0. Your TensorFlow version: %s"
% tf.__version__)
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
#glove path
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
char2id, id2char = get_char_embed()
#path for context, question, ans_span
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
#Initialise the model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, id2char, char2id)
# Some GPU settings
config = tf.ConfigProto() #set configuration for sess.run
config.gpu_options.allow_growth = True #make gpu storage usage based for condition
#different modes
if FLAGS.mode == "train":
#setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(
FLAGS.flag_values_dict(), fout
) #NoteL changed from FLAGS.__flags to FLAGS.flag_values_dict() after tensorflow 1.5
# Make bestmodel dir
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
#Added tfdbg
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
#Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
"""
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception("For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception("For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print ("Writing predictions to %s..." % FLAGS.json_out_path)
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print ("Wrote predictions to %s" % FLAGS.json_out_path)
"""
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Initialize bestmodel directory
bestmodel_dir = os.path.join(
"F:\Sid\Learnings\Data Scientist\Text Analysis\Machine Comprehension\data\ModelLogs\best_weights",
"best_checkpoint")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=10,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" % unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception("ERROR: You must use Python 2 but you are running Python %i" % sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception("You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR, FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path, FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config=tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess, qa_model, FLAGS.train_dir, expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=10, print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception("For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception("For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess, qa_model, FLAGS.ckpt_load_dir, expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id, qn_uuid_data, context_token_data, qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def _init_state(self) -> None:
"""
Initialize the state and load it from an existing checkpoint if any
"""
job_env = submitit.JobEnvironment()
if job_env.global_rank == 0:
# config_path = Path(args.save_folder) / str(job_env.job_id) / 'config.json'
os.makedirs(self._train_cfg.output_dir, exist_ok=True)
config_path = Path(self._train_cfg.output_dir) / 'config.json'
with open(config_path, "w") as g:
g.write(json.dumps(self._train_cfg._asdict()))
print(f"Setting random seed {self._train_cfg.seed}", flush=True)
random.seed(self._train_cfg.seed)
np.random.seed(self._train_cfg.seed)
torch.manual_seed(self._train_cfg.seed)
print("Create data loaders", flush=True)
tokenizer = AutoTokenizer.from_pretrained(self._train_cfg.model_name)
collate_fc = partial(rank_collate, pad_id=tokenizer.pad_token_id)
train_set = RankingDataset(tokenizer, self._train_cfg.train_file, self._train_cfg.max_seq_len, self._train_cfg.max_q_len, train=True)
train_sampler = MhopSampler(train_set, num_neg=self._train_cfg.neg_num)
batch_size_per_gpu = (1 + self._train_cfg.neg_num) * self._train_cfg.num_q_per_gpu
n_gpu = torch.cuda.device_count()
print(f"Number of GPUs: {n_gpu}", flush=True)
print(f"Batch size per node: {batch_size_per_gpu * n_gpu}", flush=True)
self._train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size_per_gpu * n_gpu, num_workers=self._train_cfg.num_workers, collate_fn=collate_fc, sampler=train_sampler)
test_set = RankingDataset(tokenizer, self._train_cfg.predict_file, self._train_cfg.max_seq_len, self._train_cfg.max_q_len)
self._test_loader = torch.utils.data.DataLoader(
test_set,
batch_size=self._train_cfg.predict_batch_size,
num_workers=self._train_cfg.num_workers, collate_fn=collate_fc
)
print("Create model", flush=True)
print(f"Local rank {job_env.local_rank}", flush=True)
bert_config = AutoConfig.from_pretrained(self._train_cfg.model_name)
model = QAModel(bert_config, self._train_cfg)
model.cuda(job_env.local_rank)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(
nd in n for nd in no_decay)], 'weight_decay': self._train_cfg.weight_decay},
{'params': [p for n, p in model.named_parameters() if any(
nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
if self._train_cfg.use_adam:
optimizer = optim.Adam(optimizer_parameters, lr=self._train_cfg.learning_rate)
else:
optimizer = AdamW(optimizer_parameters, lr=self._train_cfg.learning_rate)
# lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2)
if self._train_cfg.fp16:
model, optimizer = amp.initialize(
model, optimizer, opt_level=self._train_cfg.fp16_opt_level)
t_total = len(self._train_loader) // self._train_cfg.gradient_accumulation_steps * self._train_cfg.num_train_epochs
warmup_steps = t_total * self._train_cfg.warmup_ratio
lr_scheduler = get_linear_schedule_with_warmup(
optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total
)
model = torch.nn.DataParallel(model)
self._state = TrainerState(
epoch=0, model=model, optimizer=optimizer, lr_scheduler=lr_scheduler, global_step=0
)
self.tb_logger = SummaryWriter(self._train_cfg.output_dir.replace("logs", "tflogs"))
checkpoint_fn = osp.join(self._train_cfg.output_dir, str(job_env.job_id), "checkpoint.pth")
# checkpoint_fn = osp.join(self._train_cfg.output_dir, "checkpoint.pth")
if os.path.isfile(checkpoint_fn):
print(f"Load existing checkpoint from {checkpoint_fn}", flush=True)
self._state = TrainerState.load(
checkpoint_fn, default=self._state, gpu=job_env.local_rank)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.graph")
train_qn_path = os.path.join(FLAGS.data_dir, "train.instruction")
train_ans_path = os.path.join(FLAGS.data_dir, "train.answer")
dev_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
dev_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
dev_ans_path = os.path.join(FLAGS.data_dir, FLAGS.file_in_path + ".answer")
# Create vocabularies of the appropriate sizes for output answer.
context_vocab_path = os.path.join(
FLAGS.data_dir, "vocab%d.context" % FLAGS.context_vocabulary_size)
# ans_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d." % FLAGS.ans_vocabulary_size)
# initialize the vocabulary.
context_vocab, rev_context_vocab = create_vocabulary(
context_vocab_path, train_context_path, FLAGS.context_vocabulary_size)
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix, context_vocab,
rev_context_vocab, context_vocab)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
# with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
# json.dump(FLAGS(sys.argv), fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
""" To show a few examples without attention map.
"""
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
_, _, _, _ = qa_model.check_f1_em(
sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=FLAGS.print_num,
print_to_screen=True) #, summary_writer=summary_writer)
# summary_writer.close()
elif FLAGS.mode == "show_attention":
""" To show a few examples of attention map.
"""
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
qa_model.demo(sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=FLAGS.print_num,
print_to_screen=True,
shuffle=False) # , summary_writer=summary_writer)
elif FLAGS.mode == "official_eval":
with tf.Session(config=config) as sess:
if FLAGS.ckpt_load_dir:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
else:
# Load best model
initialize_model(sess,
qa_model,
bestmodel_dir,
expect_exists=True)
eval_context_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".graph")
eval_qn_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".instruction")
eval_ans_path = os.path.join(FLAGS.data_dir,
FLAGS.file_in_path + ".answer")
f1, em, edit_dist, rem = qa_model.check_f1_em(
sess,
eval_context_path,
eval_qn_path,
eval_ans_path,
FLAGS.file_in_path,
num_samples=0,
print_to_screen=False,
write_out=FLAGS.write_out,
file_out=FLAGS.file_out_path,
shuffle=False)
logging.info(
"F1 score: %f, EM score: %f, edit distance: %f, rough EM score: %f"
% (f1, em, edit_dist, rem))
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
def main(unused_argv):
# Print an error message if you've entered flags incorrectly
if len(unused_argv) != 1:
raise Exception("There is a problem with how you entered flags: %s" %
unused_argv)
# Check for Python 2
if sys.version_info[0] != 2:
raise Exception(
"ERROR: You must use Python 2 but you are running Python %i" %
sys.version_info[0])
# Print out Tensorflow version
print "This code was developed and tested on TensorFlow 1.4.1. Your TensorFlow version: %s" % tf.__version__
# Define train_dir
if not FLAGS.experiment_name and not FLAGS.train_dir and FLAGS.mode != "official_eval":
raise Exception(
"You need to specify either --experiment_name or --train_dir")
FLAGS.train_dir = FLAGS.train_dir or os.path.join(EXPERIMENTS_DIR,
FLAGS.experiment_name)
# Initialize bestmodel directory
bestmodel_dir = os.path.join(FLAGS.train_dir, "best_checkpoint")
# Define path for glove vecs
FLAGS.glove_path = FLAGS.glove_path or os.path.join(
DEFAULT_DATA_DIR, "glove.6B.{}d.txt".format(FLAGS.embedding_size))
# if FLAGS.mode != 'loadProbs':
# Load embedding matrix and vocab mappings
emb_matrix, word2id, id2word = get_glove(FLAGS.glove_path,
FLAGS.embedding_size)
# Get filepaths to train/dev datafiles for tokenized queries, contexts and answers
train_context_path = os.path.join(FLAGS.data_dir, "train.context")
train_qn_path = os.path.join(FLAGS.data_dir, "train.question")
train_ans_path = os.path.join(FLAGS.data_dir, "train.span")
dev_context_path = os.path.join(FLAGS.data_dir, "dev.context")
dev_qn_path = os.path.join(FLAGS.data_dir, "dev.question")
dev_ans_path = os.path.join(FLAGS.data_dir, "dev.span")
# Initialize model
qa_model = QAModel(FLAGS, id2word, word2id, emb_matrix)
# Some GPU settings
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Split by mode
if FLAGS.mode == "train":
# Setup train dir and logfile
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
file_handler = logging.FileHandler(
os.path.join(FLAGS.train_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
# Save a record of flags as a .json file in train_dir
# with open(os.path.join(FLAGS.train_dir, "flags.json"), 'w') as fout:
# json.dump(FLAGS.__flags, fout)
# Make bestmodel dir if necessary
if not os.path.exists(bestmodel_dir):
os.makedirs(bestmodel_dir)
with tf.Session(config=config) as sess:
# Load most recent model
initialize_model(sess,
qa_model,
FLAGS.train_dir,
expect_exists=False)
# Train
qa_model.train(sess, train_context_path, train_qn_path,
train_ans_path, dev_qn_path, dev_context_path,
dev_ans_path)
elif FLAGS.mode == "show_examples":
with tf.Session(config=config) as sess:
# Load best model
initialize_model(sess, qa_model, bestmodel_dir, expect_exists=True)
# Show examples with F1/EM scores
_, _ = qa_model.check_f1_em(sess,
dev_context_path,
dev_qn_path,
dev_ans_path,
"dev",
num_samples=20,
print_to_screen=True)
elif FLAGS.mode == "official_eval":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = generate_answers(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
elif FLAGS.mode == "official_eval_with_bidaf":
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict, bidaf_dict, self_dict1, self_dict2, out_dict = generate_answers_with_bidaf(
sess, qa_model, word2id, qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
print "Writing sims to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-bidaf', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(bidaf_dict, ensure_ascii=False)))
print "Wrote sims to %s" % FLAGS.json_out_path
print "Writing self sims1 to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-self1', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(self_dict1, ensure_ascii=False)))
print "Wrote self sims1 to %s" % FLAGS.json_out_path
print "Writing self sims2 to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-self2', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(self_dict2, ensure_ascii=False)))
print "Wrote self sims2 to %s" % FLAGS.json_out_path
print "Writing preds to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path + '-preds', 'w',
encoding='utf-8') as f:
f.write(unicode(json.dumps(out_dict, ensure_ascii=False)))
print "Wrote preds to %s" % FLAGS.json_out_path
elif FLAGS.mode == 'saveProbs':
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
with tf.Session(config=config) as sess:
# Load model from ckpt_load_dir
initialize_model(sess,
qa_model,
FLAGS.ckpt_load_dir,
expect_exists=True)
# Get a predicted answer for each example in the data
# Return a mapping answers_dict from uuid to answer
answers_dict = save_answer_probs(sess, qa_model, word2id,
qn_uuid_data, context_token_data,
qn_token_data)
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'wb') as f:
pickle.dump(answers_dict, f, protocol=2)
# f.write(unicode(pickle.dumps(answers_dict, ensure_ascii=False)))
# f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
elif FLAGS.mode == 'loadProbs':
if FLAGS.json_in_path == "":
raise Exception(
"For official_eval mode, you need to specify --json_in_path")
if FLAGS.ckpt_load_dir == "":
raise Exception(
"For official_eval mode, you need to specify --ckpt_load_dir")
# Read the JSON data from file
qn_uuid_data, context_token_data, qn_token_data = get_json_data(
FLAGS.json_in_path)
# word2id = pickle.load(open('word2id', 'rb'))
# pickle.dump(word2id, open('word2id', 'wb'))
print 'Loaded data'
dictLists = []
for file in os.listdir('./pickles'):
f = os.path.join('./pickles', file)
print 'Loading predictions from ', f
prob_dict = pickle.load(open(f, 'rb'))
dictLists += [prob_dict]
# mainDict = {}
# stdiDict = {}
# for probs in dictLists:
# for k in dictLists[0].keys():
# stdi = 1.0 / (np.std(np.array(probs[k][0])) + np.std(np.array(probs[k][1])) + 1e-2)
# stdiDict[k] = stdi
# try:
# mainDict[k] = (mainDict[k][0] + stdi * np.array(probs[k][0]), mainDict[k][1] + stdi* np.array(probs[k][1]))
# except KeyError:
# mainDict[k] = (stdi* np.array(probs[k][0]), stdi*np.array(probs[k][1]))
uuid2ans = {} # maps uuid to string containing predicted answer
detokenizer = MosesDetokenizer()
# for k in mainDict.keys():
# start_dist = mainDict[k][0] / stdiDict[k]
# end_dist = mainDict[k][1] / stdiDict[k]
# # Take argmax to get start_pos and end_post, both shape (batch_size)
# end_dp = np.zeros(end_dist.shape)
# # start_pos = np.argmax(start_dist)
# # end_pos = np.argmax(end_dist)
# end_dp[-1]=end_dist[-1]
# for i in range(len(end_dist)-2,-1,-1):
# end_dp[i]=np.amax([end_dist[i],end_dp[i+1]])
# start_pos=np.argmax(start_dist*end_dp)
# end_pos = start_pos + np.argmax(end_dist[start_pos:])
# uuid2ans[k] = (start_pos, end_pos)
for k in dictLists[0].keys():
spanDict = {}
for probs in dictLists:
start_dist = np.array(probs[k][0])
end_dist = np.array(probs[k][1])
# Take argmax to get start_pos and end_post, both shape (batch_size)
end_dp = np.zeros(end_dist.shape)
end_dp[-1] = end_dist[-1]
for i in range(len(end_dist) - 2, -1, -1):
end_dp[i] = np.amax([end_dist[i], end_dp[i + 1]])
start_pos = np.argmax(start_dist * end_dp)
end_pos = start_pos + np.argmax(end_dist[start_pos:])
try:
spanDict[(start_pos, end_pos)] += [
start_dist[start_pos] * end_dist[end_pos]
]
except KeyError:
spanDict[(start_pos, end_pos)] = [
start_dist[start_pos] * end_dist[end_pos]
]
best_span = (0, 0)
best_span_votes = 0
best_span_prob = 0
for span in spanDict.keys():
if len(spanDict[span]) > best_span_votes:
best_span = span
best_span_votes = len(spanDict[span])
best_span_prob = max(spanDict[span])
elif len(
spanDict[span]
) == best_span_votes and best_span_prob < max(spanDict[span]):
best_span = span
best_span_votes = len(spanDict[span])
best_span_prob = max(spanDict[span])
uuid2ans[k] = (best_span[0], best_span[1])
result = {}
data_size = len(qn_uuid_data)
num_batches = ((data_size - 1) / FLAGS.batch_size) + 1
batch_num = 0
print "Generating answers..."
for batch in get_batch_generator(word2id, qn_uuid_data,
context_token_data, qn_token_data,
FLAGS.batch_size, FLAGS.context_len,
FLAGS.question_len):
# For each example in the batch:
for ex_idx in range(FLAGS.batch_size):
# Detokenize and add to dict
try:
uuid = batch.uuids[ex_idx]
pred_start, pred_end = uuid2ans[uuid]
# Original context tokens (no UNKs or padding) for this example
context_tokens = batch.context_tokens[
ex_idx] # list of strings
# Check the predicted span is in range
assert pred_start in range(len(context_tokens))
assert pred_end in range(len(context_tokens))
# Predicted answer tokens
pred_ans_tokens = context_tokens[pred_start:pred_end +
1] # list of strings
result[uuid] = detokenizer.detokenize(pred_ans_tokens,
return_str=True)
except IndexError:
pass
batch_num += 1
if batch_num % 10 == 0:
print "Generated answers for %i/%i batches = %.2f%%" % (
batch_num, num_batches, batch_num * 100.0 / num_batches)
print "Finished generating answers for dataset."
answers_dict = result
# Write the uuid->answer mapping a to json file in root dir
print "Writing predictions to %s..." % FLAGS.json_out_path
with io.open(FLAGS.json_out_path, 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers_dict, ensure_ascii=False)))
print "Wrote predictions to %s" % FLAGS.json_out_path
else:
raise Exception("Unexpected value of FLAGS.mode: %s" % FLAGS.mode)
