def get_predictions(self, max_example, epoch, save_dir=None):
"""
This method takes some question indices as input and prints out
the paragraphs, questions, ground truth and predicted answers as text.
"""
# Process data and create batch_loader for prediction
data_dir = "/scratch/s161027/ga_reader_data/squad"
dp = DataPreprocessor()
data = dp.preprocess(question_dir=data_dir,
max_example=max_example,
use_chars=False,
only_test_run=False)
batch_loader = MiniBatchLoader(data.test,
128,
shuffle=False,
prediction_only=True)
batch_number = batch_loader.__len__()
# Logging
print(
"Loaded and processed data with {} examples. Number of batches is {}"
.format(max_example, batch_number))
# Restore Model
default_path = "/scratch/s161027/run_data/best_working_model"
if save_dir is None:
save_dir = os.path.join(default_path, "saved_models")
sess = tf.Session()
GAReader.restore(self, sess, save_dir,
epoch) # Restore model and graph
# Numeric output from the prediction is a list of tuples like:
# (document, query, answer, predicted_answer, start_probs, end_probs, attention_tensors)
# Check the 'predict' method in GAReader.py for exact output.
with sess:
# Logging
print("Generating predictions...")
numeric_output = GAReader.predict(self, sess, batch_loader)
# Get the first (and only) index of the output
# since there should be only one batch in case of prediction
numeric_output = numeric_output[0]
# Based on the dictionary, turn the embeddings back into text
# Text output is a tuple such as:
# (document, query, ground_truth, predicted_answer)
# Where the elements of this tuple are lists of strings.
text_output = self.inverse_dictionary(numeric_output, data.dictionary)
# Attentions_and_probs is a tuple such as:
# (start_probs_list, end_probs_list, attentions_list)
# Where each element of the tuple is masked to be the same shape as its respective document and query.
# As opposed to having the shape: [max_document_length] or [max_document_length x max_query_length] in case
# of the attention matrices
attentions_and_probs = self.mask_numeric_output(numeric_output)
return text_output, numeric_output, attentions_and_probs, batch_number
def test_individual_file(self, input_file, learning_rate=0.1):
current_step = self.restore_session()
dpp = DataPreprocessor()
image = dpp.load_image(input_file, 480, 320)
ground_truth = np.zeros((480, 320))
feed_dict = {
self.x: [image],
self.y: [ground_truth],
self.train_phase: 1,
self.rate: learning_rate
}
segmentation = np.squeeze(
self.session.run(self.prediction, feed_dict=feed_dict))
dp = DataPostprocessor()
dp.write_out(0, image, segmentation, current_step)
def train(args):
use_chars = args.char_dim > 0
test_run = args.test_run
# SQUAD_MOD
# Initialize session early to reserve GPU memory
sess = tf.Session()
# Condition for switching between cloze-style and span-style QA
cloze_style = args.cloze_style
if not cloze_style and not os.path.exists(
os.path.join(args.data_dir, "squad/training")):
# Parsing SQuAD data set
squad_parser(test_run)
# SQUAD_MOD
# Processing .question files and loading into memory (data in lists and tuples)
dp = DataPreprocessor()
data = dp.preprocess(
question_dir=os.path.join(args.data_dir, "squad"),
max_example=args.max_example,
use_chars=use_chars,
use_cloze_style=cloze_style,
only_test_run=test_run)
# Building the iterable batch loaders (data in numpy arrays)
train_batch_loader = MiniBatchLoader(
data.training, args.batch_size, sample=1.0, use_cloze_style=cloze_style)
valid_batch_loader = MiniBatchLoader(
data.validation, args.batch_size, shuffle=False, use_cloze_style=cloze_style)
test_batch_loader = MiniBatchLoader(
data.test, args.batch_size, shuffle=False, use_cloze_style=cloze_style)
# Fixing the max. document and query length
# Currently the max. is the same across all batches
max_doc_len = max([train_batch_loader.max_document_length,
valid_batch_loader.max_document_length,
test_batch_loader.max_document_length])
max_qry_len = max([train_batch_loader.max_query_length,
valid_batch_loader.max_query_length,
test_batch_loader.max_query_length])
if not args.resume:
# Loading the GLoVE vectors
logging.info("loading word2vec file ...")
embed_init, embed_dim = \
load_word2vec_embeddings(data.dictionary[0], None)
logging.info("embedding dim: {}".format(embed_dim))
logging.info("initialize model ...")
model = GAReader(args.n_layers, data.vocab_size, data.num_chars,
args.n_hidden, args.n_hidden_dense,
embed_dim, args.train_emb,
args.char_dim, args.use_feat,
args.gating_fn, save_attn=True,
use_cloze_style=cloze_style)
model.build_graph(args.grad_clip, embed_init, args.seed,
max_doc_len, max_qry_len, use_cloze_style=cloze_style)
init = tf.global_variables_initializer()
loc_init = tf.local_variables_initializer()
saver = tf.train.Saver(tf.global_variables())
else:
model = GAReader(args.n_layers, data.vocab_size, data.num_chars,
args.n_hidden, 100, args.train_emb, args.char_dim,
args.use_feat, args.gating_fn)
# Tensorboard
writer = tf.summary.FileWriter(args.log_dir)
# Setting GPU memory
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
with sess:
# Tensorboard
writer.add_graph(sess.graph)
# training phase
if not args.resume:
sess.run([init, loc_init])
if args.init_test:
logging.info('-' * 50)
logging.info("Initial test ...")
best_loss, best_acc = model.validate(sess, valid_batch_loader)
else:
best_acc = 0.
else:
model.restore(sess, args.save_dir)
saver = tf.train.Saver(tf.global_variables())
logging.info('-' * 100)
lr = args.init_learning_rate
logging.info("Start training ...")
epoch_range = tqdm(range(args.n_epoch), leave=True, ascii=True, ncols=100)
max_it = len(train_batch_loader)
# Start training loop
for epoch in epoch_range:
start = time.time()
it = loss = acc = n_example = 0
if epoch >= 2:
lr /= 2
for training_data in train_batch_loader:
loss_, acc_, updates_, attentions_, pred_, answer_ = \
model.train(sess, training_data, args.drop_out, lr, it, writer, epoch, max_it)
# SQUAD_MOD
# attentions_ = np.array(attentions_)
# attentions_nonzero = attentions_[-1, -1, :, :]
# attentions_nonzero = \
# attentions_nonzero[~np.all(attentions_nonzero==0, axis=1)][:, ~np.all(attentions_nonzero==0, axis=0)]
# SQUAD_MOD
loss += loss_
acc += acc_
it += 1
n_example += training_data[0].shape[0]
if it % args.print_every == 0 or \
it % max_it == 0:
spend = (time.time() - start) / 60
# Get estimated finish time in hours
eta = (spend / 60) * ((max_it - it) / args.print_every)
# SQUAD_MOD
# statement = "Size of model attentions: {}".format(attentions_.shape)
# statement += "\nFull attention matrix:\n" + str(attentions_[-1, -1, :, :])
# statement += "\nNon-zero part of attention matrix (shape: {}):".format(attentions_nonzero.shape) +\
# str(attentions_nonzero)
# SQUAD_MOD
statement = "Epoch: {}, it: {} (max: {}), " \
.format(epoch, it, max_it)
statement += "loss: {:.3f}, acc: {:.3f}, " \
.format(loss / args.print_every,
acc / n_example)
statement += "time: {:.1f}(m), " \
.format(spend)
statement += "ETA: {:.1f} hours" \
.format(eta)
logging.info(statement)
loss = acc = n_example = 0
start = time.time()
# save model
if it % args.eval_every == 0 or \
it % max_it == 0:
valid_loss, valid_acc = model.validate(
sess, valid_batch_loader)
if valid_acc >= best_acc:
logging.info("Best valid acc: {:.3f}, previous best: {:.3f}".format(
valid_acc,
best_acc))
best_acc = valid_acc
model.save(sess, saver, args.save_dir, epoch)
start = time.time()
# test model
logging.info("Final test ...")
model.validate(sess, test_batch_loader)
def train(args):
use_chars = args.char_dim > 0
# Initialize session early to reserve GPU memory
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
sess = tf.Session()
# Processing .question files and loading into memory (data in lists and tuples)
dp = DataPreprocessor()
data = dp.preprocess(question_dir=args.data_dir,
unlabeled_set=args.unlabeled_set,
training_set=args.training_set,
vocab_size=args.vocab_size,
max_example=args.max_example,
use_chars=use_chars,
only_test_run=args.test_only)
# Define the reverse word dictionary for inference
word_dict = data.dictionary[0]
inverse_word_dict = dict(zip(word_dict.values(), word_dict.keys()))
# Building the iterable batch loaders (data in numpy arrays)
train_batch_loader = MiniBatchLoader(data.training,
args.batch_size,
word_dict,
shuffle=True,
sample=1.0)
valid_batch_loader = MiniBatchLoader(data.validation,
args.batch_size,
word_dict,
shuffle=False)
test_batch_loader = MiniBatchLoader(data.test,
args.batch_size,
word_dict,
shuffle=False)
# Fixing the max. document and query length
# Currently the max. is the same across all batches
max_doc_len = max([
train_batch_loader.max_document_length,
valid_batch_loader.max_document_length,
test_batch_loader.max_document_length
])
max_qry_len = max([
train_batch_loader.max_query_length,
valid_batch_loader.max_query_length, test_batch_loader.max_query_length
])
if not args.resume:
# Loading the GLoVE vectors
logging.info("loading word2vec file ...")
embed_init, embed_dim = \
load_word2vec_embeddings(data.dictionary[0], args.embed_file)
logging.info("embedding dim: {}".format(embed_dim))
logging.info("initialize model ...")
model = Seq2Seq(args.n_layers, data.dictionary, data.vocab_size,
args.n_hidden_encoder, args.n_hidden_decoder,
embed_dim, args.train_emb, args.answer_injection,
args.batch_size, args.bi_encoder, args.use_attention,
args.use_copy_mechanism, args.max_parallel_dec,
args.gen_vocab_size, args.use_cudnn_gru)
model.build_graph(args.grad_clip, embed_init, args.seed, max_doc_len,
max_qry_len)
print("\n\nModel build successful!\n\n")
init = tf.global_variables_initializer()
loc_init = tf.local_variables_initializer()
saver = tf.train.Saver(tf.global_variables())
else:
pass
with sess:
sess.graph.finalize()
# Tensorboard. Two writers to have training and validation accuracy both on the same
# board.
writer = tf.summary.FileWriter(args.log_dir + "/tensorboard/training",
graph=sess.graph,
flush_secs=60,
filename_suffix="_train_" +
args.model_name)
writer_val = tf.summary.FileWriter(
args.log_dir + "/tensorboard/validation",
graph=sess.graph,
flush_secs=60,
filename_suffix="_validate_" + args.model_name)
# training phase
if not args.resume:
sess.run([init, loc_init])
if args.init_test:
logging.info('-' * 50)
logging.info("Initial test ...")
best_loss, best_perplexity = model.validate(
sess, valid_batch_loader)
else:
# Initialize the best validation accuracy to be 0
best_perplexity = 1e6
else:
model.restore(sess, args.save_dir, epoch=9)
saver = tf.train.Saver(tf.global_variables())
logging.info('-' * 100)
logging.info("Start training ...")
learning_rate = args.init_learning_rate
epoch_range = tqdm(range(args.n_epoch),
leave=True,
ascii=True,
ncols=100)
max_it = len(
train_batch_loader) # Max number of batches in training epoch
eval_every = max_it // args.eval_per_epoch # How often to validate
eval_list = list(range(eval_every, max_it + 1,
eval_every)) # List of validation points
eval_list[
-1] = max_it # To make sure validation happens exactly at the end of the epoch
# Define where run statistics will be saved (training loss, validation loss etc.)
dump_name = args.log_dir + "/run_stats_" + args.model_name
# Initialize lists for pickle dump of stats
train_dict_dump = {
"train_iteration": [],
"train_loss": [],
"train_perplexity": [],
"epoch": []
}
valid_dict_dump = {
"train_iteration": [],
"valid_loss": [],
"valid_perplexity": [],
"epoch": []
}
# Start training loop
for epoch in epoch_range:
start_time = time.time()
# Initialize counters at start of epoch
it = loss = num_example = 0
# From the 2. epoch and onwards we adjust the learning rate
# and re-initialize the accuracy metric
if epoch >= 2:
# Halve learning rate
learning_rate /= 2
# Count amount of times batches had to be split
split_count = 0
# Count amount of times a batch had to be skipped because it exhausted GPU memory
skip_count = 0
training_batch_range = tqdm(train_batch_loader,
leave=False,
ascii=True,
ncols=100)
for training_data in training_batch_range:
# Check the total sequence length in the current batch. If it's above a certain
# limit, then split the batch in two
total_sequence_length = np.sum(training_data[5])
try:
if total_sequence_length > 8700: # Current GPU (GTX 1080) fails at 8737
split_count += 1
# print("\nTotal sequence length in batch is too high ({}), "
# "splitting batch in two. Splits performed so far: {}".format(
# total_sequence_length, split_count))
batch_split_1, batch_split_2 = batch_splitter(
training_data)
output_split_1 = \
model.train(sess, batch_split_1, args.drop_out, learning_rate, it,
writer,
epoch, max_it)
output_split_2 = \
model.train(sess, batch_split_2, args.drop_out, learning_rate, it,
writer,
epoch, max_it)
loss_ = 0.5 * (output_split_1[0] + output_split_2[0])
else: # Pass the batch to the training method regularly
loss_, updates_ = \
model.train(sess, training_data, args.drop_out,
learning_rate, it, writer, epoch, max_it)
except tf.errors.ResourceExhaustedError:
skip_count += 1
print(
"GPU out of memory. Total sequence length in batch was {}."
"Skipping batch ({} skips so far)...".format(
total_sequence_length, skip_count))
continue
it += 1
# Cumulative loss, perplexity
loss += loss_
# Saving loss, accuracy and iteration for later pickle dump
train_dict_dump["train_loss"].append(loss_)
train_dict_dump["train_perplexity"].append(np.exp(loss_))
train_dict_dump["train_iteration"].append(epoch * max_it + it -
1)
train_dict_dump["epoch"].append(epoch)
# Adding the number of examples in the current batch
num_example += training_data[0].shape[0]
# Logging information
if it % args.print_every == 0 or \
it == max_it:
time_spent = (time.time() - start_time) / 60
# Get estimated finish time in hours
eta = (time_spent / 60) * (
(max_it - it) / args.print_every)
# Calculate current perplexity
current_loss = loss / args.print_every
current_perplexity = np.exp(current_loss)
statement = "Epoch: {}, it: {} (max: {}), " \
.format(epoch, it, max_it)
statement += "Loss: {:.3f}, Perplexity: {:.3f}, " \
.format(current_loss,
current_perplexity)
statement += "Time: {:.1f}(m), " \
.format(time_spent)
statement += "ETA: {:.1f} hours" \
.format(eta)
logging.info(statement)
loss = num_example = 0
start_time = time.time()
# Validate, and save model
if it in eval_list:
logging.info("{:-^80}".format(" Validation "))
valid_loss, valid_perplexity = \
model.validate(sess, valid_batch_loader,
inverse_word_dict, it,
writer_val, epoch, max_it)
valid_dict_dump["valid_loss"].append(valid_loss)
valid_dict_dump["valid_perplexity"].append(
valid_perplexity)
valid_dict_dump["train_iteration"].append(epoch * max_it +
it - 1)
valid_dict_dump["epoch"].append(epoch)
# Saving run statistics
logging.info("Saving run statistics in binary...")
print("Saving run statistics in binary...")
outfile = open(dump_name, "wb")
pickle.dump([train_dict_dump, valid_dict_dump], outfile)
outfile.close()
if valid_perplexity <= best_perplexity:
logging.info("Best validation perplexity: {:.3f}, "
"previous best: {:.3f}".format(
valid_perplexity, best_perplexity))
best_perplexity = valid_perplexity
best_epoch = epoch
print(
"The model reached a new best perplexity: {}, at epoch {}"
.format(best_perplexity, best_epoch))
model.save(sess, saver, args.save_dir, args.model_name,
epoch)
start_time = time.time()
# test model
logging.info("{:-^80}".format(" Final Test "))
try:
model.validate(sess, test_batch_loader, inverse_word_dict)
except tf.errors.ResourceExhaustedError:
print("GPU out of memory during testing. Skipping batch...")
# Save run statistics
logging.info("Saving run statistics in binary...")
print("Saving run statistics in binary...")
outfile = open(dump_name, "wb")
pickle.dump([train_dict_dump, valid_dict_dump], outfile)
outfile.close()
# TODO: Send e-mail with run information. Loss, epoch, validation inference example etc.
input("Script ready to finish, please press enter to exit...")
def train(args):
use_chars = args.char_dim > 0
# load data
dp = DataPreprocessor()
data = dp.preprocess(question_dir=args.data_dir,
no_training_set=False,
max_example=args.max_example,
use_chars=use_chars)
# build minibatch loader
train_batch_loader = MiniBatchLoader(data.training,
args.batch_size,
sample=1.0)
valid_batch_loader = MiniBatchLoader(data.validation,
args.batch_size,
shuffle=False)
test_batch_loader = MiniBatchLoader(data.test,
args.batch_size,
shuffle=False)
if not args.resume:
logging.info("loading word2vec file ...")
embed_init, embed_dim = \
load_word2vec_embeddings(data.dictionary[0], args.embed_file)
logging.info("embedding dim: {}".format(embed_dim))
logging.info("initialize model ...")
model = GAReader(args.n_layers, data.vocab_size, data.num_chars,
args.n_hidden, embed_dim, args.train_emb,
args.char_dim, args.use_feat, args.gating_fn)
model.build_graph(args.grad_clip, embed_init, args.seed)
init = tf.global_variables_initializer()
loc_init = tf.local_variables_initializer()
saver = tf.train.Saver(tf.global_variables())
else:
model = GAReader(
args.n_layers,
data.vocab_size,
data.num_chars,
args.n_hidden,
100,
args.train_emb, # TODO: fix embed dim to be based on variable?
args.char_dim,
args.use_feat,
args.gating_fn)
# Tensorboard
writer = tf.summary.FileWriter(args.log_dir)
with tf.Session() as sess:
# Tensorboard
writer.add_graph(sess.graph)
# training phase
if not args.resume:
sess.run([init, loc_init])
if args.init_test:
logging.info('-' * 50)
logging.info("Initial test ...")
best_loss, best_acc = model.validate(sess, valid_batch_loader)
else:
best_acc = 0.
else:
model.restore(sess, args.save_dir)
saver = tf.train.Saver(tf.global_variables())
logging.info('-' * 50)
lr = args.init_learning_rate
logging.info("Start training ...")
for epoch in range(args.n_epoch):
start = time.time()
it = loss = acc = n_example = 0
if epoch >= 2:
lr /= 2
# TODO: Put Trange here? copy from model.validate
for dw, dt, qw, qt, a, m_dw, m_qw, tt, \
tm, c, m_c, cl, fnames in train_batch_loader:
loss_, acc_ = model.train(sess, dw, dt, qw, qt, a, m_dw, m_qw,
tt, tm, c, m_c, cl, args.drop_out,
lr, it, writer)
loss += loss_
acc += acc_
it += 1
n_example += dw.shape[0]
max_it = len(train_batch_loader)
if it % args.print_every == 0 or \
it % max_it == 0:
spend = (time.time() - start) / 60
# Get estimated finish time in hours
eta = (spend / 60) * ((max_it - it) / args.print_every)
statement = "Epoch: {}, it: {} (max: {}), " \
.format(epoch, it, max_it)
statement += "loss: {:.3f}, acc: {:.3f}, " \
.format(loss / args.print_every,
acc / n_example)
statement += "time: {:.1f}(m), " \
.format(spend)
statement += "ETA: {:.1f} hours" \
.format(eta)
logging.info(statement)
loss = acc = n_example = 0
start = time.time()
# save model
if it % args.eval_every == 0 or \
it % max_it == 0:
valid_loss, valid_acc = model.validate(
sess, valid_batch_loader)
if valid_acc >= best_acc:
logging.info(
"Best valid acc: {}, previous best: {}".format(
valid_acc, best_acc))
best_acc = valid_acc
model.save(sess, saver, args.save_dir, epoch)
start = time.time()
# test model
logging.info("Final test ...")
model.validate(sess, test_batch_loader)
def train(args):
use_chars = args.char_dim > 0
# Initialize session early to reserve GPU memory
sess = tf.Session()
if not os.path.exists(os.path.join(args.data_dir, "training")):
# Parsing SQuAD data set
squad_parser()
# Processing .question files and loading into memory (data in lists and tuples)
dp = DataPreprocessor()
data = dp.preprocess(
question_dir=args.data_dir,
max_example=args.max_example,
use_chars=use_chars,
only_test_run=args.test_only)
# Building the iterable batch loaders (data in numpy arrays)
train_batch_loader = MiniBatchLoader(
data.training, args.batch_size, sample=1.0)
valid_batch_loader = MiniBatchLoader(
data.validation, args.batch_size, shuffle=False)
test_batch_loader = MiniBatchLoader(
data.test, args.batch_size, shuffle=False)
# Fixing the max. document and query length
# Currently the max. is the same across all batches
max_doc_len = max([train_batch_loader.max_document_length,
valid_batch_loader.max_document_length,
test_batch_loader.max_document_length])
max_qry_len = max([train_batch_loader.max_query_length,
valid_batch_loader.max_query_length,
test_batch_loader.max_query_length])
if not args.resume:
# Loading the GLoVE vectors
logging.info("loading word2vec file ...")
embed_init, embed_dim = \
load_word2vec_embeddings(data.dictionary[0], args.embed_file)
logging.info("embedding dim: {}".format(embed_dim))
logging.info("initialize model ...")
model = GAReader(args.n_layers, data.dictionary,
data.vocab_size, data.num_chars,
args.n_hidden, args.n_hidden_dense,
embed_dim, args.train_emb,
args.char_dim, args.use_feat,
args.gating_fn, save_attn=True)
model.build_graph(args.grad_clip, embed_init, args.seed,
max_doc_len, max_qry_len)
init = tf.global_variables_initializer()
loc_init = tf.local_variables_initializer()
saver = tf.train.Saver(tf.global_variables())
else:
model = GAReader(args.n_layers, data.vocab_size, data.num_chars,
args.n_hidden, args.n_hidden_dense, 100, args.train_emb, args.char_dim,
args.use_feat, args.gating_fn)
# Setting GPU memory
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
with sess:
# Tensorboard. Two writers to have training and validation accuracy both on the same
# board.
writer = tf.summary.FileWriter(args.log_dir+"/tensorboard/training",
graph=sess.graph,
flush_secs=60,
filename_suffix="_train_"+args.model_name)
writer_val = tf.summary.FileWriter(args.log_dir+"/tensorboard/validation",
graph=sess.graph,
flush_secs=60,
filename_suffix="_validate_"+args.model_name)
# training phase
if not args.resume:
sess.run([init, loc_init])
if args.init_test:
logging.info('-' * 50)
logging.info("Initial test ...")
best_loss, best_acc = model.validate(sess, valid_batch_loader)
else:
# Initialize the best validation accuracy to be 0
best_acc = 0.
else:
model.restore(sess, args.save_dir, epoch=9)
saver = tf.train.Saver(tf.global_variables())
logging.info('-' * 100)
logging.info("Start training ...")
learning_rate = args.init_learning_rate
epoch_range = tqdm(range(args.n_epoch), leave=True, ascii=True, ncols=100)
max_it = len(train_batch_loader)
# Initialize lists for pickle dump of stats
train_dict_dump = {"train_em_acc": [],
"train_f1_score": [],
"train_loss": [],
"train_iteration": [],
"epoch": []}
valid_dict_dump = {"train_iteration": [],
"valid_em_acc": [],
"valid_f1_score": [],
"valid_loss": [],
"epoch": []}
# Start training loop
for epoch in epoch_range:
start_time = time.time()
# Initialize counters at start of epoch
it = loss = em_acc = f1_score = num_example = 0
# From the 2. epoch and onwards we adjust the learning rate
# and re-initialize the accuracy metric
if epoch >= 2:
# Halve learning rate
learning_rate /= 2
# Reinitialize streaming accuracy metric for each epoch
# in order to measure accuracy only within one epoch
# and not across multiple epochs
running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES,
scope="em_accuracy_metric")
running_vars_initializer = tf.variables_initializer(var_list=running_vars)
sess.run(running_vars_initializer)
for training_data in train_batch_loader:
loss_, f1_score_, exact_match_accuracy_, updates_ = \
model.train(sess, training_data, args.drop_out, learning_rate, it, writer, epoch, max_it)
# Cumulative loss, exact match and F1 accuracy
loss += loss_
em_acc += exact_match_accuracy_
f1_score += f1_score_
# Saving loss, accuracy and iteration for later pickle dump
train_dict_dump["train_f1_score"].append(f1_score_)
train_dict_dump["train_em_acc"].append(exact_match_accuracy_)
train_dict_dump["train_loss"].append(loss_)
train_dict_dump["train_iteration"].append(it)
train_dict_dump["epoch"].append(epoch)
it += 1
# Adding the number of examples in the current batch
num_example += training_data[0].shape[0]
# Logging information
if it % args.print_every == 0 or \
it == max_it:
time_spent = (time.time() - start_time) / 60
# Get estimated finish time in hours
eta = (time_spent / 60) * ((max_it - it) / args.print_every)
statement = "Epoch: {}, it: {} (max: {}), " \
.format(epoch, it, max_it)
statement += "loss: {:.3f}, EM_Acc: {:.3f}, F1_Score: {:.3f}, " \
.format(loss / args.print_every,
em_acc / args.print_every,
f1_score / args.print_every)
statement += "time: {:.1f}(m), " \
.format(time_spent)
statement += "ETA: {:.1f} hours" \
.format(eta)
logging.info(statement)
loss = em_acc = f1_score = num_example = 0
start_time = time.time()
# Validate, and save model
if it % args.eval_every == 0 or \
it % max_it == 0:
# Reinitialize streaming accuracy metric for each validation
running_vars = tf.get_collection(
tf.GraphKeys.LOCAL_VARIABLES,
scope="em_valid_accuracy_metric")
running_vars_initializer = tf.variables_initializer(var_list=running_vars)
sess.run(running_vars_initializer)
logging.info("{:-^80}".format(" Validation "))
valid_loss, valid_acc, valid_f1_score = \
model.validate(sess, valid_batch_loader, it, writer_val, epoch, max_it)
valid_dict_dump["valid_f1_score"].append(f1_score_)
valid_dict_dump["valid_em_acc"].append(exact_match_accuracy_)
valid_dict_dump["valid_loss"].append(loss_)
valid_dict_dump["train_iteration"].append(epoch*(it-1)+(it-1))
valid_dict_dump["epoch"].append(epoch)
if valid_acc >= best_acc:
logging.info("Best valid em_acc: {:.3f}, previous best: {:.3f}".format(
valid_acc,
best_acc))
best_acc = valid_acc
# TODO: model saving turned off temporarily
# model.save(sess, saver, args.save_dir, epoch)
start_time = time.time()
# test model
logging.info("Final test ...")
model.validate(sess, test_batch_loader)
# Do pickle dump
logging.info("Dumping run stats...")
print("Dumping run stats...")
dump_name = args.log_dir + "/run_stats_dump_"+args.model_name
outfile = open(dump_name, "wb")
pickle.dump([train_dict_dump, valid_dict_dump], outfile)
outfile.close()
# TODO: Send e-mail warning about script ending, then pause for a while
input("Script ready to finish, please press enter to exit...")
def get_predictions(self,
max_example,
model_name,
training_set,
epoch,
test_run=True,
unlabeled=True,
save_dir=None):
"""
This method takes some question indices as input and prints out
the paragraphs, questions, ground truth and predicted questions as text.
"""
# Process data and create batch_loader for prediction
data_dir = "/scratch/s161027/ga_reader_data/ssqa_processed"
vocab_size = 10000
if unlabeled:
unlabeled_set = "small"
dp = UnlabeledDataPreprocessor()
data = dp.preprocess(data_dir,
unlabeled_set,
training_set,
vocab_size,
max_example=max_example,
use_chars=False,
only_test_run=test_run)
batch_loader_input = data.training
else:
dp = DataPreprocessor()
data = dp.preprocess(data_dir,
training_set,
vocab_size,
max_example=max_example,
use_chars=False,
only_test_run=test_run)
batch_loader_input = data.test
dictionary = data.dictionary
batch_loader = MiniBatchLoader(batch_loader_input,
32,
dictionary[0],
shuffle=True,
prediction_only=True)
batch_number = batch_loader.__len__()
# Logging
if test_run:
print(
"Loaded and processed data with {} examples. Number of batches is {}"
.format(max_example, batch_number))
else:
print(
"Loaded and processed data with all examples. Number of batches is {}"
.format(max_example, batch_number))
# Restore Model
default_path = "/scratch/s161027/run_data/SSQA/COMBINED_RUNS/"
if save_dir is None:
save_dir = os.path.join(default_path, "saved_models")
# Restore model and graph
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
Seq2Seq.restore(self, sess, save_dir, model_name, epoch)
# Numeric output from the prediction is a list of tuples like:
# (document, query, answer, predicted_question)
# Check the 'predict' method in seq2seq_model.py for exact output.
with sess:
# Logging
print("Generating predictions...")
numeric_output = Seq2Seq.predict(self, sess, batch_loader,
unlabeled)
# Get the first index of the output
# since there should be only one batch in case of prediction
if not unlabeled:
numeric_output = [numeric_output[0]]
if unlabeled:
return numeric_output, data
# Based on the dictionary, turn the embeddings back into text
# Text output is a tuple such as:
# (document, query, ground_truth, predicted_answer)
# Where the elements of this tuple are lists of strings.
text_output = []
for numeric_batch in numeric_output:
text_output.append(
self.inverse_dictionary(numeric_batch, dictionary))
return text_output, numeric_output