def load_squad_documents(dataset_name) -> Dict:
"""Loads the original SQuAD data, needed to run the official evaluation scripts"""
if dataset_name not in SQUAD_URLS:
raise ValueError(dataset_name)
src = join(config.SQUAD_SOURCE, dataset_name + ".json")
if not exists(src):
logging.info("Download SQuAD %s to %s" % (dataset_name, src))
py_utils.download_to_file(SQUAD_URLS[dataset_name], src)
return py_utils.load_json(src)["data"]
def get_dev_scores(path, cache=True, n_processes=None):
cache_file = join(path, "dev_scores.json")
if exists(cache_file) and cache:
return py_utils.load_json(cache_file)
else:
logging.info("Scoring %s..." % path)
pred = get_predictions(path, "dev", n_processes=n_processes, cache=cache)
hans = load_eval_set("dev")
score = compute_dev_score(pred, hans)
score = {"accuracy": score}
if cache:
with open(cache_file, "w") as f:
json.dump(score, f)
return score
def compute_scores(path, dataset_name, cache=True):
output_file = join(path, "%s-scores.json" % dataset_name)
if cache and exists(output_file):
return py_utils.load_json(output_file)
logging.info("Scoring on %s" % dataset_name)
docs = squad.load_squad_documents(dataset_name)
pred = get_predictions(path, dataset_name, cache=cache)
if dataset_name in ["dev", "train"]:
result = squad_v1_official_evaluation.evaluate(docs, pred)
else:
result = squad_v1_adversarial_evaluation.evaluate_adversarial(docs, pred)
if cache:
with open(output_file, "w") as f:
json.dump(result, f)
return result
def compute_scores(path, dataset_name, part, cache=True):
output_file = join(path,
"%s-scores.json" % get_cache_name(dataset_name, part))
if cache and exists(output_file):
return py_utils.load_json(output_file)
logging.info("Scoring on %s" % dataset_name)
docs = triviaqa_cp.load_triviaqa_cp(dataset_name, part)
ground_truth = {x['QuestionId']: x['Answer'] for x in docs}
pred = get_predictions(path, dataset_name, part, cache=cache)
result = triviaqa_cp_evaluation.evaluate_triviaqa(ground_truth,
pred,
mute=True)
if cache:
with open(output_file, "w") as f:
json.dump(result, f)
return result
def get_predictions(path,
dataset_name,
part,
bach_size=128,
sample=None,
cache=True):
output_file = join(
path, "%s-predictions.json" % get_cache_name(dataset_name, part))
if sample is None and cache and exists(output_file):
return py_utils.load_json(output_file)
logging.info("Computing predictions for %s on %s..." %
(path, dataset_name))
logging.info("Loading model...")
model_dir = ModelDir(path)
model = model_dir.get_model()
logging.info("Setup data...")
data = triviaqa_cp.load_annotated_triviaqa_cp(dataset_name, part)
data.sort(key=lambda x: len(x.tokens))
voc = triviaqa_cp.compute_voc(data)
model.set_vocab(voc)
tuples = triviaqa_cp.convert_to_tuples(data)
if sample is not None:
np.random.shuffle(tuples)
tuples = tuples[:sample]
with tf.Session(graph=tf.Graph()) as sess:
ds = triviaqa_cp.make_dataset(tuples)
fn = model.tensorize_fn()
ds = ds.map(fn)
ds = ds.padded_batch(bach_size, ds.output_shapes)
ds.prefetch(5)
it = ds.make_initializable_iterator()
next_op = it.get_next()
logit_op = model.apply(False, next_op, None)
logit_op = tf.nn.log_softmax(logit_op, 1)
span_op = ops.get_best_span(logit_op, 17)
logging.info("Initializing...")
if sess is None:
sess = tf.Session()
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
sess.run(it.initializer)
logging.info("Loading checkpoint...")
saver = tf.train.Saver()
saver.restore(sess, model_dir.get_latest_checkpoint())
predictions = []
pbar = tqdm(desc="classify", total=len(tuples), ncols=80)
while True:
try:
predictions.append(sess.run(span_op))
pbar.update(len(predictions[-1]))
except tf.errors.OutOfRangeError:
break
pbar.close()
predictions = np.concatenate(predictions, 0)
predicted_text = {}
for tup, (s, e) in zip(tuples, predictions):
tokens = tup[2]
predicted_text[tup[0]] = " ".join(tokens[s:e + 1])
if sample is None and cache:
with open(output_file, "w") as f:
json.dump(predicted_text, f)
return predicted_text