def get_result(checkpoint):
"""Evaluate the checkpoint on SQuAD v2.0."""
# If running eval on the TPU, you will need to specify the number of
# steps.
reader = tf.train.NewCheckpointReader(checkpoint)
global_step = reader.get_tensor(tf.GraphKeys.GLOBAL_STEP)
all_results = []
for result in estimator.predict(predict_input_fn,
yield_single_examples=True,
checkpoint_path=checkpoint):
if len(all_results) % 1000 == 0:
tf.logging.info("Processing example: %d" %
(len(all_results)))
unique_id = int(result["unique_ids"])
start_top_log_probs = ([
float(x) for x in result["start_top_log_probs"].flat
])
start_top_index = [
int(x) for x in result["start_top_index"].flat
]
end_top_log_probs = ([
float(x) for x in result["end_top_log_probs"].flat
])
end_top_index = [int(x) for x in result["end_top_index"].flat]
cls_logits = float(result["cls_logits"].flat[0])
all_results.append(
squad_utils.RawResultV2(
unique_id=unique_id,
start_top_log_probs=start_top_log_probs,
start_top_index=start_top_index,
end_top_log_probs=end_top_log_probs,
end_top_index=end_top_index,
cls_logits=cls_logits))
output_prediction_file = os.path.join(FLAGS.output_dir,
"predictions.json")
output_nbest_file = os.path.join(FLAGS.output_dir,
"nbest_predictions.json")
output_null_log_odds_file = os.path.join(FLAGS.output_dir,
"null_odds.json")
result_dict = {}
cls_dict = {}
squad_utils.accumulate_predictions_v2(
result_dict, cls_dict, eval_examples, eval_features,
all_results, FLAGS.n_best_size, FLAGS.max_answer_length,
FLAGS.start_n_top, FLAGS.end_n_top)
return squad_utils.evaluate_v2(
result_dict, cls_dict, prediction_json, eval_examples,
eval_features, all_results, FLAGS.n_best_size,
FLAGS.max_answer_length, output_prediction_file,
output_nbest_file, output_null_log_odds_file), int(global_step)
def process_output(all_results, eval_examples, eval_features, input_file,
n_best, n_best_size, max_answer_length):
with tf.gfile.Open(input_file, "r") as reader:
input_data = json.load(reader)["data"]
result_dict = {}
cls_dict = {}
#From Flags
start_n_top = 1
end_n_top = 1
output_dir = '/Users/benediktgroever/albert/albert'
output_prediction_file = os.path.join(output_dir, "predictions.json")
output_nbest_file = os.path.join(output_dir, "nbest_predictions.json")
output_null_log_odds_file = os.path.join(output_dir, "null_odds.json")
squad_utils.accumulate_predictions_v2(result_dict, cls_dict, eval_examples,
eval_features, all_results,
n_best_size, max_answer_length,
start_n_top, end_n_top)
all_predictions, all_nbest_json = squad_utils.write_predictions_v2(
result_dict,
cls_dict,
eval_examples, # all_examples
eval_features, # all_features
all_results, # all_results
n_best_size, # n_best_size
max_answer_length, #max_answer_length
output_prediction_file, # output_prediction_file,
output_nbest_file, # output_nbest_file
output_null_log_odds_file, # output_null_log_odds_file
None) # null_score_diff_threshold
re = []
for i in range(len(all_predictions)):
id_ = input_data[0]['paragraphs'][0]['qas'][i]['id']
if n_best:
re.append(
collections.OrderedDict({
"id":
id_,
"question":
input_data[0]['paragraphs'][0]['qas'][i]["question"],
"best_prediction":
all_predictions[id_],
"n_best_predictions":
all_nbest_json[id_]
}))
else:
re.append(
collections.OrderedDict({
"id":
id_,
"question":
input_data[0]['paragraphs'][0]['qas'][i]["question"],
"best_prediction":
all_predictions[id_]
}))
return re
