def process_result(result):
unique_id = int(result["unique_ids"].int64_val[0])
start_top_log_probs = ([
float(x) for x in result["start_top_log_probs"].float_val
])
start_top_index = [int(x) for x in result["start_top_index"].int_val]
end_top_log_probs = ([
float(x) for x in result["end_top_log_probs"].float_val
])
end_top_index = [int(x) for x in result["end_top_index"].int_val]
cls_logits = float(result["cls_logits"].float_val[0])
RawResultV2 = collections.namedtuple("RawResultV2", [
"unique_id", "start_top_log_probs", "start_top_index",
"end_top_log_probs", "end_top_index", "cls_logits"
])
formatted_result = squad_utils.RawResultV2(
unique_id=unique_id,
start_top_log_probs=start_top_log_probs,
start_top_index=start_top_index,
end_top_log_probs=end_top_log_probs,
end_top_index=end_top_index,
cls_logits=cls_logits)
return formatted_result
def get_result(checkpoint):
"""Evaluate the checkpoint on SQuAD v2.0."""
# If running eval on the TPU, you will need to specify the number of
# steps.
reader = tf.train.NewCheckpointReader(checkpoint)
global_step = reader.get_tensor(tf.GraphKeys.GLOBAL_STEP)
all_results = []
for result in estimator.predict(predict_input_fn,
yield_single_examples=True,
checkpoint_path=checkpoint):
if len(all_results) % 1000 == 0:
tf.logging.info("Processing example: %d" %
(len(all_results)))
unique_id = int(result["unique_ids"])
start_top_log_probs = ([
float(x) for x in result["start_top_log_probs"].flat
])
start_top_index = [
int(x) for x in result["start_top_index"].flat
]
end_top_log_probs = ([
float(x) for x in result["end_top_log_probs"].flat
])
end_top_index = [int(x) for x in result["end_top_index"].flat]
cls_logits = float(result["cls_logits"].flat[0])
all_results.append(
squad_utils.RawResultV2(
unique_id=unique_id,
start_top_log_probs=start_top_log_probs,
start_top_index=start_top_index,
end_top_log_probs=end_top_log_probs,
end_top_index=end_top_index,
cls_logits=cls_logits))
output_prediction_file = os.path.join(FLAGS.output_dir,
"predictions.json")
output_nbest_file = os.path.join(FLAGS.output_dir,
"nbest_predictions.json")
output_null_log_odds_file = os.path.join(FLAGS.output_dir,
"null_odds.json")
result_dict = {}
cls_dict = {}
squad_utils.accumulate_predictions_v2(
result_dict, cls_dict, eval_examples, eval_features,
all_results, FLAGS.n_best_size, FLAGS.max_answer_length,
FLAGS.start_n_top, FLAGS.end_n_top)
return squad_utils.evaluate_v2(
result_dict, cls_dict, prediction_json, eval_examples,
eval_features, all_results, FLAGS.n_best_size,
FLAGS.max_answer_length, output_prediction_file,
output_nbest_file, output_null_log_odds_file), int(global_step)
def get_result(checkpoint):
"""Evaluate the checkpoint on SQuAD v2.0."""
# If running eval on the TPU, you will need to specify the number of
# steps.
all_results = []
for result in estimator.predict(predict_input_fn,
yield_single_examples=True,
checkpoint_path=checkpoint):
if len(all_results) % 1000 == 0:
tf.logging.info("Processing example: %d" %
(len(all_results)))
unique_id = int(result["unique_ids"])
crf_logits = result["crf_logits"]
transition_params = result["transition_params"]
softmax = result["softmax"]
all_results.append(
squad_utils.RawResultV2(
unique_id=unique_id,
crf_logits=crf_logits,
softmax=softmax,
transition_params=transition_params,
))
output_prediction_file = os.path.join(FLAGS.output_dir,
"predictions.json")
predictions = squad_utils.write_predictions_et(
eval_examples, eval_features, all_results,
FLAGS.max_answer_length, tag_info)
import numpy
class MyEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, numpy.integer):
return int(o)
if isinstance(o, numpy.floating):
return float(o)
if isinstance(o, numpy.ndarray):
return o.tolist()
else:
return super(MyEncoder, self).default(o)
with tf.gfile.Open(output_prediction_file, 'w') as f:
json.dump(predictions, f, ensure_ascii=False, cls=MyEncoder)