def process_result(result): unique_id = int(result["unique_ids"].int64_val[0]) start_top_log_probs = ([ float(x) for x in result["start_top_log_probs"].float_val ]) start_top_index = [int(x) for x in result["start_top_index"].int_val] end_top_log_probs = ([ float(x) for x in result["end_top_log_probs"].float_val ]) end_top_index = [int(x) for x in result["end_top_index"].int_val] cls_logits = float(result["cls_logits"].float_val[0]) RawResultV2 = collections.namedtuple("RawResultV2", [ "unique_id", "start_top_log_probs", "start_top_index", "end_top_log_probs", "end_top_index", "cls_logits" ]) formatted_result = squad_utils.RawResultV2( unique_id=unique_id, start_top_log_probs=start_top_log_probs, start_top_index=start_top_index, end_top_log_probs=end_top_log_probs, end_top_index=end_top_index, cls_logits=cls_logits) return formatted_result
def get_result(checkpoint): """Evaluate the checkpoint on SQuAD v2.0.""" # If running eval on the TPU, you will need to specify the number of # steps. reader = tf.train.NewCheckpointReader(checkpoint) global_step = reader.get_tensor(tf.GraphKeys.GLOBAL_STEP) all_results = [] for result in estimator.predict(predict_input_fn, yield_single_examples=True, checkpoint_path=checkpoint): if len(all_results) % 1000 == 0: tf.logging.info("Processing example: %d" % (len(all_results))) unique_id = int(result["unique_ids"]) start_top_log_probs = ([ float(x) for x in result["start_top_log_probs"].flat ]) start_top_index = [ int(x) for x in result["start_top_index"].flat ] end_top_log_probs = ([ float(x) for x in result["end_top_log_probs"].flat ]) end_top_index = [int(x) for x in result["end_top_index"].flat] cls_logits = float(result["cls_logits"].flat[0]) all_results.append( squad_utils.RawResultV2( unique_id=unique_id, start_top_log_probs=start_top_log_probs, start_top_index=start_top_index, end_top_log_probs=end_top_log_probs, end_top_index=end_top_index, cls_logits=cls_logits)) output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") output_nbest_file = os.path.join(FLAGS.output_dir, "nbest_predictions.json") output_null_log_odds_file = os.path.join(FLAGS.output_dir, "null_odds.json") result_dict = {} cls_dict = {} squad_utils.accumulate_predictions_v2( result_dict, cls_dict, eval_examples, eval_features, all_results, FLAGS.n_best_size, FLAGS.max_answer_length, FLAGS.start_n_top, FLAGS.end_n_top) return squad_utils.evaluate_v2( result_dict, cls_dict, prediction_json, eval_examples, eval_features, all_results, FLAGS.n_best_size, FLAGS.max_answer_length, output_prediction_file, output_nbest_file, output_null_log_odds_file), int(global_step)
def get_result(checkpoint): """Evaluate the checkpoint on SQuAD v2.0.""" # If running eval on the TPU, you will need to specify the number of # steps. all_results = [] for result in estimator.predict(predict_input_fn, yield_single_examples=True, checkpoint_path=checkpoint): if len(all_results) % 1000 == 0: tf.logging.info("Processing example: %d" % (len(all_results))) unique_id = int(result["unique_ids"]) crf_logits = result["crf_logits"] transition_params = result["transition_params"] softmax = result["softmax"] all_results.append( squad_utils.RawResultV2( unique_id=unique_id, crf_logits=crf_logits, softmax=softmax, transition_params=transition_params, )) output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") predictions = squad_utils.write_predictions_et( eval_examples, eval_features, all_results, FLAGS.max_answer_length, tag_info) import numpy class MyEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, numpy.integer): return int(o) if isinstance(o, numpy.floating): return float(o) if isinstance(o, numpy.ndarray): return o.tolist() else: return super(MyEncoder, self).default(o) with tf.gfile.Open(output_prediction_file, 'w') as f: json.dump(predictions, f, ensure_ascii=False, cls=MyEncoder)