def main(_): cache_path = os.path.join(os.path.dirname(FLAGS.gold_path), 'cache') if FLAGS.cache_gold_data and os.path.exists(cache_path): logging.info('Reading from cache: %s', format(cache_path)) tydi_gold_dict = pickle.load(open(cache_path, 'r')) # pytype: disable=wrong-arg-types else: tydi_gold_dict = eval_utils.read_annotation(FLAGS.gold_path) if FLAGS.cache_gold_data: logging.info('Caching gold data for future to: %s', format(cache_path)) pickle.dump(tydi_gold_dict, open(cache_path, 'w')) # pytype: disable=wrong-arg-types total_ans_count = 0 count = 0 for ans in tydi_gold_dict.values(): count += 1 gold_has_answer = eval_utils.gold_has_minimal_answer( ans, FLAGS.minimal_non_null_threshold) total_ans_count += gold_has_answer logging.info('%d examples have minimal answers', total_ans_count) logging.info('*' * 40) tydi_pred_dict = eval_utils.read_prediction_jsonl(FLAGS.predictions_path) per_lang_gold = {} per_lang_pred = {} for ex_id, ex in tydi_gold_dict.items(): if ex[0].language in per_lang_gold: per_lang_gold[ex[0].language][ex_id] = ex else: per_lang_gold[ex[0].language] = {ex_id: ex} for ex_id, ex in tydi_pred_dict.items(): if ex.language in per_lang_pred: per_lang_pred[ex.language][ex_id] = ex else: per_lang_pred[ex.language] = {ex_id: ex} macro_avg_passage_scores = ([], [], []) macro_avg_minimal_scores = ([], [], []) language_list = [ 'english', 'arabic', 'bengali', 'finnish', 'indonesian', 'japanese', 'swahili', 'korean', 'russian', 'telugu', 'thai' ] for lang in language_list: if lang in per_lang_pred: passage_answer_stats, minimal_answer_stats = score_answers( per_lang_gold.get(lang, {}), per_lang_pred[lang]) # Passage selection task opt_result, _ = compute_pr_curves(passage_answer_stats, targets=[0.5]) f1, precision, recall, _ = opt_result if lang != 'english': macro_avg_passage_scores[0].append(f1) macro_avg_passage_scores[1].append(precision) macro_avg_passage_scores[2].append(recall) print('Passage & ' + lang + ' & ' + get_latex_str(f1, precision, recall)) # Minimal answer span task opt_result, _ = compute_pr_curves(minimal_answer_stats, targets=[0.5]) f1, precision, recall, _ = opt_result if lang != 'english': macro_avg_minimal_scores[0].append(f1) macro_avg_minimal_scores[1].append(precision) macro_avg_minimal_scores[2].append(recall) print('Minimal Answer & ' + lang + ' & ' + get_latex_str(f1, precision, recall)) if FLAGS.pretty_print: print('*' * 20) print(lang) print('Language: %s (%d)' % (lang, len(per_lang_gold.get(lang, {})))) print('*' * 20) print('PASSAGE ANSWER R@P TABLE:') print_r_at_p_table(passage_answer_stats) print('*' * 20) print('MINIMAL ANSWER R@P TABLE:') print_r_at_p_table(minimal_answer_stats) else: metrics = get_metrics_with_answer_stats( passage_answer_stats, minimal_answer_stats) print(json.dumps(metrics)) print( 'Total # examples in gold: %d, # ex. in pred: %d (including english)' % (len(tydi_gold_dict), len(tydi_pred_dict))) f1_list, precision_list, recall_list = macro_avg_passage_scores print('*** Macro Over %d Languages, excluding English **' % len(f1_list)) avg_passage_f1 = eval_utils.safe_average(f1_list) avg_passage_recall = eval_utils.safe_average(recall_list) avg_passage_precision = eval_utils.safe_average(precision_list) print('Passage F1:%.3f P:%.3f R:%3f' % (avg_passage_f1, avg_passage_precision, avg_passage_recall)) print( get_latex_str(avg_passage_f1, avg_passage_precision, avg_passage_recall)) f1_list, precision_list, recall_list = macro_avg_minimal_scores avg_minimal_f1 = eval_utils.safe_average(f1_list) avg_minimal_recall = eval_utils.safe_average(recall_list) avg_minimal_precision = eval_utils.safe_average(precision_list) print('Minimal F1:%.3f P:%.3f R:%3f' % (avg_minimal_f1, avg_minimal_precision, avg_minimal_recall)) print( get_latex_str(avg_minimal_f1, avg_minimal_precision, avg_minimal_recall)) print('*** / Aggregate Scores ****') aggregate_metrics = { 'avg_passage_f1': avg_passage_f1, 'avg_passage_recall': avg_passage_recall, 'avg_passage_precision': avg_passage_precision, 'avg_minimal_f1': avg_minimal_f1, 'avg_minimal_recall': avg_minimal_recall, 'avg_minimal_precision': avg_minimal_precision } print(json.dumps(aggregate_metrics))
def score_minimal_answer(gold_label_list, pred_label, minimal_non_null_threshold): """Scores a minimal answer. Outputs score against gold label that gives max F1. First decide if there is a gold minimal answer with FLAGS.minimal_non_null_threshold. If any of the gold label has "yes", or "no", and pred label predicted it correctly, than precision, recall, f1 is all 1.0. Args: gold_label_list: A list of TyDiLabel. pred_label: A single TyDiLabel. minimal_non_null_threshold: See FLAGS.minimal_non_null_threshold. Returns: gold_has_answer, pred_has_answer, (precision, recall, f1), score """ # There is a gold minimal answer if gold_label_list not empty and non null # answers is over the threshold (sum over annotators). gold_has_answer = eval_utils.gold_has_minimal_answer( gold_label_list, minimal_non_null_threshold) if pred_label is None: return gold_has_answer, not gold_has_answer, (0, 0, 0), 0 # There is a predicted minimal answer if the predicted minimal label span # is non-null or we have a specific predicted label (such as yes/no). pred_has_answer = ((not pred_label.minimal_answer_span.is_null_span()) or pred_label.yes_no_answer != 'none') # score is optional. score = pred_label.minimal_score # We find the closest (highest scoring) match between the system's predicted # minimal answer and one of the three gold annotations. max_f1 = 0.0 max_precision = 0.0 max_recall = 0.0 # Both sides have minimal answers, which contains yes/no questions. if gold_has_answer and pred_has_answer: if pred_label.yes_no_answer != 'none': # System predicted a yes/no answer. for gold_label in gold_label_list: if pred_label.yes_no_answer == gold_label.yes_no_answer: max_f1 = 1.0 max_precision = 1.0 max_recall = 1.0 break else: for gold_label in gold_label_list: if gold_label.minimal_answer_span.is_null_span(): continue # Compute the *micro-F1* (a partial match score for this example). # We also compute a language-wise *macro-F1* later. precision, recall, f1 = eval_utils.compute_partial_match_scores( gold_label.minimal_answer_span, pred_label.minimal_answer_span) if f1 > max_f1: max_f1 = f1 max_precision = precision max_recall = recall return (gold_has_answer, pred_has_answer, (max_precision, max_recall, max_f1), score)