def _convert_single_wtq(interaction_file, prediction_file, output_file): """Convert predictions to WikiTablequestions format.""" interactions = dict( (prediction_utils.parse_interaction_id(i.id), i) for i in prediction_utils.iterate_interactions(interaction_file)) missing_interaction_ids = set(interactions.keys()) with tf.io.gfile.GFile(output_file, 'w') as output_file: for prediction in prediction_utils.iterate_predictions( prediction_file): interaction_id = prediction['id'] if interaction_id in missing_interaction_ids: missing_interaction_ids.remove(interaction_id) else: continue coordinates = prediction_utils.parse_coordinates( prediction['answer_coordinates']) denot_pred, _ = calc_metrics_utils.execute( int(prediction.get('pred_aggr', '0')), coordinates, prediction_utils.table_to_panda_frame( interactions[interaction_id].table)) answers = '\t'.join(sorted(map(str, denot_pred))) output_file.write('{}\t{}\n'.format(interaction_id, answers)) for interaction_id in missing_interaction_ids: output_file.write('{}\n'.format(interaction_id))
def read_predictions(predictions_path, examples): """Reads predictions from a csv file.""" for row in prediction_utils.iterate_predictions(predictions_path): pred_id = '{}-{}_{}'.format(row['id'], row['annotator'], row['position']) example = examples[pred_id] example.pred_cell_coo = prediction_utils.parse_coordinates( row['answer_coordinates']) example.pred_agg_function = int(row.get('pred_aggr', '0'))
def generate_hybridqa_codalab_predictions(interaction_file, prediction_file): """Generates Codaab prediction files for HybridQA Competition. This function generates the json prediction files used to submit to HybridQA competition hosted on Codalab. (go/hybridqa-competition) Args: interaction_file: A TF record file containing the examples as interactions. prediction_file: A TSV file that is the output of the table-classifier predict job on the input interactions. Yields: An iterable of json serializable python dicts. """ vocab_file = _guess_vocab_file(interaction_file) logging.info("Vocab file: %s ", vocab_file) logging.info("Read: %s ", interaction_file) interactions = prediction_utils.iterate_interactions(interaction_file) logging.info("Read: %s ", prediction_file) predictions = prediction_utils.iterate_predictions(prediction_file) detokenizer = DeTokenizer(vocab_file) interactions_by_qid = collections.defaultdict(list) for interaction in interactions: qid = interaction.questions[0].id interactions_by_qid[_get_example_id(qid)].append(interaction) predictions_by_qid = {} for prediction in predictions: qid = prediction["question_id"] # TODO(eisenjulian): Select the best answer using model scores. predictions_by_qid[qid] = prediction for qid, candidates in interactions_by_qid.items(): answer_text = "" results = list( _get_scored_candidates( detokenizer, candidates, predictions_by_qid, )) example_id = text_utils.get_example_id(qid) if results: best_result = max(results, key=lambda result: result.score) answer_text = best_result.answer yield {"question_id": example_id, "pred": answer_text}
def evaluate_retrieval_e2e( interaction_file, prediction_file, references_file = None, vocab_file = None, ): """Computes e2e retrieval-QA metrics.""" vocab_file = vocab_file or _guess_vocab_file(interaction_file) references = None logging.info("Vocab file: %s ", vocab_file) logging.info("Read: %s ", interaction_file) interactions = prediction_utils.iterate_interactions(interaction_file) logging.info("Read: %s ", prediction_file) predictions = prediction_utils.iterate_predictions(prediction_file) return _evaluate_retrieval_e2e(vocab_file, interactions, predictions, references)
def get_predictions(prediction_file): """Yields an iterable of Prediction objects from a tsv prediction file.""" fn_map = { 'logits_cls': float, 'position': int, 'answer_coordinates': lambda x: list(prediction_utils.parse_coordinates(x)), 'answers': token_answers_from_text, 'token_probabilities': json.loads, } for prediction_dict in prediction_utils.iterate_predictions( prediction_file): for key in tuple(prediction_dict.keys()): fn = fn_map.get(key, lambda x: x) prediction_dict[key] = fn(prediction_dict[key]) yield Prediction(**prediction_dict)
def test_iterate_predictions(self): filepath = tempfile.mktemp(suffix='.tsv') predictions = [ { 'logits_cls': 0.1 }, { 'logits_cls': [3.0, 4.0] }, ] with tf.io.gfile.GFile(filepath, mode='w') as writer: writer.write('logits_cls\n') writer.write('0.1\n') writer.write('[3 4]\n') actual_predictions = list( prediction_utils.iterate_predictions(filepath)) self.assertEqual(predictions, actual_predictions)
def read_predictions(predictions_path, examples): """Reads predictions from a csv file.""" for row in prediction_utils.iterate_predictions(predictions_path): pred_id = '{}-{}_{}'.format(row['id'], row['annotator'], row['position']) example = examples[pred_id] example.pred_cell_coo = prediction_utils.parse_coordinates( row['answer_coordinates']) example.pred_agg_function = int(row.get('pred_aggr', '0')) if 'column_scores' in row: column_scores = list( filter(None, row['column_scores'][1:-1].split(' '))) removed_column_scores = [ float(score) for score in column_scores if float(score) < 0.0 ] if column_scores: example.weight = len(removed_column_scores) / len( column_scores)
def create_outputs( input_dir, prediction_files, output_dir, ): """Iterates over XML files in input dir and adds types to statements.""" prediction_dict = collections.defaultdict(list) for prediction_file in prediction_files: for row in prediction_utils.iterate_predictions(prediction_file): prediction_dict[row["question_id"]].append(row["logits_cls"]) for filename in tf.io.gfile.listdir(input_dir): if not filename.endswith("xml"): continue filepath = os.path.join(input_dir, filename) doc_id = _get_doc_id(filepath) with tf.io.gfile.GFile(filepath) as input_file: soup = bs4.BeautifulSoup(input_file.read(), "lxml") for table in soup.find_all("table"): table_id = _get_table_id(doc_id, table["id"]) for statement in table.find_all("statement"): interaction_id = _get_interaction_id( table_id, int(statement["id"]), ) question_id = _get_question_id(interaction_id) prediction_list = prediction_dict[question_id] index = _get_majority_label(prediction_list) statement["type"] = _index_to_type(index) tf.io.gfile.makedirs(output_dir) output_path = os.path.join(output_dir, filename) with tf.io.gfile.GFile(output_path, "w") as output_file: output_file.write(str(soup))
def eval_cell_selection( questions, predictions_file, ): """Evaluates cell selection results in HybridQA experiment. Args: questions: A map of Question protos by their respective ids. predictions_file: Path to a tsv file with predictions for a checkpoint. Yields: An AnswerType and its corresponding CellSelectionMetrics instance """ total = collections.Counter() total_correct = collections.Counter() total_correct_at_k = {k: collections.Counter() for k in _RECALL_KS} total_seen = collections.Counter() total_non_empty = collections.Counter() total_coordinates = collections.Counter() sum_precision = collections.defaultdict(float) for question in questions.values(): for answer_type in [AnswerType.ALL, _get_answer_type(question)]: total[answer_type] += 1 for row in prediction_utils.iterate_predictions(predictions_file): question = questions.get(row['question_id']) if question is None: # The dataset lost some examples after an update. continue gold_coordinates = {(x.row_index, x.column_index) for x in question.answer.answer_coordinates} coordinates = prediction_utils.parse_coordinates( row['answer_coordinates']) # We only care about finding one correct cell for the downstream model. correct_coordinates = len(coordinates & gold_coordinates) has_probabilities = 'token_probabilities' in row if has_probabilities: best_cells = get_best_cells(json.loads(row['token_probabilities'])) correct_at_k = { k: bool(set(best_cells[:k]) & gold_coordinates) for k in _RECALL_KS } else: correct_at_k = {} for answer_type in [AnswerType.ALL, _get_answer_type(question)]: total_coordinates[answer_type] += len(coordinates) total_correct[answer_type] += bool(correct_coordinates) total_seen[answer_type] += 1 for k, correct in correct_at_k.items(): total_correct_at_k[k][answer_type] += correct if coordinates: sum_precision[answer_type] += correct_coordinates / len( coordinates) total_non_empty[answer_type] += 1 for answer_type in AnswerType: if total[answer_type]: recall_at_k = { f'recall_at_{k}': (total_correct_at_k[k][answer_type] / total[answer_type]) if has_probabilities else None for k in _RECALL_KS } yield answer_type, CellSelectionMetrics( recall=total_correct[answer_type] / total[answer_type], non_empty=total_non_empty[answer_type] / total[answer_type], coverage=total_seen[answer_type] / total[answer_type], answer_len=total_coordinates[answer_type] / total[answer_type], precision=((sum_precision[answer_type] / total_non_empty[answer_type]) if total_non_empty[answer_type] else None), **recall_at_k, )