def __init__(self, *, validation_batched, fever_dev_path, max_evidence, max_recall_n = 15, checkpoint_dir, debug = False): """Compute the fever metrics from the batched dataset. Args: validation_batched: Batched dataset to compute metrics from. fever_dev_path: Path to fever dev data max_evidence: Max evidence to use max_recall_n: Stop computing recall after this position checkpoint_dir: If not none, then write validation predictions to disk here every epoch debug: Whether to enable debug handling of metrics since (some fail/error without full data) """ super().__init__() self._validation_batched = validation_batched self._fever_dev_path = fever_dev_path self._max_recall_n = max_recall_n self._max_evidence = max_evidence self._checkpoint_dir = checkpoint_dir self._debug = debug self._validation_flat = list(self._validation_batched.unbatch()) self._dev = util.read_jsonlines(fever_dev_path) self._verifiable_dev = [ claim for claim in self._dev if claim['label'] != constants.NOT_ENOUGH_INFO] self._verifiable_dev_lookup = { claim['id']: claim for claim in self._verifiable_dev}
def setup(self): self._claim_to_fold = {} train = util.read_jsonlines(self._fever_train_path) for claim in train: self._claim_to_fold[claim['id']] = 'train' dev = util.read_jsonlines(self._fever_dev_path) for claim in dev: self._claim_to_fold[claim['id']] = 'dev' test = util.read_jsonlines(self._fever_test_path) for claim in test: self._claim_to_fold[claim['id']] = 'test' self._wiki_db = wiki_db.WikiDatabase.from_local(self._wiki_db_path) self._wiki_titles = set(self._wiki_db.get_wikipedia_urls()) self._matcher = text_matcher.TextMatcher() self._matcher.load(self._text_matcher_params_path) drqa_scrape_table = scrape_db.ScrapeDatabase.from_local(self._drqa_db_path) # pylint: disable=unused-variable self._drqa_scrape_table = drqa_scrape_table lucene_scrape_table = scrape_db.ScrapeDatabase.from_local( self._drqa_db_path) # pylint: disable=unused-variable self._lucene_scrape_table = lucene_scrape_table self._name_to_scrape = { constants.DRQA: self._drqa_scrape_table, constants.LUCENE: self._lucene_scrape_table } ukp_claim_docs = ( util.read_jsonlines(self._ukp_docs_train_path) + util.read_jsonlines(self._ukp_docs_dev_path) + util.read_jsonlines(self._ukp_docs_test_path)) self._ukp_docs = {claim['id']: claim for claim in ukp_claim_docs}
def main(_): tf.enable_v2_behavior() flags.mark_flag_as_required('model_root') flags.mark_flag_as_required('report_dir') root = pathlib.Path(FLAGS.model_root) report_dir = pathlib.Path(FLAGS.report_dir) logging.info('Reading predictions from model_root: %s', root) logging.info('Will write analysis to: %s', report_dir) # Config() contains non-model specific configuration, which is why its # fine to use this instead of the model's configuration. conf = config.Config() dev = {c['id']: c for c in util.read_jsonlines(conf.fever_dev)} logging.info('Reading fever TFDS examples') builder = fever_tfds.FeverEvidence( data_dir=util.readahead(conf.fever_evidence_tfds_data), n_similar_negatives=FLAGS.n_similar_negatives, n_background_negatives=FLAGS.n_background_negatives, train_scrape_type=FLAGS.train_scrape_type, include_not_enough_info=True, title_in_scoring=True) val = builder.as_dataset(split='validation') val_tfds_examples = [x for x in tqdm.tqdm(val, mininterval=10)] logging.info('Reading model predictions') model_predictions = read_model_predictions(root / 'val_predictions.json') val_df = parse_fold(fold_name='val', model_predictions=model_predictions, tfds_examples=val_tfds_examples) df = pd.concat([val_df]) logging.info('Writing analysis to disk') write_summary(report_dir, df) write_per_claim_analysis(output_path=report_dir / 'claim_evidence_predictions.pickle', df=df, claim_lookup=dev)
def _generate_examples(self, filepath, **kwargs): fever_claims = util.read_jsonlines(filepath) for claim in fever_claims: claim_id = claim["id"] claim_text = claim["claim"] claim_label = claim["label"] example_id = f"{claim_id}" yield claim_id, { "example_id": example_id, "claim_text": claim_text, "evidence_text": "", "wikipedia_url": "", # Ordinarily, this would (possibly) be concatenated to the evidence # but since this is claim only, I'm using a null integer value "sentence_id": "-1", # This label doesn't matter here since its claim only "evidence_label": constants.NOT_MATCHING, "claim_label": claim_label, "scrape_type": "", "metadata": json.dumps({ "claim_id": claim_id, }) }
def _generate_examples(self, boolq_filepath, fold): boolq_claims = util.read_jsonlines(boolq_filepath) for idx, claim in enumerate(boolq_claims): example_id = f'{fold}-{idx}' example = { 'example_id': example_id, 'claim_text': claim['question'], 'evidence_text': claim['passage'], 'wikipedia_url': claim['title'], 'sentence_id': '0', # This is effectively gold evidence 'evidence_label': constants.MATCHING, 'claim_label': constants.SUPPORTS if claim['answer'] else constants.REFUTES, 'metadata': json.dumps({}) } yield example_id, example
def main(_): flags.mark_flag_as_required('out_path') flags.mark_flag_as_required('wiki_embedding_dir') flags.mark_flag_as_required('claim_id_path') flags.mark_flag_as_required('claim_embedding_path') flags.mark_flag_as_required('n_shards') tf.enable_v2_behavior() conf = config.Config() logging.info('wiki_embedding_dir: %s', FLAGS.wiki_embedding_dir) logging.info('n_shards: %s', FLAGS.n_shards) logging.info('l2_norm: %s', FLAGS.l2_norm) logging.info('claim_id_path: %s', FLAGS.claim_id_path) logging.info('claim_embedding_path: %s', FLAGS.claim_embedding_path) logging.info('copy_to_tmp: %s', FLAGS.copy_to_tmp) logging.info('batch_size: %s', FLAGS.batch_size) with util.log_time('Building index'): index = Index( wiki_embedding_dir=FLAGS.wiki_embedding_dir, n_shards=FLAGS.n_shards, l2_norm=FLAGS.l2_norm, claim_id_path=FLAGS.claim_id_path, claim_embedding_path=FLAGS.claim_embedding_path, copy_to_tmp=FLAGS.copy_to_tmp, batch_size=FLAGS.batch_size, device=FLAGS.device, ) index.build() logging.info('Reading claims from: %s', conf.fever_dev) dev = [ c for c in util.read_jsonlines(conf.fever_dev) if c['label'] != constants.NOT_ENOUGH_INFO ] logging.info('Making predictions') claim_id_to_scored_keys = index.score_claim_to_wiki(n=5) formatted_predictions = [] actual = [] for claim in tqdm.tqdm(dev): claim_id = claim['id'] predicted_evidence = [] scored_keys = claim_id_to_scored_keys[claim_id] for index_key in scored_keys['wiki_keys']: # sentence_id is a numpy int, and fever scoring script only # accepts python int. predicted_evidence.append( [index_key.wikipedia_url, int(index_key.sentence_id)]) formatted_predictions.append({ 'id': claim_id, 'predicted_label': constants.SUPPORTS, 'predicted_evidence': predicted_evidence, }) actual.append({'evidence': claim['evidence'], 'label': claim['label']}) logging.info('FEVER Metrics') strict_score, accuracy_score, precision, recall, f1 = fever_score( formatted_predictions, actual) logging.info('Strict Score: %s', strict_score) logging.info('Accuracy Score: %s', accuracy_score) logging.info('Precision: %s', precision) logging.info('Recall: %s', recall) logging.info('F1: %s', f1) logging.info('Saving predictions and metrics to: %s', FLAGS.out_path) util.write_json( { 'predictions': formatted_predictions, 'metrics': { 'strict_score': strict_score, 'accuracy_score': accuracy_score, 'precision': precision, 'recall': recall, 'f1': f1, } }, FLAGS.out_path)