def generate_embeddings(ret_trainee, input_file, out_file): with open(input_file, "r", encoding="utf-8") as f: json_data = json.load(f) source2passages, pid2passage, _ = get_passages_by_source(json_data) question_embs = [] labels = [] for example in tqdm(json_data["examples"]): pid = get_passage_id(example) passage = pid2passage[pid] labels.append('id' if is_in_distribution(passage) else 'ood') emb = ret_trainee.retriever.embed_question(get_question(example)) question_embs.append(emb) passage_header_embs = [] ood = 0 for source, passages in source2passages.items(): logger.info('embedding passages for source {}'.format(source)) for passage in tqdm(passages): if is_in_distribution(passage): emb = ret_trainee.retriever.embed_paragraph( get_passage_last_header(passage, return_error_for_ood=True)) passage_header_embs.append(emb) else: ood += 1 to_serialize = {"question_embs": question_embs, "passage_header_embs": passage_header_embs, "question_labels": labels} with open(out_file, "wb") as out_stream: pickle.dump(to_serialize, out_stream) logger.info( 'saved {} question embeddings and {} passage header embeddings ({} skipped because ' 'out-of-distribution)'.format( len(question_embs), len(passage_header_embs), ood))
def filter_user_questions(input_to_filter, faq_contents): ''' This function takes the data from the input_to_filter json file, and only returns the examples that align with the faq_contents. input_to_filer: str, filename of the json to filter faq_contents: set, all faq questions in a set. ''' with open(input_to_filter, 'r', encoding='utf-8') as in_stream: input_data = json.load(in_stream) _, pid2passage, _ = get_passages_by_source(input_data, keep_ood=True) filtered_example = [] examples = get_examples(input_data, keep_ood=True) for example in examples: related_pid = get_passage_id(example) related_passage = pid2passage[related_pid] if get_passage_last_header(related_passage) in faq_contents: filtered_example.append(example) logger.info( 'file {}: passage size {} / pre-filtering example size {} / post filtering examples size' ' {}'.format(input_to_filter, len(input_data['passages']), len(examples), len(filtered_example))) return {'examples': filtered_example, 'passages': input_data['passages']}
def generate_predictions(self, json_file, predict_to, multiple_thresholds, write_fix_report): with open(json_file, "r", encoding="utf-8") as f: json_data = json.load(f) source2passages, _, passage_id2index = get_passages_by_source( json_data) source2passages, _, _ = _encode_passages(source2passages, self.max_question_len, self.tokenizer, do_not_encode=True) res = self.compute_results(json_data, passage_id2index, source2passages) predictions, questions, sources, normalized_scores, indices_of_correct_passage = res generate_and_log_results(indices_of_correct_passage, normalized_scores, predict_to, predictions, questions, source2passages, sources, multiple_thresholds=multiple_thresholds, write_fix_report=write_fix_report, json_data=json_data)
def evaluate(model_to_evaluate, test_data): logger.info("data is composed by {} examples and {} passages".format( len(test_data["examples"]), len(test_data["passages"]))) a_start = time.time() source2passages, passage_id2source, passage_id2index = get_passages_by_source( test_data) no_ood_source2passages, _, _ = get_passages_by_source(test_data, keep_ood=False) model_to_evaluate.collect_answers(no_ood_source2passages) a_end = time.time() answer_time = a_end - a_start logger.info( "preparing {} answers: total time {:.2f} sec./ per-answer time {:.2f} sec." .format( len(test_data["passages"]), answer_time, answer_time / len(test_data["passages"]), )) correct = 0 total = 0 q_start = time.time() for example in tqdm(test_data["examples"]): question = example["question"] source = example["source"] prediction = model_to_evaluate.answer_question(question, source) index_of_correct_passage = passage_id2index[example["passage_id"]] if index_of_correct_passage == prediction: correct += 1 total += 1 q_end = time.time() question_time = q_end - q_start logger.info( "producing the answers for {} questions: total time {:.2f} sec. / " "per-question time {:.2f} sec.".format( len(test_data["examples"]), question_time, question_time / len(test_data["examples"]), )) logger.info("correct {} over {} / accuracy: {:.2f}%".format( correct, total, (correct / total) * 100))
def main(): parser = argparse.ArgumentParser(__doc__) parser.add_argument("--input", help="join files to filter", required=True) parser.add_argument("--output", help="output file", required=True) parser.add_argument("--keep-id", help="will keep id", action="store_true") parser.add_argument("--keep-ood", help="will keep ood", action="store_true") args = parser.parse_args() logging.basicConfig(level=logging.INFO) with open(args.input, 'r', encoding='utf-8') as in_stream: input_data = json.load(in_stream) filtered_examples = [] filtered_passage_ids = set() _, pid2passages, _ = get_passages_by_source(input_data) id_kept = 0 ood_kept = 0 total = 0 for example in get_examples(input_data, True): example_pid = get_passage_id(example) related_passage = pid2passages[example_pid] is_id = is_in_distribution(related_passage) if is_id and args.keep_id: filtered_examples.append(example) filtered_passage_ids.add(example_pid) id_kept += 1 elif not is_id and args.keep_ood: filtered_examples.append(example) filtered_passage_ids.add(example_pid) ood_kept += 1 total += 1 filtered_passages = [pid2passages[pid] for pid in filtered_passage_ids] logger.info('kept {} ID and {} OOD (from a total of {} examples)'.format( id_kept, ood_kept, total)) logger.info('kept {} passages (from a total of {} passages)'.format( len(filtered_passages), len(pid2passages))) with open(args.output, "w", encoding="utf-8") as ostream: json.dump( { 'examples': filtered_examples, 'passages': filtered_passages }, ostream, indent=4, ensure_ascii=False)
def generate_embeddings(ret_trainee, input_file=None, out_file=None, json_data=None, embed_passages=True): if input_file: with open(input_file, "r", encoding="utf-8") as f: json_data = json.load(f) elif json_data: pass else: raise ValueError("You should specify either the input file or the json_data") source2passages, pid2passage, _ = get_passages_by_source(json_data) question_embs = [] question_texts = [] labels = [] if json_data.get("examples"): for example in tqdm(json_data["examples"]): pid = get_passage_id(example) passage = pid2passage[pid] labels.append('id' if is_in_distribution(passage) else 'ood') question = get_question(example) emb = ret_trainee.retriever.embed_question(question) question_embs.append(emb) question_texts.append(question) passage_header_embs = [] ood = 0 passage_texts = [] if embed_passages: for source, passages in source2passages.items(): logger.info('embedding passages for source {}'.format(source)) for passage in tqdm(passages): if is_in_distribution(passage): passage_text = get_passage_last_header(passage, return_error_for_ood=True) emb = ret_trainee.retriever.embed_paragraph( passage_text) passage_header_embs.append(emb) passage_texts.append(passage_text) else: ood += 1 to_serialize = {"question_embs": question_embs, "passage_header_embs": passage_header_embs, "question_labels": labels, "passage_texts": passage_texts, "question_texts": question_texts} if out_file: with open(out_file, "wb") as out_stream: pickle.dump(to_serialize, out_stream) logger.info( 'generated {} question embeddings and {} passage header embeddings ({} skipped because ' 'out-of-distribution)'.format( len(question_embs), len(passage_header_embs), ood)) return to_serialize
def get_faq_contents(faq_file): _, faq_pid2passage, _ = get_passages_by_source(faq_file, keep_ood=True) faq_passages = list(faq_pid2passage.values()) source2faq_contents = get_passage_content2pid(faq_passages, duplicates_are_ok=True) all_source_contents = [] for source_contents in source2faq_contents.values(): all_source_contents.extend(source_contents) all_source_contents = set(all_source_contents) logger.info('passages in reference file: {}'.format( len(all_source_contents))) return all_source_contents
def __init__(self): self.model = EmbeddingBasedReRankerPlusOODDetector( "covidfaq/bert_fr_model/config.yaml", lang="fr") test_data, _ = get_latest_scrape(lang="fr") ( self.source2passages, self.passage_id2source, self.passage_id2index, ) = get_passages_by_source(test_data, keep_ood=False) self.model.collect_answers(self.source2passages) self.get_answer("quels sont les symptomes de la covid")
def __init__(self): self.model = EmbeddingBasedReRankerPlusOODDetector( "covidfaq/bert_en_model/config.yaml", lang="en") test_data, _ = get_latest_scrape(lang="en") ( self.source2passages, self.passage_id2source, self.passage_id2index, ) = get_passages_by_source(test_data, keep_ood=False) self.model.collect_answers(self.source2passages) self.get_answer("what are the symptoms of covid-19")
def main(): parser = argparse.ArgumentParser() parser.add_argument("--test-data", help="file containing the data for evaluation", required=True) parser.add_argument("--model-type", help="model to evaluate", required=True) parser.add_argument( "--eval-method", default="standard", help= 'Whether to evaluate the model using the "standard" accuracy, or with the "topk" method. By default, uses "standard".', ) parser.add_argument("--config", help="(optional) config file to initialize the model") args = parser.parse_args() logging.basicConfig(level=logging.INFO) with open(args.test_data, "r") as in_stream: test_data = json.load(in_stream) if args.model_type == "fake": model_to_evaluate = FakeReRanker() raise NotImplementedError( "Model needs to be updated to new evaluator format") elif args.model_type == "embedding_based_reranker": if args.config is None: raise ValueError( "model embedding_based_reranker requires --config") model_to_evaluate = EmbeddingBasedReRanker(args.config) elif args.model_type == "embedding_based_reranker_plus_ood": if args.config is None: raise ValueError( "model embedding_based_reranker requires --config") model_to_evaluate = EmbeddingBasedReRankerPlusOODDetector(args.config, lang="en") elif args.model_type == "cheating_model": _, _, passage_id2index = get_passages_by_source(test_data) model_to_evaluate = CheatingModel(test_data, passage_id2index) raise NotImplementedError( "Model needs to be updated to new evaluator format") elif args.model_type == "google_model": model_to_evaluate = GoogleModel() raise NotImplementedError( "Model needs to be updated to new evaluator format") elif args.model_type == "lsa": model_to_evaluate = LSA() raise NotImplementedError( "Model needs to be updated to new evaluator format") elif args.model_type == "lda": model_to_evaluate = LDAReranker() raise NotImplementedError( "Model needs to be updated to new evaluator format") elif args.model_type == "tfidf": model_to_evaluate = TFIDF() raise NotImplementedError( "Model needs to be updated to new evaluator format") else: raise ValueError("--model_type={} not supported".format( args.model_type)) if args.eval_method == "standard": evaluate(model_to_evaluate, test_data) elif args.eval_method == "topk": if not hasattr(model_to_evaluate, "topk"): raise AttributeError( "{} does not have the method 'topk'. Please use a different evaluation method." .format(args.model_type)) topk_eval(model_to_evaluate, test_data, k=1) topk_eval(model_to_evaluate, test_data, k=5) topk_eval(model_to_evaluate, test_data, k=10) topk_eval(model_to_evaluate, test_data, k=20) topk_eval(model_to_evaluate, test_data, k=50) else: raise ValueError("--eval-method={} not supported".format( args.eval_method))
def main(): parser = argparse.ArgumentParser(__doc__) parser.add_argument("--input", help="main json file", required=True) parser.add_argument("--input-ood", help="json file where to extract the ood", required=True) parser.add_argument("--output", help="output file", required=True) parser.add_argument("--max-ood", help="max amount of ood to use. -1 means all of them", type=int, default=-1) args = parser.parse_args() logging.basicConfig(level=logging.INFO) # parse input data - check there is no OOD with open(args.input, 'r', encoding='utf-8') as in_stream: input_data = json.load(in_stream) _, pid2passages, _ = get_passages_by_source(input_data) for example in input_data['examples']: related_passage_pid = get_passage_id(example) related_passage = pid2passages[related_passage_pid] if not is_in_distribution(related_passage): raise ValueError( '--input file must not have any OOD -- see example {} / passage {}'.format( example['id'], related_passage_pid)) # new pid number (for OOD) is the highest one plus 1 (so there is no collision) new_pid_for_ood = max(pid2passages.keys()) + 1 # new id number for example is the highest one plus 1 (so there is no collision) highest_id_for_example = max([ex['id'] for ex in input_data['examples']]) + 1 result = input_data with open(args.input_ood, 'r', encoding='utf-8') as in_stream: ood_data = json.load(in_stream) new_ood_passage = { "passage_id": new_pid_for_ood, "source": "ood_source", "uri": None, "reference_type": "ood", "reference": { "page_title": None, "section_headers": [], "section_content": None, "selected_span": None } } result['passages'].append(new_ood_passage) _, pid2passages, _ = get_passages_by_source(ood_data) id_count = 0 added_ood = 0 for example in ood_data['examples']: related_passage = get_passage_id(example) related_passage = pid2passages[related_passage] if is_in_distribution(related_passage): id_count += 1 else: example['passage_id'] = new_pid_for_ood example['id'] = highest_id_for_example highest_id_for_example += 1 result['examples'].append(example) added_ood += 1 if args.max_ood > -1 and added_ood >= args.max_ood: break logger.info('kept {} ood from {} (and skipped {} id from the same file)'.format( added_ood, args.input_ood, id_count )) with open(args.output, "w", encoding="utf-8") as ostream: json.dump(result, ostream, indent=4, ensure_ascii=False)
def test_get_passages_by_source__id_and_ood(): source2passages, pid2passage, pid2index = get_passages_by_source( json.loads(ID_AND_OOD_JSON)) assert pid2index == {0: 0, 12: -1}
def test_get_passages_by_source__simple(): source2passages, pid2passage, pid2index = get_passages_by_source( json.loads(SIMPLE_JSON)) assert pid2index == {0: 0}
def test_get_passages_by_source__two_sources(): source2passages, pid2passage, pid2index = get_passages_by_source( json.loads(TWO_SOURCES_JSON)) assert pid2index == {10: 0, 11: 1, 12: 0}