modelClassifier.load_state_dict(torch.load( '/Users/younesagabi/Desktop/YouTaQA/DeepLearning/Classifier/Models/BERT_ft_epoch10.model', map_location=torch.device(device)), strict=False) modelExtractor = BertForQuestionAnswering.from_pretrained( 'bert-large-uncased-whole-word-masking-finetuned-squad') txt_file = open(r"txt_file.txt", "w+") tsv_file = open("test.tsv") read_tsv = csv.reader(tsv_file, delimiter="\t") searchObject = Searcher( "/Users/younesagabi/Desktop/YouTaQA/IR/index_wiki_v7.0") Similarity = Similarity() for row in read_tsv: inputQuery = row[0] result = searchObject.multiFieldsSearch(inputQuery, BM25Similarity()) # print(result) # print("#" * 100) # print("#" * 100) content = "" list = [''] list.append(inputQuery) list.pop(0) j = 0 for i in range(len(result)): hitDoc = searchObject.searcher.doc(result[i].doc) score = result[i].score content = hitDoc.get("content_section") content = content.replace(" ##", "") content = content.replace('\\', "")
class ResultsGenerator: def __init__(self, index_dir): self.searcher = Searcher(index_dir) def get_id_section(self, request): idList = list() for i in range(len(request)): hitDoc = self.searcher.searcher.doc(request[i].doc) idList.append(hitDoc.get("id_section")) return idList def process(self, input_file, index_dir, output_dir): output_file_1 = open(output_dir + "/results_BM25_1.txt", 'a+', encoding="utf-8") output_file_2 = open(output_dir + "/results_BM25_2.txt", 'a+', encoding="utf-8") output_file_3 = open(output_dir + "/results_BM25_3.txt", 'a+', encoding="utf-8") output_file_4 = open(output_dir + "/results_BM25_4.txt", 'a+', encoding="utf-8") output_file_5 = open(output_dir + "/results_VSM_1.txt", 'a+', encoding="utf-8") output_file_6 = open(output_dir + "/results_VSM_2.txt", 'a+', encoding="utf-8") output_file_7 = open(output_dir + "/results_VSM_3.txt", 'a+', encoding="utf-8") output_file_8 = open(output_dir + "/results_VSM_4.txt", 'a+', encoding="utf-8") num_lines = 0 with open(input_file, encoding="utf-8") as json_file: data = json.load(json_file) for p in data['data']: for par in p['paragraphs']: for q in par["qas"]: num_lines += 1 with tqdm(total=num_lines) as pbar: with open(input_file, encoding="utf-8") as json_file: data = json.load(json_file) for p in data['data']: title = p["title"] for par in p['paragraphs']: for q in par["qas"]: pbar.update(1) if q["is_impossible"] is False: question_content_s_BM25 = self.searcher.simpleSearch( q["question"], BM25Similarity()) id_question_content_s_BM25 = self.get_id_section( question_content_s_BM25) question_title_content_s_BM25 = self.searcher.pairSearch( [title, q["question"]], BM25Similarity()) id_question_title_content_s_BM25 = self.get_id_section( question_title_content_s_BM25) question_content_m_BM25 = self.searcher.multiFieldsSearch( q["question"], BM25Similarity()) id_question_content_m_BM25 = self.get_id_section( question_content_m_BM25) question_title_content_m_BM25 = self.searcher.multiFieldsPairSearch( [title, q["question"]], BM25Similarity()) id_question_title_content_m_BM25 = self.get_id_section( question_title_content_m_BM25) question_content_s_TDF = self.searcher.simpleSearch( q["question"], ClassicSimilarity()) id_question_content_s_TDF = self.get_id_section( question_content_s_TDF) question_title_content_s_TDF = self.searcher.pairSearch( [title, q["question"]], ClassicSimilarity()) id_question_title_content_s_TDF = self.get_id_section( question_title_content_s_TDF) question_content_m_TDF = self.searcher.multiFieldsSearch( q["question"], ClassicSimilarity()) id_question_content_m_TDF = self.get_id_section( question_content_m_TDF) question_title_content_m_TDF = self.searcher.multiFieldsPairSearch( [title, q["question"]], ClassicSimilarity()) id_question_title_content_m_TDF = self.get_id_section( question_title_content_m_TDF) for i in range(len(question_content_s_BM25)): output_file_1.write( q["id"] + " Q0 " + str(id_question_content_s_BM25[i]) + " " + str(i + 1) + " " + str(question_content_s_BM25[i].score) + " STANDARD\n") for i in range( len(question_title_content_s_BM25)): output_file_2.write( q["id"] + " Q0 " + str(id_question_title_content_s_BM25[i] ) + " " + str(i + 1) + " " + str(question_title_content_s_BM25[i]. score) + " STANDARD\n") for i in range(len(question_content_m_BM25)): output_file_3.write( q["id"] + " Q0 " + str(id_question_content_m_BM25[i]) + " " + str(i + 1) + " " + str(question_content_m_BM25[i].score) + " STANDARD\n") for i in range( len(question_title_content_m_BM25)): output_file_4.write( q["id"] + " Q0 " + str(id_question_title_content_m_BM25[i] ) + " " + str(i + 1) + " " + str(question_title_content_m_BM25[i]. score) + " STANDARD\n") for i in range(len(question_content_s_TDF)): output_file_5.write( q["id"] + " Q0 " + str(id_question_content_s_TDF[i]) + " " + str(i + 1) + " " + str(question_content_s_TDF[i].score) + " STANDARD\n") for i in range( len(question_title_content_s_TDF)): output_file_6.write( q["id"] + " Q0 " + str(id_question_title_content_s_TDF[i]) + " " + str(i + 1) + " " + str(question_title_content_s_TDF[i]. score) + " STANDARD\n") for i in range(len(question_content_m_TDF)): output_file_7.write( q["id"] + " Q0 " + str(id_question_content_m_TDF[i]) + " " + str(i + 1) + " " + str(question_content_m_TDF[i].score) + " STANDARD\n") for i in range( len(question_title_content_m_TDF)): output_file_8.write( q["id"] + " Q0 " + str(id_question_title_content_m_TDF[i]) + " " + str(i + 1) + " " + str(question_title_content_m_TDF[i]. score) + " STANDARD\n") print("==> Results successfully created.\n")