modelClassifier.load_state_dict(torch.load(
'/Users/younesagabi/Desktop/YouTaQA/DeepLearning/Classifier/Models/BERT_ft_epoch10.model',
map_location=torch.device(device)),
strict=False)
modelExtractor = BertForQuestionAnswering.from_pretrained(
'bert-large-uncased-whole-word-masking-finetuned-squad')
txt_file = open(r"txt_file.txt", "w+")
tsv_file = open("test.tsv")
read_tsv = csv.reader(tsv_file, delimiter="\t")
searchObject = Searcher(
"/Users/younesagabi/Desktop/YouTaQA/IR/index_wiki_v7.0")
Similarity = Similarity()
for row in read_tsv:
inputQuery = row[0]
result = searchObject.multiFieldsSearch(inputQuery, BM25Similarity())
# print(result)
# print("#" * 100)
# print("#" * 100)
content = ""
list = ['']
list.append(inputQuery)
list.pop(0)
j = 0
for i in range(len(result)):
hitDoc = searchObject.searcher.doc(result[i].doc)
score = result[i].score
content = hitDoc.get("content_section")
content = content.replace(" ##", "")
content = content.replace('\\', "")
class ResultsGenerator:
def __init__(self, index_dir):
self.searcher = Searcher(index_dir)
def get_id_section(self, request):
idList = list()
for i in range(len(request)):
hitDoc = self.searcher.searcher.doc(request[i].doc)
idList.append(hitDoc.get("id_section"))
return idList
def process(self, input_file, index_dir, output_dir):
output_file_1 = open(output_dir + "/results_BM25_1.txt",
'a+',
encoding="utf-8")
output_file_2 = open(output_dir + "/results_BM25_2.txt",
'a+',
encoding="utf-8")
output_file_3 = open(output_dir + "/results_BM25_3.txt",
'a+',
encoding="utf-8")
output_file_4 = open(output_dir + "/results_BM25_4.txt",
'a+',
encoding="utf-8")
output_file_5 = open(output_dir + "/results_VSM_1.txt",
'a+',
encoding="utf-8")
output_file_6 = open(output_dir + "/results_VSM_2.txt",
'a+',
encoding="utf-8")
output_file_7 = open(output_dir + "/results_VSM_3.txt",
'a+',
encoding="utf-8")
output_file_8 = open(output_dir + "/results_VSM_4.txt",
'a+',
encoding="utf-8")
num_lines = 0
with open(input_file, encoding="utf-8") as json_file:
data = json.load(json_file)
for p in data['data']:
for par in p['paragraphs']:
for q in par["qas"]:
num_lines += 1
with tqdm(total=num_lines) as pbar:
with open(input_file, encoding="utf-8") as json_file:
data = json.load(json_file)
for p in data['data']:
title = p["title"]
for par in p['paragraphs']:
for q in par["qas"]:
pbar.update(1)
if q["is_impossible"] is False:
question_content_s_BM25 = self.searcher.simpleSearch(
q["question"], BM25Similarity())
id_question_content_s_BM25 = self.get_id_section(
question_content_s_BM25)
question_title_content_s_BM25 = self.searcher.pairSearch(
[title, q["question"]], BM25Similarity())
id_question_title_content_s_BM25 = self.get_id_section(
question_title_content_s_BM25)
question_content_m_BM25 = self.searcher.multiFieldsSearch(
q["question"], BM25Similarity())
id_question_content_m_BM25 = self.get_id_section(
question_content_m_BM25)
question_title_content_m_BM25 = self.searcher.multiFieldsPairSearch(
[title, q["question"]], BM25Similarity())
id_question_title_content_m_BM25 = self.get_id_section(
question_title_content_m_BM25)
question_content_s_TDF = self.searcher.simpleSearch(
q["question"], ClassicSimilarity())
id_question_content_s_TDF = self.get_id_section(
question_content_s_TDF)
question_title_content_s_TDF = self.searcher.pairSearch(
[title, q["question"]],
ClassicSimilarity())
id_question_title_content_s_TDF = self.get_id_section(
question_title_content_s_TDF)
question_content_m_TDF = self.searcher.multiFieldsSearch(
q["question"], ClassicSimilarity())
id_question_content_m_TDF = self.get_id_section(
question_content_m_TDF)
question_title_content_m_TDF = self.searcher.multiFieldsPairSearch(
[title, q["question"]],
ClassicSimilarity())
id_question_title_content_m_TDF = self.get_id_section(
question_title_content_m_TDF)
for i in range(len(question_content_s_BM25)):
output_file_1.write(
q["id"] + " Q0 " +
str(id_question_content_s_BM25[i]) +
" " + str(i + 1) + " " +
str(question_content_s_BM25[i].score) +
" STANDARD\n")
for i in range(
len(question_title_content_s_BM25)):
output_file_2.write(
q["id"] + " Q0 " +
str(id_question_title_content_s_BM25[i]
) + " " + str(i + 1) + " " +
str(question_title_content_s_BM25[i].
score) + " STANDARD\n")
for i in range(len(question_content_m_BM25)):
output_file_3.write(
q["id"] + " Q0 " +
str(id_question_content_m_BM25[i]) +
" " + str(i + 1) + " " +
str(question_content_m_BM25[i].score) +
" STANDARD\n")
for i in range(
len(question_title_content_m_BM25)):
output_file_4.write(
q["id"] + " Q0 " +
str(id_question_title_content_m_BM25[i]
) + " " + str(i + 1) + " " +
str(question_title_content_m_BM25[i].
score) + " STANDARD\n")
for i in range(len(question_content_s_TDF)):
output_file_5.write(
q["id"] + " Q0 " +
str(id_question_content_s_TDF[i]) +
" " + str(i + 1) + " " +
str(question_content_s_TDF[i].score) +
" STANDARD\n")
for i in range(
len(question_title_content_s_TDF)):
output_file_6.write(
q["id"] + " Q0 " +
str(id_question_title_content_s_TDF[i])
+ " " + str(i + 1) + " " +
str(question_title_content_s_TDF[i].
score) + " STANDARD\n")
for i in range(len(question_content_m_TDF)):
output_file_7.write(
q["id"] + " Q0 " +
str(id_question_content_m_TDF[i]) +
" " + str(i + 1) + " " +
str(question_content_m_TDF[i].score) +
" STANDARD\n")
for i in range(
len(question_title_content_m_TDF)):
output_file_8.write(
q["id"] + " Q0 " +
str(id_question_title_content_m_TDF[i])
+ " " + str(i + 1) + " " +
str(question_title_content_m_TDF[i].
score) + " STANDARD\n")
print("==> Results successfully created.\n")
