def test_classify_question9(self):
query = "What types of weather are experienced by people?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question7(self):
query = "How many weeks in a month"
expected_answer_type = 'NUM:count'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question8(self):
query = "Which is the warmest season of the year"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question18(self):
query = "What is the North and South Pole full of?"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question19(self):
query = "What are the four seasons of the year?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question14(self):
query = "What are the plants full of during the summer season?"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question16(self):
query = "What is the land covered by at the North and South Poles?"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question26(self):
query = "What is harvested in the autumn season?"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question2(self):
query = "What do farmers harvest in autumn"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question24(self):
query = "In which season do the birds fly south?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question25(self):
query = "What do the farmers harvest in the autumn season?"
expected_answer_type = 'ENTY:food'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question23(self):
query = "In which season do the farmers harvest their fruits and vegetables?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question22(self):
query = "In which season are birds, bugs and other animals easy to spot?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question21(self):
query = "In which season are usually many baby animals born?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question11(self):
query = "What does the color red on the map usually stand for?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question3(self):
query = "What is on the ground during the winters"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question13(self):
query = "In which continent is the North Pole located?"
expected_answer_type = 'LOC:other'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question4(self):
query = "Where is US located"
expected_answer_type = 'LOC:other'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question15(self):
query = "In which season are the animals easy to spot?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question5(self):
query = "Which animal sleeps during the winters"
expected_answer_type = 'ENTY:animal'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question17(self):
query = "What are the different kinds of weather?"
expected_answer_type = 'ENTY:event'
self.assertEqual(expected_answer_type, classify_question(query))
def test_classify_question6(self):
query = "What color on the map represents hot areas"
expected_answer_type = 'ENTY:color'
self.assertEqual(expected_answer_type, classify_question(query))
def get_answer(query):
"""
Question Ananlyzer
1. question classification
2. rephrasing question with synonyms
"""
answer_type = classify_question(query)
nlp_lib_answer_type = get_ner_type_for_answer_type[answer_type]
query = rephrase_query_with_synonyms(query)
"""
Loading the PreProcess class to pre-process queries and passages
Pre Processing query to remove punctuations, stopwords and replace words(verbs) with their lemmatized forms
"""
pre_process = PreProcess()
query_object = pre_process.pre_process(query)
# entity_tagger = AnnotatorPipeline([EntityTagger()])
# entity_tagger.process(query_object)
'''
DOC (PASSAGES)-RETRIEVAL FROM ELASTICSEARCH
'''
document_fetcher = FetchFromES()
documents = document_fetcher.fetch_document_from_es(query)
"""
joining all documents to convert into single document seperated by "\n\n" (denotes the start of a new passage)
"""
merged_document = "\n\n".join(documents)
passages = merged_document.split("\n\n")
"""
Pre Processing documents to remove punctuations, stopwords and replace words(verbs) with their lemmatized forms
"""
passage_objects = []
for passage in passages:
passage_object = pre_process.pre_process(passage)
passage_objects.append(passage_object)
"""
CANDIDATE PASSAGES EXTRACTOR
"""
passage_extractor = PassageExtraction(query_object, nlp_lib_answer_type,
passage_objects)
candidate_passages = passage_extractor.process()
"""
Checking for presence of at least one candidate passage
"""
if candidate_passages and candidate_passages[0]:
"""
SENTENCE EXTRACTION
Considering 1st passage(among candidate passages) as the required passage; using this passage to extract sentence
Sentence retrieval is on the basis of keyword percent match, similarity score and longest sub-sequence match
Final sentence selected after sorting above features
"""
sentence_extractor = SentenceExtraction(
query_object.text, nlp_lib_answer_type,
candidate_passages[0][0].text, candidate_passages[0][0].raw_text)
final_sentence = sentence_extractor.process()
return final_sentence.raw_text
else:
"""
candidate passages have no elements when at least 2 keywords from query are not present in either of the passages
"""
return "REPHRASE THE QUESTION"
#
# """
# ANSWER EXTRACTION
# """
#
# answer_extractor = AnswerExtraction(query_object,nlp_lib_answer_type,final_sentence)
# print(answer_extractor.extract_answer())
# if __name__== "__main__":
#
# data = ['LOC:other-When are the plants full of leaves, flowers and fruits?']
# for question in data:
# if question:
# query = question.split("-")[1].replace('\n','')
# print(get_answer(query))
def test_classify_question(self):
query = "What forms when snow and ice melt"
expected_answer_type = 'ENTY:product'
self.assertEqual(expected_answer_type, classify_question(query))
