def test_span_inference_result_ranking_by_confidence(bert_base_squad2,
caplog=None):
if caplog:
caplog.set_level(logging.CRITICAL)
obj_input = [
QAInput(
doc_text=
"Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created.",
questions=Question(
"Who counted the game among the best ever made?",
uid="best_id_ever"))
]
result = bert_base_squad2.inference_from_objects(obj_input,
return_json=False)[0]
# by default, result is sorted by score and not by confidence
assert all(result.prediction[i].score >= result.prediction[i + 1].score
for i in range(len(result.prediction) - 1))
assert not all(
result.prediction[i].confidence >= result.prediction[i + 1].confidence
for i in range(len(result.prediction) - 1))
# ranking can be adjusted so that result is sorted by confidence
bert_base_squad2.model.prediction_heads[
0].use_confidence_scores_for_ranking = True
result_ranked_by_confidence = bert_base_squad2.inference_from_objects(
obj_input, return_json=False)[0]
assert all(result_ranked_by_confidence.prediction[i].confidence >=
result_ranked_by_confidence.prediction[i + 1].confidence
for i in range(len(result_ranked_by_confidence.prediction) - 1))
assert not all(
result_ranked_by_confidence.prediction[i].score >=
result_ranked_by_confidence.prediction[i + 1].score
for i in range(len(result_ranked_by_confidence.prediction) - 1))
def predict_batch(self, query_doc_list: List[dict], top_k: int = None, batch_size: int = None):
"""
Use loaded QA model to find answers for a list of queries in each query's supplied list of Document.
Returns list of dictionaries containing answers sorted by (desc.) probability
:param query_doc_list: List of dictionaries containing queries with their retrieved documents
:param top_k: The maximum number of answers to return for each query
:param batch_size: Number of samples the model receives in one batch for inference
:return: List of dictionaries containing query and answers
"""
if top_k is None:
top_k = self.top_k
# convert input to FARM format
inputs = []
number_of_docs = []
labels = []
# build input objects for inference_from_objects
for query_with_docs in query_doc_list:
documents = query_with_docs["docs"]
query = query_with_docs["question"]
labels.append(query)
number_of_docs.append(len(documents))
for doc in documents:
cur = QAInput(doc_text=doc.text,
questions=Question(text=query.question,
uid=doc.id))
inputs.append(cur)
self.inferencer.batch_size = batch_size
# make predictions on all document-query pairs
predictions = self.inferencer.inference_from_objects(
objects=inputs, return_json=False, multiprocessing_chunksize=10
)
# group predictions together
grouped_predictions = []
left_idx = 0
right_idx = 0
for number in number_of_docs:
right_idx = left_idx + number
grouped_predictions.append(predictions[left_idx:right_idx])
left_idx = right_idx
result = []
for idx, group in enumerate(grouped_predictions):
answers, max_no_ans_gap = self._extract_answers_of_predictions(group, top_k)
query = group[0].question
cur_label = labels[idx]
result.append({
"query": query,
"no_ans_gap": max_no_ans_gap,
"answers": answers,
"label": cur_label
})
return result
def span_inference_result(bert_base_squad2, caplog=None):
if caplog:
caplog.set_level(logging.CRITICAL)
obj_input = [QAInput(doc_text="Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created.",
questions=Question("Who counted the game among the best ever made?", uid="best_id_ever"))]
result = bert_base_squad2.inference_from_objects(obj_input, return_json=False)[0]
return result
def no_answer_inference_result(bert_base_squad2, caplog=None):
if caplog:
caplog.set_level(logging.CRITICAL)
obj_input = [QAInput(doc_text="The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.",
questions=Question("The Amazon represents less than half of the planets remaining what?", uid="best_id_ever"))]
result = bert_base_squad2.inference_from_objects(obj_input, return_json=False)[0]
return result
def predict(self,
query: str,
documents: List[Document],
top_k: Optional[int] = None):
"""
Use loaded QA model to find answers for a query in the supplied list of Document.
Returns dictionaries containing answers sorted by (desc.) probability.
Example:
```python
|{
| 'query': 'Who is the father of Arya Stark?',
| 'answers':[
| {'answer': 'Eddard,',
| 'context': " She travels with her father, Eddard, to King's Landing when he is ",
| 'offset_answer_start': 147,
| 'offset_answer_end': 154,
| 'probability': 0.9787139466668613,
| 'score': None,
| 'document_id': '1337'
| },...
| ]
|}
```
:param query: Query string
:param documents: List of Document in which to search for the answer
:param top_k: The maximum number of answers to return
:return: Dict containing query and answers
"""
if top_k is None:
top_k = self.top_k
# convert input to FARM format
inputs = []
for doc in documents:
cur = QAInput(doc_text=doc.text,
questions=Question(text=query, uid=doc.id))
inputs.append(cur)
# get answers from QA model
# TODO: Need fix in FARM's `to_dict` function of `QAInput` class
predictions = self.inferencer.inference_from_objects(
objects=inputs, return_json=False, multiprocessing_chunksize=1)
# assemble answers from all the different documents & format them.
answers, max_no_ans_gap = self._extract_answers_of_predictions(
predictions, top_k)
result = {
"query": query,
"no_ans_gap": max_no_ans_gap,
"answers": answers
}
return result
def test_inference_different_inputs(bert_base_squad2):
qa_format_1 = [
{
"questions": ["Who counted the game among the best ever made?"],
"text": "Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created."
}]
q = Question(text="Who counted the game among the best ever made?")
qa_format_2 = QAInput(questions=[q],doc_text= "Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created.")
result1 = bert_base_squad2.inference_from_dicts(dicts=qa_format_1)
result2 = bert_base_squad2.inference_from_objects(objects=[qa_format_2])
assert result1 == result2
def predict(self,
question: str,
documents: List[Document],
top_k: Optional[int] = None):
"""
Use loaded QA model to find answers for a question in the supplied list of Document.
Returns dictionaries containing answers sorted by (desc.) probability
Example:
{'question': 'Who is the father of Arya Stark?',
'answers': [
{'answer': 'Eddard,',
'context': " She travels with her father, Eddard, to King's Landing when he is ",
'offset_answer_start': 147,
'offset_answer_end': 154,
'probability': 0.9787139466668613,
'score': None,
'document_id': '1337'
},
...
]
}
:param question: question string
:param documents: list of Document in which to search for the answer
:param top_k: the maximum number of answers to return
:return: dict containing question and answers
"""
# convert input to FARM format
inputs = []
for doc in documents:
cur = QAInput(doc_text=doc.text,
questions=Question(text=question, uid=doc.id))
inputs.append(cur)
# get answers from QA model
predictions = self.inferencer.inference_from_objects(
objects=inputs, return_json=False, multiprocessing_chunksize=1)
# assemble answers from all the different documents & format them.
answers, max_no_ans_gap = self._extract_answers_of_predictions(
predictions, top_k)
result = {
"question": question,
"no_ans_gap": max_no_ans_gap,
"answers": answers
}
return result
def predict(self,
question: str,
documents: List[Document],
top_k: Optional[int] = None):
"""
Use loaded QA model to find answers for a question in the supplied list of Document.
Returns dictionaries containing answers sorted by (desc.) probability
Example:
{'question': 'Who is the father of Arya Stark?',
'answers': [
{'answer': 'Eddard,',
'context': " She travels with her father, Eddard, to King's Landing when he is ",
'offset_answer_start': 147,
'offset_answer_end': 154,
'probability': 0.9787139466668613,
'score': None,
'document_id': '1337'
},
...
]
}
:param question: question string
:param documents: list of Document in which to search for the answer
:param top_k: the maximum number of answers to return
:return: dict containing question and answers
"""
# convert input to FARM format
inputs = []
for doc in documents:
cur = QAInput(doc_text=doc.text,
questions=Question(text=question, uid=doc.id))
inputs.append(cur)
# get answers from QA model
predictions = self.inferencer.inference_from_objects(
objects=inputs, return_json=False, multiprocessing_chunksize=1)
# assemble answers from all the different documents & format them.
# For the "no answer" option, we collect all no_ans_gaps and decide how likely
# a no answer is based on all no_ans_gaps values across all documents
answers = []
no_ans_gaps = []
best_score_answer = 0
for pred in predictions:
answers_per_document = []
no_ans_gaps.append(pred.no_answer_gap)
for ans in pred.prediction:
# skip "no answers" here
if self._check_no_answer(ans):
pass
else:
cur = {
"answer": ans.answer,
"score": ans.score,
# just a pseudo prob for now
"probability": float(expit(
np.asarray([ans.score]) / 8)), # type: ignore
"context": ans.context_window,
"offset_start": ans.offset_answer_start -
ans.offset_context_window_start,
"offset_end": ans.offset_answer_end -
ans.offset_context_window_start,
"offset_start_in_doc": ans.offset_answer_start,
"offset_end_in_doc": ans.offset_answer_end,
"document_id": pred.id
}
answers_per_document.append(cur)
if ans.score > best_score_answer:
best_score_answer = ans.score
# only take n best candidates. Answers coming back from FARM are sorted with decreasing relevance.
answers += answers_per_document[:self.top_k_per_candidate]
# Calculate the score for predicting "no answer", relative to our best positive answer score
no_ans_prediction, max_no_ans_gap = self._calc_no_answer(
no_ans_gaps, best_score_answer)
if self.return_no_answers:
answers.append(no_ans_prediction)
# sort answers by their `probability` and select top-k
answers = sorted(answers, key=lambda k: k["probability"], reverse=True)
answers = answers[:top_k]
result = {
"question": question,
"no_ans_gap": max_no_ans_gap,
"answers": answers
}
return result
from farm.infer import QAInferencer
from farm.data_handler.inputs import QAInput, Question
nlp = QAInferencer.load("deepset/roberta-base-squad2",
task_type="question_answering",
batch_size=16,
num_processes=0)
input = QAInput(doc_text="My name is Lucas and I live on Mars.",
questions=Question(text="Who lives on Mars?", uid="your-id"))
res = nlp.inference_from_objects([input], return_json=False)[0]
# High level attributes for your query
print(res.question)
print(res.context)
print(res.no_answer_gap)
# ...
# Attributes for individual predictions (= answers)
pred = res.prediction[0]
print(pred.answer)
print(pred.answer_type)
print(pred.answer_support)
print(pred.offset_answer_start)
print(pred.offset_answer_end)
