def test_onnx_conversion_and_inference(tmp_path, model_name):
AdaptiveModel.convert_to_onnx(model_name=model_name,
output_path=tmp_path / "test-onnx",
task_type="question_answering")
onnx_inferencer = Inferencer.load(tmp_path / "test-onnx",
task_type="question_answering",
num_processes=0)
qa_input = [{
"questions": ["What is the population of Berlin?"],
"text":
"Berlin is the capital and largest city of Germany by both area and population. Its 3,769,495 "
"inhabitants as of December 31, 2019 make it the most populous city of the European Union, "
"according to population within city limits.The city is also one of Germany's 16 federal states.",
}]
result_onnx = onnx_inferencer.inference_from_dicts(qa_input)[0]
assert result_onnx["predictions"][0]["answers"][0]["answer"] == "3,769,495"
pytorch_inferencer = Inferencer.load(model_name,
task_type="question_answering",
num_processes=0)
result_pytorch = pytorch_inferencer.inference_from_dicts(qa_input)[0]
for (onnx, pytorch) in zip(
result_onnx["predictions"][0]["answers"][0].items(),
result_pytorch["predictions"][0]["answers"][0].items()):
# keys
assert onnx[0] == pytorch[0]
# values
if type(onnx[1]) == float:
np.testing.assert_almost_equal(onnx[1], pytorch[1],
decimal=4) # score
else:
assert onnx[1] == pytorch[1]
def convert_to_onnx(cls,
model_name: str,
output_path: Path,
convert_to_float16: bool = False,
quantize: bool = False,
task_type: str = "question_answering",
opset_version: int = 11):
"""
Convert a PyTorch BERT model to ONNX format and write to ./onnx-export dir. The converted ONNX model
can be loaded with in the `FARMReader` using the export path as `model_name_or_path` param.
Usage:
`from haystack.reader.farm import FARMReader
from pathlib import Path
onnx_model_path = Path("roberta-onnx-model")
FARMReader.convert_to_onnx(model_name="deepset/bert-base-cased-squad2", output_path=onnx_model_path)
reader = FARMReader(onnx_model_path)`
:param model_name: transformers model name
:param output_path: Path to output the converted model
:param convert_to_float16: Many models use float32 precision by default. With the half precision of float16,
inference is faster on Nvidia GPUs with Tensor core like T4 or V100. On older GPUs,
float32 could still be be more performant.
:param quantize: convert floating point number to integers
:param task_type: Type of task for the model. Available options: "question_answering" or "embeddings".
:param opset_version: ONNX opset version
"""
AdaptiveModel.convert_to_onnx(model_name=model_name,
output_path=output_path,
task_type=task_type,
convert_to_float16=convert_to_float16,
quantize=quantize,
opset_version=opset_version)
def onnx_runtime_example():
"""
This example shows conversion of a transformers model from the Model Hub to
ONNX format & inference using ONNXRuntime.
"""
model_name_or_path = "deepset/roberta-base-squad2"
onnx_model_export_path = Path("./roberta-onnx")
AdaptiveModel.convert_to_onnx(model_name_or_path,
onnx_model_export_path,
task_type="question_answering")
# for ONNX models, the Inferencer uses ONNXRuntime under-the-hood
inferencer = Inferencer.load(model_name_or_path=onnx_model_export_path)
qa_input = [{
"questions": ["Who counted the game among the best ever made?"],
"text":
"Twilight Princess was released to universal critical acclaim and commercial success. "
"It received perfect scores from major publications such as 1UP.com, Computer and Video Games, "
"Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators "
"GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii "
"version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called "
"it one of the greatest games ever created.",
}]
results = inferencer.inference_from_dicts(qa_input)
print(results)
inferencer.close_multiprocessing_pool()
