def onnx_runtime_example():
"""
This example converts a Question Answering FARM AdaptiveModel
to ONNX format and uses ONNX Runtime for doing Inference.
"""
device = "cpu"
model_name_or_path = "deepset/bert-base-cased-squad2"
onnx_model_export_path = Path("./onnx-export")
model = AdaptiveModel.convert_from_transformers(model_name_or_path, device=device, task_type="question_answering")
model.convert_to_onnx(onnx_model_export_path)
inferencer = Inferencer.load(model_name_or_path=onnx_model_export_path)
qa_input = [
{
"qas": ["Who counted the game among the best ever made?"],
"context": "Twilight Princess was released to universal critical acclaim and commercial success. "
"It received perfect scores from major publications such as 1UP.com, Computer and Video Games, "
"Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators "
"GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii "
"version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called "
"it one of the greatest games ever created.",
}
]
results = inferencer.inference_from_dicts(qa_input)
print(results)
def convert_from_transformers():
transformers_input_name = "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
farm_output_dir = Path(
"../saved_models/farm-bert-base-german-cased-hatespeech-GermEval18Coarse"
)
# # CASE 1: MODEL
# # Load model from transformers model hub (-> continue training / compare models / ...)
model = AdaptiveModel.convert_from_transformers(
transformers_input_name, device="cpu", task_type="text_classification")
# # ... continue as in the other examples e.g. to fine-tune this QA model on your own data
#
# # CASE 2: INFERENCER
# # Load Inferencer from transformers, incl. model & tokenizer (-> just get predictions)
nlp = Inferencer.load(transformers_input_name,
task_type="text_classification")
#
# # run predictions
result = nlp.inference_from_dicts(dicts=[{
"text": "Was ein scheiß Nazi!"
}],
rest_api_schema=True)
pprint.pprint(result)
# save it
nlp.save(farm_output_dir)
def convert_from_transformers():
# CASE 1: MODEL
# Load model from transformers model hub (-> continue training / compare models / ...)
model = AdaptiveModel.convert_from_transformers(
"deepset/bert-large-uncased-whole-word-masking-squad2",
device="cpu",
task_type="question_answering")
# ... continue as in the other examples e.g. to fine-tune this QA model on your own data
# CASE 2: INFERENCER
# Load Inferencer from transformers, incl. model & tokenizer (-> just get predictions)
nlp = Inferencer.load(
"deepset/bert-large-uncased-whole-word-masking-squad2",
task_type="question_answering")
# run predictions
QA_input = [{
"questions": ["Why is model conversion important?"],
"text":
"The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."
}]
result = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True)
pprint.pprint(result)
# save it
farm_model_dir = Path("../saved_models/bert-english-qa-large")
nlp.save(farm_model_dir)
def evaluate_question_answering():
##########################
########## Settings
##########################
device, n_gpu = initialize_device_settings(use_cuda=True)
lang_model = "deepset/roberta-base-squad2"
do_lower_case = True
data_dir = Path("../data/squad20")
evaluation_filename = "dev-v2.0.json"
batch_size = 50
no_ans_boost = 0
accuracy_at = 3 # accuracy at n is useful for answers inside long documents
# 1.Create a tokenizer
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
doc_stride=128,
)
# 3. Create a DataSilo that loads dataset, provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(processor=processor, batch_size=batch_size)
# 4. Create an Evaluator
evaluator = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# 5. Load model
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
# use "load" if you want to use a local model that was trained with FARM
#model = AdaptiveModel.load(lang_model, device=device)
model.prediction_heads[0].no_ans_boost = no_ans_boost
model.prediction_heads[0].n_best = accuracy_at
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
# 6. Run the Evaluator
results = evaluator.eval(model)
f1_score = results[0]["f1"]
em_score = results[0]["EM"]
tnacc = results[0]["top_n_accuracy"]
print("F1-Score:", f1_score)
print("Exact Match Score:", em_score)
print(f"top_{accuracy_at}_accuracy:", tnacc)
def evaluate_classification():
##########################
########## Settings
##########################
device, n_gpu = initialize_device_settings(use_cuda=True)
lang_model = "deepset/bert-base-german-cased-sentiment-Germeval17"
do_lower_case = False
batch_size = 100
data_dir = Path("../data/germeval17")
evaluation_filename = "test_TIMESTAMP1.tsv"
label_list = ["negative", "neutral", "positive"]
metric = "f1_macro"
# 1.Create a tokenizer
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
# Here we load GermEval 2017 Data automaticaly if it is not available.
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=384,
label_list=label_list,
metric=metric,
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
)
# 3. Create a DataSilo that loads dataset, provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(
processor=processor,
batch_size=batch_size)
# 4. Create an Evaluator
evaluator = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device
)
# 5. Load model
model = AdaptiveModel.convert_from_transformers(lang_model, device=device, task_type="text_classification")
# use "load" if you want to use a local model that was trained with FARM
# model = AdaptiveModel.load(lang_model, device=device)
model.connect_heads_with_processor(data_silo.processor.tasks, require_labels=True)
# 6. Run the Evaluator
results = evaluator.eval(model)
f1_score = results[0]["f1_macro"]
print("Macro-averaged F1-Score:", f1_score)
def train_on_split(silo_to_use, n_fold):
logger.info(
f"############ Crossvalidation: Fold {n_fold} ############")
# fine-tune pre-trained question-answering model
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
# If positive, thjs will boost "No Answer" as prediction.
# If negative, this will prevent the model from giving "No Answer" as prediction.
model.prediction_heads[0].no_ans_boost = no_ans_boost
# Number of predictions the model will make per Question.
# The multiple predictions are used for evaluating top n recall.
model.prediction_heads[0].n_best = accuracy_at
# # or train question-answering models from scratch
# # Create an AdaptiveModel
# # a) which consists of a pretrained language model as a basis
# language_model = LanguageModel.load(lang_model)
# # b) and a prediction head on top that is suited for our task => Question-answering
# prediction_head = QuestionAnsweringHead(no_ans_boost=no_ans_boost, n_best=accuracy_at)
# model = AdaptiveModel(
# language_model=language_model,
# prediction_heads=[prediction_head],
# embeds_dropout_prob=0.1,
# lm_output_types=["per_token"],
# device=device,)
# Create an optimizer
model, optimizer, lr_schedule = initialize_optimizer(
model=model,
learning_rate=learning_rate,
device=device,
n_batches=len(silo_to_use.loaders["train"]),
n_epochs=n_epochs,
use_amp=use_amp)
# Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
# Also create an EarlyStopping instance and pass it on to the trainer
trainer = Trainer(model=model,
optimizer=optimizer,
data_silo=silo_to_use,
epochs=n_epochs,
n_gpu=n_gpu,
lr_schedule=lr_schedule,
evaluate_every=evaluate_every,
device=device,
evaluator_test=False)
# train it
trainer.train()
return trainer.model
def onnx_adaptive_model_qa(use_gpu, num_processes):
model_name_or_path = "deepset/bert-base-cased-squad2"
onnx_model_export_path = Path("benchmarks/onnx-export")
if not (onnx_model_export_path / "model.onnx").is_file():
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device="cpu", task_type="question_answering")
model.convert_to_onnx(onnx_model_export_path)
model = Inferencer.load(onnx_model_export_path,
task_type="question_answering",
batch_size=1,
num_processes=num_processes,
gpu=use_gpu)
return model
def test_conversion_adaptive_model(caplog):
if caplog:
caplog.set_level(logging.CRITICAL)
model = AdaptiveModel.convert_from_transformers(
"deepset/bert-base-cased-squad2",
device="cpu",
task_type="question_answering")
transformer_model = model.convert_to_transformers()
transformer_model2 = AutoModelForQuestionAnswering.from_pretrained(
"deepset/bert-base-cased-squad2")
# compare weights
for p1, p2 in zip(transformer_model.parameters(),
transformer_model2.parameters()):
assert (p1.data.ne(p2.data).sum() == 0)
def onnx_adaptive_model_qa(
use_gpu,
num_processes,
model_name_or_path="deepset/bert-base-cased-squad2"):
if (Path(model_name_or_path) /
"model.onnx").is_file(): # load model directly if in ONNX format
onnx_model_path = model_name_or_path
else: # convert to ONNX format
onnx_model_path = Path("benchmarks/onnx-export")
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device="cpu", task_type="question_answering")
model.convert_to_onnx(onnx_model_path)
model = Inferencer.load(onnx_model_path,
task_type="question_answering",
batch_size=1,
num_processes=num_processes,
gpu=use_gpu)
return model
config = None
with open(configFile) as file:
config = yaml.safe_load(file)
modelName = config["model"]
sqlUrl = config["sqlUrl"]
modelDir = config["modelDir"]
doc_dir = os.path.join(dirname, config["docDir"])
# download and save the model
from farm.modeling.adaptive_model import AdaptiveModel
from farm.data_handler.processor import Processor
model = AdaptiveModel.convert_from_transformers(modelName,
device="cpu",
task_type="question_answering")
processor = Processor.convert_from_transformers(modelName,
task_type="question_answering",
max_seq_len=384,
doc_stride=128)
model.save(modelDir)
processor.save(modelDir)
# try:
# //or we should config the cache_dir on from_pretrain # dont rely on cache , must save it with save()
# model = TransformersReader(model_name_or_path=modelName, use_gpu=-1)
# model = FARMReader(model_name_or_path=modelName,
# use_gpu=False, no_ans_boost=0)
# except:
# pass;
def test_evaluation():
##########################
########## Settings
##########################
lang_model = "deepset/roberta-base-squad2"
do_lower_case = False
test_assertions = True
data_dir = Path("testsave/data/squad20")
evaluation_filename = "dev-v2.0.json"
device, n_gpu = initialize_device_settings(use_cuda=True)
# loading models and evals
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
model.prediction_heads[0].no_ans_boost = 0
model.prediction_heads[0].n_best = 1
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
doc_stride=128,
)
starttime = time()
data_silo = DataSilo(processor=processor, batch_size=50)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
evaluator = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# 1. Test FARM internal evaluation
results = evaluator.eval(model)
f1_score = results[0]["f1"] * 100
em_score = results[0]["EM"] * 100
tnrecall = results[0]["top_n_recall"] * 100
elapsed = time() - starttime
print(results)
print(elapsed)
gold_EM = 77.7478
gold_f1 = 82.1557
gold_tnrecall = 84.0646 # top 1 recall
gold_elapsed = 70 # 4x V100
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=f"FARM Eval changed for EM by: {em_score-gold_EM}")
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=f"FARM Eval changed for f1 score by: {f1_score-gold_f1}")
np.testing.assert_allclose(
tnrecall,
gold_tnrecall,
rtol=0.001,
err_msg=f"FARM Eval changed for top 1 recall by: {em_score-gold_EM}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"FARM Eval speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
# 2. Test FARM predictions with outside eval script
starttime = time()
model = Inferencer(model=model,
processor=processor,
task_type="question_answering",
batch_size=50,
gpu=device.type == "cuda")
filename = data_dir / evaluation_filename
result = model.inference_from_file(file=filename)
elapsed = time() - starttime
os.makedirs("../testsave", exist_ok=True)
write_squad_predictions(predictions=result,
predictions_filename=filename,
out_filename="testsave/predictions.json")
script_params = {
"data_file": filename,
"pred_file": "testsave/predictions.json",
"na_prob_thresh": 1,
"na_prob_file": False,
"out_file": False
}
results_official = squad_evaluation.main(OPTS=DotMap(script_params))
f1_score = results_official["f1"]
em_score = results_official["exact"]
gold_EM = 78.4890
gold_f1 = 81.7104
gold_elapsed = 66 # 4x V100
print(elapsed)
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=
f"Eval with official script changed for EM by: {em_score - gold_EM}"
)
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=
f"Eval with official script changed for f1 score by: {f1_score - gold_f1}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"Inference speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
def load(cls,
model_name_or_path,
batch_size=4,
gpu=False,
task_type=None,
return_class_probs=False,
strict=True,
max_seq_len=256):
"""
Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by
1. specifying a public name from transformers' model hub (https://huggingface.co/models)
2. or pointing to a local directory it is saved in.
:param model_name_or_path: Local directory or public name of the model to load.
:type model_name_or_path: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param task_type: Type of task the model should be used for. Currently supporting:
"embeddings", "question_answering", "text_classification". More coming soon...
:param task_type: str
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
use_amp=None)
name = os.path.basename(model_name_or_path)
# a) either from local dir
if os.path.exists(model_name_or_path):
model = AdaptiveModel.load(model_name_or_path,
device,
strict=strict)
if task_type == "embeddings":
processor = InferenceProcessor.load_from_dir(
model_name_or_path)
else:
processor = Processor.load_from_dir(model_name_or_path)
# b) or from remote transformers model hub
else:
logger.info(
f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..."
)
if not task_type:
raise ValueError(
"Please specify the 'task_type' of the model you want to load from transformers. "
"Valid options for arg `task_type`:"
"'question_answering', 'embeddings', 'text_classification'"
)
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device, task_type)
config = AutoConfig.from_pretrained(model_name_or_path)
tokenizer = Tokenizer.load(model_name_or_path)
# TODO infer task_type automatically from config (if possible)
if task_type == "question_answering":
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
label_list=["start_token", "end_token"],
metric="squad",
data_dir=None,
)
elif task_type == "embeddings":
processor = InferenceProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len)
elif task_type == "text_classification":
label_list = list(config.id2label[id]
for id in range(len(config.id2label)))
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=None,
label_list=label_list,
label_column_name="label",
metric="acc",
quote_char='"',
)
# elif task_type == "multilabel-classification":
# # label_list = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
# label_list = list(config.label2id.keys())
#
# processor = TextClassificationProcessor(tokenizer=tokenizer,
# max_seq_len=max_seq_len,
# data_dir=None,
# label_list=label_list,
# label_column_name="label",
# metric="acc",
# quote_char='"',
# multilabel=True,
# )
elif task_type == "ner":
label_list = list(config.label2id.keys())
processor = NERProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir=None,
metric="seq_f1",
label_list=label_list)
else:
raise ValueError(
f"`task_type` {task_type} is not supported yet. "
f"Valid options for arg `task_type`: 'question_answering', 'embeddings', 'text_classification'"
)
return cls(
model,
processor,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
)
def load(
cls,
model_name_or_path,
batch_size=4,
gpu=False,
task_type=None,
return_class_probs=False,
strict=True,
max_seq_len=256,
doc_stride=128,
extraction_layer=None,
extraction_strategy=None,
s3e_stats=None,
num_processes=None,
disable_tqdm=False,
tokenizer_class=None,
use_fast=False,
tokenizer_args=None,
dummy_ph=False,
benchmarking=False,
):
"""
Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by
1. specifying a public name from transformers' model hub (https://huggingface.co/models)
2. or pointing to a local directory it is saved in.
:param model_name_or_path: Local directory or public name of the model to load.
:type model_name_or_path: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param task_type: Type of task the model should be used for. Currently supporting:
"embeddings", "question_answering", "text_classification", "ner". More coming soon...
:param task_type: str
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:param max_seq_len: maximum length of one text sample
:type max_seq_len: int
:param doc_stride: Only QA: When input text is longer than max_seq_len it gets split into parts, strided by doc_stride
:type doc_stride: int
:param extraction_strategy: Strategy to extract vectors. Choices: 'cls_token' (sentence vector), 'reduce_mean'
(sentence vector), reduce_max (sentence vector), 'per_token' (individual token vectors)
:type extraction_strategy: str
:param extraction_layer: number of layer from which the embeddings shall be extracted. Default: -1 (very last layer).
:type extraction_layer: int
:param s3e_stats: Stats of a fitted S3E model as returned by `fit_s3e_on_corpus()`
(only needed for task_type="embeddings" and extraction_strategy = "s3e")
:type s3e_stats: dict
:param num_processes: the number of processes for `multiprocessing.Pool`. Set to value of 0 to disable
multiprocessing. Set to None to let Inferencer use all CPU cores minus one. If you want to
debug the Language Model, you might need to disable multiprocessing!
**Warning!** If you use multiprocessing you have to close the
`multiprocessing.Pool` again! To do so call
:func:`~farm.infer.Inferencer.close_multiprocessing_pool` after you are
done using this class. The garbage collector will not do this for you!
:type num_processes: int
:param disable_tqdm: Whether to disable tqdm logging (can get very verbose in multiprocessing)
:type disable_tqdm: bool
:param tokenizer_class: (Optional) Name of the tokenizer class to load (e.g. `BertTokenizer`)
:type tokenizer_class: str
:param use_fast: (Optional, False by default) Indicate if FARM should try to load the fast version of the tokenizer (True) or
use the Python one (False).
:param tokenizer_args: (Optional) Will be passed to the Tokenizer ``__init__`` method.
See https://huggingface.co/transformers/main_classes/tokenizer.html and detailed tokenizer documentation
on `Hugging Face Transformers <https://huggingface.co/transformers/>`_.
:type tokenizer_args: dict
:type use_fast: bool
:param dummy_ph: If True, methods of the prediction head will be replaced
with a dummy method. This is used to isolate lm run time from ph run time.
:type dummy_ph: bool
:param benchmarking: If True, a benchmarking object will be initialised within the class and
certain parts of the code will be timed for benchmarking. Should be kept
False if not benchmarking since these timing checkpoints require synchronization
of the asynchronous Pytorch operations and may slow down the model.
:type benchmarking: bool
:return: An instance of the Inferencer.
"""
if tokenizer_args is None:
tokenizer_args = {}
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
use_amp=None)
name = os.path.basename(model_name_or_path)
# a) either from local dir
if os.path.exists(model_name_or_path):
model = BaseAdaptiveModel.load(load_dir=model_name_or_path,
device=device,
strict=strict)
if task_type == "embeddings":
processor = InferenceProcessor.load_from_dir(
model_name_or_path)
else:
processor = Processor.load_from_dir(model_name_or_path)
# override processor attributes loaded from config file with inferencer params
processor.max_seq_len = max_seq_len
if hasattr(processor, "doc_stride"):
processor.doc_stride = doc_stride
# b) or from remote transformers model hub
else:
logger.info(
f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..."
)
if not task_type:
raise ValueError(
"Please specify the 'task_type' of the model you want to load from transformers. "
"Valid options for arg `task_type`:"
"'question_answering', 'embeddings', 'text_classification', 'ner'"
)
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device, task_type)
config = AutoConfig.from_pretrained(model_name_or_path)
tokenizer = Tokenizer.load(
model_name_or_path,
tokenizer_class=tokenizer_class,
use_fast=use_fast,
**tokenizer_args,
)
# TODO infer task_type automatically from config (if possible)
if task_type == "question_answering":
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
label_list=["start_token", "end_token"],
metric="squad",
data_dir="data",
doc_stride=doc_stride)
elif task_type == "embeddings":
processor = InferenceProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len)
elif task_type == "text_classification":
label_list = list(config.id2label[id]
for id in range(len(config.id2label)))
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir="data",
label_list=label_list,
label_column_name="label",
metric="acc",
quote_char='"',
)
elif task_type == "ner":
label_list = list(config.label2id.keys())
processor = NERProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir="data",
metric="seq_f1",
label_list=label_list)
else:
raise ValueError(
f"`task_type` {task_type} is not supported yet. "
f"Valid options for arg `task_type`: 'question_answering', "
f"'embeddings', 'text_classification', 'ner'")
if not isinstance(model, ONNXAdaptiveModel):
model, _ = optimize_model(model=model,
device=device,
local_rank=-1,
optimizer=None)
return cls(model,
processor,
task_type=task_type,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
extraction_strategy=extraction_strategy,
extraction_layer=extraction_layer,
s3e_stats=s3e_stats,
num_processes=num_processes,
disable_tqdm=disable_tqdm,
benchmarking=benchmarking,
dummy_ph=dummy_ph)
def load(
cls,
model_name_or_path,
revision=None,
batch_size=4,
gpu=False,
task_type=None,
return_class_probs=False,
strict=True,
max_seq_len=256,
doc_stride=128,
extraction_layer=None,
extraction_strategy=None,
s3e_stats=None,
num_processes=None,
disable_tqdm=False,
tokenizer_class=None,
use_fast=True,
tokenizer_args=None,
multithreading_rust=True,
dummy_ph=False,
benchmarking=False,
):
"""
Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by
1. specifying a public name from transformers' model hub (https://huggingface.co/models)
2. or pointing to a local directory it is saved in.
:param model_name_or_path: Local directory or public name of the model to load.
:type model_name_or_path: str
:param revision: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
:type revision: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param task_type: Type of task the model should be used for. Currently supporting:
"embeddings", "question_answering", "text_classification", "ner". More coming soon...
:param task_type: str
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:param max_seq_len: maximum length of one text sample
:type max_seq_len: int
:param doc_stride: Only QA: When input text is longer than max_seq_len it gets split into parts, strided by doc_stride
:type doc_stride: int
:param extraction_strategy: Strategy to extract vectors. Choices: 'cls_token' (sentence vector), 'reduce_mean'
(sentence vector), reduce_max (sentence vector), 'per_token' (individual token vectors)
:type extraction_strategy: str
:param extraction_layer: number of layer from which the embeddings shall be extracted. Default: -1 (very last layer).
:type extraction_layer: int
:param s3e_stats: Stats of a fitted S3E model as returned by `fit_s3e_on_corpus()`
(only needed for task_type="embeddings" and extraction_strategy = "s3e")
:type s3e_stats: dict
:param num_processes: the number of processes for `multiprocessing.Pool`. Set to value of 0 to disable
multiprocessing. Set to None to let Inferencer use all CPU cores minus one. If you want to
debug the Language Model, you might need to disable multiprocessing!
**Warning!** If you use multiprocessing you have to close the
`multiprocessing.Pool` again! To do so call
:func:`~farm.infer.Inferencer.close_multiprocessing_pool` after you are
done using this class. The garbage collector will not do this for you!
:type num_processes: int
:param disable_tqdm: Whether to disable tqdm logging (can get very verbose in multiprocessing)
:type disable_tqdm: bool
:param tokenizer_class: (Optional) Name of the tokenizer class to load (e.g. `BertTokenizer`)
:type tokenizer_class: str
:param use_fast: (Optional, True by default) Indicate if FARM should try to load the fast version of the tokenizer (True) or
use the Python one (False).
:type use_fast: bool
:param tokenizer_args: (Optional) Will be passed to the Tokenizer ``__init__`` method.
See https://huggingface.co/transformers/main_classes/tokenizer.html and detailed tokenizer documentation
on `Hugging Face Transformers <https://huggingface.co/transformers/>`_.
:type tokenizer_args: dict
:param multithreading_rust: Whether to allow multithreading in Rust, e.g. for FastTokenizers.
Note: Enabling multithreading in Rust AND multiprocessing in python might cause
deadlocks.
:type multithreading_rust: bool
:param dummy_ph: If True, methods of the prediction head will be replaced
with a dummy method. This is used to isolate lm run time from ph run time.
:type dummy_ph: bool
:param benchmarking: If True, a benchmarking object will be initialised within the class and
certain parts of the code will be timed for benchmarking. Should be kept
False if not benchmarking since these timing checkpoints require synchronization
of the asynchronous Pytorch operations and may slow down the model.
:type benchmarking: bool
:return: An instance of the Inferencer.
"""
if tokenizer_args is None:
tokenizer_args = {}
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
use_amp=None)
name = os.path.basename(model_name_or_path)
# a) either from local dir
if os.path.exists(model_name_or_path):
model = BaseAdaptiveModel.load(load_dir=model_name_or_path,
device=device,
strict=strict)
if task_type == "embeddings":
processor = InferenceProcessor.load_from_dir(
model_name_or_path)
else:
processor = Processor.load_from_dir(model_name_or_path)
# b) or from remote transformers model hub
else:
if not task_type:
raise ValueError(
"Please specify the 'task_type' of the model you want to load from transformers. "
"Valid options for arg `task_type`:"
"'question_answering', 'embeddings', 'text_classification', 'ner'"
)
model = AdaptiveModel.convert_from_transformers(
model_name_or_path,
revision=revision,
device=device,
task_type=task_type)
processor = Processor.convert_from_transformers(
model_name_or_path,
revision=revision,
task_type=task_type,
max_seq_len=max_seq_len,
doc_stride=doc_stride,
tokenizer_class=tokenizer_class,
tokenizer_args=tokenizer_args,
use_fast=use_fast)
# override processor attributes loaded from config or HF with inferencer params
processor.max_seq_len = max_seq_len
processor.multithreading_rust = multithreading_rust
if hasattr(processor, "doc_stride"):
assert doc_stride < max_seq_len, "doc_stride is longer than max_seq_len. This means that there will be gaps " \
"as the passage windows slide, causing the model to skip over parts of the document. " \
"Please set a lower value for doc_stride (Suggestions: doc_stride=128, max_seq_len=384) "
processor.doc_stride = doc_stride
return cls(model,
processor,
task_type=task_type,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
extraction_strategy=extraction_strategy,
extraction_layer=extraction_layer,
s3e_stats=s3e_stats,
num_processes=num_processes,
disable_tqdm=disable_tqdm,
benchmarking=benchmarking,
dummy_ph=dummy_ph)
def test_evaluation():
##########################
########## Settings
##########################
lang_model = "deepset/roberta-base-squad2"
do_lower_case = False
test_assertions = False
data_dir = Path("testsave/data/squad20")
evaluation_filename = "dev-v2.0.json"
device, n_gpu = initialize_device_settings(use_cuda=True)
# loading models and evals
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
model.prediction_heads[0].no_ans_boost = 0
model.prediction_heads[0].n_best = 1
model.prediction_heads[0].n_best_per_sample = 1
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
doc_stride=128,
)
starttime = time()
data_silo = DataSilo(processor=processor, batch_size=40 * n_gpu_factor)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
model, _ = optimize_model(model=model,
device=device,
local_rank=-1,
optimizer=None,
distributed=False,
use_amp=None)
evaluator = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# 1. Test FARM internal evaluation
results = evaluator.eval(model)
f1_score = results[0]["f1"] * 100
em_score = results[0]["EM"] * 100
tnacc = results[0]["top_n_accuracy"] * 100
elapsed = time() - starttime
print(results)
print(elapsed)
gold_EM = 78.4721
gold_f1 = 82.6671
gold_tnacc = 84.3594 # top 1 recall
gold_elapsed = 40 # 4x V100
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=f"FARM Eval changed for EM by: {em_score-gold_EM}")
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=f"FARM Eval changed for f1 score by: {f1_score-gold_f1}")
np.testing.assert_allclose(
tnacc,
gold_tnacc,
rtol=0.001,
err_msg=
f"FARM Eval changed for top 1 accuracy by: {tnacc-gold_tnacc}")
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"FARM Eval speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
if not np.allclose(f1_score, gold_f1, rtol=0.001):
error_messages.append(
f"FARM Eval changed for f1 score by: {round(f1_score - gold_f1, 4)}"
)
if not np.allclose(em_score, gold_EM, rtol=0.001):
error_messages.append(
f"FARM Eval changed for EM by: {round(em_score - gold_EM, 4)}")
if not np.allclose(tnacc, gold_tnacc, rtol=0.001):
error_messages.append(
f"FARM Eval changed for top 1 accuracy by: {round(tnacc-gold_tnacc, 4)}"
)
if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
error_messages.append(
f"FARM Eval speed changed significantly by: {round(elapsed - gold_elapsed, 4)} seconds"
)
benchmark_result = [{
"run": "FARM internal evaluation",
"f1_change": round(f1_score - gold_f1, 4),
"em_change": round(em_score - gold_EM, 4),
"tnacc_change": round(tnacc - gold_tnacc, 4),
"elapsed_change": round(elapsed - gold_elapsed, 4),
"f1": f1_score,
"em": em_score,
"tnacc": round(tnacc, 4),
"elapsed": elapsed,
"f1_gold": gold_f1,
"em_gold": gold_EM,
"tnacc_gold": gold_tnacc,
"elapsed_gold": gold_elapsed
}]
logger.info("\n\n" + pformat(benchmark_result[0]) + "\n")
# # 2. Test FARM predictions with outside eval script
starttime = time()
model = Inferencer(model=model,
processor=processor,
task_type="question_answering",
batch_size=40 * n_gpu_factor,
gpu=device.type == "cuda")
filename = data_dir / evaluation_filename
result = model.inference_from_file(file=filename,
return_json=False,
multiprocessing_chunksize=80)
results_squad = [x.to_squad_eval() for x in result]
model.close_multiprocessing_pool()
elapsed = time() - starttime
os.makedirs("../testsave", exist_ok=True)
write_squad_predictions(predictions=results_squad,
predictions_filename=filename,
out_filename="testsave/predictions.json")
script_params = {
"data_file": filename,
"pred_file": "testsave/predictions.json",
"na_prob_thresh": 1,
"na_prob_file": False,
"out_file": False
}
results_official = squad_evaluation.main(OPTS=DotMap(script_params))
f1_score = results_official["f1"]
em_score = results_official["exact"]
gold_EM = 79.878
gold_f1 = 82.917
gold_elapsed = 27 # 4x V100
print(elapsed)
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=
f"Eval with official script changed for EM by: {em_score - gold_EM}"
)
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=
f"Eval with official script changed for f1 score by: {f1_score - gold_f1}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"Inference speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
if not np.allclose(f1_score, gold_f1, rtol=0.001):
error_messages.append(
f"Eval with official script changed for f1 score by: {round(f1_score - gold_f1, 4)}"
)
if not np.allclose(em_score, gold_EM, rtol=0.001):
error_messages.append(
f"Eval with official script changed for EM by: {round(em_score - gold_EM, 4)}"
)
if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
error_messages.append(
f"Inference speed changed significantly by: {round(elapsed - gold_elapsed,4)} seconds"
)
benchmark_result.append({
"run": "outside eval script",
"f1_change": round(f1_score - gold_f1, 4),
"em_change": round(em_score - gold_EM, 4),
"tnacc_change": "-",
"elapsed_change": round(elapsed - gold_elapsed, 4),
"f1": f1_score,
"em": em_score,
"tnacc": "-",
"elapsed": elapsed,
"f1_gold": gold_f1,
"em_gold": gold_EM,
"tnacc_gold": "-",
"elapsed_gold": gold_elapsed
})
logger.info("\n\n" + pformat(benchmark_result[1]) + "\n")
return benchmark_result
def load(cls,
model_name_or_path,
batch_size=4,
gpu=False,
task_type=None,
return_class_probs=False,
strict=True,
max_seq_len=256,
doc_stride=128,
extraction_layer=None,
extraction_strategy=None,
s3e_stats=None,
num_processes=None,
disable_tqdm=False):
"""
Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by
1. specifying a public name from transformers' model hub (https://huggingface.co/models)
2. or pointing to a local directory it is saved in.
:param model_name_or_path: Local directory or public name of the model to load.
:type model_name_or_path: str
:param batch_size: Number of samples computed once per batch
:type batch_size: int
:param gpu: If GPU shall be used
:type gpu: bool
:param task_type: Type of task the model should be used for. Currently supporting:
"embeddings", "question_answering", "text_classification", "ner". More coming soon...
:param task_type: str
:param strict: whether to strictly enforce that the keys loaded from saved model match the ones in
the PredictionHead (see torch.nn.module.load_state_dict()).
Set to `False` for backwards compatibility with PHs saved with older version of FARM.
:type strict: bool
:param max_seq_len: maximum length of one text sample
:type max_seq_len: int
:param doc_stride: Only QA: When input text is longer than max_seq_len it gets split into parts, strided by doc_stride
:type doc_stride: int
:param extraction_strategy: Strategy to extract vectors. Choices: 'cls_token' (sentence vector), 'reduce_mean'
(sentence vector), reduce_max (sentence vector), 'per_token' (individual token vectors)
:type extraction_strategy: str
:param extraction_layer: number of layer from which the embeddings shall be extracted. Default: -1 (very last layer).
:type extraction_layer: int
:param s3e_stats: Stats of a fitted S3E model as returned by `fit_s3e_on_corpus()`
(only needed for task_type="embeddings" and extraction_strategy = "s3e")
:type s3e_stats: dict
:param num_processes: the number of processes for `multiprocessing.Pool`. Set to value of 0 to disable
multiprocessing. Set to None to let Inferencer use all CPU cores. If you want to
debug the Language Model, you might need to disable multiprocessing!
:type num_processes: int
:param disable_tqdm: Whether to disable tqdm logging (can get very verbose in multiprocessing)
:type disable_tqdm: bool
:return: An instance of the Inferencer.
"""
device, n_gpu = initialize_device_settings(use_cuda=gpu,
local_rank=-1,
use_amp=None)
name = os.path.basename(model_name_or_path)
# a) either from local dir
if os.path.exists(model_name_or_path):
model = BaseAdaptiveModel.load(load_dir=model_name_or_path,
device=device,
strict=strict)
if task_type == "embeddings":
processor = InferenceProcessor.load_from_dir(
model_name_or_path)
else:
processor = Processor.load_from_dir(model_name_or_path)
# override processor attributes loaded from config file with inferencer params
processor.max_seq_len = max_seq_len
if hasattr(processor, "doc_stride"):
processor.doc_stride = doc_stride
# b) or from remote transformers model hub
else:
logger.info(
f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..."
)
if not task_type:
raise ValueError(
"Please specify the 'task_type' of the model you want to load from transformers. "
"Valid options for arg `task_type`:"
"'question_answering', 'embeddings', 'text_classification', 'ner'"
)
model = AdaptiveModel.convert_from_transformers(
model_name_or_path, device, task_type)
config = AutoConfig.from_pretrained(model_name_or_path)
tokenizer = Tokenizer.load(model_name_or_path)
# TODO infer task_type automatically from config (if possible)
if task_type == "question_answering":
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
label_list=["start_token", "end_token"],
metric="squad",
data_dir="data",
doc_stride=doc_stride)
elif task_type == "embeddings":
processor = InferenceProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len)
elif task_type == "text_classification":
label_list = list(config.id2label[id]
for id in range(len(config.id2label)))
processor = TextClassificationProcessor(
tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir="data",
label_list=label_list,
label_column_name="label",
metric="acc",
quote_char='"',
)
elif task_type == "ner":
label_list = list(config.label2id.keys())
processor = NERProcessor(tokenizer=tokenizer,
max_seq_len=max_seq_len,
data_dir="data",
metric="seq_f1",
label_list=label_list)
else:
raise ValueError(
f"`task_type` {task_type} is not supported yet. "
f"Valid options for arg `task_type`: 'question_answering', "
f"'embeddings', 'text_classification', 'ner'")
return cls(model,
processor,
task_type=task_type,
batch_size=batch_size,
gpu=gpu,
name=name,
return_class_probs=return_class_probs,
extraction_strategy=extraction_strategy,
extraction_layer=extraction_layer,
s3e_stats=s3e_stats,
num_processes=num_processes,
disable_tqdm=disable_tqdm)
def question_answering_confidence():
##########################
########## Logging
##########################
logger = logging.getLogger(__name__)
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO)
# reduce verbosity from transformers library
logging.getLogger('transformers').setLevel(logging.WARNING)
##########################
########## Settings
##########################
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=True)
lang_model = "deepset/roberta-base-squad2"
do_lower_case = False
batch_size = 80
data_dir = Path("../data/squad20")
# We use the same file for dev and test set only for demo purposes
dev_filename = "dev-v2.0.json"
test_filename = "dev-v2.0.json"
accuracy_at = 3 # accuracy at n is useful for answers inside long documents
# 1.Create a tokenizer
tokenizer = Tokenizer.load(
pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
# 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=384,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=dev_filename,
test_filename=test_filename,
data_dir=data_dir,
doc_stride=192,
)
# 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets
data_silo = DataSilo(
processor=processor,
batch_size=batch_size)
# 4. Load pre-trained question-answering model
model = AdaptiveModel.convert_from_transformers(lang_model, device=device, task_type="question_answering")
model.connect_heads_with_processor(data_silo.processor.tasks, require_labels=True)
# Number of predictions the model will make per Question.
# The multiple predictions are used for evaluating top n recall.
model.prediction_heads[0].n_best = accuracy_at
# 5. The calibration of model confidence scores sets one parameter, which is called temperature and can be accessed through the prediction_head.
# This temperature is applied to each logit in the forward pass, where each logit is divided by the temperature.
# A softmax function is applied to the logits afterward to get confidence scores in the range [0,1].
# A temperature larger than 1 decreases the model’s confidence scores.
logger.info(f"Parameter used for temperature scaling of model confidence scores: {model.prediction_heads[0].temperature_for_confidence}")
# 6a. We can either manually set the temperature (default value is 1.0)...
model.prediction_heads[0].temperature_for_confidence = torch.nn.Parameter((torch.ones(1) * 1.0).to(device=device))
# 6b. ...or we can run the evaluator on the dev set and use it to calibrate confidence scores with a technique called temperature scaling.
# It will align the confidence scores with the model's accuracy based on the dev set data by tuning the temperature parameter.
# During the calibration, this parameter is automatically set internally as an attribute of the prediction head.
evaluator_dev = Evaluator(
data_loader=data_silo.get_data_loader("dev"),
tasks=data_silo.processor.tasks,
device=device
)
result_dev = evaluator_dev.eval(model, return_preds_and_labels=True, calibrate_conf_scores=True)
# evaluator_dev.log_results(result_dev, "Dev", logging=False, steps=len(data_silo.get_data_loader("dev")))
# 7. Optionally, run the evaluator on the test set to see how well the confidence scores are aligned with the model's accuracy
evaluator_test = Evaluator(
data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device
)
result_test = evaluator_test.eval(model, return_preds_and_labels=True)[0]
logger.info("Grouping predictions by confidence score and calculating metrics for each bin.")
em_per_bin, confidence_per_bin, count_per_bin = metrics_per_bin(result_test["preds"], result_test["labels"], num_bins=10)
for bin_number in range(10):
logger.info(f"Bin {bin_number} - exact match: {em_per_bin[bin_number]}, average confidence score: {confidence_per_bin[bin_number]}")
# 8. Hooray! You have a model with calibrated confidence scores.
# Store the model and the temperature parameter will be stored automatically as an attribute of the prediction head.
save_dir = Path("../saved_models/qa-confidence-tutorial")
model.save(save_dir)
processor.save(save_dir)
# 9. When making a prediction with the calibrated model, we could filter out predictions where the model is not confident enough
# To this end, load the stored model, which will automatically load the stored temperature parameter.
# The confidence scores are automatically adjusted based on this temperature parameter.
# For each prediction, we can check the model's confidence and decide whether to output the prediction or not.
inferencer = QAInferencer.load(save_dir, batch_size=40, gpu=True)
logger.info(f"Loaded model with stored temperature: {inferencer.model.prediction_heads[0].temperature_for_confidence}")
QA_input = [
{
"questions": ["Who counted the game among the best ever made?"],
"text": "Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created."
}]
result = inferencer.inference_from_dicts(dicts=QA_input, return_json=False)[0]
if result.prediction[0].confidence > 0.9:
print(result.prediction[0].answer)
else:
print("The confidence is not high enough to give an answer.")
import shutil
# get documents that we want to query
dirname = os.path.dirname(__file__)
configFile = os.path.join(dirname,'config.yaml')
import yaml
config = None
with open(configFile) as file:
config = yaml.safe_load(file)
sqlUrlFAQ = config["sqlUrlFAQ"]
model = AdaptiveModel.convert_from_transformers(
"deepset/sentence_bert", device="cpu", task_type="embeddings")
processor = Processor.convert_from_transformers(
"deepset/sentence_bert", task_type="embeddings", max_seq_len=384, doc_stride=128)
model.save("sentence_bert-saved")
processor.save("sentence_bert-saved")
document_store = FAISSDocumentStore(sql_url=sqlUrlFAQ)
# from haystack.retriever.dense import DensePassageRetriever
# retriever = DensePassageRetriever(document_store=document_store,
# query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
# passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
# max_seq_len_query=64,
# max_seq_len_passage=256,
