def import_downstream_models():
####################### loads a SQUAD finetuned model
# saves it as a FARM adaptive model
device, n_gpu = initialize_device_settings(use_cuda=True)
model = "bert-large-uncased-whole-word-masking-finetuned-squad"
save_dir = "saved_models/FARM-bert-large-uncased-whole-word-masking-finetuned-squad"
lm = Bert.load(model)
ph = QuestionAnsweringHead.load(model)
am = AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
am.save(save_dir)
# saves the processor associated with it, so you can use it in inference mode
# TODO load HF's tokenizer_config.json and adjust settings
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=model)
label_list = ["start_token", "end_token"]
metric = "squad"
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=label_list,
metric=metric,
data_dir="../data/squad20",
)
processor.save(save_dir)
def convert_from_transformers(cls, model_name_or_path, device, task_type):
"""
Load a (downstream) model from huggingface's transformers format. Use cases:
- continue training in FARM (e.g. take a squad QA model and fine-tune on your own data)
- compare models without switching frameworks
- use model directly for inference
:param model_name_or_path: local path of a saved model or name of a public one.
Exemplary public names:
- distilbert-base-uncased-distilled-squad
- deepset/bert-large-uncased-whole-word-masking-squad2
See https://huggingface.co/models for full list
:param device: "cpu" or "cuda"
:param task_type: One of :
- 'question_answering'
- 'text_classification'
- 'embeddings'
More tasks coming soon ...
:return: AdaptiveModel
"""
lm = LanguageModel.load(model_name_or_path)
#TODO Infer type of head automatically from config
if task_type == "question_answering":
ph = QuestionAnsweringHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
elif task_type == "text_classification":
ph = TextClassificationHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_sequence",
device=device)
elif task_type == "ner":
ph = TokenClassificationHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
elif task_type == "embeddings":
adaptive_model = cls(language_model=lm,
prediction_heads=[],
embeds_dropout_prob=0.1,
lm_output_types=["per_token", "per_sequence"],
device=device)
else:
raise NotImplementedError(
f"Huggingface's transformer models of type {task_type} are not supported yet"
)
return adaptive_model
def convert_from_transformers(cls, model_name_or_path, device, task_type, processor=None):
"""
Load a (downstream) model from huggingface's transformers format. Use cases:
- continue training in FARM (e.g. take a squad QA model and fine-tune on your own data)
- compare models without switching frameworks
- use model directly for inference
:param model_name_or_path: local path of a saved model or name of a public one.
Exemplary public names:
- distilbert-base-uncased-distilled-squad
- deepset/bert-large-uncased-whole-word-masking-squad2
See https://huggingface.co/models for full list
:param device: "cpu" or "cuda"
:param task_type: One of :
- 'question_answering'
- 'text_classification'
- 'embeddings'
More tasks coming soon ...
:param processor: populates prediction head with information coming from tasks
:type processor: Processor
:return: AdaptiveModel
"""
lm = LanguageModel.load(model_name_or_path)
# TODO Infer type of head automatically from config
if task_type == "question_answering":
ph = QuestionAnsweringHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm, prediction_heads=[ph], embeds_dropout_prob=0.1,
lm_output_types="per_token", device=device)
elif task_type == "text_classification":
if "roberta" in model_name_or_path:
# The RobertaClassificationhead has components: input2dense, dropout, tanh, dense2output
# The tanh function cannot be mapped to current FARM style linear Feed Forward PredictionHeads.
logger.error(
"Conversion for Text Classification with Roberta or XLMRoberta not possible at the moment.")
raise NotImplementedError
ph = TextClassificationHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm, prediction_heads=[ph], embeds_dropout_prob=0.1,
lm_output_types="per_sequence", device=device)
elif task_type == "ner":
ph = TokenClassificationHead.load(model_name_or_path)
adaptive_model = cls(language_model=lm, prediction_heads=[ph], embeds_dropout_prob=0.1,
lm_output_types="per_token", device=device)
elif task_type == "embeddings":
adaptive_model = cls(language_model=lm, prediction_heads=[], embeds_dropout_prob=0.1,
lm_output_types=["per_token", "per_sequence"], device=device)
else:
raise NotImplementedError(f"Huggingface's transformer models of type {task_type} are not supported yet")
if processor:
adaptive_model.connect_heads_with_processor(processor.tasks)
return adaptive_model
def convert_from_transformers(model_name_or_path,
device,
revision=None,
task_type=None,
processor=None,
**kwargs):
"""
Load a (downstream) model from huggingface's transformers format. Use cases:
- continue training in FARM (e.g. take a squad QA model and fine-tune on your own data)
- compare models without switching frameworks
- use model directly for inference
:param model_name_or_path: local path of a saved model or name of a public one.
Exemplary public names:
- distilbert-base-uncased-distilled-squad
- deepset/bert-large-uncased-whole-word-masking-squad2
See https://huggingface.co/models for full list
:param device: "cpu" or "cuda"
:param revision: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
:type revision: str
:param task_type: One of :
- 'question_answering'
- 'text_classification'
- 'embeddings'
More tasks coming soon ...
:param processor: populates prediction head with information coming from tasks
:type processor: Processor
:return: AdaptiveModel
"""
lm = LanguageModel.load(model_name_or_path,
revision=revision,
**kwargs)
if task_type is None:
# Infer task type from config
architecture = lm.model.config.architectures[0]
if "MaskedLM" in architecture:
task_type = "lm"
elif "QuestionAnswering" in architecture:
task_type = "question_answering"
elif "SequenceClassification" in architecture:
if lm.model.config.num_labels == 1:
task_type = "regression"
else:
task_type = "text_classification"
elif "TokenClassification" in architecture:
task_type = "ner"
else:
logger.error(
"Could not infer task type from model config. Please provide task type manually. "
"('lm', 'question_answering', 'regression', 'text_classification', 'ner' or 'embeddings')"
)
if task_type == "lm":
ph = BertLMHead.load(model_name_or_path,
revision=revision,
**kwargs)
adaptive_model = am.AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
elif task_type == "question_answering":
ph = QuestionAnsweringHead.load(model_name_or_path,
revision=revision,
**kwargs)
adaptive_model = am.AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
elif task_type == "regression":
if "roberta" in model_name_or_path:
# The RobertaClassificationHead has components: input2dense, dropout, tanh, dense2output
# The tanh function cannot be mapped to current FARM style linear Feed Forward PredictionHeads.
logger.error(
"Conversion for Regression with Roberta or XLMRoberta not possible at the moment."
)
raise NotImplementedError
ph = RegressionHead.load(model_name_or_path, **kwargs)
adaptive_model = am.AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_sequence",
device=device)
elif task_type == "text_classification":
if "roberta" in model_name_or_path:
# The RobertaClassificationHead has components: input2dense, dropout, tanh, dense2output
# The tanh function cannot be mapped to current FARM style linear Feed Forward PredictionHeads.
logger.error(
"Conversion for Text Classification with Roberta or XLMRoberta not possible at the moment."
)
raise NotImplementedError
ph = TextClassificationHead.load(model_name_or_path,
revision=revision,
**kwargs)
adaptive_model = am.AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_sequence",
device=device)
elif task_type == "ner":
ph = TokenClassificationHead.load(model_name_or_path,
revision=revision,
**kwargs)
adaptive_model = am.AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
elif task_type == "embeddings":
adaptive_model = am.AdaptiveModel(
language_model=lm,
prediction_heads=[],
embeds_dropout_prob=0.1,
lm_output_types=["per_token", "per_sequence"],
device=device)
if processor:
adaptive_model.connect_heads_with_processor(processor.tasks)
return adaptive_model
from farm.modeling.adaptive_model import AdaptiveModel
from farm.modeling.language_model import Bert
from farm.modeling.tokenization import BertTokenizer
from farm.modeling.prediction_head import QuestionAnsweringHead
from farm.data_handler.processor import SquadProcessor
from farm.utils import initialize_device_settings
####################### loads a SQUAD finetuned model
# saves it as a FARM adaptive model
device, n_gpu = initialize_device_settings(use_cuda=True)
model = "bert-large-uncased-whole-word-masking-finetuned-squad"
save_dir = "saved_models/FARM-bert-large-uncased-whole-word-masking-finetuned-squad"
lm = Bert.load(model)
ph = QuestionAnsweringHead.load(model)
am = AdaptiveModel(language_model=lm,
prediction_heads=[ph],
embeds_dropout_prob=0.1,
lm_output_types="per_token",
device=device)
am.save(save_dir)
# saves the processor associated with it, so you can use it in inference mode
# TODO load HF's tokenizer_config.json and adjust settings
tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path=model)
label_list = ["start_token", "end_token"]
metric = "squad"
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=label_list,
metric=metric,
data_dir="../data/squad20",