def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): config = kwargs.pop("config", None) if not isinstance(config, PretrainedConfig): config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) for config_class, model_class in MODEL_FOR_MULTITASK_SEQUENCE_CLASSIFICATION_MAPPING.items( ): if isinstance(config, config_class): return model_class.from_pretrained( pretrained_model_name_or_path, *model_args, config=config, **kwargs) raise ValueError( "Unrecognized configuration class {} for this kind of AutoModel: {}.\n" "Model type should be one of {}.".format( config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_MULTITASK_SEQUENCE_CLASSIFICATION_MAPPING. keys()), ))
def test_tokenizer_from_tokenizer_class(self): config = AutoConfig.from_pretrained(DUMMY_DIFF_TOKENIZER_IDENTIFIER) self.assertIsInstance(config, RobertaConfig) # Check that tokenizer_type ≠ model_type tokenizer = AutoTokenizer.from_pretrained(DUMMY_DIFF_TOKENIZER_IDENTIFIER, config=config) self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast)) self.assertEqual(tokenizer.vocab_size, 12)
def load(cls, model_name: str, tokenizer_name: str, cache_model: bool = True, adapter_size: int = 8, pretrained: bool = True) -> AutoModel: has_adapter = False if model_name.startswith("adapter"): has_adapter = True _, model_name = model_name.split("_") if model_name in cls._cache: return PretrainedAutoModel._cache[model_name] pretrained_config = AutoConfig.from_pretrained( model_name, output_hidden_states=True) if has_adapter: from src.modules.modeling_adapter_bert import AdapterBertModel pretrained_config.adapter_size = adapter_size model = AdapterBertModel.from_pretrained(model_name, config=pretrained_config) else: if pretrained: model = AutoModel.from_pretrained(model_name, config=pretrained_config) else: model = AutoModel.from_config(config=pretrained_config) if cache_model: cls._cache[model_name] = model return model
def from_pretrained(cls, pretrained_model_name_or_path, num_labels, *model_args, **kwargs): config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items(): if isinstance(config, config_class): config.num_labels = num_labels print(config_class) return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
def __init__(self, args): super().__init__() if not isinstance(args, argparse.Namespace): # eval mode assert isinstance(args, dict) args = argparse.Namespace(**args) # compute other fields according to args train_dataset = DependencyDataset(file_path=os.path.join( args.data_dir, f"train.{args.data_format}"), bert=args.bert_dir) # save these information to args to convene evaluation. args.pos_tags = train_dataset.pos_tags args.dep_tags = train_dataset.dep_tags args.ignore_pos_tags = train_dataset.ignore_pos_tags if args.ignore_punct else set( ) args.num_gpus = len( [x for x in str(args.gpus).split(",") if x.strip()]) if "," in args.gpus else int(args.gpus) args.t_total = (len(train_dataset) // (args.accumulate_grad_batches * args.num_gpus) + 1) * args.max_epochs self.save_hyperparameters(args) self.args = args bert_config = AutoConfig.from_pretrained(args.bert_dir) self.model_config = BiaffineDependencyConfig( bert_config=bert_config, pos_tags=args.pos_tags, dep_tags=args.dep_tags, tag_representation_dim=args.tag_representation_dim, arc_representation_dim=args.arc_representation_dim, pos_dim=args.pos_dim, biaf_dropout=args.biaf_dropout, additional_layer=args.additional_layer, additional_layer_type=args.additional_layer_type, additional_layer_dim=args.additional_layer_dim) self.model = BiaffineDependencyParser(args.bert_dir, config=self.model_config) if args.freeze_bert: for param in self.model.bert.parameters(): param.requires_grad = False self.train_stat = AttachmentScores() self.val_stat = AttachmentScores() self.test_stat = AttachmentScores() self.ignore_pos_tags = list(args.ignore_pos_tags)
def __init__( self, vocab: Vocabulary, model_name: str, num_labels: int, translation_factor: float = 0.5, seq_decoder: SeqDecoder = None, decoding_dim: int = 512, target_embedding_dim: int = 512, load_classifier: bool = False, transformer_trainable: bool = True, classifier_traninable: bool = True, dropout: float = 0.1, index: str = "transformer", label_namespace: str = "label", initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ) -> None: super().__init__(vocab, regularizer) if not num_labels: num_labels = vocab.get_vocab_size(namespace=label_namespace) config = AutoConfig.from_pretrained(model_name) config.num_labels = num_labels self.transformer = AutoModel.from_pretrained(model_name, config=config) for param in self.transformer.parameters(): param.requires_grad = transformer_trainable # Only BERT supports loading classifier layer currently if load_classifier: self.classifier = BertForNextSentencePrediction.from_pretrained( model_name, config=config).cls for param in self.classifier.parameters(): param.requires_grad = classifier_traninable else: classifier = torch.nn.Linear(config.hidden_size, config.num_labels) initializer(classifier) self.classifier = torch.nn.Sequential(torch.nn.Dropout(dropout), classifier) self._accuracy = CategoricalAccuracy() self._loss = torch.nn.CrossEntropyLoss() self._index = index self._label_namespace = label_namespace self._translation_factor = translation_factor self._seq_decoder = seq_decoder
def load(cls, model_name_or_path, batch_size=4, gpu=False, task_type=None, return_class_probs=False, strict=True, max_seq_len=256): """ Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by 1. specifying a public name from transformers' model hub (https://huggingface.co/models) 2. or pointing to a local directory it is saved in. :param model_name_or_path: Local directory or public name of the model to load. :type model_name_or_path: str :param batch_size: Number of samples computed once per batch :type batch_size: int :param gpu: If GPU shall be used :type gpu: bool :param task_type: Type of task the model should be used for. Currently supporting: "embeddings", "question_answering", "text_classification". More coming soon... :param task_type: str :param strict: whether to strictly enforce that the keys loaded from saved model match the ones in the PredictionHead (see torch.nn.module.load_state_dict()). Set to `False` for backwards compatibility with PHs saved with older version of FARM. :type strict: bool :return: An instance of the Inferencer. """ device, n_gpu = initialize_device_settings(use_cuda=gpu, local_rank=-1, use_amp=None) name = os.path.basename(model_name_or_path) # a) either from local dir if os.path.exists(model_name_or_path): model = AdaptiveModel.load(model_name_or_path, device, strict=strict) if task_type == "embeddings": processor = InferenceProcessor.load_from_dir( model_name_or_path) else: processor = Processor.load_from_dir(model_name_or_path) # b) or from remote transformers model hub else: logger.info( f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..." ) if not task_type: raise ValueError( "Please specify the 'task_type' of the model you want to load from transformers. " "Valid options for arg `task_type`:" "'question_answering', 'embeddings', 'text_classification'" ) model = AdaptiveModel.convert_from_transformers( model_name_or_path, device, task_type) config = AutoConfig.from_pretrained(model_name_or_path) tokenizer = Tokenizer.load(model_name_or_path) # TODO infer task_type automatically from config (if possible) if task_type == "question_answering": processor = SquadProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, label_list=["start_token", "end_token"], metric="squad", data_dir=None, ) elif task_type == "embeddings": processor = InferenceProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len) elif task_type == "text_classification": label_list = list(config.id2label[id] for id in range(len(config.id2label))) processor = TextClassificationProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir=None, label_list=label_list, label_column_name="label", metric="acc", quote_char='"', ) # elif task_type == "multilabel-classification": # # label_list = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"] # label_list = list(config.label2id.keys()) # # processor = TextClassificationProcessor(tokenizer=tokenizer, # max_seq_len=max_seq_len, # data_dir=None, # label_list=label_list, # label_column_name="label", # metric="acc", # quote_char='"', # multilabel=True, # ) elif task_type == "ner": label_list = list(config.label2id.keys()) processor = NERProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir=None, metric="seq_f1", label_list=label_list) else: raise ValueError( f"`task_type` {task_type} is not supported yet. " f"Valid options for arg `task_type`: 'question_answering', 'embeddings', 'text_classification'" ) return cls( model, processor, batch_size=batch_size, gpu=gpu, name=name, return_class_probs=return_class_probs, )
def from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs): r""" Instantiates one of the base model classes of the library from a pre-trained model configuration. The `from_pretrained()` method takes care of returning the correct model class instance based on the `model_type` property of the config object, or when it's missing, falling back to using pattern matching on the `pretrained_model_name_or_path` string: - `t5`: TFT5Model (T5 model) - `distilbert`: TFDistilBertModel (DistilBERT model) - `roberta`: TFRobertaModel (RoBERTa model) - `bert`: TFTFBertModel (Bert model) - `openai-gpt`: TFOpenAIGPTModel (OpenAI GPT model) - `gpt2`: TFGPT2Model (OpenAI GPT-2 model) - `transfo-xl`: TFTransfoXLModel (Transformer-XL model) - `xlnet`: TFXLNetModel (XLNet model) - `ctrl`: TFCTRLModel (CTRL model) Params: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. from_pt: (`Optional`) Boolean Set to True if the Checkpoint is a PyTorch checkpoint. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model configuration should be cached if the standard cache should not be used. force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. resume_download: (`optional`) boolean, default False: Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. output_loading_info: (`optional`) boolean: Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. kwargs: (`optional`) Remaining dictionary of keyword arguments: Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: model = TFAutoModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. model = TFAutoModel.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` model = TFAutoModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') model = TFAutoModel.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ config = kwargs.pop("config", None) if not isinstance(config, PretrainedConfig): config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) for config_class, model_class in TF_MODEL_MAPPING.items(): if isinstance(config, config_class): return from_pretrained_detailed(model_class, pretrained_model_name_or_path, *model_args, config=config, **kwargs) raise ValueError( "Unrecognized configuration class {} for this kind of TFAutoModel.\n" "Model type should be one of {}.".format( config.__class__, ", ".join(c.__name__ for c in TF_MODEL_MAPPING.keys())))
def load(cls, model_name_or_path, batch_size=4, gpu=False, task_type=None, return_class_probs=False, strict=True, max_seq_len=256, doc_stride=128, extraction_layer=None, extraction_strategy=None, s3e_stats=None, num_processes=None, disable_tqdm=False): """ Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by 1. specifying a public name from transformers' model hub (https://huggingface.co/models) 2. or pointing to a local directory it is saved in. :param model_name_or_path: Local directory or public name of the model to load. :type model_name_or_path: str :param batch_size: Number of samples computed once per batch :type batch_size: int :param gpu: If GPU shall be used :type gpu: bool :param task_type: Type of task the model should be used for. Currently supporting: "embeddings", "question_answering", "text_classification", "ner". More coming soon... :param task_type: str :param strict: whether to strictly enforce that the keys loaded from saved model match the ones in the PredictionHead (see torch.nn.module.load_state_dict()). Set to `False` for backwards compatibility with PHs saved with older version of FARM. :type strict: bool :param max_seq_len: maximum length of one text sample :type max_seq_len: int :param doc_stride: Only QA: When input text is longer than max_seq_len it gets split into parts, strided by doc_stride :type doc_stride: int :param extraction_strategy: Strategy to extract vectors. Choices: 'cls_token' (sentence vector), 'reduce_mean' (sentence vector), reduce_max (sentence vector), 'per_token' (individual token vectors) :type extraction_strategy: str :param extraction_layer: number of layer from which the embeddings shall be extracted. Default: -1 (very last layer). :type extraction_layer: int :param s3e_stats: Stats of a fitted S3E model as returned by `fit_s3e_on_corpus()` (only needed for task_type="embeddings" and extraction_strategy = "s3e") :type s3e_stats: dict :param num_processes: the number of processes for `multiprocessing.Pool`. Set to value of 0 to disable multiprocessing. Set to None to let Inferencer use all CPU cores. If you want to debug the Language Model, you might need to disable multiprocessing! :type num_processes: int :param disable_tqdm: Whether to disable tqdm logging (can get very verbose in multiprocessing) :type disable_tqdm: bool :return: An instance of the Inferencer. """ device, n_gpu = initialize_device_settings(use_cuda=gpu, local_rank=-1, use_amp=None) name = os.path.basename(model_name_or_path) # a) either from local dir if os.path.exists(model_name_or_path): model = BaseAdaptiveModel.load(load_dir=model_name_or_path, device=device, strict=strict) if task_type == "embeddings": processor = InferenceProcessor.load_from_dir( model_name_or_path) else: processor = Processor.load_from_dir(model_name_or_path) # override processor attributes loaded from config file with inferencer params processor.max_seq_len = max_seq_len if hasattr(processor, "doc_stride"): processor.doc_stride = doc_stride # b) or from remote transformers model hub else: logger.info( f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..." ) if not task_type: raise ValueError( "Please specify the 'task_type' of the model you want to load from transformers. " "Valid options for arg `task_type`:" "'question_answering', 'embeddings', 'text_classification', 'ner'" ) model = AdaptiveModel.convert_from_transformers( model_name_or_path, device, task_type) config = AutoConfig.from_pretrained(model_name_or_path) tokenizer = Tokenizer.load(model_name_or_path) # TODO infer task_type automatically from config (if possible) if task_type == "question_answering": processor = SquadProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, label_list=["start_token", "end_token"], metric="squad", data_dir="data", doc_stride=doc_stride) elif task_type == "embeddings": processor = InferenceProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len) elif task_type == "text_classification": label_list = list(config.id2label[id] for id in range(len(config.id2label))) processor = TextClassificationProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir="data", label_list=label_list, label_column_name="label", metric="acc", quote_char='"', ) elif task_type == "ner": label_list = list(config.label2id.keys()) processor = NERProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir="data", metric="seq_f1", label_list=label_list) else: raise ValueError( f"`task_type` {task_type} is not supported yet. " f"Valid options for arg `task_type`: 'question_answering', " f"'embeddings', 'text_classification', 'ner'") return cls(model, processor, task_type=task_type, batch_size=batch_size, gpu=gpu, name=name, return_class_probs=return_class_probs, extraction_strategy=extraction_strategy, extraction_layer=extraction_layer, s3e_stats=s3e_stats, num_processes=num_processes, disable_tqdm=disable_tqdm)
def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): r""" Instantiates one of the sequence classification model classes of the library from a pre-trained model configuration. See multimodal_transformers.py for supported transformer models The `from_pretrained()` method takes care of returning the correct model class instance based on the `model_type` property of the config object, or when it's missing, falling back to using pattern matching on the `pretrained_model_name_or_path` string: The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` Args: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaining positional arguments will be passed to the underlying model's ``__init__`` method config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model configuration should be cached if the standard cache should not be used. force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. resume_download: (`optional`) boolean, default False: Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. output_loading_info: (`optional`) boolean: Set to ``True`` to also return a dictionary containing missing keys, unexpected keys and error messages. kwargs: (`optional`) Remaining dictionary of keyword arguments: These arguments will be passed to the configuration and the model. Examples:: model = AutoModelWithTabular.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. model = AutoModelWithTabular.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') model = AutoModelWithTabular.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ config = kwargs.pop("config", None) if not isinstance(config, PretrainedConfig): config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) for config_class, model_class in MODEL_FOR_SEQUENCE_W_TABULAR_CLASSIFICATION_MAPPING.items( ): if isinstance(config, config_class): return model_class.from_pretrained( pretrained_model_name_or_path, *model_args, config=config, **kwargs) raise ValueError( "Unrecognized configuration class {} for this kind of AutoModel: {}.\n" "Model type should be one of {}.".format( config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_SEQUENCE_W_TABULAR_CLASSIFICATION_MAPPING. keys()), ))
def test_config_model_type_from_local_file(self): config = AutoConfig.from_pretrained(SAMPLE_ROBERTA_CONFIG) self.assertIsInstance(config, RobertaConfig)
def test_config_from_model_shortcut(self): config = AutoConfig.from_pretrained("bert-base-uncased") self.assertIsInstance(config, BertConfig)
def load( cls, model_name_or_path, batch_size=4, gpu=False, task_type=None, return_class_probs=False, strict=True, max_seq_len=256, doc_stride=128, extraction_layer=None, extraction_strategy=None, s3e_stats=None, num_processes=None, disable_tqdm=False, tokenizer_class=None, use_fast=False, tokenizer_args=None, dummy_ph=False, benchmarking=False, ): """ Load an Inferencer incl. all relevant components (model, tokenizer, processor ...) either by 1. specifying a public name from transformers' model hub (https://huggingface.co/models) 2. or pointing to a local directory it is saved in. :param model_name_or_path: Local directory or public name of the model to load. :type model_name_or_path: str :param batch_size: Number of samples computed once per batch :type batch_size: int :param gpu: If GPU shall be used :type gpu: bool :param task_type: Type of task the model should be used for. Currently supporting: "embeddings", "question_answering", "text_classification", "ner". More coming soon... :param task_type: str :param strict: whether to strictly enforce that the keys loaded from saved model match the ones in the PredictionHead (see torch.nn.module.load_state_dict()). Set to `False` for backwards compatibility with PHs saved with older version of FARM. :type strict: bool :param max_seq_len: maximum length of one text sample :type max_seq_len: int :param doc_stride: Only QA: When input text is longer than max_seq_len it gets split into parts, strided by doc_stride :type doc_stride: int :param extraction_strategy: Strategy to extract vectors. Choices: 'cls_token' (sentence vector), 'reduce_mean' (sentence vector), reduce_max (sentence vector), 'per_token' (individual token vectors) :type extraction_strategy: str :param extraction_layer: number of layer from which the embeddings shall be extracted. Default: -1 (very last layer). :type extraction_layer: int :param s3e_stats: Stats of a fitted S3E model as returned by `fit_s3e_on_corpus()` (only needed for task_type="embeddings" and extraction_strategy = "s3e") :type s3e_stats: dict :param num_processes: the number of processes for `multiprocessing.Pool`. Set to value of 0 to disable multiprocessing. Set to None to let Inferencer use all CPU cores minus one. If you want to debug the Language Model, you might need to disable multiprocessing! **Warning!** If you use multiprocessing you have to close the `multiprocessing.Pool` again! To do so call :func:`~farm.infer.Inferencer.close_multiprocessing_pool` after you are done using this class. The garbage collector will not do this for you! :type num_processes: int :param disable_tqdm: Whether to disable tqdm logging (can get very verbose in multiprocessing) :type disable_tqdm: bool :param tokenizer_class: (Optional) Name of the tokenizer class to load (e.g. `BertTokenizer`) :type tokenizer_class: str :param use_fast: (Optional, False by default) Indicate if FARM should try to load the fast version of the tokenizer (True) or use the Python one (False). :param tokenizer_args: (Optional) Will be passed to the Tokenizer ``__init__`` method. See https://huggingface.co/transformers/main_classes/tokenizer.html and detailed tokenizer documentation on `Hugging Face Transformers <https://huggingface.co/transformers/>`_. :type tokenizer_args: dict :type use_fast: bool :param dummy_ph: If True, methods of the prediction head will be replaced with a dummy method. This is used to isolate lm run time from ph run time. :type dummy_ph: bool :param benchmarking: If True, a benchmarking object will be initialised within the class and certain parts of the code will be timed for benchmarking. Should be kept False if not benchmarking since these timing checkpoints require synchronization of the asynchronous Pytorch operations and may slow down the model. :type benchmarking: bool :return: An instance of the Inferencer. """ if tokenizer_args is None: tokenizer_args = {} device, n_gpu = initialize_device_settings(use_cuda=gpu, local_rank=-1, use_amp=None) name = os.path.basename(model_name_or_path) # a) either from local dir if os.path.exists(model_name_or_path): model = BaseAdaptiveModel.load(load_dir=model_name_or_path, device=device, strict=strict) if task_type == "embeddings": processor = InferenceProcessor.load_from_dir( model_name_or_path) else: processor = Processor.load_from_dir(model_name_or_path) # override processor attributes loaded from config file with inferencer params processor.max_seq_len = max_seq_len if hasattr(processor, "doc_stride"): assert doc_stride < max_seq_len, "doc_stride is longer than max_seq_len. This means that there will be gaps " \ "as the passage windows slide, causing the model to skip over parts of the document. "\ "Please set a lower value for doc_stride (Suggestions: doc_stride=128, max_seq_len=384) " processor.doc_stride = doc_stride # b) or from remote transformers model hub else: logger.info( f"Could not find `{model_name_or_path}` locally. Try to download from model hub ..." ) if not task_type: raise ValueError( "Please specify the 'task_type' of the model you want to load from transformers. " "Valid options for arg `task_type`:" "'question_answering', 'embeddings', 'text_classification', 'ner'" ) model = AdaptiveModel.convert_from_transformers( model_name_or_path, device, task_type) config = AutoConfig.from_pretrained(model_name_or_path) tokenizer = Tokenizer.load( model_name_or_path, tokenizer_class=tokenizer_class, use_fast=use_fast, **tokenizer_args, ) # TODO infer task_type automatically from config (if possible) if task_type == "question_answering": processor = SquadProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, label_list=["start_token", "end_token"], metric="squad", data_dir="data", doc_stride=doc_stride) elif task_type == "embeddings": processor = InferenceProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len) elif task_type == "text_classification": label_list = list(config.id2label[id] for id in range(len(config.id2label))) processor = TextClassificationProcessor( tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir="data", label_list=label_list, label_column_name="label", metric="acc", quote_char='"', ) elif task_type == "ner": label_list = list(config.label2id.keys()) processor = NERProcessor(tokenizer=tokenizer, max_seq_len=max_seq_len, data_dir="data", metric="seq_f1", label_list=label_list) else: raise ValueError( f"`task_type` {task_type} is not supported yet. " f"Valid options for arg `task_type`: 'question_answering', " f"'embeddings', 'text_classification', 'ner'") if not isinstance(model, ONNXAdaptiveModel): model, _ = optimize_model(model=model, device=device, local_rank=-1, optimizer=None) return cls(model, processor, task_type=task_type, batch_size=batch_size, gpu=gpu, name=name, return_class_probs=return_class_probs, extraction_strategy=extraction_strategy, extraction_layer=extraction_layer, s3e_stats=s3e_stats, num_processes=num_processes, disable_tqdm=disable_tqdm, benchmarking=benchmarking, dummy_ph=dummy_ph)
def convert_to_onnx(cls, model_name, output_path, task_type, convert_to_float16=False, quantize=False, opset_version=11): """ Convert a PyTorch model from transformers hub to an ONNX Model. :param model_name: transformers model name :type model_name: str :param output_path: output Path to write the converted to :type output_path: Path :param task_type: Type of task for the model. Available options: "embeddings", "question_answering", "text_classification", "ner". :param convert_to_float16: By default, the model use float32 precision. With half precision of flaot16, inference should be faster on Nvidia GPUs with Tensor core like T4 or V100. On older GPUs, float32 might be more performant. :type convert_to_float16: bool :param quantize: convert floating point number to integers :type quantize: bool :param opset_version: ONNX opset version :type opset_version: int :return: """ language_model_class = LanguageModel.get_language_model_class( model_name) if language_model_class not in ["Bert", "Roberta", "XLMRoberta"]: raise Exception( "The current ONNX conversion only support 'BERT', 'RoBERTa', and 'XLMRoberta' models." ) task_type_to_pipeline_map = { "question_answering": "question-answering", "embeddings": "feature-extraction", "ner": "ner" } convert(pipeline_name=task_type_to_pipeline_map[task_type], framework="pt", model=model_name, output=output_path / "model.onnx", opset=opset_version, use_external_format=True if language_model_class is "XLMRoberta" else False) # save processor & model config files that are needed when loading the model with the FARM Inferencer processor = Processor.convert_from_transformers( tokenizer_name_or_path=model_name, task_type=task_type, max_seq_len=256, doc_stride=128, use_fast=True) processor.save(output_path) model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type=task_type) model.save(output_path) os.remove( output_path / "language_model.bin" ) # remove the actual PyTorch model(only configs are required) onnx_model_config = { "task_type": task_type, "onnx_opset_version": opset_version, "language_model_class": language_model_class, "language": model.language_model.language } with open(output_path / "onnx_model_config.json", "w") as f: json.dump(onnx_model_config, f) if convert_to_float16: from onnxruntime_tools import optimizer config = AutoConfig.from_pretrained(model_name) optimized_model = optimizer.optimize_model( input=str(output_path / "model.onnx"), model_type='bert', num_heads=config.num_hidden_layers, hidden_size=config.hidden_size) optimized_model.convert_model_float32_to_float16() optimized_model.save_model_to_file("model.onnx") if quantize: quantize_model(output_path / "model.onnx")
def test_config_model_type_from_model_identifier(self): config = AutoConfig.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER) self.assertIsInstance(config, RobertaConfig)
def pipeline(task: str, model: Optional = None, config: Optional[Union[str, PretrainedConfig]] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, modelcard: Optional[Union[str, ModelCard]] = None, device=torch.device("cpu"), **kwargs) -> Pipeline: """ Utility factory method to build a pipeline. Pipeline are made of: A Tokenizer instance in charge of mapping raw textual input to token A Model instance Some (optional) post processing for enhancing model's output Examples: pipeline('sentiment-analysis') """ # Register all the supported task here SUPPORTED_TASKS = { "sentiment-analysis": { "impl": TextClassificationPipelineMod, "pt": AutoModelForSequenceClassification, # if is_torch_available() else None, "default": { "model": { "pt": "distilbert-base-uncased-finetuned-sst-2-english", }, "config": "distilbert-base-uncased-finetuned-sst-2-english", "tokenizer": "distilbert-base-uncased", }, }, } # Retrieve the task if task not in SUPPORTED_TASKS: raise KeyError("Unknown task {}, available tasks are {}".format( task, list(SUPPORTED_TASKS.keys()))) framework = "pt" #get_framework(model) targeted_task = SUPPORTED_TASKS[task] task, model_class = targeted_task["impl"], targeted_task[framework] # Use default model/config/tokenizer for the task if no model is provided if model is None: models, config, tokenizer = tuple(targeted_task["default"].values()) model = models[framework] # Try to infer tokenizer from model or config name (if provided as str) if tokenizer is None: if isinstance(model, str) and model in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: tokenizer = model elif isinstance(config, str) and config in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: tokenizer = config else: # Impossible to guest what is the right tokenizer here raise Exception( "Impossible to guess which tokenizer to use. " "Please provided a PretrainedTokenizer class or a path/url/shortcut name to a pretrained tokenizer." ) # Try to infer modelcard from model or config name (if provided as str) if modelcard is None: # Try to fallback on one of the provided string for model or config (will replace the suffix) if isinstance(model, str): modelcard = model elif isinstance(config, str): modelcard = config # Instantiate tokenizer if needed if isinstance(tokenizer, str): tokenizer = AutoTokenizer.from_pretrained(tokenizer) # Instantiate config if needed if isinstance(config, str): config = AutoConfig.from_pretrained(config) # Instantiate modelcard if needed if isinstance(modelcard, str): modelcard = ModelCard.from_pretrained(modelcard) # Instantiate model if needed if isinstance(model, str): # Handle transparent TF/PT model conversion model_kwargs = {} if framework == "pt" and model.endswith(".h5"): model_kwargs["from_tf"] = True logger.warning( "Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. " "Trying to load the model with PyTorch.") model = model_class.from_pretrained(model, config=config, **model_kwargs) model = model.to(device) model.device = device return task(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, **kwargs)
def test_config_for_model_str(self): config = AutoConfig.for_model("roberta") self.assertIsInstance(config, RobertaConfig)
def _build_encoder_config(self, config): return AutoConfig.from_pretrained( self.config.bert_model_name, **OmegaConf.to_container(self.config))
def pipeline(task: str, model: Optional = None, config: Optional[Union[str, PretrainedConfig]] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, framework: Optional[str] = None, **kwargs) -> Pipeline: """ Utility factory method to build a :class:`~transformers.Pipeline`. Pipelines are made of: - A :doc:`tokenizer <tokenizer>` in charge of mapping raw textual input to token. - A :doc:`model <model>` to make predictions from the inputs. - Some (optional) post processing for enhancing model's output. Args: task (:obj:`str`): The task defining which pipeline will be returned. Currently accepted tasks are: - :obj:`"feature-extraction"`: will return a :class:`~transformers.FeatureExtractionPipeline`. - :obj:`"sentiment-analysis"`: will return a :class:`~transformers.TextClassificationPipeline`. - :obj:`"ner"`: will return a :class:`~transformers.TokenClassificationPipeline`. - :obj:`"question-answering"`: will return a :class:`~transformers.QuestionAnsweringPipeline`. - :obj:`"fill-mask"`: will return a :class:`~transformers.FillMaskPipeline`. - :obj:`"summarization"`: will return a :class:`~transformers.SummarizationPipeline`. - :obj:`"translation_xx_to_yy"`: will return a :class:`~transformers.TranslationPipeline`. - :obj:`"text-generation"`: will return a :class:`~transformers.TextGenerationPipeline`. - :obj:`"conversation"`: will return a :class:`~transformers.ConversationalPipeline`. model (:obj:`str` or :obj:`~transformers.PreTrainedModel` or :obj:`~transformers.TFPreTrainedModel`, `optional`): The model that will be used by the pipeline to make predictions. This can be a model identifier or an actual instance of a pretrained model inheriting from :class:`~transformers.PreTrainedModel` (for PyTorch) or :class:`~transformers.TFPreTrainedModel` (for TensorFlow). If not provided, the default for the :obj:`task` will be loaded. config (:obj:`str` or :obj:`~transformers.PretrainedConfig`, `optional`): The configuration that will be used by the pipeline to instantiate the model. This can be a model identifier or an actual pretrained model configuration inheriting from :class:`~transformers.PretrainedConfig`. If not provided, the default for the :obj:`task` will be loaded. tokenizer (:obj:`str` or :obj:`~transformers.PreTrainedTokenizer`, `optional`): The tokenizer that will be used by the pipeline to encode data for the model. This can be a model identifier or an actual pretrained tokenizer inheriting from :class:`~transformers.PreTrainedTokenizer`. If not provided, the default for the :obj:`task` will be loaded. framework (:obj:`str`, `optional`): The framework to use, either :obj:`"pt"` for PyTorch or :obj:`"tf"` for TensorFlow. The specified framework must be installed. If no framework is specified, will default to the one currently installed. If no framework is specified and both frameworks are installed, will default to the framework of the :obj:`model`, or to PyTorch if no model is provided. kwargs: Additional keyword arguments passed along to the specific pipeline init (see the documentation for the corresponding pipeline class for possible values). Returns: :class:`~transformers.Pipeline`: A suitable pipeline for the task. Examples:: >>> from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer >>> # Sentiment analysis pipeline >>> pipeline('sentiment-analysis') >>> # Question answering pipeline, specifying the checkpoint identifier >>> pipeline('question-answering', model='distilbert-base-cased-distilled-squad', tokenizer='bert-base-cased') >>> # Named entity recognition pipeline, passing in a specific model and tokenizer >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english") >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") >>> pipeline('ner', model=model, tokenizer=tokenizer) """ framework = 'pt' task_class, model_class = QuestionAnsweringPipeline, AutoModelForQuestionAnswering # Use default model/config/tokenizer for the task if no model is provided if model is None: model = 'distilbert-base-cased-distilled-squad' # Try to infer tokenizer from model or config name (if provided as str) if tokenizer is None: if isinstance(model, str): tokenizer = model elif isinstance(config, str): tokenizer = config else: # Impossible to guest what is the right tokenizer here raise Exception( "Impossible to guess which tokenizer to use. " "Please provided a PretrainedTokenizer class or a path/identifier to a pretrained tokenizer." ) modelcard = None # Try to infer modelcard from model or config name (if provided as str) if isinstance(model, str): modelcard = model elif isinstance(config, str): modelcard = config # Instantiate tokenizer if needed if isinstance(tokenizer, (str, tuple)): if isinstance(tokenizer, tuple): # For tuple we have (tokenizer name, {kwargs}) tokenizer = AutoTokenizer.from_pretrained(tokenizer[0], **tokenizer[1]) else: tokenizer = AutoTokenizer.from_pretrained(tokenizer) # Instantiate config if needed if isinstance(config, str): config = AutoConfig.from_pretrained(config) # Instantiate modelcard if needed if isinstance(modelcard, str): modelcard = ModelCard.from_pretrained(modelcard) # Instantiate model if needed if isinstance(model, str): # Handle transparent TF/PT model conversion model_kwargs = {} if framework == "pt" and model.endswith(".h5"): model_kwargs["from_tf"] = True logger.warning( "Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. " "Trying to load the model with PyTorch.") elif framework == "tf" and model.endswith(".bin"): model_kwargs["from_pt"] = True logger.warning( "Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. " "Trying to load the model with Tensorflow.") model = model_class.from_pretrained(model, config=config, **model_kwargs) return task_class(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, task=task, **kwargs)