def test_fold_long_sequences(self): # Let's just say [PAD] is 0, [CLS] is 1, and [SEP] is 2 token_ids = torch.LongTensor([ [1, 101, 102, 103, 104, 2, 1, 105, 106, 107, 108, 2, 1, 109, 2], [1, 201, 202, 203, 204, 2, 1, 205, 206, 207, 208, 2, 0, 0, 0], [1, 301, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ]) # Shape: [3, 15] segment_concat_mask = (token_ids > 0).long() folded_token_ids = torch.LongTensor([ [1, 101, 102, 103, 104, 2], [1, 105, 106, 107, 108, 2], [1, 109, 2, 0, 0, 0], [1, 201, 202, 203, 204, 2], [1, 205, 206, 207, 208, 2], [0, 0, 0, 0, 0, 0], [1, 301, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ]) folded_segment_concat_mask = (folded_token_ids > 0).long() token_embedder = PretrainedTransformerEmbedder("bert-base-uncased", max_length=6) ( folded_token_ids_out, folded_segment_concat_mask_out, _, ) = token_embedder._fold_long_sequences(token_ids, segment_concat_mask) assert (folded_token_ids_out == folded_token_ids).all() assert (folded_segment_concat_mask_out == folded_segment_concat_mask ).all()
def test_unfold_long_sequences(self): # Let's just say [PAD] is 0, [CLS] is xxx1, and [SEP] is xxx2 # We assume embeddings are 1-dim and are the same as indices embeddings = torch.LongTensor([ [1001, 101, 102, 103, 104, 1002], [1011, 105, 106, 107, 108, 1012], [1021, 109, 1022, 0, 0, 0], [2001, 201, 202, 203, 204, 2002], [2011, 205, 206, 207, 208, 2012], [0, 0, 0, 0, 0, 0], [3001, 301, 3002, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ]).unsqueeze(-1) mask = (embeddings > 0).long() unfolded_embeddings = torch.LongTensor([ [1001, 101, 102, 103, 104, 105, 106, 107, 108, 109, 1022], [2001, 201, 202, 203, 204, 205, 206, 207, 208, 2012, 0], [3001, 301, 3002, 0, 0, 0, 0, 0, 0, 0, 0], ]).unsqueeze(-1) token_embedder = PretrainedTransformerEmbedder("bert-base-uncased", max_length=6) unfolded_embeddings_out = token_embedder._unfold_long_sequences( embeddings, mask, unfolded_embeddings.size(0), 15) assert (unfolded_embeddings_out == unfolded_embeddings).all()
def __init__( self, model_name: str, max_length: int = None, train_parameters: bool = True, last_layer_only: bool = True, override_weights_file: Optional[str] = None, override_weights_strip_prefix: Optional[str] = None, load_weights: bool = True, gradient_checkpointing: Optional[bool] = None, tokenizer_kwargs: Optional[Dict[str, Any]] = None, transformer_kwargs: Optional[Dict[str, Any]] = None, sub_token_mode: Optional[str] = "avg", ) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length=max_length, train_parameters=train_parameters, last_layer_only=last_layer_only, override_weights_file=override_weights_file, override_weights_strip_prefix=override_weights_strip_prefix, load_weights=load_weights, gradient_checkpointing=gradient_checkpointing, tokenizer_kwargs=tokenizer_kwargs, transformer_kwargs=transformer_kwargs, ) self.sub_token_mode = sub_token_mode
def __init__(self, model_name: str, max_length: int = None, requires_grad: bool = True) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length, requires_grad)
def __init__(self, model_name: str, max_length: int = None, train_parameters: bool = True) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length=max_length, train_parameters=train_parameters)
def test_embeddings_resize(self): regular_token_embedder = PretrainedTransformerEmbedder( "bert-base-cased") assert (regular_token_embedder.transformer_model.embeddings. word_embeddings.num_embeddings == 28996) tokenizer_kwargs = {"additional_special_tokens": ["<NEW_TOKEN>"]} enhanced_token_embedder = PretrainedTransformerEmbedder( "bert-base-cased", tokenizer_kwargs=tokenizer_kwargs) assert (enhanced_token_embedder.transformer_model.embeddings. word_embeddings.num_embeddings == 28997)
def __init__(self, vocab, num_labels): super().__init__(vocab) self.bert_embedder = PretrainedTransformerEmbedder('bert-base-uncased') self.pooler = ClsPooler(self.bert_embedder.get_output_dim()) self.linear = torch.nn.Sequential( torch.nn.Dropout(0.1), torch.nn.Linear(in_features=768, out_features=num_labels)) self.accuracy = CategoricalAccuracy() self.loss_function = torch.nn.CrossEntropyLoss()
def emb_returner(config): if config.bert_name == 'japanese-bert': huggingface_model = 'cl-tohoku/bert-base-japanese' else: huggingface_model = 'dummy' print(config.bert_name, 'are not supported') exit() bert_embedder = PretrainedTransformerEmbedder( model_name="cl-tohoku/bert-base-japanese") return bert_embedder, bert_embedder.get_output_dim( ), BasicTextFieldEmbedder({'tokens': bert_embedder})
def emb_returner(self): if self.args.bert_name == 'bert-base-uncased': huggingface_model = 'bert-base-uncased' else: huggingface_model = 'dummy' print(self.args.bert_name, 'are not supported') exit() bert_embedder = PretrainedTransformerEmbedder( model_name=huggingface_model) return bert_embedder, bert_embedder.get_output_dim( ), BasicTextFieldEmbedder({'tokens': bert_embedder}, allow_unmatched_keys=True)
def from_params(cls, vocab: Vocabulary, params: Params, constructor_to_call=None, constructor_to_inspect=None) -> 'BertModel': #initialize the class using JSON params embedder_params = params.pop("text_field_embedder") token_params = embedder_params.pop("tokens") embedding = PretrainedTransformerEmbedder.from_params( vocab=vocab, params=token_params) text_field_embedder = BasicTextFieldEmbedder( token_embedders={'tokens': embedding}) # text_field_embedder = TextFieldEmbedder.from_params(vocab, embedder_params) seq2vec_encoder_params = params.pop("seq2vec_encoder") seq2vec_encoder = Seq2VecEncoder.from_params(seq2vec_encoder_params) initializer = InitializerApplicator( ) #.from_params(params.pop("initializer", [])) params.assert_empty(cls.__name__) # print(cls) return cls(vocab=vocab, text_field_embedder=text_field_embedder, seq2vec_encoder=seq2vec_encoder, initializer=initializer)
def __init__( self, vocab: Vocabulary, transformer_model: str = "roberta-large", override_weights_file: Optional[str] = None, override_weights_strip_prefix: Optional[str] = None, **kwargs ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = PretrainedTransformerEmbedder( transformer_model, override_weights_file=override_weights_file, override_weights_strip_prefix=override_weights_strip_prefix, ) self._text_field_embedder = BasicTextFieldEmbedder( {"tokens": self._text_field_embedder}) self._pooler = BertPooler( transformer_model, override_weights_file=override_weights_file, override_weights_strip_prefix=override_weights_strip_prefix, dropout=0.1, ) self._linear_layer = torch.nn.Linear( self._text_field_embedder.get_output_dim(), 1) self._linear_layer.weight.data.normal_(mean=0.0, std=0.02) self._linear_layer.bias.data.zero_() self._loss = torch.nn.CrossEntropyLoss() self._accuracy = CategoricalAccuracy()
def build_model(vocab: Vocabulary, wbrun: Any) -> Model: """ Build the Model object, along with the embedder and encoder. :param vocab: The pre-instantiated vocabulary object. :return Model: The model object itself. """ log.debug("Building the model.") # vocab_size = vocab.get_vocab_size("tokens") # TokenEmbedder object. bert_embedder = PretrainedTransformerEmbedder("bert-base-uncased") # TextFieldEmbedder that wraps TokenEmbedder objects. Each # TokenEmbedder output from one TokenIndexer--the data produced # by a TextField is a dict {names:representations}, hence # TokenEmbedders have corresponding names. embedder: TextFieldEmbedder = BasicTextFieldEmbedder( {"tokens": bert_embedder} ) log.debug("Embedder built.") encoder = BertPooler("bert-base-uncased", requires_grad=True) # encoder = PytorchSeq2VecWrapper(torch.nn.LSTM(768,20,batch_first=True)) log.debug("Encoder built.") return BertLinearClassifier(vocab, embedder, encoder, wbrun).cuda(0)
def build_model(vocab: Vocabulary, bert_model: str = None) -> Model: if bert_model: embedder = BasicTextFieldEmbedder({"bert": PretrainedTransformerEmbedder(model_name=bert_model, train_parameters=True)}) encoder = BertPooler(pretrained_model=bert_model, requires_grad=True) else: # (3) How to get vectors for each Token ID: # (3.1) embed each token token_embedding = Embedding(embedding_dim=10, num_embeddings=vocab.get_vocab_size("token_vocab")) # pretrained_file='https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz' # (3.2) embed each character in each token character_embedding = Embedding(embedding_dim=3, num_embeddings=vocab.get_vocab_size("character_vocab")) cnn_encoder = CnnEncoder(embedding_dim=3, num_filters=4, ngram_filter_sizes=[3,]) token_encoder = TokenCharactersEncoder(character_embedding, cnn_encoder) # (3.3) embed the POS of each token pos_tag_embedding = Embedding(embedding_dim=10, num_embeddings=vocab.get_vocab_size("pos_tag_vocab")) # Each TokenEmbedders embeds its input, and the result is concatenated in an arbitrary (but consistent) order # cf: https://docs.allennlp.org/master/api/modules/text_field_embedders/basic_text_field_embedder/ embedder = BasicTextFieldEmbedder( token_embedders={"tokens": token_embedding, "token_characters": token_encoder, "pos_tags": pos_tag_embedding} ) # emb_dim = 10 + 4 + 10 = 24 encoder = BagOfEmbeddingsEncoder(embedding_dim=24, averaged=True) # ^ # average the embeddings across time, rather than simply summing # (ie. we will divide the summed embeddings by the length of the sentence). return SimpleClassifier(vocab, embedder, encoder)
def test_xlnet_token_type_ids(self): token_embedder = PretrainedTransformerEmbedder("xlnet-base-cased") token_ids = torch.LongTensor([[1, 2, 3], [2, 3, 4]]) mask = torch.ones_like(token_ids).bool() type_ids = torch.zeros_like(token_ids) type_ids[1, 1] = 1 token_embedder(token_ids, mask, type_ids)
def build_adversarial_transformer_model(vocab: Vocabulary, transformer_model: str) -> Model: print("Building the model") vocab_size = vocab.get_vocab_size("tokens") embedding = PretrainedTransformerEmbedder(model_name=transformer_model) embedder = BasicTextFieldEmbedder(token_embedders={'bert_tokens': embedding}) encoder = BertPooler(transformer_model) return SimpleClassifier(vocab, embedder, encoder)
def build_pool_transformer_model(vocab: Vocabulary, transformer_model: str) -> Model: print("Building the model") vocab_size = vocab.get_vocab_size("tokens") embedding = PretrainedTransformerEmbedder(model_name=transformer_model) embedder = BasicTextFieldEmbedder(token_embedders={'bert_tokens': embedding}) encoder = BagOfEmbeddingsEncoder(embedding_dim=embedder.get_output_dim(), averaged=True) #encoder = ClsPooler(embedding_dim=embedder.get_output_dim()) return SimpleClassifier(vocab, embedder, encoder)
def __init__( self, model_name: str, max_length: int = None, train_parameters: bool = True, last_layer_only: bool = True, gradient_checkpointing: Optional[bool] = None, ) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length=max_length, train_parameters=train_parameters, last_layer_only=last_layer_only, gradient_checkpointing=gradient_checkpointing, )
def test_big_token_type_ids(self): token_embedder = PretrainedTransformerEmbedder("roberta-base") token_ids = torch.LongTensor([[1, 2, 3], [2, 3, 4]]) mask = torch.ones_like(token_ids) type_ids = torch.zeros_like(token_ids) type_ids[1, 1] = 1 with pytest.raises(ValueError): token_embedder(token_ids, mask, type_ids)
def test_forward_runs_when_initialized_from_params(self): # This code just passes things off to pytorch-transformers, so we only have a very simple # test. params = Params({'model_name': 'bert-base-uncased'}) embedder = PretrainedTransformerEmbedder.from_params(params) tensor = torch.randint(0, 100, (1, 4)) output = embedder(tensor) assert tuple(output.size()) == (1, 4, 768)
def test_forward_runs_when_initialized_from_params(self): # This code just passes things off to `transformers`, so we only have a very simple # test. params = Params({"model_name": "bert-base-uncased"}) embedder = PretrainedTransformerEmbedder.from_params(params) token_ids = torch.randint(0, 100, (1, 4)) mask = torch.randint(0, 2, (1, 4)) output = embedder(token_ids=token_ids, mask=mask) assert tuple(output.size()) == (1, 4, 768)
def __init__( self, vocab: Vocabulary, span_extractor: SpanExtractor, transformer_model_name_or_archive_path: str = "bert-base-uncased", encoder: Optional[Seq2SeqEncoder] = None, freeze: bool = False, smoothing: bool = False, drop_out: float = 0.0, **kwargs) -> None: super().__init__(vocab, **kwargs) self._span_extractor = span_extractor # text_field_embedder if "model.tar.gz" in transformer_model_name_or_archive_path: archive = load_archive(transformer_model_name_or_archive_path) self._text_field_embedder = archive.extract_module( "_text_field_embedder", freeze) else: self._text_field_embedder = BasicTextFieldEmbedder({ "tokens": PretrainedTransformerEmbedder( transformer_model_name_or_archive_path) }) if freeze: for parameter in self._text_field_embedder.parameters( ): # type: ignore parameter.requires_grad_(not freeze) # encoder if encoder is None: self._encoder = None else: self._encoder = encoder # linear self._linear = nn.Linear(self._span_extractor.get_output_dim(), 1) # drop out self._drop_out = nn.Dropout(drop_out) # loss self._smoothing = smoothing if not smoothing: self._loss = nn.CrossEntropyLoss() else: self._loss = nn.KLDivLoss(reduction="batchmean") # metric self._span_accuracy = CategoricalAccuracy() self._processed_span_accuracy = CategoricalAccuracy() self._candidate_jaccard = Jaccard()
def __init__( self, vocab: Vocabulary, transformer_model_name: str = "bert-base-cased", **kwargs ) -> None: super().__init__(vocab, **kwargs) self._text_field_embedder = BasicTextFieldEmbedder( {"tokens": PretrainedTransformerEmbedder(transformer_model_name)} ) self._linear_layer = nn.Linear(self._text_field_embedder.get_output_dim(), 2) self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._per_instance_metrics = SquadEmAndF1()
def __init__(self, model_name, use_pretrained_embeddings=False): super().__init__() # or some flag that indicates the bart encoder in it's entirety could be used. if use_pretrained_embeddings: # will use the entire bart encoder including all embeddings bart = PretrainedTransformerEmbedder(model_name, sub_module="encoder") else: bart = BartModel.from_pretrained(model_name) self.bart_encoder.embed_tokens = lambda x: x self.bart_encoder.embed_positions = lambda x: torch.zeros( (x.shape[0], x.shape[1], self.hidden_dim), dtype=torch.float32) self.hidden_dim = bart.config.hidden_size self.bart_encoder = bart.transformer_model
def test_eval_mode(self): token_embedder = PretrainedTransformerEmbedder( "epwalsh/bert-xsmall-dummy", train_parameters=False) assert token_embedder.training and not token_embedder.transformer_model.training class TrainableModule(torch.nn.Module): def __init__(self, fixed_module): super().__init__() self.fixed_module = fixed_module trainable = TrainableModule(token_embedder) assert (trainable.training and trainable.fixed_module.training and not trainable.fixed_module.transformer_model.training) trainable.train() assert not trainable.fixed_module.transformer_model.training
def __init__( self, word_embedding_dropout: float = 0.05, bert_model_name: str = 'japanese_bert', word_embedder: BasicTextFieldEmbedder = BasicTextFieldEmbedder({ 'tokens': PretrainedTransformerEmbedder( model_name='cl-tohoku/bert-base-japanese') })): super(BertPoolerForMention, self).__init__() self.bert_model_name = bert_model_name self.huggingface_nameloader() self.bertpooler_sec2vec = BertPooler( pretrained_model=self.bert_weight_filepath) self.word_embedder = word_embedder self.word_embedding_dropout = nn.Dropout(word_embedding_dropout)
def build_model_Transformer(vocab: Vocabulary, use_reg: bool = True) -> Model: print("Building the model") vocab_size = vocab.get_vocab_size("tokens") EMBED_DIMS = 300 # turn the tokens into 300 dim embedding. Then, turn the embeddings into encodings embedder = PretrainedTransformerEmbedder(BERT_MODEL_NAME) encoder = BertPooler( BERT_MODEL_NAME ) # num_filters is a tad bit dangerous: the reason is that we have this many filters for EACH ngram f # encoder = BertPooler("bert-base-cased") # the output dim is just the num filters *len(ngram_filter_sizes) # construct the regularizer applicator regularizer_applicator = None if use_reg: l2_reg = L2Regularizer() regexes = [("embedder", l2_reg), ("encoder", l2_reg), ("classifier", l2_reg)] regularizer_applicator = RegularizerApplicator(regexes) return MortalityClassifier(vocab, embedder, encoder, regularizer_applicator)
class BertClassifier(Model): def __init__(self, vocab, num_labels): super().__init__(vocab) self.bert_embedder = PretrainedTransformerEmbedder('bert-base-uncased') self.pooler = ClsPooler(self.bert_embedder.get_output_dim()) self.linear = torch.nn.Sequential( torch.nn.Dropout(0.1), torch.nn.Linear(in_features=768, out_features=num_labels)) self.accuracy = CategoricalAccuracy() self.loss_function = torch.nn.CrossEntropyLoss() def forward(self, sent, label=None): bert_embeddings = self.bert_embedder( token_ids=sent['tokens']['token_ids'], type_ids=sent['tokens']['type_ids'], mask=sent['tokens']['mask']) bert_vec = self.pooler(bert_embeddings) logits = self.linear(bert_vec) output = {"logits": logits, "probs": F.softmax(logits, dim=1)} if label is not None: self.accuracy(logits, label) output["loss"] = self.loss_function(logits, label) return output def get_metrics(self, reset=False): return {'accuracy': self.accuracy.get_metric(reset)} def get_optimizer(self): optimizer = AdamW(self.parameters(), lr=2e-5, eps=1e-8) # get_linear_schedule_with_warmup( # optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total # ) return optimizer
class PretrainedTransformerMismatchedEmbedder(TokenEmbedder): """ Use this embedder to embed wordpieces given by `PretrainedTransformerMismatchedIndexer` and to pool the resulting vectors to get word-level representations. Registered as a `TokenEmbedder` with name "pretrained_transformer_mismatchd". # Parameters model_name : `str` The name of the `transformers` model to use. Should be the same as the corresponding `PretrainedTransformerMismatchedIndexer`. max_length : `int`, optional (default = `None`) If positive, folds input token IDs into multiple segments of this length, pass them through the transformer model independently, and concatenate the final representations. Should be set to the same value as the `max_length` option on the `PretrainedTransformerMismatchedIndexer`. train_parameters: `bool`, optional (default = `True`) If this is `True`, the transformer weights get updated during training. """ def __init__(self, model_name: str, max_length: int = None, train_parameters: bool = True) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length=max_length, train_parameters=train_parameters) @overrides def get_output_dim(self): return self._matched_embedder.get_output_dim() @overrides def forward( self, token_ids: torch.LongTensor, mask: torch.BoolTensor, offsets: torch.LongTensor, wordpiece_mask: torch.BoolTensor, type_ids: Optional[torch.LongTensor] = None, segment_concat_mask: Optional[torch.BoolTensor] = None, ) -> torch.Tensor: # type: ignore """ # Parameters token_ids: `torch.LongTensor` Shape: [batch_size, num_wordpieces] (for exception see `PretrainedTransformerEmbedder`). mask: `torch.BoolTensor` Shape: [batch_size, num_orig_tokens]. offsets: `torch.LongTensor` Shape: [batch_size, num_orig_tokens, 2]. Maps indices for the original tokens, i.e. those given as input to the indexer, to a span in token_ids. `token_ids[i][offsets[i][j][0]:offsets[i][j][1] + 1]` corresponds to the original j-th token from the i-th batch. wordpiece_mask: `torch.BoolTensor` Shape: [batch_size, num_wordpieces]. type_ids: `Optional[torch.LongTensor]` Shape: [batch_size, num_wordpieces]. segment_concat_mask: `Optional[torch.BoolTensor]` See `PretrainedTransformerEmbedder`. # Returns `torch.Tensor` Shape: [batch_size, num_orig_tokens, embedding_size]. """ # Shape: [batch_size, num_wordpieces, embedding_size]. embeddings = self._matched_embedder( token_ids, wordpiece_mask, type_ids=type_ids, segment_concat_mask=segment_concat_mask) # span_embeddings: (batch_size, num_orig_tokens, max_span_length, embedding_size) # span_mask: (batch_size, num_orig_tokens, max_span_length) span_embeddings, span_mask = util.batched_span_select( embeddings.contiguous(), offsets) span_mask = span_mask.unsqueeze(-1) span_embeddings *= span_mask # zero out paddings span_embeddings_sum = span_embeddings.sum(2) span_embeddings_len = span_mask.sum(2) # Shape: (batch_size, num_orig_tokens, embedding_size) orig_embeddings = span_embeddings_sum / span_embeddings_len # All the places where the span length is zero, write in zeros. orig_embeddings[(span_embeddings_len == 0).expand( orig_embeddings.shape)] = 0 return orig_embeddings
def bert_emb_returner(): return BasicTextFieldEmbedder({ 'tokens': PretrainedTransformerEmbedder( model_name='cl-tohoku/bert-base-japanese') })
def test_encoder_decoder_model(self): token_embedder = PretrainedTransformerEmbedder("facebook/bart-large", sub_module="encoder") token_ids = torch.LongTensor([[1, 2, 3], [2, 3, 4]]) mask = torch.ones_like(token_ids).bool() token_embedder(token_ids, mask)