def test_encoder_decoder_model(self): token_embedder = PretrainedTransformerEmbedder("facebook/bart-large", sub_module="encoder") token_ids = torch.LongTensor([[1, 2, 3], [2, 3, 4]]) mask = torch.ones_like(token_ids).bool() token_embedder(token_ids, mask)
def __init__(self, model_name: str, max_length: int = None) -> None: super().__init__() # The matched version v.s. mismatched self._matched_embedder = PretrainedTransformerEmbedder( model_name, max_length)
def __init__( self, vocab: Vocabulary, transformer_model_name: str = "bert-base-uncased", feedforward: Optional[FeedForward] = None, smoothing: bool = False, smooth_alpha: float = 0.7, sentiment_task: bool = False, sentiment_task_weight: float = 1.0, sentiment_classification_with_label: bool = True, sentiment_seq2vec: Optional[Seq2VecEncoder] = None, candidate_span_task: bool = False, candidate_span_task_weight: float = 1.0, candidate_delay: int = 30000, candidate_span_num: int = 5, candidate_classification_layer_units: int = 128, candidate_span_extractor: Optional[SpanExtractor] = None, candidate_span_with_logits: bool = False, dropout: Optional[float] = None, **kwargs, ) -> None: super().__init__(vocab, **kwargs) if "BERTweet" not in transformer_model_name: self._text_field_embedder = BasicTextFieldEmbedder({ "tokens": PretrainedTransformerEmbedder(transformer_model_name) }) else: self._text_field_embedder = BasicTextFieldEmbedder( {"tokens": TweetBertEmbedder(transformer_model_name)}) # span start & end task if feedforward is None: self._linear_layer = nn.Sequential( nn.Linear(self._text_field_embedder.get_output_dim(), 128), nn.ReLU(), nn.Linear(128, 2), ) else: self._linear_layer = feedforward self._span_start_accuracy = CategoricalAccuracy() self._span_end_accuracy = CategoricalAccuracy() self._span_accuracy = BooleanAccuracy() self._jaccard = Jaccard() self._candidate_delay = candidate_delay self._delay = 0 self._smoothing = smoothing self._smooth_alpha = smooth_alpha if smoothing: self._loss = nn.KLDivLoss(reduction="batchmean") else: self._loss = nn.CrossEntropyLoss() # sentiment task self._sentiment_task = sentiment_task if self._sentiment_task: self._sentiment_classification_accuracy = CategoricalAccuracy() self._sentiment_loss_log = LossLog() self.register_buffer("sentiment_task_weight", torch.tensor(sentiment_task_weight)) self._sentiment_classification_with_label = ( sentiment_classification_with_label) if sentiment_seq2vec is None: raise ConfigurationError( "sentiment task is True, we need a sentiment seq2vec encoder" ) else: self._sentiment_encoder = sentiment_seq2vec self._sentiment_linear = nn.Linear( self._sentiment_encoder.get_output_dim(), vocab.get_vocab_size("labels"), ) # candidate span task self._candidate_span_task = candidate_span_task if candidate_span_task: assert candidate_span_num > 0 assert candidate_span_task_weight > 0 assert candidate_classification_layer_units > 0 self._candidate_span_num = candidate_span_num self.register_buffer("candidate_span_task_weight", torch.tensor(candidate_span_task_weight)) self._candidate_classification_layer_units = ( candidate_classification_layer_units) self._span_classification_accuracy = CategoricalAccuracy() self._candidate_loss_log = LossLog() self._candidate_span_linear = nn.Linear( self._text_field_embedder.get_output_dim(), self._candidate_classification_layer_units, ) if candidate_span_extractor is None: self._candidate_span_extractor = EndpointSpanExtractor( input_dim=self._candidate_classification_layer_units) else: self._candidate_span_extractor = candidate_span_extractor if candidate_span_with_logits: self._candidate_with_logits = True self._candidate_span_vec_linear = nn.Linear( self._candidate_span_extractor.get_output_dim() + 1, 1) else: self._candidate_with_logits = False self._candidate_span_vec_linear = nn.Linear( self._candidate_span_extractor.get_output_dim(), 1) self._candidate_jaccard = Jaccard() if sentiment_task or candidate_span_task: self._base_loss_log = LossLog() else: self._base_loss_log = None if dropout is not None: self._dropout = nn.Dropout(dropout) else: self._dropout = None
parser.add_argument('--cache-path', type=str) parser.add_argument('--batch-size', type=int, default=12) parser.add_argument('--model', type=str, default='bert-base-cased') parser.add_argument('--file-suffix', type=str, choices=['.json', '.comm', '.concrete'], default='.json') parser.add_argument('--normalize-token', action='store_true') parser.add_argument('--cuda', action='store_true') args = parser.parse_args() tokenizer = PretrainedTransformerTokenizer(model_name=args.model) token_indexers = { 'token': PretrainedTransformerIndexer(model_name=args.model) } model = PretrainedTransformerEmbedder(model_name=args.model) if args.cuda: device = 'cuda' model.to('cuda') else: device = 'cpu' o = h5py.File(args.cache_path, 'w') write_to_hdf5( cache_handler=o, embed_stream=debatch_embedding_stream( embed_stream=get_embedding_stream( embedder=model, batch_stream=compose_batch_stream( ins_stream=instance_stream(file_path=args.input_path, tokenizer=tokenizer,
# Again, it's easier to just run the data code to get the right output. # We're using the smallest transformer model we can here, so that it runs on # binder. transformer_model = 'google/reformer-crime-and-punishment' tokenizer = PretrainedTransformerTokenizer(model_name=transformer_model) token_indexer = PretrainedTransformerIndexer(model_name=transformer_model) text = "Some text with an extraordinarily long identifier." tokens = tokenizer.tokenize(text) print("Transformer tokens:", tokens) text_field = TextField(tokens, {'bert_tokens': token_indexer}) text_field.index(vocab) token_tensor = text_field.as_tensor(text_field.get_padding_lengths()) print("Transformer tensors:", token_tensor) embedding = PretrainedTransformerEmbedder(model_name=transformer_model) embedder = BasicTextFieldEmbedder(token_embedders={'bert_tokens': embedding}) tensor_dict = text_field.batch_tensors([token_tensor]) embedded_tokens = embedder(tensor_dict) print("Transformer embedded tokens:", embedded_tokens) # Use GloVe # This is what gets created by TextField.as_tensor with a SingleIdTokenIndexer; # see the exercises above. token_tensor = {'tokens': {'tokens': torch.LongTensor([1, 3, 2, 1, 4, 3])}} vocab = Vocabulary() vocab.add_tokens_to_namespace(['This', 'is', 'some', 'text', '.'], namespace='token_vocab')