def __init__(self, config: LukeConfig): super(LukeModel, self).__init__() self.config = config self.encoder = BertEncoder(config) self.pooler = BertPooler(config) if self.config.bert_model_name and "roberta" in self.config.bert_model_name: self.embeddings = RobertaEmbeddings(config) self.embeddings.token_type_embeddings.requires_grad = False else: self.embeddings = BertEmbeddings(config) self.entity_embeddings = EntityEmbeddings(config)
def test_create_position_ids_from_inputs_embeds(self): """ Ensure that the default position ids only assign a sequential . This is a regression test for https://github.com/huggingface/transformers/issues/1761 The position ids should be masked with the embedding object's padding index. Therefore, the first available non-padding position index is RobertaEmbeddings.padding_idx + 1 """ config = self.model_tester.prepare_config_and_inputs()[0] embeddings = RobertaEmbeddings(config=config) inputs_embeds = torch.Tensor(2, 4, 30) expected_single_positions = [ 0 + embeddings.padding_idx + 1, 1 + embeddings.padding_idx + 1, 2 + embeddings.padding_idx + 1, 3 + embeddings.padding_idx + 1, ] expected_positions = torch.as_tensor( [expected_single_positions, expected_single_positions]) position_ids = embeddings.create_position_ids_from_inputs_embeds( inputs_embeds) self.assertEqual(position_ids.shape, expected_positions.shape) self.assertTrue(torch.all(torch.eq(position_ids, expected_positions)))
def __init__(self, config): super().__init__(config) self.embeddings = RobertaEmbeddings(config) self.init_weights()
def __init__(self, config): super(RobertaModel4TokenMix, self).__init__(config) self.embeddings = RobertaEmbeddings(config) self.init_weights()