def __init__(self, config: LukeConfig):
super(LukeModel, self).__init__()
self.config = config
self.encoder = BertEncoder(config)
self.pooler = BertPooler(config)
if self.config.bert_model_name and "roberta" in self.config.bert_model_name:
self.embeddings = RobertaEmbeddings(config)
self.embeddings.token_type_embeddings.requires_grad = False
else:
self.embeddings = BertEmbeddings(config)
self.entity_embeddings = EntityEmbeddings(config)
def test_create_position_ids_from_inputs_embeds(self):
""" Ensure that the default position ids only assign a sequential . This is a regression
test for https://github.com/huggingface/transformers/issues/1761
The position ids should be masked with the embedding object's padding index. Therefore, the
first available non-padding position index is RobertaEmbeddings.padding_idx + 1
"""
config = self.model_tester.prepare_config_and_inputs()[0]
embeddings = RobertaEmbeddings(config=config)
inputs_embeds = torch.Tensor(2, 4, 30)
expected_single_positions = [
0 + embeddings.padding_idx + 1,
1 + embeddings.padding_idx + 1,
2 + embeddings.padding_idx + 1,
3 + embeddings.padding_idx + 1,
]
expected_positions = torch.as_tensor(
[expected_single_positions, expected_single_positions])
position_ids = embeddings.create_position_ids_from_inputs_embeds(
inputs_embeds)
self.assertEqual(position_ids.shape, expected_positions.shape)
self.assertTrue(torch.all(torch.eq(position_ids, expected_positions)))
def __init__(self, config):
super().__init__(config)
self.embeddings = RobertaEmbeddings(config)
self.init_weights()
def __init__(self, config):
super(RobertaModel4TokenMix, self).__init__(config)
self.embeddings = RobertaEmbeddings(config)
self.init_weights()