def test_multilingual_transformer_sentence_encoder(self):
input_tuple = (
self.tokens,
self.pad_mask,
self.lengths,
self.segment_labels,
self.positions,
)
sentence_encoder = TransformerSentenceEncoder.from_config(
TransformerSentenceEncoder.Config(
embedding_dim=self.embedding_dim,
num_encoder_layers=self.num_encoder_layers,
multilingual=True,
),
output_encoded_layers=True,
padding_idx=self.padding_idx,
vocab_size=self.vocab_size,
)
encoded_layers, pooled_outputs = sentence_encoder(input_tuple)
# Check sizes for pooled output
self.assertEqual(pooled_outputs.size()[0], self.batch_size)
self.assertEqual(pooled_outputs.size()[1], self.embedding_dim)
# Check sizes for encoded_layers
self.assertEqual(encoded_layers.__len__(), self.num_encoder_layers + 1)
self.assertEqual(encoded_layers[-1].size()[0], self.batch_size)
self.assertEqual(encoded_layers[-1].size()[1], self.num_tokens)
self.assertEqual(encoded_layers[-1].size()[2], self.embedding_dim)
def __init__(self,
config: TransformerSentenceEncoder.Config,
padding_idx: int,
vocab_size: int,
projection_dim: int = 0,
*args,
**kwarg):
TransformerSentenceEncoder.__init__(self, config, False, padding_idx,
vocab_size, *args, **kwarg)
assert config.embedding_dim > 0, 'Encoder hidden_size can\'t be zero'
self.encode_proj = nn.Linear(
config.embedding_dim,
projection_dim) if projection_dim != 0 else None
class Config(BaseModel.Config):
class InputConfig(ConfigBase):
tokens: BERTTensorizer.Config = BERTTensorizer.Config(max_seq_len=128)
inputs: InputConfig = InputConfig()
encoder: TransformerSentenceEncoderBase.Config = TransformerSentenceEncoder.Config()
decoder: MLPDecoder.Config = MLPDecoder.Config()
output_layer: LMOutputLayer.Config = LMOutputLayer.Config()
mask_prob: float = 0.15
mask_bos: bool = False
# masking
masking_strategy: MaskingStrategy = MaskingStrategy.RANDOM
# tie weights determines whether the input embedding weights are used
# in the output vocabulary projection as well
tie_weights: bool = True
def get_pytext_bert_base_cfg():
cfg = TransformerSentenceEncoder.Config()
cfg.embedding_dim = 768
cfg.ffn_embedding_dim = 3072
cfg.num_encoder_layers = 12
cfg.num_attention_heads = 12
cfg.num_segments = 2
cfg.use_position_embeddings = True
cfg.offset_positions_by_padding = True
cfg.apply_bert_init = True
cfg.encoder_normalize_before = True
cfg.activation_fn = "gelu"
cfg.projection_dim = 0
cfg.max_seq_len = 512
cfg.multilingual = False
cfg.freeze_embeddings = False
cfg.n_trans_layers_to_freeze = 0
cfg.use_torchscript = False
return cfg