def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(hparams=hparams)
self.load_pretrained_config(pretrained_model_name, cache_dir)
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Segment embedding for each type of tokens
self.segment_embedder = None
if self._hparams.get('type_vocab_size', 0) > 0:
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.pooler = nn.Sequential(
nn.Linear(self._hparams.hidden_size, self._hparams.hidden_size),
nn.Tanh())
self.init_pretrained_weights()
def test_word_embedder_trainable(self):
"""Tests freezing the embedding parameters.
"""
init_value = np.expand_dims(np.arange(5), 1)
embedder = WordEmbedder(init_value=init_value,
hparams={"trainable": False})
self.assertEqual(len(embedder.trainable_variables), 0)
embedder = WordEmbedder(init_value=init_value)
self.assertEqual(len(embedder.trainable_variables), 1)
def _test_word_embedder(self, hparams):
"""Tests :class:`texar.torch.modules.WordEmbedder`.
"""
embedder = WordEmbedder(vocab_size=100, hparams=hparams)
inputs = torch.randint(embedder.vocab_size, (64, 16), dtype=torch.long)
outputs = embedder(inputs)
inputs_soft = torch.randn((64, 16, embedder.vocab_size),
dtype=torch.float32)
outputs_soft = embedder(soft_ids=inputs_soft)
if isinstance(embedder.dim, (list, tuple)):
emb_dim = tuple(embedder.dim)
else:
emb_dim = (embedder.dim, )
if isinstance(hparams["dim"], (list, tuple)):
hparams_dim = tuple(hparams["dim"])
else:
hparams_dim = (hparams["dim"], )
self.assertEqual(outputs.size(), (64, 16) + emb_dim)
self.assertEqual(outputs.size(-1), embedder.output_size)
self.assertEqual(outputs_soft.size(), (64, 16) + emb_dim)
self.assertEqual(outputs_soft.size(-1), embedder.output_size)
self.assertEqual(emb_dim, hparams_dim)
self.assertEqual(embedder.vocab_size, 100)
self.assertEqual(outputs.size(), (64, 16) + emb_dim)
self.assertEqual(outputs_soft.size(), (64, 16) + emb_dim)
def setUp(self):
self._vocab_size = 4
self._max_time = 8
self._batch_size = 16
self._emb_dim = 20
self._inputs = torch.randint(
self._vocab_size, size=(self._batch_size, self._max_time))
embedding = torch.rand(
self._vocab_size, self._emb_dim, dtype=torch.float)
self._embedder = WordEmbedder(init_value=embedding)
self._hparams = HParams(None, BasicRNNDecoder.default_hparams())
def test_word_embedder_soft_ids(self):
"""Tests the correctness of using soft ids.
"""
init_value = np.expand_dims(np.arange(5), 1)
embedder = WordEmbedder(init_value=init_value)
ids = torch.tensor([3])
soft_ids = torch.tensor([0, 0, 0, 1, 0], dtype=torch.float)
outputs = embedder(ids=ids)
soft_outputs = embedder(soft_ids=soft_ids)
self.assertEqual(outputs, soft_outputs)
def setUp(self):
self._vocab_size = 10
self._max_time = 16
self._batch_size = 8
self._emb_dim = 20
self._attention_dim = 256
self._inputs = torch.randint(
self._vocab_size, size=(self._batch_size, self._max_time))
embedding = torch.rand(
self._vocab_size, self._emb_dim, dtype=torch.float)
self._embedder = WordEmbedder(init_value=embedding)
self._encoder_output = torch.rand(
self._batch_size, self._max_time, 64)
self._test_hparams = {} # (cell_type, is_multi) -> hparams
for cell_type in ["RNNCell", "LSTMCell", "GRUCell"]:
hparams = {
"rnn_cell": {
'type': cell_type,
'kwargs': {
'num_units': 256,
},
},
"attention": {
"kwargs": {
"num_units": self._attention_dim
},
}
}
self._test_hparams[(cell_type, False)] = HParams(
hparams, AttentionRNNDecoder.default_hparams())
hparams = {
"rnn_cell": {
'type': 'LSTMCell',
'kwargs': {
'num_units': 256,
},
'num_layers': 3,
},
"attention": {
"kwargs": {
"num_units": self._attention_dim
},
}
}
self._test_hparams[("LSTMCell", True)] = HParams(
hparams, AttentionRNNDecoder.default_hparams())
def test_embedder_multi_calls(self):
"""Tests embedders called by multiple times.
"""
hparams = {"dim": 26, "dropout_rate": 0.3, "dropout_strategy": "item"}
embedder = WordEmbedder(vocab_size=100, hparams=hparams)
inputs = torch.randint(embedder.vocab_size, (64, 16), dtype=torch.long)
outputs = embedder(inputs)
if isinstance(embedder.dim, (list, tuple)):
emb_dim = tuple(embedder.dim)
else:
emb_dim = (embedder.dim, )
self.assertEqual(outputs.size(), (64, 16) + emb_dim)
# Call with inputs in a different shape
inputs = torch.randint(embedder.vocab_size, (64, 10, 20),
dtype=torch.long)
outputs = embedder(inputs)
self.assertEqual(outputs.size(), (64, 10, 20) + emb_dim)
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(hparams=hparams)
self.load_pretrained_config(pretrained_model_name, cache_dir)
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# The encoder (a TransformerEncoder)
self.encoder = T5Encoder(hparams=self._hparams.encoder)
# The decoder (a TransformerDecoder)
self.decoder = T5Decoder(token_embedder=self._embedding_fn,
output_layer=Identity(),
hparams=self._hparams.decoder)
self.init_pretrained_weights()
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(hparams=hparams)
self.load_pretrained_config(pretrained_model_name, cache_dir)
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The GPT2 encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.init_pretrained_weights(load_output_layer=False)
def __init__(self, hparams=None):
super().__init__(hparams=hparams)
# Segment embedding for each type of tokens
self.segment_embedder = None
if self._hparams.get('type_vocab_size', 0) > 0:
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.pooler = nn.Sequential(
nn.Linear(self._hparams.encoder.dim, self._hparams.hidden_size),
nn.Tanh())
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
self.load_pretrained_config(pretrained_model_name, cache_dir, hparams)
# Word embedding
word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Position embedding
position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The GPT2 encoder (a TransformerEncoder)
super().__init__(hparams=None)
# Register modules after `__init__` is called.
self.word_embedder = word_embedder
self.position_embedder = position_embedder
self.init_pretrained_weights(load_output_layer=False)