def test_output_layer(self):
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=None)
self.assertIsInstance(decoder, TransformerDecoder)
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
output_layer=identity)
self.assertIsInstance(decoder, TransformerDecoder)
tensor = torch.rand(self._vocab_size, self._emb_dim, dtype=torch.float)
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
output_layer=tensor)
self.assertIsInstance(decoder, TransformerDecoder)
self.assertEqual(decoder.vocab_size, self._vocab_size)
def test_greedy_embedding_helper(self):
"""Tests with tf.contrib.seq2seq.GreedyEmbeddingHelper
"""
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=self._output_layer)
decoder.eval()
helper = decoder_helpers.GreedyEmbeddingHelper(self._start_tokens,
self._end_token)
outputs, length = decoder(
memory=self._memory,
memory_sequence_length=self._memory_sequence_length,
memory_attention_bias=None,
helper=helper,
max_decoding_length=self._max_decode_len)
self.assertIsInstance(outputs, TransformerDecoderOutput)
def test_infer_greedy_with_context_without_memory(self):
"""Tests train_greedy with context
"""
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=self._output_layer)
decoder.eval()
outputs, length = decoder(memory=None,
memory_sequence_length=None,
memory_attention_bias=None,
inputs=None,
decoding_strategy='infer_greedy',
context=self._context,
context_sequence_length=self._context_length,
end_token=self._end_token,
max_decoding_length=self._max_decode_len)
self.assertIsInstance(outputs, TransformerDecoderOutput)
def test_decode_infer_sample(self):
"""Tests infer_sample
"""
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=self._output_layer)
decoder.eval()
helper = decoder_helpers.SampleEmbeddingHelper(self._start_tokens,
self._end_token)
outputs, length = decoder(
memory=self._memory,
memory_sequence_length=self._memory_sequence_length,
memory_attention_bias=None,
inputs=None,
helper=helper,
max_decoding_length=self._max_decode_len)
self.assertIsInstance(outputs, TransformerDecoderOutput)
def test_decode_train(self):
"""Tests train_greedy
"""
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=self._output_layer)
decoder.train()
# 6 blocks
# -self multihead_attention: 4 dense without bias + 2 layer norm vars
# -encdec multihead_attention: 4 dense without bias + 2 layer norm vars
# -poswise_network: Dense with bias, Dense with bias + 2 layer norm vars
# 2 layer norm vars
outputs = decoder(memory=self._memory,
memory_sequence_length=self._memory_sequence_length,
memory_attention_bias=None,
inputs=self._inputs,
decoding_strategy='train_greedy')
# print(decoder)
# for name, _ in decoder.named_parameters():
# print(name)
self.assertEqual(len(decoder.trainable_variables), 110)
self.assertIsInstance(outputs, TransformerDecoderOutput)
def test_beam_search(self):
"""Tests beam_search
"""
decoder = TransformerDecoder(token_pos_embedder=self._embedding_fn,
vocab_size=self._vocab_size,
output_layer=self._output_layer)
decoder.eval()
beam_width = 5
outputs = decoder(memory=self._memory,
memory_sequence_length=self._memory_sequence_length,
memory_attention_bias=None,
inputs=None,
beam_width=beam_width,
start_tokens=self._start_tokens,
end_token=self._end_token,
max_decoding_length=self._max_decode_len)
self.assertEqual(outputs['log_prob'].size(),
(self._batch_size, beam_width))
self.assertEqual(outputs['sample_id'].size(0), self._batch_size)
self.assertLessEqual(outputs['sample_id'].size(2),
self._max_decode_len)
self.assertEqual(outputs['sample_id'].size(2), beam_width)
def default_hparams():
r"""Returns a dictionary of hyperparameters with default values.
* The decoder arch is determined by the constructor argument
:attr:`pretrained_model_name` if it's specified. In this case,
`hparams` are ignored.
* Otherwise, the encoder arch is determined by
`hparams['pretrained_model_name']` if it's specified. All other
configurations in `hparams` are ignored.
* If the above two are `None`, the encoder arch is defined by the
configurations in `hparams` and weights are randomly initialized.
.. code-block:: python
{
"name": "gpt2_decoder",
"pretrained_model_name": "gpt2-small",
"vocab_size": 50257,
"context_size": 1024,
"embedding_size": 768,
"embed": {
"dim": 768,
"name": "word_embeddings"
},
"position_size": 1024,
"position_embed": {
"dim": 768,
"name": "position_embeddings"
},
# hparams for TransformerDecoder
"decoder": {
"dim": 768,
"num_blocks": 12,
"use_gpt_config": True,
"embedding_dropout": 0,
"residual_dropout": 0,
"multihead_attention": {
"use_bias": True,
"num_units": 768,
"num_heads": 12,
"dropout_rate": 0.0,
"output_dim": 768
},
"initializer": {
"type": "variance_scaling_initializer",
"kwargs": {
"factor": 1.0,
"mode": "FAN_AVG",
"uniform": True
}
},
"poswise_feedforward": {
"layers": [
{
"type": "Linear",
"kwargs": {
"in_features": 768,
"out_features": 3072,
"bias": True
}
},
{
"type": "GPTGELU",
"kwargs": {}
},
{
"type": "Linear",
"kwargs": {
"in_features": 3072,
"out_features": 768,
"bias": True
}
}
],
"name": "ffn"
}
},
}
Here:
The default parameters are values for 124M GPT2 model.
`"pretrained_model_name"`: str or None
The name of the pre-trained GPT2 model. If None, the model
will be randomly initialized.
`"embed"`: dict
Hyperparameters for word embedding layer.
`"vocab_size"`: int
The vocabulary size of `inputs` in `GPT2Model`.
`"position_embed"`: dict
Hyperparameters for position embedding layer.
`"position_size"`: int
The maximum sequence length that this model might ever be used with.
`"name"`: str
Name of the module.
"""
return {
**TransformerDecoder.default_hparams(),
'dim': 768,
'num_blocks': 12,
'use_gpt_config': True,
'embedding_dropout': 0,
'residual_dropout': 0,
'multihead_attention': {
'use_bias': True,
'num_units': 768,
'num_heads': 12,
"dropout_rate": 0.0,
'output_dim': 768
},
'initializer': {
'type': 'variance_scaling_initializer',
'kwargs': {
'factor': 1.0,
'mode': 'FAN_AVG',
'uniform': True
}
},
'poswise_feedforward': {
'layers': [{
'type': 'Linear',
'kwargs': {
'in_features': 768,
'out_features': 3072,
'bias': True
}
}, {
'type': 'GPTGELU',
'kwargs': {}
}, {
'type': 'Linear',
'kwargs': {
'in_features': 3072,
'out_features': 768,
'bias': True
}
}],
'name':
'ffn'
},
'pretrained_model_name': 'gpt2-small',
'vocab_size': 50257,
'context_size': 1024,
'embedding_size': 768,
'embed': {
'dim': 768,
'name': 'word_embeddings'
},
'position_size': 1024,
'position_embed': {
'dim': 768,
'name': 'position_embeddings'
},
'name': 'gpt2_decoder',
'@no_typecheck': ['pretrained_model_name'],
}