def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(hparams=hparams)
self.load_pretrained_config(pretrained_model_name, cache_dir)
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Segment embedding for each type of tokens
self.segment_embedder = None
if self._hparams.get('type_vocab_size', 0) > 0:
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.pooler = nn.Sequential(
nn.Linear(self._hparams.hidden_size, self._hparams.hidden_size),
nn.Tanh())
self.init_pretrained_weights()
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(hparams=hparams)
self.load_pretrained_config(pretrained_model_name, cache_dir)
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The GPT2 encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.init_pretrained_weights(load_output_layer=False)
def __init__(self, hparams=None):
super().__init__(hparams=hparams)
# Segment embedding for each type of tokens
self.segment_embedder = None
if self._hparams.get('type_vocab_size', 0) > 0:
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.pooler = nn.Sequential(
nn.Linear(self._hparams.encoder.dim, self._hparams.hidden_size),
nn.Tanh())
def default_hparams():
r"""Returns a dictionary of hyperparameters with default values.
* The encoder arch is determined by the constructor argument
:attr:`pretrained_model_name` if it's specified. In this case,
`hparams` are ignored.
* Otherwise, the encoder arch is determined by
`hparams['pretrained_model_name']` if it's specified. All other
configurations in `hparams` are ignored.
* If the above two are `None`, the encoder arch is defined by the
configurations in `hparams` and weights are randomly initialized.
.. code-block:: python
{
"pretrained_model_name": "gpt2-small",
"vocab_size": 50257,
"context_size": 1024,
"embedding_size": 768,
"embed": {
"dim": 768,
"name": "word_embeddings"
},
"position_size": 1024,
"position_embed": {
"dim": 768,
"name": "position_embeddings"
},
"decoder": {
"dim": 768,
"num_blocks": 12,
"use_gpt_config": True,
"embedding_dropout": 0,
"residual_dropout": 0,
"multihead_attention": {
"use_bias": True,
"num_units": 768,
"num_heads": 12,
"output_dim": 768
},
"initializer": {
"type": "variance_scaling_initializer",
"kwargs": {
"factor": 1.0,
"mode": "FAN_AVG",
"uniform": True
}
},
"poswise_feedforward": {
"layers": [
{
"type": "Linear",
"kwargs": {
"in_features": 768,
"out_features": 3072,
"bias": True
}
},
{
"type": "GPTGELU",
"kwargs": {}
},
{
"type": "Linear",
"kwargs": {
"in_features": 3072,
"out_features": 768,
"bias": True
}
}
],
"name": "ffn"
}
},
"initializer": None,
"name": "gpt2_encoder",
}
Here:
The default parameters are values for 124M GPT2 model.
`"pretrained_model_name"`: str or None
The name of the pre-trained GPT2 model. If None, the model
will be randomly initialized.
`"embed"`: dict
Hyperparameters for word embedding layer.
`"vocab_size"`: int
The vocabulary size of `inputs` in `GPT2Model`.
`"position_embed"`: dict
Hyperparameters for position embedding layer.
`"position_size"`: int
The maximum sequence length that this model might ever be used with.
`"decoder"`: dict
Hyperparameters for the TransformerDecoder.
See :func:`~texar.torch.modules.TransformerDecoder.default_hparams`
for details.
`"initializer"`: dict, optional
Hyperparameters of the default initializer that initializes
variables created in this module.
See :func:`~texar.torch.core.get_initializer` for details.
`"name"`: str
Name of the module.
"""
return {
**TransformerEncoder.default_hparams(),
'dim': 768,
'num_blocks': 12,
'use_bert_config': False,
'use_gpt_config': True,
'embedding_dropout': 0,
'residual_dropout': 0,
'multihead_attention': {
'use_bias': True,
'num_units': 768,
'num_heads': 12,
'output_dim': 768
},
'initializer': {
'type': 'variance_scaling_initializer',
'kwargs': {
'factor': 1.0,
'mode': 'FAN_AVG',
'uniform': True
}
},
'poswise_feedforward': {
'layers': [{
'type': 'Linear',
'kwargs': {
'in_features': 768,
'out_features': 3072,
'bias': True
}
}, {
'type': 'GPTGELU',
'kwargs': {}
}, {
'type': 'Linear',
'kwargs': {
'in_features': 3072,
'out_features': 768,
'bias': True
}
}],
'name':
'ffn'
},
'pretrained_model_name': 'gpt2-small',
'vocab_size': 50257,
'context_size': 1024,
'embedding_size': 768,
'embed': {
'dim': 768,
'name': 'word_embeddings'
},
'position_size': 1024,
'position_embed': {
'dim': 768,
'name': 'position_embeddings'
},
'name': 'gpt2_encoder',
'@no_typecheck': ['pretrained_model_name'],
}