def __init__(self,
embedding_size,
hidden_size,
num_layers,
num_heads,
total_key_depth,
total_value_depth,
filter_size,
max_length=1000,
input_dropout=0.0,
layer_dropout=0.0,
attention_dropout=0.0,
relu_dropout=0.0,
use_mask=False,
universal=False,
concept=False):
"""
Parameters:
embedding_size: Size of embeddings
hidden_size: Hidden size
num_layers: Total layers in the Encoder 2
num_heads: Number of attention heads 2
total_key_depth: Size of last dimension of keys. Must be divisible by num_head 40
total_value_depth: Size of last dimension of values. Must be divisible by num_head 40
output_depth: Size last dimension of the final output
filter_size: Hidden size of the middle layer in FFN 50
max_length: Max sequence length (required for timing signal)
input_dropout: Dropout just after embedding
layer_dropout: Dropout for each layer
attention_dropout: Dropout probability after attention (Should be non-zero only during training)
relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
use_mask: Set to True to turn on future value masking
"""
super(Semantic_Encoder, self).__init__()
self.universal = universal
self.num_layers = num_layers
self.timing_signal = _gen_timing_signal(max_length, hidden_size)
if (self.universal):
## for t
self.position_signal = _gen_timing_signal(num_layers, hidden_size)
params = (hidden_size, total_key_depth
or hidden_size, total_value_depth
or hidden_size, filter_size, num_heads,
_gen_bias_mask(max_length) if use_mask else None,
layer_dropout, attention_dropout, relu_dropout)
self.embedding_proj = nn.Linear(embedding_size,
hidden_size,
bias=False)
if (self.universal):
self.enc = EncoderLayer(*params)
else:
self.enc = nn.ModuleList(
[EncoderLayer(*params) for _ in range(num_layers)])
self.layer_norm = LayerNorm(hidden_size)
self.input_dropout = nn.Dropout(input_dropout)
def __init__(self, embedding_size, hidden_size, num_layers, num_heads, total_key_depth, total_value_depth,
filter_size, max_length=1000, input_dropout=0.0, layer_dropout=0.0,
attention_dropout=0.0, relu_dropout=0.0, universal=False):
"""
Parameters:
embedding_size: Size of embeddings
hidden_size: Hidden size
num_layers: Total layers in the Encoder
num_heads: Number of attention heads
total_key_depth: Size of last dimension of keys. Must be divisible by num_head
total_value_depth: Size of last dimension of values. Must be divisible by num_head
output_depth: Size last dimension of the final output
filter_size: Hidden size of the middle layer in FFN
max_length: Max sequence length (required for timing signal)
input_dropout: Dropout just after embedding
layer_dropout: Dropout for each layer
attention_dropout: Dropout probability after attention (Should be non-zero only during training)
relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
"""
super(Decoder, self).__init__()
self.universal = universal
self.num_layers = num_layers
self.timing_signal = _gen_timing_signal(max_length, hidden_size)
if(self.universal):
## for t
self.position_signal = _gen_timing_signal(num_layers, hidden_size)
self.mask = _get_attn_subsequent_mask(max_length)
params =(hidden_size,
total_key_depth or hidden_size,
total_value_depth or hidden_size,
filter_size,
num_heads,
_gen_bias_mask(max_length), # mandatory
layer_dropout,
attention_dropout,
relu_dropout)
if(self.universal):
self.dec = DecoderLayer(*params)
else:
self.dec = nn.Sequential(*[DecoderLayer(*params) for l in range(num_layers)])
self.embedding_proj = nn.Linear(embedding_size, hidden_size, bias=False)
self.layer_norm = LayerNorm(hidden_size)
self.input_dropout = nn.Dropout(input_dropout)