def __init__(self,
width: int,
input_size: int,
hidden_size: int,
n_heads: int,
n_layers: int,
n_highway: int,
use_position: bool = False,
use_relative_position: bool = False,
dropout: float = 0.0):
super(SelfAttentiveLBLBiLMV3, self).__init__()
self.use_position = use_position
self.use_relative_position_weights = use_relative_position
self.n_layers = n_layers
self.n_highway = n_highway
self.n_heads = n_heads
self.input_size = input_size
self.width = width
self.hidden_size = hidden_size
forward_attns, backward_attns = [], []
forward_blocks, backward_blocks = [], []
for _ in range(n_layers):
if self.use_relative_position_weights:
forward_attn = MultiHeadedAttentionWithRelativePositionEmbeddings(
n_heads,
hidden_size,
width=width + 1,
left_to_right=True,
dropout=dropout)
backward_attn = MultiHeadedAttentionWithRelativePositionEmbeddings(
n_heads,
hidden_size,
width=width + 1,
left_to_right=False,
dropout=dropout)
else:
forward_attn = MultiHeadedAttention(n_heads, hidden_size,
dropout)
backward_attn = MultiHeadedAttention(n_heads, hidden_size,
dropout)
forward_attns.append(forward_attn)
backward_attns.append(backward_attn)
forward_blocks.append(Highway(hidden_size, n_highway))
backward_blocks.append(Highway(hidden_size, n_highway))
self.forward_attns = torch.nn.ModuleList(forward_attns)
self.backward_attns = torch.nn.ModuleList(backward_attns)
self.forward_blocks = torch.nn.ModuleList(forward_blocks)
self.backward_blocks = torch.nn.ModuleList(backward_blocks)
if self.use_position:
self.position = PositionalEncoding(hidden_size)
def __init__(
self,
decoding_dim: int,
target_embedding_dim: int,
feedforward_hidden_dim: int,
num_layers: int,
num_attention_heads: int,
use_positional_encoding: bool = True,
positional_encoding_max_steps: int = 5000,
dropout_prob: float = 0.1,
residual_dropout_prob: float = 0.2,
attention_dropout_prob: float = 0.1,
) -> None:
super().__init__(decoding_dim=decoding_dim,
target_embedding_dim=target_embedding_dim,
decodes_parallel=True)
attn = MultiHeadedAttention(num_attention_heads, decoding_dim,
attention_dropout_prob)
feed_forward = PositionwiseFeedForward(decoding_dim,
feedforward_hidden_dim,
dropout_prob)
self._embed_scale = math.sqrt(decoding_dim)
self._positional_embedder = PositionalEncoding(decoding_dim,
positional_encoding_max_steps) \
if use_positional_encoding else None
self._dropout = nn.Dropout(dropout_prob)
self._self_attention = Decoder(
DecoderLayer(decoding_dim, deepcopy(attn), deepcopy(attn),
feed_forward, residual_dropout_prob), num_layers)
def __init__(
self,
num_layers: int,
decoding_dim: int,
target_embedding_dim: int,
feedforward_hidden_dim: int,
num_attention_heads: int,
combiner: TransformerCombiner,
num_sources: int,
use_positional_encoding: bool = True,
positional_encoding_max_steps: int = 5000,
dropout_prob: float = 0.1,
residual_dropout_prob: float = 0.2,
attention_dropout_prob: float = 0.2,
) -> None:
super().__init__(decoding_dim,
target_embedding_dim,
decodes_parallel=True)
self._decoding_dim = decoding_dim
self._embed_scale = math.sqrt(decoding_dim)
self._positional_embedder = (PositionalEncoding(
input_dim=decoding_dim, max_len=positional_encoding_max_steps)
if use_positional_encoding else None)
self._dropout = nn.Dropout(dropout_prob)
generic_attn = MultiHeadedAttention(num_attention_heads, decoding_dim,
attention_dropout_prob)
combined_attn = AttentionCombiner(num_sources, generic_attn, combiner)
feed_forward = PositionwiseFeedForward(decoding_dim,
feedforward_hidden_dim,
dropout_prob)
layer = DecoderLayer(size=decoding_dim,
self_attn=deepcopy(generic_attn),
src_attn=deepcopy(combined_attn),
feed_forward=feed_forward,
dropout=residual_dropout_prob)
self._self_attention_layers = _clones(layer, num_layers)
self.norm = nn.LayerNorm(layer.size)
def __init__(self,
width: int,
input_size: int,
hidden_size: int,
n_heads: int,
n_layers: int,
n_highway: int,
use_position: bool = False,
use_relative_position: bool = False,
dropout: float = 0.0):
super(SelfAttentiveLBLBiLM, self).__init__()
self.use_position = use_position
self.use_relative_position_weights = use_relative_position
self.n_layers = n_layers
self.n_highway = n_highway
self.n_heads = n_heads
self.input_size = input_size
self.width = width
self.hidden_size = hidden_size
forward_attns, backward_attns = [], []
forward_paddings, backward_paddings = [], []
forward_blocks, backward_blocks = [], []
forward_weights, backward_weights = [], []
for _ in range(n_layers):
forward_attns.append(
MultiHeadedAttention(n_heads, hidden_size, dropout))
backward_attns.append(
MultiHeadedAttention(n_heads, hidden_size, dropout))
forward_paddings.append(
torch.nn.Parameter(
torch.randn(width, hidden_size) / np.sqrt(hidden_size)))
backward_paddings.append(
torch.nn.Parameter(
torch.randn(width, hidden_size) / np.sqrt(hidden_size)))
forward_blocks.append(Highway(hidden_size, n_highway))
backward_blocks.append(Highway(hidden_size, n_highway))
if self.use_relative_position_weights:
forward_weights.append(
torch.nn.Parameter(torch.randn(width + 1)))
backward_weights.append(
torch.nn.Parameter(torch.randn(width + 1)))
self.forward_attns = torch.nn.ModuleList(forward_attns)
self.backward_attns = torch.nn.ModuleList(backward_attns)
self.forward_paddings = torch.nn.ParameterList(forward_paddings)
self.backward_paddings = torch.nn.ParameterList(backward_paddings)
self.forward_blocks = torch.nn.ModuleList(forward_blocks)
self.backward_blocks = torch.nn.ModuleList(backward_blocks)
if self.use_relative_position_weights:
self.forward_weights = torch.nn.ParameterList(forward_weights)
self.backward_weights = torch.nn.ParameterList(backward_weights)
if self.use_position:
self.position = PositionalEncoding(hidden_size)