def __init__(self, input_dim, enc_type, attn_type, n_heads, n_layers,
n_layers_sub1, n_layers_sub2, d_model, d_ff, last_proj_dim,
pe_type, layer_norm_eps, ffn_activation, dropout_in, dropout,
dropout_att, dropout_layer, n_stacks, n_splices,
conv_in_channel, conv_channels, conv_kernel_sizes,
conv_strides, conv_poolings, conv_batch_norm, conv_layer_norm,
conv_bottleneck_dim, conv_param_init, task_specific_layer,
param_init, chunk_size_left, chunk_size_current,
chunk_size_right):
super(TransformerEncoder, self).__init__()
if n_layers_sub1 < 0 or (n_layers_sub1 > 1
and n_layers < n_layers_sub1):
raise ValueError('Set n_layers_sub1 between 1 to n_layers.')
if n_layers_sub2 < 0 or (n_layers_sub2 > 1
and n_layers_sub1 < n_layers_sub2):
raise ValueError('Set n_layers_sub2 between 1 to n_layers_sub1.')
self.d_model = d_model
self.n_layers = n_layers
self.n_heads = n_heads
self.pe_type = pe_type
# for streaming TransformerXL encoder
self.N_l = chunk_size_left
self.N_c = chunk_size_current
self.N_r = chunk_size_right
self.latency_controlled = chunk_size_left > 0 or chunk_size_current > 0 or chunk_size_right > 0
self.memory_transformer = ('transformer_xl' in enc_type)
self.mem_len = chunk_size_left
self.scale = math.sqrt(d_model)
if self.memory_transformer:
assert pe_type == 'none'
assert chunk_size_left > 0
assert chunk_size_current > 0
# for hierarchical encoder
self.n_layers_sub1 = n_layers_sub1
self.n_layers_sub2 = n_layers_sub2
self.task_specific_layer = task_specific_layer
# for bridge layers
self.bridge = None
self.bridge_sub1 = None
self.bridge_sub2 = None
# for attention plot
self.aws_dict = {}
self.data_dict = {}
# Setting for CNNs
if conv_channels:
assert n_stacks == 1 and n_splices == 1
self.conv = ConvEncoder(input_dim,
in_channel=conv_in_channel,
channels=conv_channels,
kernel_sizes=conv_kernel_sizes,
strides=conv_strides,
poolings=conv_poolings,
dropout=0.,
batch_norm=conv_batch_norm,
layer_norm=conv_layer_norm,
layer_norm_eps=layer_norm_eps,
residual=False,
bottleneck_dim=d_model,
param_init=conv_param_init)
self._odim = self.conv.output_dim
else:
self.conv = None
self._odim = input_dim * n_splices * n_stacks
self.embed = nn.Linear(self._odim, d_model)
# calculate subsampling factor
self._factor = 1
if self.conv is not None:
self._factor *= self.conv.subsampling_factor
if self.memory_transformer:
self.pos_emb = XLPositionalEmbedding(d_model, dropout)
self.u = nn.Parameter(
torch.Tensor(self.n_heads, self.d_model // self.n_heads))
self.v = nn.Parameter(
torch.Tensor(self.n_heads, self.d_model // self.n_heads))
# NOTE: u and v are global parameters
self.pos_enc = PositionalEncoding(d_model, dropout_in, pe_type,
param_init)
self.layers = nn.ModuleList([
copy.deepcopy(
TransformerEncoderBlock(
d_model,
d_ff,
attn_type,
n_heads,
dropout,
dropout_att,
dropout_layer,
layer_norm_eps,
ffn_activation,
param_init,
memory_transformer=self.memory_transformer))
for _ in range(n_layers)
])
self.norm_out = nn.LayerNorm(d_model, eps=layer_norm_eps)
self._odim = d_model
if n_layers_sub1 > 0:
if task_specific_layer:
self.layer_sub1 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_layer, layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub1 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim > 0 and last_proj_dim != self.output_dim:
self.bridge_sub1 = nn.Linear(self._odim, last_proj_dim)
if n_layers_sub2 > 0:
if task_specific_layer:
self.layer_sub2 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_layer, layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub2 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim > 0 and last_proj_dim != self.output_dim:
self.bridge_sub2 = nn.Linear(self._odim, last_proj_dim)
if last_proj_dim > 0 and last_proj_dim != self.output_dim:
self.bridge = nn.Linear(self._odim, last_proj_dim)
self._odim = last_proj_dim
self.reset_parameters(param_init)
def __init__(self,
input_dim,
attn_type,
attn_n_heads,
n_layers,
d_model,
d_ff,
pe_type='add',
layer_norm_eps=1e-6,
dropout_in=0,
dropout=0,
dropout_att=0,
last_proj_dim=0,
n_stacks=1,
n_splices=1,
conv_in_channel=1,
conv_channels=0,
conv_kernel_sizes=[],
conv_strides=[],
conv_poolings=[],
conv_batch_norm=False,
conv_residual=False,
conv_bottleneck_dim=0,
param_init=0.1):
super(TransformerEncoder, self).__init__()
logger = logging.getLogger("training")
self.d_model = d_model
self.n_layers = n_layers
self.attn_n_heads = attn_n_heads
self.pe_type = pe_type
# Setting for CNNs before RNNs
if conv_channels:
channels = [int(c) for c in conv_channels.split('_')
] if len(conv_channels) > 0 else []
kernel_sizes = [[
int(c.split(',')[0].replace('(', '')),
int(c.split(',')[1].replace(')', ''))
] for c in conv_kernel_sizes.split('_')
] if len(conv_kernel_sizes) > 0 else []
strides = [[
int(c.split(',')[0].replace('(', '')),
int(c.split(',')[1].replace(')', ''))
] for c in conv_strides.split('_')
] if len(conv_strides) > 0 else []
poolings = [[
int(c.split(',')[0].replace('(', '')),
int(c.split(',')[1].replace(')', ''))
] for c in conv_poolings.split('_')
] if len(conv_poolings) > 0 else []
else:
channels = []
kernel_sizes = []
strides = []
poolings = []
logger.warning(
'Subsampling is automatically ignored because CNN layers are used before RNN layers.'
)
if len(channels) > 0:
assert n_stacks == 1 and n_splices == 1
self.conv = ConvEncoder(input_dim,
in_channel=conv_in_channel,
channels=channels,
kernel_sizes=kernel_sizes,
strides=strides,
poolings=poolings,
dropout=0,
batch_norm=conv_batch_norm,
residual=conv_residual,
bottleneck_dim=d_model,
param_init=param_init)
self._output_dim = self.conv.output_dim
else:
self._output_dim = input_dim * n_splices * n_stacks
self.conv = None
self.embed = LinearND(self._output_dim, d_model,
dropout=0) # NOTE: do not apply dropout here
self.pos_enc = PositionalEncoding(d_model, dropout_in, pe_type)
self.layers = nn.ModuleList([
TransformerEncoderBlock(d_model, d_ff, attn_type, attn_n_heads,
dropout, dropout_att, layer_norm_eps)
for l in range(n_layers)
])
self.norm_out = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge = LinearND(self._output_dim,
last_proj_dim,
dropout=dropout)
self._output_dim = last_proj_dim
else:
self.bridge = None
self._output_dim = d_model
# Initialize parameters
self.reset_parameters()
def __init__(self, input_dim, enc_type, attn_type, n_heads, n_layers,
n_layers_sub1, n_layers_sub2, d_model, d_ff, last_proj_dim,
pe_type, layer_norm_eps, ffn_activation, dropout_in, dropout,
dropout_att, dropout_residual, n_stacks, n_splices,
conv_in_channel, conv_channels, conv_kernel_sizes,
conv_strides, conv_poolings, conv_batch_norm, conv_layer_norm,
conv_bottleneck_dim, conv_param_init, task_specific_layer,
param_init, chunk_size_left, chunk_size_current,
chunk_size_right, n_layers_rnn):
super(TransformerEncoder, self).__init__()
if n_layers_sub1 < 0 or (n_layers_sub1 > 1
and n_layers < n_layers_sub1):
raise ValueError('Set n_layers_sub1 between 1 to n_layers.')
if n_layers_sub2 < 0 or (n_layers_sub2 > 1
and n_layers_sub1 < n_layers_sub2):
raise ValueError('Set n_layers_sub2 between 1 to n_layers_sub1.')
self.d_model = d_model
self.n_layers = n_layers
self.n_heads = n_heads
self.pe_type = pe_type
# for streaming TransformerXL encoder
self.chunk_size_left = chunk_size_left
self.chunk_size_cur = chunk_size_current
self.chunk_size_right = chunk_size_right
self.latency_controlled = chunk_size_left > 0 or chunk_size_current > 0 or chunk_size_right > 0
self.memory_transformer = ('transformer_xl' in enc_type)
self.mem_len = chunk_size_left
self.scale = math.sqrt(d_model)
if self.memory_transformer:
assert pe_type == 'none'
assert chunk_size_left > 0
assert chunk_size_current > 0
if self.latency_controlled:
assert pe_type == 'none'
# for hybrid RNN-Transformer encoder
self.hybrid_rnn = n_layers_rnn > 0
self.n_layers_rnn = n_layers_rnn
self.proj = None
# for hierarchical encoder
self.n_layers_sub1 = n_layers_sub1
self.n_layers_sub2 = n_layers_sub2
self.task_specific_layer = task_specific_layer
# for bridge layers
self.bridge = None
self.bridge_sub1 = None
self.bridge_sub2 = None
# for attention plot
self.aws_dict = {}
self.data_dict = {}
# Setting for CNNs before RNNs
if conv_channels:
assert n_stacks == 1 and n_splices == 1
self.conv = ConvEncoder(input_dim,
in_channel=conv_in_channel,
channels=conv_channels,
kernel_sizes=conv_kernel_sizes,
strides=conv_strides,
poolings=conv_poolings,
dropout=0.,
batch_norm=conv_batch_norm,
layer_norm=conv_layer_norm,
layer_norm_eps=layer_norm_eps,
residual=False,
bottleneck_dim=d_model,
param_init=conv_param_init)
self._odim = self.conv.output_dim
else:
self.conv = None
self._odim = input_dim * n_splices * n_stacks
self.embed = nn.Linear(self._odim, d_model)
# Hybrid RNN-Transformer
if self.hybrid_rnn:
assert pe_type == 'none'
self.rnn = nn.ModuleList()
self.rnn_bwd = nn.ModuleList()
self.dropout_rnn = nn.Dropout(p=dropout)
assert ('blstm' in enc_type or 'bgru' in enc_type)
# NOTE: support bidirectional only
self.bidir_sum = True
for _ in range(n_layers_rnn):
if 'blstm' in enc_type:
rnn_i = nn.LSTM
elif 'bgru' in enc_type:
rnn_i = nn.GRU
else:
raise ValueError(
'rnn_type must be "(conv_)blstm_transformer(_xl)" or "(conv_)bgru_transformer(_xl)".'
)
self.rnn += [rnn_i(self._odim, d_model, 1, batch_first=True)]
self.rnn_bwd += [
rnn_i(self._odim, d_model, 1, batch_first=True)
]
self._odim = d_model if self.bidir_sum else d_model * self.n_dirs
if self._odim != d_model:
self.proj = nn.Linear(self._odim, d_model)
if self.memory_transformer:
self.pos_emb = XLPositionalEmbedding(d_model, dropout)
self.u = nn.Parameter(
torch.Tensor(self.n_heads, self.d_model // self.n_heads))
self.v = nn.Parameter(
torch.Tensor(self.n_heads, self.d_model // self.n_heads))
# NOTE: u and v are global parameters
self.pos_enc = PositionalEncoding(d_model, dropout_in, pe_type,
param_init)
# TODO: replace dropout_in with dropout
self.layers = nn.ModuleList([
copy.deepcopy(
TransformerEncoderBlock(
d_model,
d_ff,
attn_type,
n_heads,
dropout,
dropout_att,
dropout_residual * (lth + 1) / n_layers,
layer_norm_eps,
ffn_activation,
param_init,
memory_transformer=self.memory_transformer))
for lth in range(n_layers)
])
self.norm_out = nn.LayerNorm(d_model, eps=layer_norm_eps)
self._odim = d_model
if n_layers_sub1 > 0:
if task_specific_layer:
self.layer_sub1 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_residual * n_layers_sub1 / n_layers,
layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub1 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge_sub1 = nn.Linear(self._odim, last_proj_dim)
if n_layers_sub2 > 0:
if task_specific_layer:
self.layer_sub2 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_residual * n_layers_sub2 / n_layers,
layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub2 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge_sub2 = nn.Linear(self._odim, last_proj_dim)
if last_proj_dim != self.output_dim:
self.bridge = nn.Linear(self._odim, last_proj_dim)
self._odim = last_proj_dim
# calculate subsampling factor
self._factor = 1
if self.conv is not None:
self._factor *= self.conv.subsampling_factor
self.reset_parameters(param_init)
def __init__(self, input_dim, enc_type, attn_type, n_heads, n_layers,
n_layers_sub1, n_layers_sub2, d_model, d_ff, last_proj_dim,
pe_type, layer_norm_eps, ffn_activation, dropout_in, dropout,
dropout_att, dropout_residual, n_stacks, n_splices,
conv_in_channel, conv_channels, conv_kernel_sizes,
conv_strides, conv_poolings, conv_batch_norm, conv_layer_norm,
conv_bottleneck_dim, conv_param_init, task_specific_layer,
param_init, chunk_size_left, chunk_size_current,
chunk_size_right):
super(TransformerEncoder, self).__init__()
if n_layers_sub1 < 0 or (n_layers_sub1 > 1
and n_layers < n_layers_sub1):
raise ValueError('Set n_layers_sub1 between 1 to n_layers.')
if n_layers_sub2 < 0 or (n_layers_sub2 > 1
and n_layers_sub1 < n_layers_sub2):
raise ValueError('Set n_layers_sub2 between 1 to n_layers_sub1.')
self.d_model = d_model
self.n_layers = n_layers
self.n_heads = n_heads
self.pe_type = pe_type
# for latency-controlled
self.chunk_size_left = chunk_size_left
self.chunk_size_cur = chunk_size_current
self.chunk_size_right = chunk_size_right
# for hierarchical encoder
self.n_layers_sub1 = n_layers_sub1
self.n_layers_sub2 = n_layers_sub2
self.task_specific_layer = task_specific_layer
# for bridge layers
self.bridge = None
self.bridge_sub1 = None
self.bridge_sub2 = None
# for attention plot
self.aws_dict = {}
self.data_dict = {}
# Setting for CNNs before RNNs
if conv_channels:
assert n_stacks == 1 and n_splices == 1
self.conv = ConvEncoder(input_dim,
in_channel=conv_in_channel,
channels=conv_channels,
kernel_sizes=conv_kernel_sizes,
strides=conv_strides,
poolings=conv_poolings,
dropout=0.,
batch_norm=conv_batch_norm,
layer_norm=conv_layer_norm,
layer_norm_eps=layer_norm_eps,
residual=False,
bottleneck_dim=d_model,
param_init=conv_param_init)
self._odim = self.conv.output_dim
else:
self.conv = None
self._odim = input_dim * n_splices * n_stacks
self.embed = nn.Linear(self._odim, d_model)
self.pos_enc = PositionalEncoding(d_model, dropout_in, pe_type)
self.layers = nn.ModuleList([
copy.deepcopy(
TransformerEncoderBlock(d_model, d_ff, attn_type, n_heads,
dropout, dropout_att,
dropout_residual * (l + 1) / n_layers,
layer_norm_eps, ffn_activation,
param_init)) for l in range(n_layers)
])
self.norm_out = nn.LayerNorm(d_model, eps=layer_norm_eps)
self._odim = d_model
if n_layers_sub1 > 0:
if task_specific_layer:
self.layer_sub1 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_residual * n_layers_sub1 / n_layers,
layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub1 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge_sub1 = nn.Linear(self._odim, last_proj_dim)
if n_layers_sub2 > 0:
if task_specific_layer:
self.layer_sub2 = TransformerEncoderBlock(
d_model, d_ff, attn_type, n_heads, dropout, dropout_att,
dropout_residual * n_layers_sub2 / n_layers,
layer_norm_eps, ffn_activation, param_init)
self.norm_out_sub2 = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge_sub2 = nn.Linear(self._odim, last_proj_dim)
if last_proj_dim != self.output_dim:
self.bridge = nn.Linear(self._odim, last_proj_dim)
self._odim = last_proj_dim
# calculate subsampling factor
self._factor = 1
if self.conv is not None:
self._factor *= self.conv.subsampling_factor()
if param_init == 'xavier_uniform':
self.reset_parameters()
def __init__(self, input_dim, attn_type, n_heads, n_layers, d_model, d_ff,
last_proj_dim, pe_type, layer_norm_eps, ffn_activation,
dropout_in, dropout, dropout_att, n_stacks, n_splices,
conv_in_channel, conv_channels, conv_kernel_sizes,
conv_strides, conv_poolings, conv_batch_norm, conv_layer_norm,
conv_bottleneck_dim, conv_param_init, param_init,
chunk_size_left, chunk_size_current, chunk_size_right):
super(TransformerEncoder, self).__init__()
self.d_model = d_model
self.n_layers = n_layers
self.n_heads = n_heads
self.pe_type = pe_type
self.chunk_size_left = chunk_size_left
self.chunk_size_current = chunk_size_current
self.chunk_size_right = chunk_size_right
# Setting for CNNs before RNNs
if conv_channels:
assert n_stacks == 1 and n_splices == 1
self.conv = ConvEncoder(input_dim,
in_channel=conv_in_channel,
channels=conv_channels,
kernel_sizes=conv_kernel_sizes,
strides=conv_strides,
poolings=conv_poolings,
dropout=0.,
batch_norm=conv_batch_norm,
layer_norm=conv_layer_norm,
layer_norm_eps=layer_norm_eps,
residual=False,
bottleneck_dim=d_model,
param_init=conv_param_init)
self._odim = self.conv.output_dim
else:
self.conv = None
self._odim = input_dim * n_splices * n_stacks
self.embed = nn.Linear(self._odim, d_model)
self.pos_enc = PositionalEncoding(d_model, dropout_in, pe_type)
self.layers = repeat(
TransformerEncoderBlock(d_model, d_ff, attn_type, n_heads, dropout,
dropout_att, layer_norm_eps,
ffn_activation, param_init), n_layers)
self.norm_out = nn.LayerNorm(d_model, eps=layer_norm_eps)
if last_proj_dim != self.output_dim:
self.bridge = nn.Linear(self._odim, last_proj_dim)
self._odim = last_proj_dim
else:
self.bridge = None
self._odim = d_model
# calculate subsampling factor
self._factor = 1
if self.conv is not None:
self._factor *= self.conv.subsampling_factor()
if param_init == 'xavier_uniform':
self.reset_parameters()