def __init__(
self, dictionary, embed_dim=512, hidden_size=512, out_embed_dim=512,
num_layers=1, dropout_in=0.1, dropout_out=0.1, encoder_output_units=0,
attn_type=None, attn_dim=0, need_attn=False, residual=False, pretrained_embed=None,
share_input_output_embed=False, adaptive_softmax_cutoff=None,
):
super().__init__(dictionary)
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.hidden_size = hidden_size
self.share_input_output_embed = share_input_output_embed
if attn_type is None or attn_type.lower() == 'none':
# no attention, no encoder output needed (language model case)
need_attn = False
encoder_output_units = 0
self.need_attn = need_attn
self.residual = residual
self.adaptive_softmax = None
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
if pretrained_embed is None:
self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
else:
self.embed_tokens = pretrained_embed
self.encoder_output_units = encoder_output_units
self.layers = nn.ModuleList([
LSTMCell(
input_size=encoder_output_units + (embed_dim if layer == 0 else hidden_size),
hidden_size=hidden_size,
)
for layer in range(num_layers)
])
if attn_type is None or attn_type.lower() == 'none':
self.attention = None
elif attn_type.lower() == 'bahdanau':
self.attention = speech_attention.BahdanauAttention(
hidden_size, encoder_output_units, attn_dim,
)
elif attn_type.lower() == 'luong':
self.attention = speech_attention.LuongAttention(
hidden_size, encoder_output_units,
)
else:
raise ValueError('unrecognized attention type.')
if hidden_size + encoder_output_units != out_embed_dim:
self.additional_fc = Linear(hidden_size + encoder_output_units, out_embed_dim)
if adaptive_softmax_cutoff is not None:
# setting adaptive_softmax dropout to dropout_out for now but can be redefined
self.adaptive_softmax = AdaptiveSoftmax(num_embeddings, hidden_size, adaptive_softmax_cutoff,
dropout=dropout_out)
elif not self.share_input_output_embed:
self.fc_out = Linear(out_embed_dim, num_embeddings, dropout=dropout_out)
def __init__(
self,
dictionary,
embed_dim=512,
hidden_size=512,
out_embed_dim=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
encoder_output_units=0,
attn_type=None,
attn_dim=0,
need_attn=False,
residual=False,
pretrained_embed=None,
share_input_output_embed=False,
adaptive_softmax_cutoff=None,
max_target_positions=DEFAULT_MAX_TARGET_POSITIONS,
scheduled_sampling_rate_scheduler=None,
):
super().__init__(dictionary)
self.dropout_in_module = FairseqDropout(
dropout_in * 1.0, module_name=self.__class__.__name__
)
self.dropout_out_module = FairseqDropout(
dropout_out * 1.0, module_name=self.__class__.__name__
)
self.hidden_size = hidden_size
self.share_input_output_embed = share_input_output_embed
if attn_type is None or str(attn_type).lower() == "none":
# no attention, no encoder output needed (language model case)
need_attn = False
encoder_output_units = 0
self.need_attn = need_attn
self.residual = residual
self.max_target_positions = max_target_positions
self.num_layers = num_layers
self.adaptive_softmax = None
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
if pretrained_embed is None:
self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
else:
self.embed_tokens = pretrained_embed
self.encoder_output_units = encoder_output_units
self.layers = nn.ModuleList(
[
LSTMCell(
input_size=encoder_output_units
+ (embed_dim if layer == 0 else hidden_size),
hidden_size=hidden_size,
)
for layer in range(num_layers)
]
)
if attn_type is None or str(attn_type).lower() == "none":
self.attention = None
elif str(attn_type).lower() == "bahdanau":
self.attention = speech_attention.BahdanauAttention(
hidden_size,
encoder_output_units,
attn_dim,
)
elif str(attn_type).lower() == "luong":
self.attention = speech_attention.LuongAttention(
hidden_size,
encoder_output_units,
)
else:
raise ValueError("unrecognized attention type.")
if hidden_size + encoder_output_units != out_embed_dim:
self.additional_fc = Linear(
hidden_size + encoder_output_units, out_embed_dim
)
if adaptive_softmax_cutoff is not None:
# setting adaptive_softmax dropout to dropout_out for now but can be redefined
self.adaptive_softmax = AdaptiveSoftmax(
num_embeddings,
hidden_size,
adaptive_softmax_cutoff,
dropout=dropout_out,
)
elif not self.share_input_output_embed:
self.fc_out = Linear(out_embed_dim, num_embeddings, dropout=dropout_out)
self.scheduled_sampling_rate_scheduler = scheduled_sampling_rate_scheduler