def __init__(
self,
dictionary,
word_dropout_params=None,
embed_dim=512,
freeze_embed=False,
hidden_dim=512,
num_layers=1,
cell_type="lstm",
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
pretrained_embed=None,
padding_value=0,
left_pad=True,
):
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.output_units = hidden_dim # fairseq LSTM compatibility
self.bidirectional = bidirectional
num_embeddings = len(dictionary)
self.padding_idx = dictionary.pad()
self.padding_value = padding_value
self.left_pad = left_pad
if pretrained_embed is not None and type(pretrained_embed) is not str:
self.embed_tokens = pretrained_embed
else:
self.embed_tokens = Embedding(
num_embeddings=num_embeddings,
embedding_dim=embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
pretrained_embed=pretrained_embed,
)
self.word_dim = embed_dim
self.cell_type = cell_type
self.layers = nn.ModuleList([])
for layer in range(num_layers):
self.layers.append(
RNNLayer(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim,
self.cell_type,
True if bidirectional and layer == 0 else False,
))
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (word_dropout_params and
word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params)
def __init__(
self,
dictionary,
embed_dim=512,
freeze_embed=False,
cell_type="lstm",
hidden_dim=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
word_dropout_params=None,
padding_value=0,
):
assert cell_type == "lstm", 'sequence-lstm requires cell_type="lstm"'
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
num_embeddings = len(dictionary)
self.padding_idx = dictionary.pad()
self.padding_value = padding_value
self.embed_tokens = Embedding(
num_embeddings=num_embeddings,
embedding_dim=embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
self.word_dim = embed_dim
self.layers = nn.ModuleList([])
for layer in range(num_layers):
is_layer_bidirectional = self.bidirectional and layer == 0
self.layers.append(
LSTMSequenceEncoder.LSTM(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim // 2 if is_layer_bidirectional else hidden_dim,
num_layers=1,
dropout=self.dropout_out,
bidirectional=is_layer_bidirectional,
)
)
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (
word_dropout_params
and word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0
):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params
)
def test_apply_probabilistic_unking(self):
word_dropout_params = {
"word_dropout_freq_threshold": 3,
"word_dropout_smoothing_alpha": 1,
}
test_args = test_utils.ModelParamsDict(cell_type="rnn")
test_args.word_dropout_params = word_dropout_params
samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
word_dropout_module = word_dropout.WordDropout(src_dict, word_dropout_params)
word_dropout_module.apply_probabilistic_unking(3)
def __init__(
self,
dictionary,
word_dropout_params=None,
embed_dim=512,
freeze_embed=False,
hidden_dim=512,
num_layers=1,
cell_type='lstm',
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
add_encoder_output_as_decoder_input=False,
):
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
self.add_encoder_output_as_decoder_input = (
add_encoder_output_as_decoder_input
)
num_embeddings = len(dictionary)
self.padding_idx = dictionary.pad()
self.embed_tokens = Embedding(
num_embeddings=num_embeddings,
embedding_dim=embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
self.cell_type = cell_type
self.layers = nn.ModuleList([])
for layer in range(num_layers):
self.layers.append(
RNNLayer(
embed_dim if layer == 0 else hidden_dim,
hidden_dim,
self.cell_type,
True if bidirectional and layer == 0 else False
)
)
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (
word_dropout_params and
word_dropout_params['word_dropout_freq_threshold'] is not None and
word_dropout_params['word_dropout_freq_threshold'] > 0
):
self.word_dropout_module = (
word_dropout.WordDropout(dictionary, word_dropout_params)
)
def __init__(
self,
dictionary,
num_chars=50,
embed_dim=32,
token_embed_dim=256,
freeze_embed=False,
char_cnn_params="[(128, 3), (128, 5)]",
char_cnn_output_dim=256,
char_cnn_nonlinear_fn="tanh",
char_cnn_pool_type="max",
char_cnn_num_highway_layers=0,
hidden_dim=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
word_dropout_params=None,
):
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
convolutions_params = literal_eval(char_cnn_params)
self.char_cnn_encoder = char_encoder.CharCNNModel(
dictionary,
num_chars,
embed_dim,
convolutions_params,
char_cnn_nonlinear_fn,
char_cnn_pool_type,
char_cnn_num_highway_layers,
)
self.embed_tokens = None
num_tokens = len(dictionary)
self.padding_idx = dictionary.pad()
if token_embed_dim > 0:
self.embed_tokens = rnn.Embedding(
num_embeddings=num_tokens,
embedding_dim=token_embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
self.word_dim = (sum(out_dim for (out_dim, _) in convolutions_params) +
token_embed_dim)
self.layers = nn.ModuleList([])
for layer in range(num_layers):
is_layer_bidirectional = self.bidirectional and layer == 0
if is_layer_bidirectional:
assert hidden_dim % 2 == 0, (
"encoder_hidden_dim must be even if encoder_bidirectional "
"(to be divided evenly between directions)"
)
self.layers.append(
rnn.LSTMSequenceEncoder.LSTM(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim // 2 if is_layer_bidirectional else hidden_dim,
num_layers=1,
dropout=self.dropout_out,
bidirectional=is_layer_bidirectional,
)
)
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (
word_dropout_params
and word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0
):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params
)
def __init__(
self,
dictionary,
num_chars,
char_embed_dim,
token_embed_dim,
freeze_embed=False,
char_rnn_units=256,
char_rnn_layers=1,
hidden_dim=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
word_dropout_params=None,
):
super().__init__(dictionary)
self.dictionary = dictionary
self.num_chars = num_chars
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
num_tokens = len(dictionary)
self.padding_idx = dictionary.pad()
self.embed_chars = rnn.Embedding(
num_embeddings=num_chars,
embedding_dim=char_embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
assert (
char_rnn_units % 2 == 0
), "char_rnn_units must be even (to be divided evenly between directions)"
self.char_lstm_encoder = rnn.LSTMSequenceEncoder.LSTM(
char_embed_dim,
char_rnn_units // 2,
num_layers=char_rnn_layers,
bidirectional=True,
)
self.embed_tokens = None
if token_embed_dim > 0:
self.embed_tokens = rnn.Embedding(
num_embeddings=num_tokens,
embedding_dim=token_embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
self.word_dim = char_rnn_units + token_embed_dim
self.layers = nn.ModuleList([])
for layer in range(num_layers):
is_layer_bidirectional = self.bidirectional and layer == 0
if is_layer_bidirectional:
assert hidden_dim % 2 == 0, (
"encoder_hidden_dim must be even if encoder_bidirectional "
"(to be divided evenly between directions)"
)
self.layers.append(
rnn.LSTMSequenceEncoder.LSTM(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim // 2 if is_layer_bidirectional else hidden_dim,
num_layers=1,
dropout=self.dropout_out,
bidirectional=is_layer_bidirectional,
)
)
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (
word_dropout_params
and word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0
):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params
)
# disables sorting and word-length thresholding if True
# (enables ONNX tracing of length-sorted input with batch_size = 1)
self.onnx_export_model = False
def __init__(
self,
dictionary,
embed_dim=512,
freeze_embed=False,
cell_type="lstm",
hidden_dim=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
pretrained_embed=None,
word_dropout_params=None,
padding_value=0,
left_pad=True,
):
assert cell_type == "lstm", 'sequence-lstm requires cell_type="lstm"'
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
num_embeddings = len(dictionary)
self.padding_idx = dictionary.pad()
self.padding_value = padding_value
self.left_pad = left_pad
self.embed_tokens = Embedding(
num_embeddings=num_embeddings,
embedding_dim=embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
pytorch_translate_utils.load_embedding(
embedding=self.embed_tokens,
dictionary=dictionary,
pretrained_embed=pretrained_embed,
)
self.word_dim = embed_dim
self.layers = nn.ModuleList([])
for layer in range(num_layers):
is_layer_bidirectional = self.bidirectional and layer == 0
self.layers.append(
LSTMSequenceEncoder.LSTM(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim // 2 if is_layer_bidirectional else hidden_dim,
num_layers=1,
dropout=self.dropout_out,
bidirectional=is_layer_bidirectional,
))
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (word_dropout_params and
word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params)
# Variable tracker
self.tracker = VariableTracker()
# Initialize adversarial mode
self.set_gradient_tracking_mode(False)
def __init__(
self,
dictionary,
word_dropout_params=None,
embed_dim=512,
freeze_embed=False,
hidden_dim=512,
num_layers=1,
cell_type="lstm",
dropout_in=0.1,
dropout_out=0.1,
residual_level=None,
bidirectional=False,
add_encoder_output_as_decoder_input=False,
char_rnn_params=None,
):
super().__init__(dictionary)
self.dictionary = dictionary
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.residual_level = residual_level
self.hidden_dim = hidden_dim
self.bidirectional = bidirectional
self.add_encoder_output_as_decoder_input = add_encoder_output_as_decoder_input
num_embeddings = len(dictionary)
self.padding_idx = dictionary.pad()
self.char_rnn_params = char_rnn_params
if self.char_rnn_params is not None:
self.char_rnn_encoder = char_rnn_encoder.CharRNN(
dictionary=self.dictionary,
embed_dim=embed_dim,
hidden_dim=char_rnn_params["char_rnn_units"],
num_layers=char_rnn_params["char_rnn_layers"],
bidirectional=True,
word_delimiter=char_rnn_params["word_delimiter"],
)
self.word_dim = char_rnn_params["char_rnn_units"]
else:
self.embed_tokens = Embedding(
num_embeddings=num_embeddings,
embedding_dim=embed_dim,
padding_idx=self.padding_idx,
freeze_embed=freeze_embed,
)
self.word_dim = embed_dim
self.cell_type = cell_type
self.layers = nn.ModuleList([])
for layer in range(num_layers):
self.layers.append(
RNNLayer(
self.word_dim if layer == 0 else hidden_dim,
hidden_dim,
self.cell_type,
True if bidirectional and layer == 0 else False,
))
self.num_layers = len(self.layers)
self.word_dropout_module = None
if (word_dropout_params and
word_dropout_params["word_dropout_freq_threshold"] is not None
and word_dropout_params["word_dropout_freq_threshold"] > 0):
self.word_dropout_module = word_dropout.WordDropout(
dictionary, word_dropout_params)
