def __init__( self, dictionary, word_dropout_params=None, embed_dim=512, freeze_embed=False, hidden_dim=512, num_layers=1, cell_type="lstm", dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, pretrained_embed=None, padding_value=0, left_pad=True, ): super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.output_units = hidden_dim # fairseq LSTM compatibility self.bidirectional = bidirectional num_embeddings = len(dictionary) self.padding_idx = dictionary.pad() self.padding_value = padding_value self.left_pad = left_pad if pretrained_embed is not None and type(pretrained_embed) is not str: self.embed_tokens = pretrained_embed else: self.embed_tokens = Embedding( num_embeddings=num_embeddings, embedding_dim=embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, pretrained_embed=pretrained_embed, ) self.word_dim = embed_dim self.cell_type = cell_type self.layers = nn.ModuleList([]) for layer in range(num_layers): self.layers.append( RNNLayer( self.word_dim if layer == 0 else hidden_dim, hidden_dim, self.cell_type, True if bidirectional and layer == 0 else False, )) self.num_layers = len(self.layers) self.word_dropout_module = None if (word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params)
def __init__( self, dictionary, embed_dim=512, freeze_embed=False, cell_type="lstm", hidden_dim=512, num_layers=1, dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, word_dropout_params=None, padding_value=0, ): assert cell_type == "lstm", 'sequence-lstm requires cell_type="lstm"' super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional num_embeddings = len(dictionary) self.padding_idx = dictionary.pad() self.padding_value = padding_value self.embed_tokens = Embedding( num_embeddings=num_embeddings, embedding_dim=embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) self.word_dim = embed_dim self.layers = nn.ModuleList([]) for layer in range(num_layers): is_layer_bidirectional = self.bidirectional and layer == 0 self.layers.append( LSTMSequenceEncoder.LSTM( self.word_dim if layer == 0 else hidden_dim, hidden_dim // 2 if is_layer_bidirectional else hidden_dim, num_layers=1, dropout=self.dropout_out, bidirectional=is_layer_bidirectional, ) ) self.num_layers = len(self.layers) self.word_dropout_module = None if ( word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0 ): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params )
def test_apply_probabilistic_unking(self): word_dropout_params = { "word_dropout_freq_threshold": 3, "word_dropout_smoothing_alpha": 1, } test_args = test_utils.ModelParamsDict(cell_type="rnn") test_args.word_dropout_params = word_dropout_params samples, src_dict, tgt_dict = test_utils.prepare_inputs(test_args) word_dropout_module = word_dropout.WordDropout(src_dict, word_dropout_params) word_dropout_module.apply_probabilistic_unking(3)
def __init__( self, dictionary, word_dropout_params=None, embed_dim=512, freeze_embed=False, hidden_dim=512, num_layers=1, cell_type='lstm', dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, add_encoder_output_as_decoder_input=False, ): super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional self.add_encoder_output_as_decoder_input = ( add_encoder_output_as_decoder_input ) num_embeddings = len(dictionary) self.padding_idx = dictionary.pad() self.embed_tokens = Embedding( num_embeddings=num_embeddings, embedding_dim=embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) self.cell_type = cell_type self.layers = nn.ModuleList([]) for layer in range(num_layers): self.layers.append( RNNLayer( embed_dim if layer == 0 else hidden_dim, hidden_dim, self.cell_type, True if bidirectional and layer == 0 else False ) ) self.num_layers = len(self.layers) self.word_dropout_module = None if ( word_dropout_params and word_dropout_params['word_dropout_freq_threshold'] is not None and word_dropout_params['word_dropout_freq_threshold'] > 0 ): self.word_dropout_module = ( word_dropout.WordDropout(dictionary, word_dropout_params) )
def __init__( self, dictionary, num_chars=50, embed_dim=32, token_embed_dim=256, freeze_embed=False, char_cnn_params="[(128, 3), (128, 5)]", char_cnn_output_dim=256, char_cnn_nonlinear_fn="tanh", char_cnn_pool_type="max", char_cnn_num_highway_layers=0, hidden_dim=512, num_layers=1, dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, word_dropout_params=None, ): super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional convolutions_params = literal_eval(char_cnn_params) self.char_cnn_encoder = char_encoder.CharCNNModel( dictionary, num_chars, embed_dim, convolutions_params, char_cnn_nonlinear_fn, char_cnn_pool_type, char_cnn_num_highway_layers, ) self.embed_tokens = None num_tokens = len(dictionary) self.padding_idx = dictionary.pad() if token_embed_dim > 0: self.embed_tokens = rnn.Embedding( num_embeddings=num_tokens, embedding_dim=token_embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) self.word_dim = (sum(out_dim for (out_dim, _) in convolutions_params) + token_embed_dim) self.layers = nn.ModuleList([]) for layer in range(num_layers): is_layer_bidirectional = self.bidirectional and layer == 0 if is_layer_bidirectional: assert hidden_dim % 2 == 0, ( "encoder_hidden_dim must be even if encoder_bidirectional " "(to be divided evenly between directions)" ) self.layers.append( rnn.LSTMSequenceEncoder.LSTM( self.word_dim if layer == 0 else hidden_dim, hidden_dim // 2 if is_layer_bidirectional else hidden_dim, num_layers=1, dropout=self.dropout_out, bidirectional=is_layer_bidirectional, ) ) self.num_layers = len(self.layers) self.word_dropout_module = None if ( word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0 ): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params )
def __init__( self, dictionary, num_chars, char_embed_dim, token_embed_dim, freeze_embed=False, char_rnn_units=256, char_rnn_layers=1, hidden_dim=512, num_layers=1, dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, word_dropout_params=None, ): super().__init__(dictionary) self.dictionary = dictionary self.num_chars = num_chars self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional num_tokens = len(dictionary) self.padding_idx = dictionary.pad() self.embed_chars = rnn.Embedding( num_embeddings=num_chars, embedding_dim=char_embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) assert ( char_rnn_units % 2 == 0 ), "char_rnn_units must be even (to be divided evenly between directions)" self.char_lstm_encoder = rnn.LSTMSequenceEncoder.LSTM( char_embed_dim, char_rnn_units // 2, num_layers=char_rnn_layers, bidirectional=True, ) self.embed_tokens = None if token_embed_dim > 0: self.embed_tokens = rnn.Embedding( num_embeddings=num_tokens, embedding_dim=token_embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) self.word_dim = char_rnn_units + token_embed_dim self.layers = nn.ModuleList([]) for layer in range(num_layers): is_layer_bidirectional = self.bidirectional and layer == 0 if is_layer_bidirectional: assert hidden_dim % 2 == 0, ( "encoder_hidden_dim must be even if encoder_bidirectional " "(to be divided evenly between directions)" ) self.layers.append( rnn.LSTMSequenceEncoder.LSTM( self.word_dim if layer == 0 else hidden_dim, hidden_dim // 2 if is_layer_bidirectional else hidden_dim, num_layers=1, dropout=self.dropout_out, bidirectional=is_layer_bidirectional, ) ) self.num_layers = len(self.layers) self.word_dropout_module = None if ( word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0 ): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params ) # disables sorting and word-length thresholding if True # (enables ONNX tracing of length-sorted input with batch_size = 1) self.onnx_export_model = False
def __init__( self, dictionary, embed_dim=512, freeze_embed=False, cell_type="lstm", hidden_dim=512, num_layers=1, dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, pretrained_embed=None, word_dropout_params=None, padding_value=0, left_pad=True, ): assert cell_type == "lstm", 'sequence-lstm requires cell_type="lstm"' super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional num_embeddings = len(dictionary) self.padding_idx = dictionary.pad() self.padding_value = padding_value self.left_pad = left_pad self.embed_tokens = Embedding( num_embeddings=num_embeddings, embedding_dim=embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) pytorch_translate_utils.load_embedding( embedding=self.embed_tokens, dictionary=dictionary, pretrained_embed=pretrained_embed, ) self.word_dim = embed_dim self.layers = nn.ModuleList([]) for layer in range(num_layers): is_layer_bidirectional = self.bidirectional and layer == 0 self.layers.append( LSTMSequenceEncoder.LSTM( self.word_dim if layer == 0 else hidden_dim, hidden_dim // 2 if is_layer_bidirectional else hidden_dim, num_layers=1, dropout=self.dropout_out, bidirectional=is_layer_bidirectional, )) self.num_layers = len(self.layers) self.word_dropout_module = None if (word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params) # Variable tracker self.tracker = VariableTracker() # Initialize adversarial mode self.set_gradient_tracking_mode(False)
def __init__( self, dictionary, word_dropout_params=None, embed_dim=512, freeze_embed=False, hidden_dim=512, num_layers=1, cell_type="lstm", dropout_in=0.1, dropout_out=0.1, residual_level=None, bidirectional=False, add_encoder_output_as_decoder_input=False, char_rnn_params=None, ): super().__init__(dictionary) self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out self.residual_level = residual_level self.hidden_dim = hidden_dim self.bidirectional = bidirectional self.add_encoder_output_as_decoder_input = add_encoder_output_as_decoder_input num_embeddings = len(dictionary) self.padding_idx = dictionary.pad() self.char_rnn_params = char_rnn_params if self.char_rnn_params is not None: self.char_rnn_encoder = char_rnn_encoder.CharRNN( dictionary=self.dictionary, embed_dim=embed_dim, hidden_dim=char_rnn_params["char_rnn_units"], num_layers=char_rnn_params["char_rnn_layers"], bidirectional=True, word_delimiter=char_rnn_params["word_delimiter"], ) self.word_dim = char_rnn_params["char_rnn_units"] else: self.embed_tokens = Embedding( num_embeddings=num_embeddings, embedding_dim=embed_dim, padding_idx=self.padding_idx, freeze_embed=freeze_embed, ) self.word_dim = embed_dim self.cell_type = cell_type self.layers = nn.ModuleList([]) for layer in range(num_layers): self.layers.append( RNNLayer( self.word_dim if layer == 0 else hidden_dim, hidden_dim, self.cell_type, True if bidirectional and layer == 0 else False, )) self.num_layers = len(self.layers) self.word_dropout_module = None if (word_dropout_params and word_dropout_params["word_dropout_freq_threshold"] is not None and word_dropout_params["word_dropout_freq_threshold"] > 0): self.word_dropout_module = word_dropout.WordDropout( dictionary, word_dropout_params)