def __init__(self, config, tokenizer, label_tokenizer): super(AttnEDSeqLabeler, self).__init__(config, tokenizer, label_tokenizer) # Attributes # Attributes from config self.dial_encoder_hidden_dim = config.dial_encoder_hidden_dim self.n_dial_encoder_layers = config.n_dial_encoder_layers self.attention_type = config.attention_type # Encoding components self.dial_encoder = EncoderRNN( input_dim=self.sent_encoder_hidden_dim, hidden_dim=self.dial_encoder_hidden_dim, n_layers=self.n_dial_encoder_layers, dropout_emb=self.dropout, dropout_input=self.dropout, dropout_hidden=self.dropout, dropout_output=self.dropout, bidirectional=False, rnn_type=self.rnn_type, ) # Decoding components self.enc2dec_hidden_fc = nn.Linear( self.dial_encoder_hidden_dim, self.n_decoder_layers*self.decoder_hidden_dim if self.rnn_type == "gru" else self.n_decoder_layers*self.decoder_hidden_dim*2 ) self.decoder = DecoderRNN( vocab_size=self.label_vocab_size, input_dim=self.attr_embedding_dim, hidden_dim=self.decoder_hidden_dim, feat_dim=self.sent_encoder_hidden_dim, n_layers=self.n_decoder_layers, bos_token_id=self.bos_label_id, eos_token_id=self.eos_label_id, pad_token_id=self.pad_label_id, max_len=self.decode_max_len, dropout_emb=self.dropout, dropout_input=self.dropout, dropout_hidden=self.dropout, dropout_output=self.dropout, embedding=self.label_embedding, tie_weights=self.tie_weights, rnn_type=self.rnn_type, use_attention=True, attn_dim=self.sent_encoder_hidden_dim ) self.floor_encoder = RelFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim )
def __init__(self, config, tokenizer): super(HRED, self).__init__() # Attributes # Attributes from config self.word_embedding_dim = config.word_embedding_dim self.attr_embedding_dim = config.attr_embedding_dim self.sent_encoder_hidden_dim = config.sent_encoder_hidden_dim self.n_sent_encoder_layers = config.n_sent_encoder_layers self.dial_encoder_hidden_dim = config.dial_encoder_hidden_dim self.n_dial_encoder_layers = config.n_dial_encoder_layers self.decoder_hidden_dim = config.decoder_hidden_dim self.n_decoder_layers = config.n_decoder_layers self.use_attention = config.use_attention self.decode_max_len = config.decode_max_len self.tie_weights = config.tie_weights self.rnn_type = config.rnn_type self.gen_type = config.gen_type self.top_k = config.top_k self.top_p = config.top_p self.temp = config.temp self.word_embedding_path = config.word_embedding_path self.floor_encoder_type = config.floor_encoder # Optional attributes from config self.dropout_emb = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_input = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_hidden = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_output = config.dropout if hasattr(config, "dropout") else 0.0 self.use_pretrained_word_embedding = config.use_pretrained_word_embedding if hasattr( config, "use_pretrained_word_embedding") else False # Other attributes self.word2id = tokenizer.word2id self.id2word = tokenizer.id2word self.vocab_size = len(tokenizer.word2id) self.pad_token_id = tokenizer.pad_token_id self.bos_token_id = tokenizer.bos_token_id self.eos_token_id = tokenizer.eos_token_id # Input components self.word_embedding = nn.Embedding( self.vocab_size, self.word_embedding_dim, padding_idx=self.pad_token_id, _weight=init_word_embedding( load_pretrained_word_embedding=self. use_pretrained_word_embedding, pretrained_word_embedding_path=self.word_embedding_path, id2word=self.id2word, word_embedding_dim=self.word_embedding_dim, vocab_size=self.vocab_size, pad_token_id=self.pad_token_id), ) # Encoding components self.sent_encoder = EncoderRNN( input_dim=self.word_embedding_dim, hidden_dim=self.sent_encoder_hidden_dim, n_layers=self.n_sent_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=True, embedding=self.word_embedding, rnn_type=self.rnn_type, ) self.dial_encoder = EncoderRNN( input_dim=self.sent_encoder_hidden_dim, hidden_dim=self.dial_encoder_hidden_dim, n_layers=self.n_dial_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=False, rnn_type=self.rnn_type, ) # Decoding components self.enc2dec_hidden_fc = nn.Linear( self.dial_encoder_hidden_dim, self.n_decoder_layers * self.decoder_hidden_dim if self.rnn_type == "gru" else self.n_decoder_layers * self.decoder_hidden_dim * 2) self.decoder = DecoderRNN(vocab_size=len(self.word2id), input_dim=self.word_embedding_dim, hidden_dim=self.decoder_hidden_dim, feat_dim=self.dial_encoder_hidden_dim, n_layers=self.n_decoder_layers, bos_token_id=self.bos_token_id, eos_token_id=self.eos_token_id, pad_token_id=self.pad_token_id, max_len=self.decode_max_len, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, embedding=self.word_embedding, tie_weights=self.tie_weights, rnn_type=self.rnn_type, use_attention=self.use_attention, attn_dim=self.sent_encoder_hidden_dim) # Extra components # floor encoding if self.floor_encoder_type == "abs": self.floor_encoder = AbsFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim) elif self.floor_encoder_type == "rel": self.floor_encoder = RelFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim) else: self.floor_encoder = None # Initialization self._init_weights()
def __init__(self, config, tokenizer): super(RNNLM, self).__init__() # Attributes # Attributes from config self.word_embedding_dim = config.word_embedding_dim self.decoder_hidden_dim = config.decoder_hidden_dim self.n_decoder_layers = config.n_decoder_layers self.decode_max_len = config.decode_max_len self.tie_weights = config.tie_weights self.rnn_type = config.rnn_type # Optional attributes from config self.gen_type = config.gen_type if hasattr(config, "gen_type") else "greedy" self.top_k = config.top_k if hasattr(config, "top_k") else 0 self.top_p = config.top_p if hasattr(config, "top_p") else 0.0 self.temp = config.temp if hasattr(config, "temp") else 1.0 self.dropout_emb = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_input = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_hidden = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_output = config.dropout if hasattr(config, "dropout") else 0.0 self.use_pretrained_word_embedding = config.use_pretrained_word_embedding if hasattr(config, "use_pretrained_word_embedding") else False self.word_embedding_path = config.word_embedding_path if hasattr(config, "word_embedding_path") else None # Other attributes self.tokenizer = tokenizer self.word2id = tokenizer.word2id self.id2word = tokenizer.id2word self.vocab_size = len(tokenizer.word2id) self.pad_token_id = tokenizer.pad_token_id self.bos_token_id = tokenizer.bos_token_id self.eos_token_id = tokenizer.eos_token_id # Components self.word_embedding = nn.Embedding( self.vocab_size, self.word_embedding_dim, padding_idx=self.pad_token_id, _weight=init_word_embedding( load_pretrained_word_embedding=self.use_pretrained_word_embedding, pretrained_word_embedding_path=self.word_embedding_path, id2word=self.id2word, word_embedding_dim=self.word_embedding_dim, vocab_size=self.vocab_size, pad_token_id=self.pad_token_id ), ) self.decoder = DecoderRNN( vocab_size=len(self.word2id), input_dim=self.word_embedding_dim, hidden_dim=self.decoder_hidden_dim, n_layers=self.n_decoder_layers, bos_token_id=self.bos_token_id, eos_token_id=self.eos_token_id, pad_token_id=self.pad_token_id, max_len=self.decode_max_len, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, embedding=self.word_embedding, tie_weights=self.tie_weights, rnn_type=self.rnn_type, ) # Initialization self._init_weights()
def __init__(self, config, tokenizer): super(VHRED, self).__init__() # Attributes # Attributes from config self.word_embedding_dim = config.word_embedding_dim self.attr_embedding_dim = config.attr_embedding_dim self.sent_encoder_hidden_dim = config.sent_encoder_hidden_dim self.n_sent_encoder_layers = config.n_sent_encoder_layers self.dial_encoder_hidden_dim = config.dial_encoder_hidden_dim self.n_dial_encoder_layers = config.n_dial_encoder_layers self.latent_variable_dim = config.latent_dim self.decoder_hidden_dim = config.decoder_hidden_dim self.n_decoder_layers = config.n_decoder_layers self.use_attention = config.use_attention self.decode_max_len = config.decode_max_len self.tie_weights = config.tie_weights self.rnn_type = config.rnn_type self.gen_type = config.gen_type self.top_k = config.top_k self.top_p = config.top_p self.temp = config.temp self.word_embedding_path = config.word_embedding_path self.floor_encoder_type = config.floor_encoder self.gaussian_mix_type = config.gaussian_mix_type # Optional attributes from config self.use_bow_loss = config.use_bow_loss if hasattr( config, "use_bow_loss") else True self.dropout = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_emb = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_input = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_hidden = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_output = config.dropout if hasattr(config, "dropout") else 0.0 self.use_pretrained_word_embedding = config.use_pretrained_word_embedding if hasattr( config, "use_pretrained_word_embedding") else False self.n_step_annealing = config.n_step_annealing if hasattr( config, "n_step_annealing") else 1 self.n_components = config.n_components if hasattr( config, "n_components") else 1 # Other attributes self.vocab_size = len(tokenizer.word2id) self.word2id = tokenizer.word2id self.id2word = tokenizer.id2word self.pad_token_id = tokenizer.pad_token_id self.bos_token_id = tokenizer.bos_token_id self.eos_token_id = tokenizer.eos_token_id # Input components self.word_embedding = nn.Embedding( self.vocab_size, self.word_embedding_dim, padding_idx=self.pad_token_id, _weight=init_word_embedding( load_pretrained_word_embedding=self. use_pretrained_word_embedding, pretrained_word_embedding_path=self.word_embedding_path, id2word=self.id2word, word_embedding_dim=self.word_embedding_dim, vocab_size=self.vocab_size, pad_token_id=self.pad_token_id), ) # Encoding components self.sent_encoder = EncoderRNN( input_dim=self.word_embedding_dim, hidden_dim=self.sent_encoder_hidden_dim, n_layers=self.n_sent_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=True, embedding=self.word_embedding, rnn_type=self.rnn_type, ) self.dial_encoder = EncoderRNN( input_dim=self.sent_encoder_hidden_dim, hidden_dim=self.dial_encoder_hidden_dim, n_layers=self.n_dial_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=False, rnn_type=self.rnn_type, ) # Variational components if config.n_components == 1: self.prior_net = GaussianVariation( input_dim=self.dial_encoder_hidden_dim, z_dim=self.latent_variable_dim, # large_mlp=True ) elif config.n_components > 1: if self.gaussian_mix_type == "gmm": self.prior_net = GMMVariation( input_dim=self.dial_encoder_hidden_dim, z_dim=self.latent_variable_dim, n_components=self.n_components, ) elif self.gaussian_mix_type == "lgm": self.prior_net = LGMVariation( input_dim=self.dial_encoder_hidden_dim, z_dim=self.latent_variable_dim, n_components=self.n_components, ) self.post_net = GaussianVariation( input_dim=self.sent_encoder_hidden_dim + self.dial_encoder_hidden_dim, z_dim=self.latent_variable_dim, ) self.latent_to_bow = nn.Sequential( nn.Linear(self.latent_variable_dim + self.dial_encoder_hidden_dim, self.latent_variable_dim), nn.Tanh(), nn.Dropout(self.dropout), nn.Linear(self.latent_variable_dim, self.vocab_size)) self.ctx_fc = nn.Sequential( nn.Linear( self.latent_variable_dim + self.dial_encoder_hidden_dim, self.dial_encoder_hidden_dim, ), nn.Tanh(), nn.Dropout(self.dropout)) # Decoding components self.enc2dec_hidden_fc = nn.Linear( self.dial_encoder_hidden_dim, self.n_decoder_layers * self.decoder_hidden_dim if self.rnn_type == "gru" else self.n_decoder_layers * self.decoder_hidden_dim * 2) self.decoder = DecoderRNN(vocab_size=len(self.word2id), input_dim=self.word_embedding_dim, hidden_dim=self.decoder_hidden_dim, feat_dim=self.dial_encoder_hidden_dim, n_layers=self.n_decoder_layers, bos_token_id=self.bos_token_id, eos_token_id=self.eos_token_id, pad_token_id=self.pad_token_id, max_len=self.decode_max_len, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, embedding=self.word_embedding, tie_weights=self.tie_weights, rnn_type=self.rnn_type, use_attention=self.use_attention, attn_dim=self.sent_encoder_hidden_dim) # Extra components # Floor encoding if self.floor_encoder_type == "abs": self.floor_encoder = AbsFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim) elif self.floor_encoder_type == "rel": self.floor_encoder = RelFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim) else: self.floor_encoder = None # Initialization self._init_weights()
def __init__(self, config, tokenizer): super(VHCR, self).__init__() # Attributes # Attributes from config self.word_embedding_dim = config.word_embedding_dim self.attr_embedding_dim = config.attr_embedding_dim self.sent_encoder_hidden_dim = config.sent_encoder_hidden_dim self.n_sent_encoder_layers = config.n_sent_encoder_layers self.dial_encoder_hidden_dim = config.dial_encoder_hidden_dim self.n_dial_encoder_layers = config.n_dial_encoder_layers self.latent_variable_dim = config.latent_dim self.decoder_hidden_dim = config.decoder_hidden_dim self.n_decoder_layers = config.n_decoder_layers self.use_attention = config.use_attention self.decode_max_len = config.decode_max_len self.tie_weights = config.tie_weights self.rnn_type = config.rnn_type self.gen_type = config.gen_type self.top_k = config.top_k self.top_p = config.top_p self.temp = config.temp self.word_embedding_path = config.word_embedding_path self.floor_encoder_type = config.floor_encoder # Optional attributes from config self.dropout_emb = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_input = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_hidden = config.dropout if hasattr(config, "dropout") else 0.0 self.dropout_output = config.dropout if hasattr(config, "dropout") else 0.0 self.use_pretrained_word_embedding = config.use_pretrained_word_embedding if hasattr(config, "use_pretrained_word_embedding") else False self.n_step_annealing = config.n_step_annealing if hasattr(config, "n_step_annealing") else 0 # Other attributes self.vocab_size = len(tokenizer.word2id) self.word2id = tokenizer.word2id self.id2word = tokenizer.id2word self.pad_token_id = tokenizer.pad_token_id self.bos_token_id = tokenizer.bos_token_id self.eos_token_id = tokenizer.eos_token_id self.dropout_sent = 0.25 # Input components self.word_embedding = nn.Embedding( self.vocab_size, self.word_embedding_dim, padding_idx=self.pad_token_id, _weight=init_word_embedding( load_pretrained_word_embedding=self.use_pretrained_word_embedding, pretrained_word_embedding_path=self.word_embedding_path, id2word=self.id2word, word_embedding_dim=self.word_embedding_dim, vocab_size=self.vocab_size, pad_token_id=self.pad_token_id ), ) # Encoding components self.sent_encoder = EncoderRNN( input_dim=self.word_embedding_dim, hidden_dim=self.sent_encoder_hidden_dim, n_layers=self.n_sent_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=True, embedding=self.word_embedding, rnn_type=self.rnn_type, ) self.dial_encoder = EncoderRNN( input_dim=self.sent_encoder_hidden_dim+self.latent_variable_dim, hidden_dim=self.dial_encoder_hidden_dim, n_layers=self.n_dial_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=False, rnn_type=self.rnn_type, ) self.dial_infer_encoder = EncoderRNN( input_dim=self.sent_encoder_hidden_dim, hidden_dim=self.dial_encoder_hidden_dim, n_layers=self.n_dial_encoder_layers, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, bidirectional=True, rnn_type=self.rnn_type, ) # Variational components self.dial_post_net = GaussianVariation( input_dim=self.dial_encoder_hidden_dim, z_dim=self.latent_variable_dim ) self.sent_prior_net = GaussianVariation( input_dim=self.dial_encoder_hidden_dim+self.latent_variable_dim, z_dim=self.latent_variable_dim ) self.sent_post_net = GaussianVariation( input_dim=self.sent_encoder_hidden_dim+self.dial_encoder_hidden_dim+self.latent_variable_dim, z_dim=self.latent_variable_dim ) self.unk_sent_vec = nn.Parameter(torch.randn(self.sent_encoder_hidden_dim)).to(DEVICE) # Decoding components self.ctx_fc = nn.Linear( 2*self.latent_variable_dim+self.dial_encoder_hidden_dim, self.dial_encoder_hidden_dim ) self.enc2dec_hidden_fc = nn.Linear( self.dial_encoder_hidden_dim, self.n_decoder_layers*self.decoder_hidden_dim if self.rnn_type == "gru" else self.n_decoder_layers*self.decoder_hidden_dim*2 ) self.decoder = DecoderRNN( vocab_size=len(self.word2id), input_dim=self.word_embedding_dim, hidden_dim=self.decoder_hidden_dim, feat_dim=self.dial_encoder_hidden_dim, n_layers=self.n_decoder_layers, bos_token_id=self.bos_token_id, eos_token_id=self.eos_token_id, pad_token_id=self.pad_token_id, max_len=self.decode_max_len, dropout_emb=self.dropout_emb, dropout_input=self.dropout_input, dropout_hidden=self.dropout_hidden, dropout_output=self.dropout_output, embedding=self.word_embedding, tie_weights=self.tie_weights, rnn_type=self.rnn_type, use_attention=self.use_attention, attn_dim=self.sent_encoder_hidden_dim ) # Extra components # Floor encoding if self.floor_encoder_type == "abs": self.floor_encoder = AbsFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim ) elif self.floor_encoder_type == "rel": self.floor_encoder = RelFloorEmbEncoder( input_dim=self.sent_encoder_hidden_dim, embedding_dim=self.attr_embedding_dim ) else: self.floor_encoder = None # Hidden initialization self.dial_z2dial_enc_hidden_fc = nn.Linear( self.latent_variable_dim, self.n_dial_encoder_layers*self.dial_encoder_hidden_dim if self.rnn_type == "gru" else self.n_dial_encoder_layers*self.dial_encoder_hidden_dim*2 ) # Initialization self._init_weights()