def __init__(self, vocab, decoder_number, model_file_path=None, is_eval=False, load_optim=False): super(Transformer, self).__init__() self.vocab = vocab self.vocab_size = vocab.n_words self.embedding = share_embedding(self.vocab,config.pretrain_emb) self.encoder = Encoder(config.emb_dim, config.hidden_dim, num_layers=config.hop, num_heads=config.heads, total_key_depth=config.depth, total_value_depth=config.depth, filter_size=config.filter,universal=config.universal) ## multiple decoders self.decoder = Decoder(config.emb_dim, hidden_size = config.hidden_dim, num_layers=config.hop, num_heads=config.heads, total_key_depth=config.depth,total_value_depth=config.depth, filter_size=config.filter) self.decoder_key = nn.Linear(config.hidden_dim ,decoder_number, bias=False) self.generator = Generator(config.hidden_dim, self.vocab_size) if config.weight_sharing: # Share the weight matrix between target word embedding & the final logit dense layer self.generator.proj.weight = self.embedding.lut.weight self.criterion = nn.NLLLoss(ignore_index=config.PAD_idx) if config.label_smoothing: self.criterion = LabelSmoothing(size=self.vocab_size, padding_idx=config.PAD_idx, smoothing=0.1) self.criterion_ppl = nn.NLLLoss(ignore_index=config.PAD_idx) self.optimizer = torch.optim.Adam(self.parameters(), lr=config.lr) if config.noam: self.optimizer = NoamOpt(config.hidden_dim, 1, 8000, torch.optim.Adam(self.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) if model_file_path is not None: print("loading weights") state = torch.load(model_file_path, map_location=lambda storage, location: storage) self.encoder.load_state_dict(state['encoder_state_dict']) self.decoder.load_state_dict(state['decoder_state_dict']) self.generator.load_state_dict(state['generator_dict']) self.embedding.load_state_dict(state['embedding_dict']) self.decoder_key.load_state_dict(state['decoder_key_state_dict']) if load_optim: self.optimizer.load_state_dict(state['optimizer']) self.eval() self.model_dir = config.save_path if not os.path.exists(self.model_dir): os.makedirs(self.model_dir) self.best_path = ""
def __init__(self, vocab, embedding_size, hidden_size, num_layers, layer_dropout=0.4): super(Semantic_Discriminator, self).__init__() self.vocab = vocab self.vocab_size = vocab.n_words self.embedding = share_embedding(self.vocab, config.pretrain_emb) self.lstm = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers, dropout=(0 if num_layers == 1 else layer_dropout), bidirectional=False, batch_first=True) self.kernel_size = 2 # means capture 2-gram features self.bias = nn.Parameter(torch.randn(1, hidden_size)) self.hidden_size = hidden_size self.logits_dense = nn.Linear(int(self.hidden_size / 2), 1) self.dense = nn.Sequential( nn.Linear(self.hidden_size, int(self.hidden_size / 2)), nn.ReLU() )
def __init__(self, vocab, emotion_number, model_file_path=None, is_eval=False, load_optim=False): ''' :param decoder_number: the number of emotion labels, i.e., 32 ''' super(Empdg_G, self).__init__() self.vocab = vocab self.vocab_size = vocab.n_words self.embedding = share_embedding(self.vocab, config.pretrain_emb) self.semantic_und = Semantic_Encoder(config.emb_dim, config.hidden_dim, num_layers=config.hop, num_heads=config.heads, total_key_depth=config.depth, total_value_depth=config.depth, filter_size=config.filter, universal=config.universal) self.emotion_pec = Emotion_Encoder(config.emb_dim, config.hidden_dim, num_layers=config.hop, num_heads=config.heads, total_key_depth=config.depth, total_value_depth=config.depth, filter_size=config.filter, universal=config.universal) self.map_emo = { 0: 'surprised', 1: 'excited', 2: 'annoyed', 3: 'proud', 4: 'angry', 5: 'sad', 6: 'grateful', 7: 'lonely', 8: 'impressed', 9: 'afraid', 10: 'disgusted', 11: 'confident', 12: 'terrified', 13: 'hopeful', 14: 'anxious', 15: 'disappointed', 16: 'joyful', 17: 'prepared', 18: 'guilty', 19: 'furious', 20: 'nostalgic', 21: 'jealous', 22: 'anticipating', 23: 'embarrassed', 24: 'content', 25: 'devastated', 26: 'sentimental', 27: 'caring', 28: 'trusting', 29: 'ashamed', 30: 'apprehensive', 31: 'faithful' } ## emotional signal distilling self.identify = nn.Linear(config.emb_dim, emotion_number, bias=False) self.identify_new = nn.Linear(2 * config.emb_dim, emotion_number, bias=False) self.activation = nn.Softmax(dim=1) ## decoders self.emotion_embedding = nn.Linear(emotion_number, config.emb_dim) self.decoder = Decoder(config.emb_dim, hidden_size=config.hidden_dim, num_layers=config.hop, num_heads=config.heads, total_key_depth=config.depth, total_value_depth=config.depth, filter_size=config.filter) self.decoder_key = nn.Linear(config.hidden_dim, emotion_number, bias=False) self.generator = Generator(config.hidden_dim, self.vocab_size) if config.weight_sharing: # Share the weight matrix between target word embedding & the final logit dense layer self.generator.proj.weight = self.embedding.lut.weight self.criterion = nn.NLLLoss(ignore_index=config.PAD_idx) if config.label_smoothing: self.criterion = LabelSmoothing(size=self.vocab_size, padding_idx=config.PAD_idx, smoothing=0.1) self.criterion_ppl = nn.NLLLoss(ignore_index=config.PAD_idx) self.optimizer = torch.optim.Adam(self.parameters(), lr=config.lr) if config.noam: self.optimizer = NoamOpt( config.hidden_dim, 1, 8000, torch.optim.Adam(self.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) if model_file_path is not None: print("loading weights") state = torch.load(model_file_path, map_location=lambda storage, location: storage) self.encoder.load_state_dict(state['encoder_state_dict']) self.decoder.load_state_dict(state['decoder_state_dict']) self.generator.load_state_dict(state['generator_dict']) self.embedding.load_state_dict(state['embedding_dict']) self.decoder_key.load_state_dict(state['decoder_key_state_dict']) if load_optim: self.optimizer.load_state_dict(state['optimizer']) self.eval() self.model_dir = config.save_path if not os.path.exists(self.model_dir): os.makedirs(self.model_dir) self.best_path = ""