def __init__(self, vocab, decoder_number, model_file_path=None, is_eval=False, load_optim=False):
super(Transformer, self).__init__()
self.vocab = vocab
self.vocab_size = vocab.n_words
self.embedding = share_embedding(self.vocab,config.pretrain_emb)
self.encoder = Encoder(config.emb_dim, config.hidden_dim, num_layers=config.hop, num_heads=config.heads,
total_key_depth=config.depth, total_value_depth=config.depth,
filter_size=config.filter,universal=config.universal)
## multiple decoders
self.decoder = Decoder(config.emb_dim, hidden_size = config.hidden_dim, num_layers=config.hop, num_heads=config.heads,
total_key_depth=config.depth,total_value_depth=config.depth,
filter_size=config.filter)
self.decoder_key = nn.Linear(config.hidden_dim ,decoder_number, bias=False)
self.generator = Generator(config.hidden_dim, self.vocab_size)
if config.weight_sharing:
# Share the weight matrix between target word embedding & the final logit dense layer
self.generator.proj.weight = self.embedding.lut.weight
self.criterion = nn.NLLLoss(ignore_index=config.PAD_idx)
if config.label_smoothing:
self.criterion = LabelSmoothing(size=self.vocab_size, padding_idx=config.PAD_idx, smoothing=0.1)
self.criterion_ppl = nn.NLLLoss(ignore_index=config.PAD_idx)
self.optimizer = torch.optim.Adam(self.parameters(), lr=config.lr)
if config.noam:
self.optimizer = NoamOpt(config.hidden_dim, 1, 8000, torch.optim.Adam(self.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
if model_file_path is not None:
print("loading weights")
state = torch.load(model_file_path, map_location=lambda storage, location: storage)
self.encoder.load_state_dict(state['encoder_state_dict'])
self.decoder.load_state_dict(state['decoder_state_dict'])
self.generator.load_state_dict(state['generator_dict'])
self.embedding.load_state_dict(state['embedding_dict'])
self.decoder_key.load_state_dict(state['decoder_key_state_dict'])
if load_optim:
self.optimizer.load_state_dict(state['optimizer'])
self.eval()
self.model_dir = config.save_path
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
self.best_path = ""
def __init__(self, vocab, embedding_size, hidden_size, num_layers, layer_dropout=0.4):
super(Semantic_Discriminator, self).__init__()
self.vocab = vocab
self.vocab_size = vocab.n_words
self.embedding = share_embedding(self.vocab, config.pretrain_emb)
self.lstm = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers,
dropout=(0 if num_layers == 1 else layer_dropout), bidirectional=False, batch_first=True)
self.kernel_size = 2 # means capture 2-gram features
self.bias = nn.Parameter(torch.randn(1, hidden_size))
self.hidden_size = hidden_size
self.logits_dense = nn.Linear(int(self.hidden_size / 2), 1)
self.dense = nn.Sequential(
nn.Linear(self.hidden_size, int(self.hidden_size / 2)),
nn.ReLU()
)
def __init__(self,
vocab,
emotion_number,
model_file_path=None,
is_eval=False,
load_optim=False):
'''
:param decoder_number: the number of emotion labels, i.e., 32
'''
super(Empdg_G, self).__init__()
self.vocab = vocab
self.vocab_size = vocab.n_words
self.embedding = share_embedding(self.vocab, config.pretrain_emb)
self.semantic_und = Semantic_Encoder(config.emb_dim,
config.hidden_dim,
num_layers=config.hop,
num_heads=config.heads,
total_key_depth=config.depth,
total_value_depth=config.depth,
filter_size=config.filter,
universal=config.universal)
self.emotion_pec = Emotion_Encoder(config.emb_dim,
config.hidden_dim,
num_layers=config.hop,
num_heads=config.heads,
total_key_depth=config.depth,
total_value_depth=config.depth,
filter_size=config.filter,
universal=config.universal)
self.map_emo = {
0: 'surprised',
1: 'excited',
2: 'annoyed',
3: 'proud',
4: 'angry',
5: 'sad',
6: 'grateful',
7: 'lonely',
8: 'impressed',
9: 'afraid',
10: 'disgusted',
11: 'confident',
12: 'terrified',
13: 'hopeful',
14: 'anxious',
15: 'disappointed',
16: 'joyful',
17: 'prepared',
18: 'guilty',
19: 'furious',
20: 'nostalgic',
21: 'jealous',
22: 'anticipating',
23: 'embarrassed',
24: 'content',
25: 'devastated',
26: 'sentimental',
27: 'caring',
28: 'trusting',
29: 'ashamed',
30: 'apprehensive',
31: 'faithful'
}
## emotional signal distilling
self.identify = nn.Linear(config.emb_dim, emotion_number, bias=False)
self.identify_new = nn.Linear(2 * config.emb_dim,
emotion_number,
bias=False)
self.activation = nn.Softmax(dim=1)
## decoders
self.emotion_embedding = nn.Linear(emotion_number, config.emb_dim)
self.decoder = Decoder(config.emb_dim,
hidden_size=config.hidden_dim,
num_layers=config.hop,
num_heads=config.heads,
total_key_depth=config.depth,
total_value_depth=config.depth,
filter_size=config.filter)
self.decoder_key = nn.Linear(config.hidden_dim,
emotion_number,
bias=False)
self.generator = Generator(config.hidden_dim, self.vocab_size)
if config.weight_sharing:
# Share the weight matrix between target word embedding & the final logit dense layer
self.generator.proj.weight = self.embedding.lut.weight
self.criterion = nn.NLLLoss(ignore_index=config.PAD_idx)
if config.label_smoothing:
self.criterion = LabelSmoothing(size=self.vocab_size,
padding_idx=config.PAD_idx,
smoothing=0.1)
self.criterion_ppl = nn.NLLLoss(ignore_index=config.PAD_idx)
self.optimizer = torch.optim.Adam(self.parameters(), lr=config.lr)
if config.noam:
self.optimizer = NoamOpt(
config.hidden_dim, 1, 8000,
torch.optim.Adam(self.parameters(),
lr=0,
betas=(0.9, 0.98),
eps=1e-9))
if model_file_path is not None:
print("loading weights")
state = torch.load(model_file_path,
map_location=lambda storage, location: storage)
self.encoder.load_state_dict(state['encoder_state_dict'])
self.decoder.load_state_dict(state['decoder_state_dict'])
self.generator.load_state_dict(state['generator_dict'])
self.embedding.load_state_dict(state['embedding_dict'])
self.decoder_key.load_state_dict(state['decoder_key_state_dict'])
if load_optim:
self.optimizer.load_state_dict(state['optimizer'])
self.eval()
self.model_dir = config.save_path
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
self.best_path = ""