def main(beam_size):
word2id, id2word = build_dict()
pad_len = 30
batch_size = 20
emb_dim = 300
dim = 600
vocab_size = len(word2id)
model = Seq2SeqAttentionSharedEmbedding(
emb_dim=emb_dim,
vocab_size=vocab_size,
src_hidden_dim=dim,
trg_hidden_dim=dim,
ctx_hidden_dim=dim,
attention_mode='dot',
batch_size=batch_size,
bidirectional=False,
pad_token_src=word2id['<pad>'],
pad_token_trg=word2id['<pad>'],
nlayers=2,
nlayers_trg=2,
dropout=0.,
).cuda()
# model = nn.DataParallel(model).cuda()
model_path = 'checkpoint/new_vocab_epoch_17.model'
from collections import OrderedDict
new_state_dict = OrderedDict()
state_dict = torch.load(model_path)
# trick for loading models from DataParallel
for k, v in state_dict.items():
name = k[7:] # remove `module.`
new_state_dict[name] = v
# load params
model.load_state_dict(new_state_dict)
# model = nn.DataParallel(model).cuda()
# model.load_state_dict(torch.load(
# model_path
# ))
# model = nn.DataParallel(model).cuda()
df = pd.read_csv('data_6_remove_dup_test.csv')
X, y, tag = df['source'], df['target'], df['tag']
test_set = EmotionDataLoader(X, y, tag, pad_len, word2id)
test_loader = DataLoader(test_set, batch_size=batch_size)
for __emo in range(9):
decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size,
word2id, id2word)
decoder.translate(__emo)
def main(beam_size):
num_emotions = 9
word2id, id2word = build_dict()
word2id, id2word = add_emo_token(word2id, id2word, num_emotions)
pad_len = 30
batch_size = 2000
emb_dim = 300
dim = 600
vocab_size = len(word2id)
from sklearn.model_selection import ShuffleSplit
model = Seq2SeqAttentionSharedEmbedding(
emb_dim=emb_dim,
vocab_size=vocab_size,
src_hidden_dim=dim,
trg_hidden_dim=dim,
ctx_hidden_dim=dim,
attention_mode='dot',
batch_size=batch_size,
bidirectional=False,
pad_token_src=word2id['<pad>'],
pad_token_trg=word2id['<pad>'],
nlayers=2,
nlayers_trg=2,
dropout=0.,
)
model.cuda()
model_path = 'checkpoint/new_simple_start_epoch_22.model'
model.load_state_dict(torch.load(
model_path
))
df = pd.read_csv('data_6_remove_dup_test.csv')
X, y, tag = df['source'], df['target'], df['tag']
__emo = 0
test_set = EmotionDataLoaderStart(X, y, __emo, pad_len, word2id)
test_loader = DataLoader(test_set, batch_size=batch_size)
for __emo in range(9):
decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size, word2id, id2word)
decoder.translate(__emo)
def main(beam_size):
word2id, id2word = build_dict()
pad_len = 30
batch_size = 500
emb_dim = 300
dim = 600
vocab_size = len(word2id)
model = PersonaSeq2SeqAttentionSharedEmbedding(
emb_dim=emb_dim,
vocab_size=vocab_size,
src_hidden_dim=dim,
trg_hidden_dim=dim,
ctx_hidden_dim=dim,
attention_mode='dot',
batch_size=batch_size,
bidirectional=False,
pad_token_src=word2id['<pad>'],
pad_token_trg=word2id['<pad>'],
nlayers=2,
nlayers_trg=2,
dropout=0.,
).cuda()
model_path = 'checkpoint/new_persona_epoch_22.model'
model.load_state_dict(torch.load(
model_path
))
df = pd.read_csv('data_6_remove_dup_test.csv')
X, y, tag = df['source'], df['target'], df['tag']
test_set = EmotionDataLoader(X, y, tag, pad_len, word2id)
test_loader = DataLoader(test_set, batch_size=batch_size)
for __emo in range(9):
decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size, word2id, id2word)
decoder.translate(__emo)
trg = [self.word2id[x] for x in self.target[idx].split()]
if len(trg) > self.pad_len - 2:
trg = trg[:self.pad_len-2]
trg = [self.start_int] + trg + [self.eos_int] + [self.pad_int] * (self.pad_len - len(trg) - 2)
if not len(src) == len(trg) == self.pad_len:
print(src, trg)
assert len(src) == len(trg) == self.pad_len
if self.tag[idx] == 'Nan':
tag = NUM_EMO
else:
tag = int(self.tag[idx])
return torch.LongTensor(src), torch.LongTensor(trg), torch.LongTensor([tag])
if __name__ == '__main__':
word2id, id2word = build_dict()
pad_len = 30
batch_size = 200
emb_dim = 300
dim = 450
vocab_size = len(word2id)
from sklearn.model_selection import ShuffleSplit
split_ratio = 0.05
sss = ShuffleSplit(n_splits=1, test_size=split_ratio, random_state=0)
df = pd.read_csv('data_6_remove_dup_train.csv')
X, y, tag = df['source'], df['target'], df['tag']
# temp_up = 10000
# X = X[:temp_up]