def main(beam_size): word2id, id2word = build_dict() pad_len = 30 batch_size = 20 emb_dim = 300 dim = 600 vocab_size = len(word2id) model = Seq2SeqAttentionSharedEmbedding( emb_dim=emb_dim, vocab_size=vocab_size, src_hidden_dim=dim, trg_hidden_dim=dim, ctx_hidden_dim=dim, attention_mode='dot', batch_size=batch_size, bidirectional=False, pad_token_src=word2id['<pad>'], pad_token_trg=word2id['<pad>'], nlayers=2, nlayers_trg=2, dropout=0., ).cuda() # model = nn.DataParallel(model).cuda() model_path = 'checkpoint/new_vocab_epoch_17.model' from collections import OrderedDict new_state_dict = OrderedDict() state_dict = torch.load(model_path) # trick for loading models from DataParallel for k, v in state_dict.items(): name = k[7:] # remove `module.` new_state_dict[name] = v # load params model.load_state_dict(new_state_dict) # model = nn.DataParallel(model).cuda() # model.load_state_dict(torch.load( # model_path # )) # model = nn.DataParallel(model).cuda() df = pd.read_csv('data_6_remove_dup_test.csv') X, y, tag = df['source'], df['target'], df['tag'] test_set = EmotionDataLoader(X, y, tag, pad_len, word2id) test_loader = DataLoader(test_set, batch_size=batch_size) for __emo in range(9): decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size, word2id, id2word) decoder.translate(__emo)
def main(beam_size): num_emotions = 9 word2id, id2word = build_dict() word2id, id2word = add_emo_token(word2id, id2word, num_emotions) pad_len = 30 batch_size = 2000 emb_dim = 300 dim = 600 vocab_size = len(word2id) from sklearn.model_selection import ShuffleSplit model = Seq2SeqAttentionSharedEmbedding( emb_dim=emb_dim, vocab_size=vocab_size, src_hidden_dim=dim, trg_hidden_dim=dim, ctx_hidden_dim=dim, attention_mode='dot', batch_size=batch_size, bidirectional=False, pad_token_src=word2id['<pad>'], pad_token_trg=word2id['<pad>'], nlayers=2, nlayers_trg=2, dropout=0., ) model.cuda() model_path = 'checkpoint/new_simple_start_epoch_22.model' model.load_state_dict(torch.load( model_path )) df = pd.read_csv('data_6_remove_dup_test.csv') X, y, tag = df['source'], df['target'], df['tag'] __emo = 0 test_set = EmotionDataLoaderStart(X, y, __emo, pad_len, word2id) test_loader = DataLoader(test_set, batch_size=batch_size) for __emo in range(9): decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size, word2id, id2word) decoder.translate(__emo)
def main(beam_size): word2id, id2word = build_dict() pad_len = 30 batch_size = 500 emb_dim = 300 dim = 600 vocab_size = len(word2id) model = PersonaSeq2SeqAttentionSharedEmbedding( emb_dim=emb_dim, vocab_size=vocab_size, src_hidden_dim=dim, trg_hidden_dim=dim, ctx_hidden_dim=dim, attention_mode='dot', batch_size=batch_size, bidirectional=False, pad_token_src=word2id['<pad>'], pad_token_trg=word2id['<pad>'], nlayers=2, nlayers_trg=2, dropout=0., ).cuda() model_path = 'checkpoint/new_persona_epoch_22.model' model.load_state_dict(torch.load( model_path )) df = pd.read_csv('data_6_remove_dup_test.csv') X, y, tag = df['source'], df['target'], df['tag'] test_set = EmotionDataLoader(X, y, tag, pad_len, word2id) test_loader = DataLoader(test_set, batch_size=batch_size) for __emo in range(9): decoder = BeamSearchDecoder(model, test_loader, pad_len, beam_size, word2id, id2word) decoder.translate(__emo)
trg = [self.word2id[x] for x in self.target[idx].split()] if len(trg) > self.pad_len - 2: trg = trg[:self.pad_len-2] trg = [self.start_int] + trg + [self.eos_int] + [self.pad_int] * (self.pad_len - len(trg) - 2) if not len(src) == len(trg) == self.pad_len: print(src, trg) assert len(src) == len(trg) == self.pad_len if self.tag[idx] == 'Nan': tag = NUM_EMO else: tag = int(self.tag[idx]) return torch.LongTensor(src), torch.LongTensor(trg), torch.LongTensor([tag]) if __name__ == '__main__': word2id, id2word = build_dict() pad_len = 30 batch_size = 200 emb_dim = 300 dim = 450 vocab_size = len(word2id) from sklearn.model_selection import ShuffleSplit split_ratio = 0.05 sss = ShuffleSplit(n_splits=1, test_size=split_ratio, random_state=0) df = pd.read_csv('data_6_remove_dup_train.csv') X, y, tag = df['source'], df['target'], df['tag'] # temp_up = 10000 # X = X[:temp_up]