def test_run(data_path, model_path, n_show=10): """ Test function. Input is training data. Output have to be the sentence which is correct data in training phase. :return: """ corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # run an interpreter for num, input_sentence in enumerate(corpus.posts): id_sequence = input_sentence.copy() # input_sentence.reverse() # input_sentence.insert(0, corpus.dic.token2id["<eos>"]) model.initialize() # initialize cell sentence = model.generate(input_sentence, sentence_limit=len(input_sentence) + 30, word2id=corpus.dic.token2id, id2word=corpus.dic) print("teacher : ", " ".join([corpus.dic[w_id] for w_id in id_sequence])) print("correct :", " ".join([corpus.dic[w_id] for w_id in corpus.cmnts[num]])) print("-> ", sentence) print('') if num == n_show: break
def show_heatmap(data_path, model_path): import seaborn import matplotlib seaborn.set() matplotlib.rc('font', family='sans-serif') # call dictionary class if args.lang == 'en': corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) elif args.lang == 'ja': corpus = JaConvCorpus(file_path=None) corpus.load(load_dir=data_path) else: print( 'You gave wrong argument to this system. Check out your argument about languages.' ) raise ValueError print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, label_num=args.label_num, label_embed_num=args.label_embed, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # sentiment matrix in the decoder sentiment_mat = model.dec.le.W.data cmap = seaborn.diverging_palette( 220, 10, as_cmap=True) # Generate a custom diverging colormap seaborn.heatmap(sentiment_mat, cmap=cmap, center=0, linewidths=.5, xticklabels=False ) # square=True, cbar_kws={"orientation": "horizontal"}) plt.xlabel("Dimension (=" + str(sentiment_mat.shape[1]) + ")") plt.ylabel("Sentiment") plt.savefig('./data/sentiment_matrix.png') # encoder embedding matrix encode_mat = model.enc.xe.W seaborn.heatmap(encode_mat, cmap=cmap, center=0, linewidths=.5, xticklabels=False ) # square=True, cbar_kws={"orientation": "horizontal"}) plt.xlabel("Dimension (=" + str(sentiment_mat.shape[1]) + ")") plt.ylabel("Sentiment") plt.savefig('./data/sentiment_matrix.png')
def interpreter(data_path, model_path): """ Run this function, if you want to talk to seq2seq model. if you type "exit", finish to talk. :param data_path: the path of corpus you made model learn :param model_path: the path of model you made learn :return: """ # call dictionary class corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # run conversation system print('The system is ready to run, please talk to me!') print('( If you want to end a talk, please type "exit". )') print('') while True: print('>> ', end='') sentence = input() if sentence == 'exit': print('See you again!') break input_vocab = [ unicodedata.normalize('NFKC', word.lower()) for word in word_tokenize(sentence) ] input_vocab.reverse() input_vocab.insert(0, "<eos>") # convert word into ID input_sentence = [ corpus.dic.token2id[word] for word in input_vocab if not corpus.dic.token2id.get(word) is None ] model.initialize() # initialize cell sentence = model.generate(input_sentence, sentence_limit=len(input_sentence) + 30, word2id=corpus.dic.token2id, id2word=corpus.dic) print("-> ", sentence) print('')
def main(): ########################### #### create dictionary #### ########################### if os.path.exists('./data/corpus/dictionary.dict'): if args.lang == 'ja': corpus = JaConvCorpus(file_path=None, batch_size=batchsize, size_filter=True) else: corpus = ConvCorpus(file_path=None, batch_size=batchsize) corpus.load(load_dir='./data/corpus/') else: if args.lang == 'ja': corpus = JaConvCorpus(file_path=data_file, batch_size=batchsize, size_filter=True) else: corpus = ConvCorpus(file_path=data_file, batch_size=batchsize) corpus.save(save_dir='./data/corpus/') print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) ###################### #### create model #### ###################### model = Seq2Seq(len(corpus.dic.token2id), feature_num=feature_num, hidden_num=hidden_num, label_num=label_num, label_embed_num=label_embed_num, batch_size=batchsize, gpu_flg=args.gpu) if args.gpu >= 0: model.to_gpu() optimizer = optimizers.Adam(alpha=0.001) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(5)) # optimizer.add_hook(chainer.optimizer.WeightDecay(0.0001)) ########################## #### create ID corpus #### ########################## input_mat = [] output_mat = [] label_mat = [] max_input_ren = max_output_ren = 0 for input_text, output_text in zip(corpus.posts, corpus.cmnts): # convert to list # input_text.reverse() # encode words in a reverse order # input_text.insert(0, corpus.dic.token2id["<eos>"]) output_text.append(corpus.dic.token2id["<eos>"]) # update max sentence length max_input_ren = max(max_input_ren, len(input_text)) max_output_ren = max(max_output_ren, len(output_text)) # listのlistを作る(要修正) label = input_text.pop(-1) if label == corpus.dic.token2id["__label__1"]: label_mat.append([0 for _ in range(len(output_text))]) elif label == corpus.dic.token2id["__label__2"]: label_mat.append([1 for _ in range(len(output_text))]) else: print('label error!: ', label) raise ValueError input_mat.append(input_text) output_mat.append(output_text) # padding for li in input_mat: insert_num = max_input_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) # そのままの入力順序にする場合 # li.insert(0, corpus.dic.token2id['<pad>']) # 入力順序を逆にする場合 for li in output_mat: insert_num = max_output_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) for li in label_mat: insert_num = max_output_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) # create batch matrix input_mat = np.array(input_mat, dtype=np.int32).T output_mat = np.array(output_mat, dtype=np.int32).T label_mat = np.array(label_mat, dtype=np.int32).T # separate corpus into Train and Test perm = np.random.permutation(len(corpus.posts)) test_input_mat = input_mat[:, perm[0:0 + testsize]] test_output_mat = output_mat[:, perm[0:0 + testsize]] test_label_mat = label_mat[:, perm[0:0 + testsize]] train_input_mat = input_mat[:, perm[testsize:]] train_output_mat = output_mat[:, perm[testsize:]] train_label_mat = label_mat[:, perm[testsize:]] list_of_references = [] for text_ndarray in test_output_mat.T: reference = text_ndarray.tolist() references = [[w_id for w_id in reference if w_id is not -1]] list_of_references.append(references) ############################# #### train seq2seq model #### ############################# accum_loss = 0 train_loss_data = [] test_loss_data = [] bleu_score_data = [] wer_score_data = [] for num, epoch in enumerate(range(n_epoch)): total_loss = test_loss = 0 batch_num = 0 perm = np.random.permutation(len(corpus.posts) - testsize) # for training for i in range(0, len(corpus.posts) - testsize, batchsize): # select batch data input_batch = train_input_mat[:, perm[i:i + batchsize]] output_batch = train_output_mat[:, perm[i:i + batchsize]] label_batch = train_label_mat[:, perm[i:i + batchsize]] # Encode a sentence model.initialize() # initialize cell model.encode(input_batch, train=True) # encode (output: hidden Variable) # Decode from encoded context next_ids = xp.array([corpus.dic.token2id["<eos>"] for _ in range(batchsize)]) accum_loss = 0 for w_ids, l_ids in zip(output_batch, label_batch): loss, predict_mat = model.decode(next_ids, w_ids, l_ids, train=True) next_ids = w_ids accum_loss += loss # learn model model.cleargrads() # initialize all grad to zero accum_loss.backward() # back propagation optimizer.update() total_loss += float(accum_loss.data) batch_num += 1 print('Epoch: ', num, 'Batch_num', batch_num, 'batch loss: {:.2f}'.format(float(accum_loss.data))) # for testing list_of_hypotheses = [] for i in range(0, testsize, batchsize): # select test batch data input_batch = test_input_mat[:, i:i + batchsize] output_batch = test_output_mat[:, i:i + batchsize] label_batch = test_label_mat[:, i:i + batchsize] # Encode a sentence model.initialize() # initialize cell model.encode(input_batch, train=True) # encode (output: hidden Variable) # Decode from encoded context next_ids = xp.array([corpus.dic.token2id["<start>"] for _ in range(batchsize)]) if args.gpu >= 0: hypotheses = [cuda.to_cpu(next_ids)] else: hypotheses = [next_ids] for w_ids, l_ids in zip(output_batch, label_batch): loss, predict_mat = model.decode(next_ids, w_ids, l_ids, train=True) next_ids = xp.argmax(predict_mat.data, axis=1) test_loss += loss.data if args.gpu >= 0: hypotheses.append(cuda.to_cpu(next_ids)) else: hypotheses.append(next_ids) # collect hypotheses for calculating BLEU score hypotheses = np.array(hypotheses).T for hypothesis in hypotheses: text_list = hypothesis.tolist() list_of_hypotheses.append([w_id for w_id in text_list if w_id is not -1]) # calculate BLEU score from test (develop) data bleu_score = nltk.translate.bleu_score.corpus_bleu(list_of_references, list_of_hypotheses, weights=(0.25, 0.25, 0.25, 0.25)) bleu_score_data.append(bleu_score) print('Epoch: ', num, 'BLEU SCORE: ', bleu_score) # calculate WER score from test (develop) data wer_score = 0 for index, references in enumerate(list_of_references): wer_score += wer(references[0], list_of_hypotheses[index]) wer_score /= len(list_of_references) wer_score_data.append(wer_score) print('Epoch: ', num, 'WER SCORE: ', wer_score) # save model and optimizer if (epoch + 1) % 10 == 0: print('-----', epoch + 1, ' times -----') print('save the model and optimizer') serializers.save_hdf5('data/' + str(epoch) + '.model', model) serializers.save_hdf5('data/' + str(epoch) + '.state', optimizer) # display the on-going status print('Epoch: ', num, 'Train loss: {:.2f}'.format(total_loss), 'Test loss: {:.2f}'.format(float(test_loss))) train_loss_data.append(float(total_loss / batch_num)) test_loss_data.append(float(test_loss)) # evaluate a test loss check_loss = test_loss_data[-10:] # check out the last 10 loss data end_flg = [j for j in range(len(check_loss) - 1) if check_loss[j] < check_loss[j + 1]] if len(end_flg) > 9: print('Probably it is over-fitting. So stop to learn...') break # save loss data with open('./data/loss_train_data.pkl', 'wb') as f: pickle.dump(train_loss_data, f) with open('./data/loss_test_data.pkl', 'wb') as f: pickle.dump(test_loss_data, f) with open('./data/bleu_score_data.pkl', 'wb') as f: pickle.dump(bleu_score_data, f) with open('./data/wer_score_data.pkl', 'wb') as f: pickle.dump(wer_score_data, f)
def main(): ########################### #### create dictionary #### ########################### if os.path.exists('./data/corpus/dictionary.dict'): if args.lang == 'ja': corpus = JaConvCorpus(file_path=None, batch_size=batchsize, size_filter=True) else: corpus = ConvCorpus(file_path=None, batch_size=batchsize, size_filter=True) corpus.load(load_dir='./data/corpus/') else: if args.lang == 'ja': corpus = JaConvCorpus(file_path=data_file, batch_size=batchsize, size_filter=True) else: corpus = ConvCorpus(file_path=data_file, batch_size=batchsize, size_filter=True) corpus.save(save_dir='./data/corpus/') print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) ###################### #### create model #### ###################### model = Seq2Seq(vocab_size=len(corpus.dic.token2id), feature_num=feature_num, hidden_num=hidden_num, batch_size=batchsize, gpu_flg=args.gpu) if args.gpu >= 0: model.to_gpu() optimizer = optimizers.Adam(alpha=0.001) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(5)) ########################## #### create ID corpus #### ########################## input_mat = [] output_mat = [] input_mat_rev = [] # output_wp_mat = [] max_input_ren = max_output_ren = 0 for input_text, output_text in zip(corpus.posts, corpus.cmnts): output_text.append(corpus.dic.token2id["<eos>"]) # update max sentence length max_input_ren = max(max_input_ren, len(input_text)) max_output_ren = max(max_output_ren, len(output_text)) input_mat.append(input_text) output_mat.append(output_text) # # create word prediction matrix # wp = [] # for wid in output_text: # if wid not in wp: # wp.append(wid) # output_wp_mat.append(wp) # make reverse corpus for input_text in input_mat: input_mat_rev.append(input_text[::-1]) # padding for li in input_mat: insert_num = max_input_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) for li in output_mat: insert_num = max_output_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) for li in input_mat_rev: insert_num = max_input_ren - len(li) for _ in range(insert_num): li.insert(0, corpus.dic.token2id['<pad>']) # create batch matrix input_mat = np.array(input_mat, dtype=np.int32).T input_mat_rev = np.array(input_mat_rev, dtype=np.int32).T output_mat = np.array(output_mat, dtype=np.int32).T # separate corpus into Train and Test perm = np.random.permutation(len(corpus.posts)) test_input_mat = input_mat[:, perm[0:0 + testsize]] test_output_mat = output_mat[:, perm[0:0 + testsize]] test_input_mat_rev = input_mat_rev[:, perm[0:0 + testsize]] train_input_mat = input_mat[:, perm[testsize:]] train_output_mat = output_mat[:, perm[testsize:]] train_input_mat_rev = input_mat_rev[:, perm[testsize:]] # train_output_wp_mat = [] # for index in perm[testsize:]: # train_output_wp_mat.append(output_wp_mat[index]) ############################# #### train seq2seq model #### ############################# accum_loss = 0 train_loss_data = [] for num, epoch in enumerate(range(n_epoch)): total_loss = 0 batch_num = 0 perm = np.random.permutation(len(corpus.posts) - testsize) # for training for i in range(0, len(corpus.posts) - testsize, batchsize): # select batch data input_batch = remove_extra_padding( train_input_mat[:, perm[i:i + batchsize]], reverse_flg=False) input_batch_rev = remove_extra_padding( train_input_mat_rev[:, perm[i:i + batchsize]], reverse_flg=True) output_batch = remove_extra_padding( train_output_mat[:, perm[i:i + batchsize]], reverse_flg=False) # output_wp_batch = [] # for index in perm[i:i + batchsize]: # output_wp_batch.append(train_output_wp_mat[index]) # output_wp_batch = create_wp_batch(vocab_size=len(corpus.dic.token2id), # wp_lists=output_wp_batch) # Encode a sentence model.initialize(batch_size=input_batch.shape[1]) model.encode(input_batch, input_batch_rev, train=True) # Decode from encoded context end_batch = xp.array([ corpus.dic.token2id["<start>"] for _ in range(input_batch.shape[1]) ]) first_words = output_batch[0] loss, predict_mat = model.decode(end_batch, first_words, train=True) next_ids = first_words accum_loss += loss for w_ids in output_batch[1:]: loss, predict_mat = model.decode(next_ids, w_ids, train=True) next_ids = w_ids accum_loss += loss # learn model model.cleargrads() # initialize all grad to zero accum_loss.backward() # back propagation optimizer.update() total_loss += float(accum_loss.data) batch_num += 1 print('Epoch: ', num, 'Batch_num', batch_num, 'batch loss: {:.2f}'.format(float(accum_loss.data))) accum_loss = 0 train_loss_data.append(float(total_loss / batch_num)) # save model and optimizer if (epoch + 1) % 5 == 0: print('-----', epoch + 1, ' times -----') print('save the model and optimizer') serializers.save_hdf5('data/' + str(epoch) + '.model', model) serializers.save_hdf5('data/' + str(epoch) + '.state', optimizer) # save loss data with open('./data/loss_train_data.pkl', 'wb') as f: pickle.dump(train_loss_data, f)
def main(): ########################### #### create dictionary #### ########################### if os.path.exists('./data/corpus/dictionary.dict'): corpus = ConvCorpus(file_path=None, batch_size=batchsize) corpus.load(load_dir='./data/corpus/') else: corpus = ConvCorpus(file_path=args.data, batch_size=batchsize) corpus.save(save_dir='./data/corpus/') print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) ###################### #### create model #### ###################### model = Seq2Seq(len(corpus.dic.token2id), feature_num=feature_num, hidden_num=hidden_num, batch_size=batchsize, gpu_flg=args.gpu) if args.gpu >= 0: model.to_gpu() optimizer = optimizers.Adam(alpha=0.001) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(5)) # optimizer.add_hook(chainer.optimizer.WeightDecay(0.0001)) ########################## #### create ID corpus #### ########################## input_mat = [] output_mat = [] max_input_ren = max_output_ren = 0 for input_text, output_text in zip(corpus.posts, corpus.cmnts): # convert to list input_text.reverse() # encode words in a reverse order input_text.insert(0, corpus.dic.token2id["<eos>"]) output_text.append(corpus.dic.token2id["<eos>"]) # update max sentence length max_input_ren = max(max_input_ren, len(input_text)) max_output_ren = max(max_output_ren, len(output_text)) input_mat.append(input_text) output_mat.append(output_text) # padding for li in input_mat: insert_num = max_input_ren - len(li) for _ in range(insert_num): li.insert(0, corpus.dic.token2id['<pad>']) for li in output_mat: insert_num = max_output_ren - len(li) for _ in range(insert_num): li.append(corpus.dic.token2id['<pad>']) # create batch matrix input_mat = np.array(input_mat, dtype=np.int32).T output_mat = np.array(output_mat, dtype=np.int32).T # separate corpus into Train and Test perm = np.random.permutation(len(corpus.posts)) test_input_mat = input_mat[:, perm[0:0 + batchsize]] test_output_mat = output_mat[:, perm[0:0 + batchsize]] train_input_mat = input_mat[:, perm[batchsize:]] train_output_mat = output_mat[:, perm[batchsize:]] ############################# #### train seq2seq model #### ############################# accum_loss = 0 train_loss_data = [] test_loss_data = [] for num, epoch in enumerate(range(n_epoch)): total_loss = test_loss = 0 batch_num = 0 perm = np.random.permutation(len(corpus.posts) - batchsize) for i in range(0, len(corpus.posts) - batchsize, batchsize): # select batch data input_batch = train_input_mat[:, perm[i:i + batchsize]] output_batch = train_output_mat[:, perm[i:i + batchsize]] # Encode a sentence model.initialize() # initialize cell model.encode(input_batch, train=True) # encode (output: hidden Variable) # Decode from encoded context end_batch = xp.array( [corpus.dic.token2id["<start>"] for _ in range(batchsize)]) first_words = output_batch[0] loss, predict_mat = model.decode(end_batch, first_words, train=True) next_ids = first_words accum_loss += loss for w_ids in output_batch[1:]: loss, predict_mat = model.decode(next_ids, w_ids, train=True) next_ids = w_ids accum_loss += loss # learn model model.cleargrads() # initialize all grad to zero accum_loss.backward() # back propagation optimizer.update() total_loss += float(accum_loss.data) batch_num += 1 print('Epoch: ', num, 'Batch_num', batch_num, 'batch loss: {:.2f}'.format(float(accum_loss.data))) accum_loss = 0 else: # select last batch data input_batch = test_input_mat output_batch = test_output_mat # Encode a sentence model.initialize() # initialize cell model.encode(input_batch, train=True) # encode (output: hidden Variable) # Decode from encoded context end_batch = xp.array( [corpus.dic.token2id["<start>"] for _ in range(batchsize)]) first_words = output_batch[0] loss, predict_mat = model.decode(end_batch, first_words, train=True) next_ids = xp.argmax(predict_mat.data, axis=1) test_loss += loss for w_ids in output_batch[1:]: loss, predict_mat = model.decode(next_ids, w_ids, train=True) next_ids = xp.argmax(predict_mat.data, axis=1) test_loss += loss # save model and optimizer if (epoch + 1) % 10 == 0: print('-----', epoch + 1, ' times -----') print('save the model and optimizer') serializers.save_hdf5('data/' + str(epoch) + '.model', model) serializers.save_hdf5('data/' + str(epoch) + '.state', optimizer) # display the on-going status print('Epoch: ', num, 'Train loss: {:.2f}'.format(total_loss), 'Test loss: {:.2f}'.format(float(test_loss.data))) train_loss_data.append(float(total_loss / batch_num)) test_loss_data.append(float(test_loss.data)) # evaluate a test loss check_loss = test_loss_data[-10:] # check out the last 10 loss data end_flg = [ j for j in range(len(check_loss) - 1) if check_loss[j] < check_loss[j + 1] ] if len(end_flg) > 7: print('Probably it is over-fitting. So stop to learn...') break # save loss data with open('./data/loss_train_data.pkl', 'wb') as f: pickle.dump(train_loss_data, f) with open('./data/loss_test_data.pkl', 'wb') as f: pickle.dump(test_loss_data, f)
def interpreter(data_path, model_path): """ Run this function, if you want to talk to seq2seq model. if you type "exit", finish to talk. :param data_path: the path of corpus you made model learn :param model_path: the path of model you made learn :return: """ # call dictionary class if args.lang == 'en': corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) elif args.lang == 'ja': corpus = JaConvCorpus(file_path=None) corpus.load(load_dir=data_path) else: print( 'You gave wrong argument to this system. Check out your argument about languages.' ) raise ValueError print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, label_num=args.label_num, label_embed_num=args.label_embed, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # run conversation system print('The system is ready to run, please talk to me!') print('( If you want to end a talk, please type "exit". )') print('') while True: print('>> ', end='') sentence = input() if sentence == 'exit': print('See you again!') break # convert to a list if args.lang == 'en': input_vocab = [ unicodedata.normalize('NFKC', word.lower()) for word in word_tokenize(sentence) ] elif args.lang == 'ja': input_vocab = parse_ja_text(sentence) else: print("Sorry, but your language is not supported...") raise ValueError # check a sentiment tag label_id = -1 if len(input_vocab) == 0: print('caution: you donot set any words!)') pass elif input_vocab[-1] == '2': del input_vocab[-1] label_id = 1 elif input_vocab[-1] == '1': del input_vocab[-1] label_id = 0 else: print('caution: you donot set any sentiment tags!') break # input_vocab.reverse() # input_vocab.insert(0, "<eos>") # convert word into ID input_sentence = [ corpus.dic.token2id[word] for word in input_vocab if not corpus.dic.token2id.get(word) is None ] model.initialize() # initialize cell sentence = model.generate(input_sentence, sentence_limit=len(input_sentence) + 30, word2id=corpus.dic.token2id, id2word=corpus.dic, label_id=label_id) print("-> ", sentence) print('')
def calculate_embedding_vectors(data_path, model_path): # call dictionary class if args.lang == 'en': corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) elif args.lang == 'ja': corpus = JaConvCorpus(file_path=None) corpus.load(load_dir=data_path) else: print( 'You gave wrong argument to this system. Check out your argument about languages.' ) raise ValueError print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, label_num=args.label_num, label_embed_num=args.label_embed, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # get embedding vectors embed_mat = model.dec.ye.W.data sentiment_mat = model.dec.le.W.data neg_vec = np.array([sentiment_mat[0, :]]) pos_vec = np.array([sentiment_mat[1, :]]) # calculate cos similarity neg_sim_dic = {} pos_sim_dic = {} for i in range(embed_mat.shape[0]): word_vec = np.array([embed_mat[i, :]]) neg_sim_dic[i] = cosine_similarity(word_vec, neg_vec) pos_sim_dic[i] = cosine_similarity(word_vec, pos_vec) # if cosine_similarity(word_vec, pos_vec) > cosine_similarity(word_vec, neg_vec): # print('pos: ', corpus.dic[i]) # elif cosine_similarity(word_vec, pos_vec) < cosine_similarity(word_vec, neg_vec): # print('neg: ', corpus.dic[i]) # else: # print('???: ', corpus.dic[i]) # raise ValueError # sort in descending order neg_ordered = collections.OrderedDict( sorted(neg_sim_dic.items(), key=lambda x: x[1], reverse=True)) pos_ordered = collections.OrderedDict( sorted(pos_sim_dic.items(), key=lambda x: x[1], reverse=True)) # show TOP50 words print('------- The words which is similar to a NEGATIVE tag --------') for index, w_index in enumerate(neg_ordered): print(corpus.dic[w_index], ': ', neg_ordered[w_index][0, 0]) if index == 49: break print('------- The words which is similar to a POSITIVE tag --------') for index, w_index in enumerate(pos_ordered): print(corpus.dic[w_index], ': ', pos_ordered[w_index][0, 0]) if index == 49: break
def interpreter(data_path, model_path): """ Run this function, if you want to talk to seq2seq model. if you type "exit", finish to talk. :param data_path: the path of corpus you made model learn :param model_path: the path of model you made learn :return: """ # call dictionary class if args.lang == 'en': corpus = ConvCorpus(file_path=None) corpus.load(load_dir=data_path) elif args.lang == 'ja': corpus = JaConvCorpus(file_path=None) corpus.load(load_dir=data_path) else: print( 'You gave wrong argument to this system. Check out your argument about languages.' ) raise ValueError print('Vocabulary Size (number of words) :', len(corpus.dic.token2id)) print('') # rebuild seq2seq model model = Seq2Seq(len(corpus.dic.token2id), feature_num=args.feature_num, hidden_num=args.hidden_num, batch_size=1, gpu_flg=args.gpu) serializers.load_hdf5(model_path, model) # run conversation system print('The system is ready to run, please talk to me!') print('( If you want to end a talk, please type "exit". )') print('') while True: print('>> ', end='') sentence = input() if sentence == 'exit': print('See you again!') break if args.lang == 'en': input_vocab = [ unicodedata.normalize('NFKC', word.lower()) for word in word_tokenize(sentence) ] elif args.lang == 'ja': input_vocab = [ unicodedata.normalize('NFKC', word.lower()) for word in parse_ja_text(sentence) ] input_vocab_rev = input_vocab[::-1] # convert word into ID input_sentence = [ corpus.dic.token2id[word] for word in input_vocab if not corpus.dic.token2id.get(word) is None ] input_sentence_rev = [ corpus.dic.token2id[word] for word in input_vocab_rev if not corpus.dic.token2id.get(word) is None ] model.initialize(batch_size=1) # initialize cell sentence = model.generate(input_sentence, input_sentence_rev, sentence_limit=len(input_sentence) + 30, word2id=corpus.dic.token2id, id2word=corpus.dic) print("-> ", sentence) print('')