def main(): corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() # model.load_params('Rnnlm.pkl') start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt) print('DONE')
def main(word): corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() # start文字とskip文字の設定 start_word = word start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] # 文章生成 word_ids = model.generate(start_id, skip_ids=skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt)
def main(): curpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(curpus) model = RnnlmGen() # model.load_params('../ch06_RNNWithGate/Rnnlm.pkl') # start文字とskip文字の設定 start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] # 文書生成 word_ids = model.generate(start_id, skip_ids) # サンプルサイズ100 txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt)
# coding: utf-8 import sys sys.path.append('/home/hiromasa/deep-learning-from-scratch-2') from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() model.load_params('/home/hiromasa/deep-learning-from-scratch-2/ch06/Rnnlm.pkl') # start文字とskip文字の設定 start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] # 文章生成 word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt)
# coding: utf-8 import sys sys.path.append('..') from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() model.load_params('../ch06/Rnnlm.pkl') start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt)
import numpy as np from rnnlm_gen import RnnlmGen as BetterRnnlmGen from dataset import ptb def get_perplexity(model): from common.util import eval_perplexity corpus_val, _, _ = ptb.load_data('val') ppl = eval_perplexity(model, corpus_val) return ppl corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = BetterRnnlmGen() model.load_params('../chapter6/BetterRnnlm.pkl') # print(get_perplexity(model)) # start文字とskip文字の設定 start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] # 文章生成 word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace(' <eos>', '.\n') print(txt)
import sys sys.path.append('..') from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() #model.load_params('../ch06/Rnnlm.pkl') # 앞 장에서 학습을 끝낸 가중치 매개변수 # 시작(start) 문자와 건너뜀(skip) 문자 설정 start_word = 'you' # 첫 문자로 'you'를 사용 start_id = word_to_id[start_word] # 'you'의 ID skip_words = ['N', '<unk>', '$'] # 샘플링하지 않을 단어 지정 skip_ids = [word_to_id[w] for w in skip_words] # 문자 생성 word_ids = model.generate(start_id, skip_ids) # 문장을 생성하는 generate() 는 단어 ID들을 배열 형태로 반환함 txt = ' '.join([id_to_word[i] for i in word_ids]) txt = replace(' <eos>', '.\n') print(txt)
import sys sys.path.append('..') from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace('<eos>', '.\n') print(txt)
config.GPU = True from rnnlm_gen import RnnlmGen #import rnnlm_train #import train_better_lstm import train_simple_lstmlm #import txt #from tangoatume import preprocess #file_name = "./txt/koi_z.txt" #file = open(file_name,encoding='utf-8') corpus, word_to_id, id_to_word = train_simple_lstmlm.corpus, train_simple_lstmlm.word_to_id, train_simple_lstmlm.id_to_word #print(word_to_id) vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen(vocab_size, train_simple_lstmlm.wordvec_size, train_simple_lstmlm.hidden_size) model.load_params("./pkl/SimpleLstmlm_epoch50.pkl") #start文字とskip文字の設定 start_word = 'コロナウイルス' start_id = word_to_id[start_word] skip_words = [''] skip_ids = [word_to_id[w] for w in skip_words] #文章生成 word_ids = model.generate(start_id, skip_ids) #print(word_ids) ''' map_result = map(str, word_ids) result = ' '.join(map_result) for i in result
import sys sys.path.append("..") from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() model.load_params( '../ch06_게이트가_추가된_RNN/Rnnlm.pkl') # 미리 학습된 가중치 불러와 성능 높임 # 시작 (start) 문자와 건너뛸 (skip) 문자 설정 start_word = 'you' start_id = word_to_id[start_word] skip_words = ['N', '<unk>', '$'] skip_ids = [word_to_id[w] for w in skip_words] # 문장 생성 word_ids = model.generate(start_id, skip_ids) txt = ' '.join([id_to_word[i] for i in word_ids]) txt = txt.replace('<eos>', '.\n') print(txt)
# coding: utf-8 import sys sys.path.append('..') # from rnnlm_gen_test import RnnlmGen from rnnlm_gen import RnnlmGen from dataset import ptb corpus, word_to_id, id_to_word = ptb.load_data('train') vocab_size = len(word_to_id) corpus_size = len(corpus) model = RnnlmGen() # 가중치 매개변수 : 무작위 초깃값(학습 X) # model.load_params('../ch06/Rnnlm.pkl') # 앞 단원에서 학습을 끝낸 가중치 매개변수를 읽어들임 # start 문자와 skip 문자 설정 start_word = 'you' # 첫 단어를 you 로 설정 start_id = word_to_id[start_word] # you 에 해당하는 id 를 start_id 로 설정 skip_words = ['N', '<unk>', '$'] # 샘플링하지 않을 단어 : ['N', '<unk>', '$'] 로 설정 skip_ids = [word_to_id[w] for w in skip_words] # 샘플링 하지 않을 단어에 해당하는 skip_ids # 문장 생성 word_ids = model.generate(start_id, skip_ids) # 문장 생성 -> 단어 ID 들을 배열 형태로 반환 txt = ' '.join([id_to_word[i] for i in word_ids]) # 각 배열 요소 사이에 ' ' 넣어서 문장 형태로 변환 txt = txt.replace(' <eos>', '.\n') # <eos> 는 줄바꿈 문자로. print(txt)