예제 #1
0
def main():
    corpus, word_to_id, id_to_word = ptb.load_data('train')
    vocab_size = len(word_to_id)
    corpus_size = len(corpus)

    model = RnnlmGen()
    # model.load_params('Rnnlm.pkl')

    start_word = 'you'
    start_id = word_to_id[start_word]
    skip_words = ['N', '<unk>', '$']
    skip_ids = [word_to_id[w] for w in skip_words]

    word_ids = model.generate(start_id, skip_ids)
    txt = ' '.join([id_to_word[i] for i in word_ids])
    txt = txt.replace(' <eos>', '.\n')
    print(txt)
    print('DONE')
예제 #2
0
def main(word):
    corpus, word_to_id, id_to_word = ptb.load_data('train')
    vocab_size = len(word_to_id)
    corpus_size = len(corpus)

    model = RnnlmGen()

    # start文字とskip文字の設定
    start_word = word
    start_id = word_to_id[start_word]
    skip_words = ['N', '<unk>', '$']
    skip_ids = [word_to_id[w] for w in skip_words]

    # 文章生成
    word_ids = model.generate(start_id, skip_ids=skip_ids)
    txt = ' '.join([id_to_word[i] for i in word_ids])
    txt = txt.replace(' <eos>', '.\n')
    print(txt)
def main():
    curpus, word_to_id, id_to_word = ptb.load_data('train')
    vocab_size = len(word_to_id)
    corpus_size = len(curpus)

    model = RnnlmGen()
    # model.load_params('../ch06_RNNWithGate/Rnnlm.pkl')

    # start文字とskip文字の設定
    start_word = 'you'
    start_id = word_to_id[start_word]
    skip_words = ['N', '<unk>', '$']
    skip_ids = [word_to_id[w] for w in skip_words]

    # 文書生成
    word_ids = model.generate(start_id, skip_ids)  # サンプルサイズ100
    txt = ' '.join([id_to_word[i] for i in word_ids])
    txt = txt.replace(' <eos>', '.\n')
    print(txt)
# coding: utf-8
import sys
sys.path.append('/home/hiromasa/deep-learning-from-scratch-2')
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()
model.load_params('/home/hiromasa/deep-learning-from-scratch-2/ch06/Rnnlm.pkl')

# start文字とskip文字の設定
start_word = 'you'
start_id = word_to_id[start_word]
skip_words = ['N', '<unk>', '$']
skip_ids = [word_to_id[w] for w in skip_words]
# 文章生成
word_ids = model.generate(start_id, skip_ids)
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = txt.replace(' <eos>', '.\n')
print(txt)
예제 #5
0
# coding: utf-8
import sys
sys.path.append('..')
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()
model.load_params('../ch06/Rnnlm.pkl')

start_word = 'you'
start_id = word_to_id[start_word]
skip_words = ['N', '<unk>', '$']
skip_ids = [word_to_id[w] for w in skip_words]

word_ids = model.generate(start_id, skip_ids)
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = txt.replace(' <eos>', '.\n')
print(txt)
예제 #6
0
import numpy as np
from rnnlm_gen import RnnlmGen as BetterRnnlmGen
from dataset import ptb

def get_perplexity(model):
    from common.util import eval_perplexity
    corpus_val, _, _ = ptb.load_data('val')
    ppl = eval_perplexity(model, corpus_val)
    return ppl

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)


model = BetterRnnlmGen()
model.load_params('../chapter6/BetterRnnlm.pkl')

# print(get_perplexity(model))

# start文字とskip文字の設定
start_word = 'you'
start_id = word_to_id[start_word]
skip_words = ['N', '<unk>', '$']
skip_ids = [word_to_id[w] for w in skip_words]
# 文章生成
word_ids = model.generate(start_id, skip_ids)
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = txt.replace(' <eos>', '.\n')

print(txt)
예제 #7
0
import sys
sys.path.append('..')
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()
#model.load_params('../ch06/Rnnlm.pkl') # 앞 장에서 학습을 끝낸 가중치 매개변수

# 시작(start) 문자와 건너뜀(skip) 문자 설정
start_word = 'you'  # 첫 문자로 'you'를 사용
start_id = word_to_id[start_word]  # 'you'의 ID
skip_words = ['N', '<unk>', '$']  # 샘플링하지 않을 단어 지정
skip_ids = [word_to_id[w] for w in skip_words]

# 문자 생성
word_ids = model.generate(start_id,
                          skip_ids)  # 문장을 생성하는 generate() 는 단어 ID들을 배열 형태로 반환함
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = replace(' <eos>', '.\n')
print(txt)
예제 #8
0
import sys
sys.path.append('..')
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()

start_word = 'you'
start_id = word_to_id[start_word]

skip_words = ['N', '<unk>', '$']
skip_ids = [word_to_id[w] for w in skip_words]

word_ids = model.generate(start_id, skip_ids)
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = txt.replace('<eos>', '.\n')
print(txt)
예제 #9
0
config.GPU = True
from rnnlm_gen import RnnlmGen
#import rnnlm_train
#import train_better_lstm
import train_simple_lstmlm
#import txt
#from tangoatume import preprocess

#file_name = "./txt/koi_z.txt"
#file = open(file_name,encoding='utf-8')
corpus, word_to_id, id_to_word = train_simple_lstmlm.corpus, train_simple_lstmlm.word_to_id, train_simple_lstmlm.id_to_word
#print(word_to_id)
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen(vocab_size, train_simple_lstmlm.wordvec_size,
                 train_simple_lstmlm.hidden_size)
model.load_params("./pkl/SimpleLstmlm_epoch50.pkl")

#start文字とskip文字の設定
start_word = 'コロナウイルス'
start_id = word_to_id[start_word]
skip_words = ['']
skip_ids = [word_to_id[w] for w in skip_words]

#文章生成
word_ids = model.generate(start_id, skip_ids)
#print(word_ids)
'''
map_result = map(str, word_ids)
result = ' '.join(map_result)
for i in result
예제 #10
0
import sys
sys.path.append("..")
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()
model.load_params(
    '../ch06_게이트가_추가된_RNN/Rnnlm.pkl')  # 미리 학습된 가중치 불러와 성능 높임

# 시작 (start) 문자와 건너뛸 (skip) 문자 설정
start_word = 'you'
start_id = word_to_id[start_word]
skip_words = ['N', '<unk>', '$']
skip_ids = [word_to_id[w] for w in skip_words]

# 문장 생성
word_ids = model.generate(start_id, skip_ids)
txt = ' '.join([id_to_word[i] for i in word_ids])
txt = txt.replace('<eos>', '.\n')
print(txt)
예제 #11
0
# coding: utf-8
import sys
sys.path.append('..')
# from rnnlm_gen_test import RnnlmGen
from rnnlm_gen import RnnlmGen
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
corpus_size = len(corpus)

model = RnnlmGen()  # 가중치 매개변수 : 무작위 초깃값(학습 X)
# model.load_params('../ch06/Rnnlm.pkl')      # 앞 단원에서 학습을 끝낸 가중치 매개변수를 읽어들임

# start 문자와 skip 문자 설정
start_word = 'you'  #  첫 단어를 you 로 설정
start_id = word_to_id[start_word]  # you 에 해당하는 id 를 start_id 로 설정
skip_words = ['N', '<unk>', '$']  # 샘플링하지 않을 단어 : ['N', '<unk>', '$'] 로 설정
skip_ids = [word_to_id[w] for w in skip_words]  # 샘플링 하지 않을 단어에 해당하는 skip_ids
# 문장 생성
word_ids = model.generate(start_id, skip_ids)  # 문장 생성 -> 단어 ID 들을 배열 형태로 반환
txt = ' '.join([id_to_word[i]
                for i in word_ids])  # 각 배열 요소 사이에 ' ' 넣어서 문장 형태로 변환
txt = txt.replace(' <eos>', '.\n')  # <eos> 는 줄바꿈 문자로.
print(txt)