def gen(**kwargs):
"""
提供命令行接口，用以生成相应的诗
"""
for k, v in kwargs.items():
setattr(opt, k, v)
data, word2ix, ix2word = get_data(opt)
model = PoetryModel(len(word2ix), 128, 256);
map_location = lambda s, l: s
state_dict = t.load(opt.model_path, map_location=map_location)
model.load_state_dict(state_dict)
if opt.use_gpu:
model.cuda()
if sys.version_info.major == 3:
if opt.start_words.isprintable():
start_words = opt.start_words
prefix_words = opt.prefix_words if opt.prefix_words else None
else:
start_words = opt.start_words.encode('ascii', 'surrogateescape').decode('utf8')
prefix_words = opt.prefix_words.encode('ascii', 'surrogateescape').decode(
'utf8') if opt.prefix_words else None
else:
start_words = opt.start_words.decode('utf8')
prefix_words = opt.prefix_words.decode('utf8') if opt.prefix_words else None
start_words = start_words.replace(',', u'，') \
.replace('.', u'。') \
.replace('?', u'？')
gen_poetry = gen_acrostic if opt.acrostic else generate
result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words)
print(''.join(result))
def train(opt):
data, word2ix, ix2word = get_data(opt)
data = torch.from_numpy(data)
dataloader = td.DataLoader(data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
model = PoetryModel(len(word2ix), 128, 256)
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.use_gpu:
model.cuda()
criterion.cuda()
for epoch in range(opt.epoch):
for step, data_ in enumerate(dataloader):
data_ = data_.long().transpose(1, 0).contiguous()
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()
input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :])
output, _ = model(input_)
loss = criterion(output, target.view(-1))
loss.backward()
optimizer.step()
if (1 + step) % 10 == 0:
print("current loss", loss.data)
t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def gen(**kwargs):
'''
提供命令行接口，用以生成相应的诗
'''
for k, v in kwargs.items():
setattr(opt, k, v)
data, word2ix, ix2word = get_data(opt)
model = PoetryModel(len(word2ix), 128, 256)
map_location = lambda s, l: s
state_dict = t.load(opt.model_path, map_location=map_location)
model.load_state_dict(state_dict)
if opt.use_gpu:
model.cuda()
# if sys.version_info.major == 3:
# start_words = opt.start_words.encode('ascii', 'surrogateescape').decode('utf8')
# prefix_words = opt.prefix_words.encode('ascii', 'surrogateescape').decode('utf8') if opt.prefix_words else None
# else:
# start_words = opt.start_words.decode('utf8')
# prefix_words = opt.prefix_words.decode('utf8') if opt.prefix_words else None
start_words = opt.start_words
prefix_words = opt.prefix_words
start_words= start_words.replace(',',u'，')\
.replace('.',u'。')\
.replace('?',u'？')
gen_poetry = gen_acrostic if opt.acrostic else generate
result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words)
print(''.join(result))
def gen(**kwargs):
"""
提供命令行接口，用以生成相应的诗
"""
for k, v in kwargs.items():
print(k, v)
setattr(opt, k, v)
data, word2ix, ix2word = get_data(opt)
model = PoetryModel(len(word2ix), 128, 256)
map_location = lambda s, l: s
state_dict = t.load(opt.model_path, map_location=map_location)
model.load_state_dict(state_dict)
if opt.use_gpu:
model.cuda()
# python2和python3 字符串兼容
if sys.version_info.major == 3:
if opt.start_words.isprintable():
start_words = opt.start_words
prefix_words = opt.prefix_words if opt.prefix_words else None
else:
start_words = opt.start_words.encode(
'ascii', 'surrogateescape').decode('utf8')
prefix_words = opt.prefix_words.encode(
'ascii',
'surrogateescape').decode('utf8') if opt.prefix_words else None
else:
start_words = opt.start_words.decode('utf8')
prefix_words = opt.prefix_words.decode(
'utf8') if opt.prefix_words else None
prefix_words = prefix_words.replace(',', u'，') \
.replace('.', u'。') \
.replace('?', u'？')
start_words = start_words.replace(',', u'，') \
.replace('.', u'。') \
.replace('?', u'？')
gen_poetry = gen_acrostic if opt.acrostic else generate
result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words)
with open('result.txt', 'w') as f:
f.writelines(result)
print(''.join(result))
def train(**kwargs:dict) -> None:
for k, v in kwargs.items():
setattr(opt, k, v)
#vis = Visdom(env=opt.env)
#data get
data, word2ix, ix2word = get_data(opt)
data = t.from_numpy(data)
dataloader = t.utils.data.DataLoader(
data,
batch_size=opt.batch_size,
shuffle = True,
)
model = PoetryModel(len(word2ix), 2, 2)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.user_gpu:
model.cuda()
criterion.cuda()
for epoch in range(opt.epoch):
for ii, data_ in tqdm.tqdm(enumerate(dataloader)):
data_ = data.long().transpose(1, 0).contiguous()
if opt.user_gpu : data_ = data_.cuda()
optimizer.zero_grad()
input_, target = V(data_[:-1, :]), V(data[1:, :])
ouput, _ = model(input_)
loss = criterion(ouput, target.view(-1))
loss.backward()
optimizer.step()
def gen(**kwargs):
"""
提供命令行接口，用以生成相应的诗
"""
print(kwargs)
for k, v in kwargs.items():
setattr(opt, k, v)
data, word2ix, ix2word = get_data(opt)
model = PoetryModel(len(word2ix), 128, 256)
map_location = lambda s, l: s
state_dict = t.load(opt.model_path, map_location=map_location)
model.load_state_dict(state_dict)
if opt.use_gpu:
model.cuda()
# read names from file
with open(opt.start_words_file) as f:
json_data = json.load(f)
names = json_data["names"]
prefix_words_list = json_data["prefix_words_list"]
gen_poetry = gen_acrostic if opt.acrostic else generate
for staff in names:
result = gen_poetry(
model, staff['name'], ix2word, word2ix,
prefix_words_list[random.randint(1,
len(prefix_words_list) - 1)])
print(''.join(result))
result_data[staff['name']] = ''.join(result)
# 写入文件
# {key: value}
# for, 多个人名
# 读写相关配置文件，生成各人名对应的诗
with open('data.json', 'w', encoding='utf-8') as outfile:
json.dump(result_data, outfile, ensure_ascii=False)
print "data_size", len(data)
for i in range(len(data)):
data[i] = toList(data[i])
data[i].append("<EOP>")
# save the word dic for sample method
p.dump(word_to_ix, file('wordDic', 'w'))
# save all avaible word
# wordList = open('wordList','w')
# for w in word_to_ix:
# wordList.write(w.encode('utf-8'))
# wordList.close()
model = PoetryModel(len(word_to_ix), 256, 256)
model.cuda(
) # running on GPU,if you want to run it on CPU,delete all .cuda() usage.
optimizer = optim.RMSprop(model.parameters(), lr=0.01, weight_decay=0.0001)
criterion = nn.NLLLoss()
one_hot_var_target = {}
for w in word_to_ix:
one_hot_var_target.setdefault(w, make_one_hot_vec_target(w, word_to_ix))
epochNum = 10
TRAINSIZE = len(data)
batch = 100
def test():
v = int(TRAINSIZE / batch)
loss = 0
def train(**kwargs):
for k, v in kwargs.items():
setattr(opt, k, v)
vis = Visualizer(env=opt.env)
# 获取数据
data, word2ix, ix2word = get_data(opt)
data = t.from_numpy(data)
dataloader = t.utils.data.DataLoader(data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
# 模型定义
model = PoetryModel(len(word2ix), 128, 256)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.use_gpu:
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for ii, data_ in tqdm.tqdm(enumerate(dataloader)):
# 训练
data_ = data_.long().transpose(1, 0).contiguous()
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()
input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :])
output, _ = model(input_)
loss = criterion(output, target.view(-1))
loss.backward()
optimizer.step()
loss_meter.add(loss.data[0])
# 可视化
if (1 + ii) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
# 诗歌原文
poetrys = [[ix2word[_word] for _word in data_[:, _iii]]
for _iii in range(data_.size(1))][:16]
vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]), win=u'origin_poem')
gen_poetries = []
# 分别以这几个字作为诗歌的第一个字，生成8首诗
for word in list(u'春江花月夜凉如水'):
gen_poetry = ''.join(generate(model, word, ix2word, word2ix))
gen_poetries.append(gen_poetry)
vis.text('</br>'.join([''.join(poetry) for poetry in gen_poetries]), win=u'gen_poem')
t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs):
for k, v in kwargs.items():
setattr(opt, k, v)
vis = Visualizer(env=opt.env)
#获取数据
data, word2ix, ix2word = get_data(opt)
data = t.from_numpy(data)
dataloader = t.utils.data.DataLoader(data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
#模型定义
model = PoetryModel(len(word2ix), 128, 256)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.use_gpu:
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for li, data_ in tqdm.tqdm(enumerate(dataloader)):
#训练
data_ = data_.long().transpose(1, 0).contiguous()
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()
##输入和目标错开
input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :])
output, _ = model(input_)
loss = criterion(output, target.view(-1))
loss.backward()
optimizer.step()
loss_meter.add(loss.data[0])
# 可视化
if (1 + ii) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
#诗歌原文
poetrys = [[ix2word[_word] for _word in data_[:, -iii]]
for _iii in range(data_.size(1))][:16]
vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]),
win=u'origin_poem')
gen_poetries = []
#分别以这几个字作为诗歌的第一个字，生成8首诗
for word in list(u'春江花月夜凉如水'):
gen_poetry = ''.join(
generate(model, word, ix2word, word2ix))
gen_poetries.append(gen_poetry)
vis.text('</br>'.join(
[''.join(poetry) for poetry in gen_poetries]),
win=u'gen_poem')
t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs):
for k,v in kwargs.items():
setattr(opt,k,v)
vis = Visualizer(env=opt.env)
# 获取数据
data,word2ix,ix2word = get_data(opt)
data = t.from_numpy(data)#把数据类型转为tensor
dataloader = t.utils.data.DataLoader(data,#初始化Dataloader类实例
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
# 模型定义
model = PoetryModel(len(word2ix), 128, 256)#(vocab_size, embedding_dim, hidden_dim)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()#损失函数定义为交叉熵
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.use_gpu:
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for ii,data_ in tqdm.tqdm(enumerate(dataloader)): #tqdm进度条工具
#取一个batch的数据
# 训练
#data_.size:(batch_size,maxlen)
data_ = data_.long().transpose(1,0).contiguous()#转置后返回一个内存连续的有相同数据的tensor
# if epoch==0 and ii ==0:
# print('size of data_ after transpose: \n',data_.size())
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()#梯度清零
input_,target = Variable(data_[:-1,:]),Variable(data_[1:,:])#input_是所有句子的前maxlen-1个item的集合，
#target是所有句子的后maxlen-１个item的集合
#以＂床前明月光＂为例，输入是＂床前明月＂，要预测＂前明月光＂
output,_ = model(input_)
#Tensor.view(-1)按照第０个维度逐个元素读取将张量展开成数组
loss = criterion(output,target.view(-1))
loss.backward()
optimizer.step()
loss_meter.add(loss.data[0])
# 可视化
if (1+ii)%opt.plot_every==0:
if os.path.exists(opt.debug_file):#如果存在调试文件，
#则进入调试模式
ipdb.set_trace()
vis.plot('loss',loss_meter.value()[0])
# 诗歌原文
poetrys=[ [ix2word[_word] for _word in data_[:,_iii]] #每一个句子（诗歌）的每一个item（id）要转换成文本
for _iii in range(data_.size(1))][:16]#_iii的取值范围[,127]
vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]),win=u'origin_poem')
#在visdom中输出这些句子（诗歌）中的前１６个
gen_poetries = []
# 分别以这几个字作为诗歌的第一个字，生成8首诗
for word in list(u'春江花月夜凉如水'):
gen_poetry = ''.join(generate(model,word,ix2word,word2ix))
gen_poetries.append(gen_poetry)
vis.text('</br>'.join([''.join(poetry) for poetry in gen_poetries]),win=u'gen_poem')
t.save(model.state_dict(),'%s_%s.pth' %(opt.model_prefix,epoch))
def train_torch_lstm(conf, args=None):
pdata = PoemData()
pdata.read_data(conf)
pdata.get_vocab()
if conf.use_gpu:
device = torch.device('cuda')
else:
device = torch.device('cpu')
model = PoetryModel(pdata.vocab_size, conf, device)
train_data = pdata.train_data
test_data = pdata.test_data
train_data = torch.from_numpy(np.array(train_data['pad_words']))
dev_data = torch.from_numpy(np.array(test_data['pad_words']))
dataloader = DataLoader(train_data,
batch_size=conf.batch_size,
shuffle=True,
num_workers=conf.num_workers)
devloader = DataLoader(dev_data,
batch_size=conf.batch_size,
shuffle=True,
num_workers=conf.num_workers)
optimizer = Adam(model.parameters(), lr=conf.learning_rate)
criterion = nn.CrossEntropyLoss()
loss_meter = meter.AverageValueMeter()
if conf.load_best_model:
model.load_state_dict(torch.load(conf.beat_model_path))
if conf.use_gpu:
model.cuda()
criterion.cuda()
step = 0
bestppl = 1e9
early_stop_controller = 0
for epoch in range(conf.n_epochs):
losses = []
loss_meter.reset()
model.train()
for i, data in enumerate(dataloader):
data = data.long().transpose(1, 0).contiguous()
if conf.use_gpu:
data = data.cuda()
input, target = data[:-1, :], data[1:, :]
optimizer.zero_grad()
output, _ = model(input)
loss = criterion(output, target.contiguous().view(-1))
loss.backward()
optimizer.step()
losses.append(loss.item())
loss_meter.add(loss.item())
step += 1
if step % 100 == 0:
print("epoch_%d_step_%d_loss:%0.4f" %
(epoch + 1, step, loss.item()))
train_loss = float(loss_meter.value()[0])
model.eval()
for i, data in enumerate(devloader):
data = data.long().transpose(1, 0).contiguous()
if conf.use_gpu:
data = data.cuda()
input, target = data[:-1, :], data[1:, :]
output, _ = model(input)
loss = criterion(output, target.view(-1))
loss_meter.add(loss.item())
ppl = math.exp(loss_meter.value()[0])
print("epoch_%d_loss:%0.4f , ppl:%0.4f" % (epoch + 1, train_loss, ppl))
if epoch % conf.save_every == 0:
torch.save(model.state_dict(),
"{0}_{1}".format(conf.model_prefix, epoch))
fout = open("{0}out_{1}".format(conf.out_path, epoch),
'w',
encoding='utf-8')
for word in list('日红山夜湖海月'):
gen_poetry = generate_poet(model, word, pdata.vocab, conf)
fout.write("".join(gen_poetry) + '\n\n')
fout.close()
if ppl < bestppl:
bestppl = ppl
early_stop_controller = 0
torch.save(model.state_dict(),
"{0}_{1}".format(conf.best_model_path, "best_model"))
else:
early_stop_controller += 1
if early_stop_controller > conf.patience:
print("early stop.")
break
