def gen(**kwargs): """ 提供命令行接口,用以生成相应的诗 """ for k, v in kwargs.items(): setattr(opt, k, v) data, word2ix, ix2word = get_data(opt) model = PoetryModel(len(word2ix), 128, 256); map_location = lambda s, l: s state_dict = t.load(opt.model_path, map_location=map_location) model.load_state_dict(state_dict) if opt.use_gpu: model.cuda() if sys.version_info.major == 3: if opt.start_words.isprintable(): start_words = opt.start_words prefix_words = opt.prefix_words if opt.prefix_words else None else: start_words = opt.start_words.encode('ascii', 'surrogateescape').decode('utf8') prefix_words = opt.prefix_words.encode('ascii', 'surrogateescape').decode( 'utf8') if opt.prefix_words else None else: start_words = opt.start_words.decode('utf8') prefix_words = opt.prefix_words.decode('utf8') if opt.prefix_words else None start_words = start_words.replace(',', u',') \ .replace('.', u'。') \ .replace('?', u'?') gen_poetry = gen_acrostic if opt.acrostic else generate result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words) print(''.join(result))
def train(opt): data, word2ix, ix2word = get_data(opt) data = torch.from_numpy(data) dataloader = td.DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=1) model = PoetryModel(len(word2ix), 128, 256) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() if opt.use_gpu: model.cuda() criterion.cuda() for epoch in range(opt.epoch): for step, data_ in enumerate(dataloader): data_ = data_.long().transpose(1, 0).contiguous() if opt.use_gpu: data_ = data_.cuda() optimizer.zero_grad() input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :]) output, _ = model(input_) loss = criterion(output, target.view(-1)) loss.backward() optimizer.step() if (1 + step) % 10 == 0: print("current loss", loss.data) t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def gen(**kwargs): ''' 提供命令行接口,用以生成相应的诗 ''' for k, v in kwargs.items(): setattr(opt, k, v) data, word2ix, ix2word = get_data(opt) model = PoetryModel(len(word2ix), 128, 256) map_location = lambda s, l: s state_dict = t.load(opt.model_path, map_location=map_location) model.load_state_dict(state_dict) if opt.use_gpu: model.cuda() # if sys.version_info.major == 3: # start_words = opt.start_words.encode('ascii', 'surrogateescape').decode('utf8') # prefix_words = opt.prefix_words.encode('ascii', 'surrogateescape').decode('utf8') if opt.prefix_words else None # else: # start_words = opt.start_words.decode('utf8') # prefix_words = opt.prefix_words.decode('utf8') if opt.prefix_words else None start_words = opt.start_words prefix_words = opt.prefix_words start_words= start_words.replace(',',u',')\ .replace('.',u'。')\ .replace('?',u'?') gen_poetry = gen_acrostic if opt.acrostic else generate result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words) print(''.join(result))
def gen(**kwargs): """ 提供命令行接口,用以生成相应的诗 """ for k, v in kwargs.items(): print(k, v) setattr(opt, k, v) data, word2ix, ix2word = get_data(opt) model = PoetryModel(len(word2ix), 128, 256) map_location = lambda s, l: s state_dict = t.load(opt.model_path, map_location=map_location) model.load_state_dict(state_dict) if opt.use_gpu: model.cuda() # python2和python3 字符串兼容 if sys.version_info.major == 3: if opt.start_words.isprintable(): start_words = opt.start_words prefix_words = opt.prefix_words if opt.prefix_words else None else: start_words = opt.start_words.encode( 'ascii', 'surrogateescape').decode('utf8') prefix_words = opt.prefix_words.encode( 'ascii', 'surrogateescape').decode('utf8') if opt.prefix_words else None else: start_words = opt.start_words.decode('utf8') prefix_words = opt.prefix_words.decode( 'utf8') if opt.prefix_words else None prefix_words = prefix_words.replace(',', u',') \ .replace('.', u'。') \ .replace('?', u'?') start_words = start_words.replace(',', u',') \ .replace('.', u'。') \ .replace('?', u'?') gen_poetry = gen_acrostic if opt.acrostic else generate result = gen_poetry(model, start_words, ix2word, word2ix, prefix_words) with open('result.txt', 'w') as f: f.writelines(result) print(''.join(result))
def train(**kwargs:dict) -> None: for k, v in kwargs.items(): setattr(opt, k, v) #vis = Visdom(env=opt.env) #data get data, word2ix, ix2word = get_data(opt) data = t.from_numpy(data) dataloader = t.utils.data.DataLoader( data, batch_size=opt.batch_size, shuffle = True, ) model = PoetryModel(len(word2ix), 2, 2) optimizer = t.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() if opt.model_path: model.load_state_dict(t.load(opt.model_path)) if opt.user_gpu: model.cuda() criterion.cuda() for epoch in range(opt.epoch): for ii, data_ in tqdm.tqdm(enumerate(dataloader)): data_ = data.long().transpose(1, 0).contiguous() if opt.user_gpu : data_ = data_.cuda() optimizer.zero_grad() input_, target = V(data_[:-1, :]), V(data[1:, :]) ouput, _ = model(input_) loss = criterion(ouput, target.view(-1)) loss.backward() optimizer.step()
def gen(**kwargs): """ 提供命令行接口,用以生成相应的诗 """ print(kwargs) for k, v in kwargs.items(): setattr(opt, k, v) data, word2ix, ix2word = get_data(opt) model = PoetryModel(len(word2ix), 128, 256) map_location = lambda s, l: s state_dict = t.load(opt.model_path, map_location=map_location) model.load_state_dict(state_dict) if opt.use_gpu: model.cuda() # read names from file with open(opt.start_words_file) as f: json_data = json.load(f) names = json_data["names"] prefix_words_list = json_data["prefix_words_list"] gen_poetry = gen_acrostic if opt.acrostic else generate for staff in names: result = gen_poetry( model, staff['name'], ix2word, word2ix, prefix_words_list[random.randint(1, len(prefix_words_list) - 1)]) print(''.join(result)) result_data[staff['name']] = ''.join(result) # 写入文件 # {key: value} # for, 多个人名 # 读写相关配置文件,生成各人名对应的诗 with open('data.json', 'w', encoding='utf-8') as outfile: json.dump(result_data, outfile, ensure_ascii=False)
print "data_size", len(data) for i in range(len(data)): data[i] = toList(data[i]) data[i].append("<EOP>") # save the word dic for sample method p.dump(word_to_ix, file('wordDic', 'w')) # save all avaible word # wordList = open('wordList','w') # for w in word_to_ix: # wordList.write(w.encode('utf-8')) # wordList.close() model = PoetryModel(len(word_to_ix), 256, 256) model.cuda( ) # running on GPU,if you want to run it on CPU,delete all .cuda() usage. optimizer = optim.RMSprop(model.parameters(), lr=0.01, weight_decay=0.0001) criterion = nn.NLLLoss() one_hot_var_target = {} for w in word_to_ix: one_hot_var_target.setdefault(w, make_one_hot_vec_target(w, word_to_ix)) epochNum = 10 TRAINSIZE = len(data) batch = 100 def test(): v = int(TRAINSIZE / batch) loss = 0
def train(**kwargs): for k, v in kwargs.items(): setattr(opt, k, v) vis = Visualizer(env=opt.env) # 获取数据 data, word2ix, ix2word = get_data(opt) data = t.from_numpy(data) dataloader = t.utils.data.DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=1) # 模型定义 model = PoetryModel(len(word2ix), 128, 256) optimizer = t.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() if opt.model_path: model.load_state_dict(t.load(opt.model_path)) if opt.use_gpu: model.cuda() criterion.cuda() loss_meter = meter.AverageValueMeter() for epoch in range(opt.epoch): loss_meter.reset() for ii, data_ in tqdm.tqdm(enumerate(dataloader)): # 训练 data_ = data_.long().transpose(1, 0).contiguous() if opt.use_gpu: data_ = data_.cuda() optimizer.zero_grad() input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :]) output, _ = model(input_) loss = criterion(output, target.view(-1)) loss.backward() optimizer.step() loss_meter.add(loss.data[0]) # 可视化 if (1 + ii) % opt.plot_every == 0: if os.path.exists(opt.debug_file): ipdb.set_trace() vis.plot('loss', loss_meter.value()[0]) # 诗歌原文 poetrys = [[ix2word[_word] for _word in data_[:, _iii]] for _iii in range(data_.size(1))][:16] vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]), win=u'origin_poem') gen_poetries = [] # 分别以这几个字作为诗歌的第一个字,生成8首诗 for word in list(u'春江花月夜凉如水'): gen_poetry = ''.join(generate(model, word, ix2word, word2ix)) gen_poetries.append(gen_poetry) vis.text('</br>'.join([''.join(poetry) for poetry in gen_poetries]), win=u'gen_poem') t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs): for k, v in kwargs.items(): setattr(opt, k, v) vis = Visualizer(env=opt.env) #获取数据 data, word2ix, ix2word = get_data(opt) data = t.from_numpy(data) dataloader = t.utils.data.DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=1) #模型定义 model = PoetryModel(len(word2ix), 128, 256) optimizer = t.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() if opt.model_path: model.load_state_dict(t.load(opt.model_path)) if opt.use_gpu: model.cuda() criterion.cuda() loss_meter = meter.AverageValueMeter() for epoch in range(opt.epoch): loss_meter.reset() for li, data_ in tqdm.tqdm(enumerate(dataloader)): #训练 data_ = data_.long().transpose(1, 0).contiguous() if opt.use_gpu: data_ = data_.cuda() optimizer.zero_grad() ##输入和目标错开 input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :]) output, _ = model(input_) loss = criterion(output, target.view(-1)) loss.backward() optimizer.step() loss_meter.add(loss.data[0]) # 可视化 if (1 + ii) % opt.plot_every == 0: if os.path.exists(opt.debug_file): ipdb.set_trace() vis.plot('loss', loss_meter.value()[0]) #诗歌原文 poetrys = [[ix2word[_word] for _word in data_[:, -iii]] for _iii in range(data_.size(1))][:16] vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]), win=u'origin_poem') gen_poetries = [] #分别以这几个字作为诗歌的第一个字,生成8首诗 for word in list(u'春江花月夜凉如水'): gen_poetry = ''.join( generate(model, word, ix2word, word2ix)) gen_poetries.append(gen_poetry) vis.text('</br>'.join( [''.join(poetry) for poetry in gen_poetries]), win=u'gen_poem') t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs): for k,v in kwargs.items(): setattr(opt,k,v) vis = Visualizer(env=opt.env) # 获取数据 data,word2ix,ix2word = get_data(opt) data = t.from_numpy(data)#把数据类型转为tensor dataloader = t.utils.data.DataLoader(data,#初始化Dataloader类实例 batch_size=opt.batch_size, shuffle=True, num_workers=1) # 模型定义 model = PoetryModel(len(word2ix), 128, 256)#(vocab_size, embedding_dim, hidden_dim) optimizer = t.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss()#损失函数定义为交叉熵 if opt.model_path: model.load_state_dict(t.load(opt.model_path)) if opt.use_gpu: model.cuda() criterion.cuda() loss_meter = meter.AverageValueMeter() for epoch in range(opt.epoch): loss_meter.reset() for ii,data_ in tqdm.tqdm(enumerate(dataloader)): #tqdm进度条工具 #取一个batch的数据 # 训练 #data_.size:(batch_size,maxlen) data_ = data_.long().transpose(1,0).contiguous()#转置后返回一个内存连续的有相同数据的tensor # if epoch==0 and ii ==0: # print('size of data_ after transpose: \n',data_.size()) if opt.use_gpu: data_ = data_.cuda() optimizer.zero_grad()#梯度清零 input_,target = Variable(data_[:-1,:]),Variable(data_[1:,:])#input_是所有句子的前maxlen-1个item的集合, #target是所有句子的后maxlen-1个item的集合 #以"床前明月光"为例,输入是"床前明月",要预测"前明月光" output,_ = model(input_) #Tensor.view(-1)按照第0个维度逐个元素读取将张量展开成数组 loss = criterion(output,target.view(-1)) loss.backward() optimizer.step() loss_meter.add(loss.data[0]) # 可视化 if (1+ii)%opt.plot_every==0: if os.path.exists(opt.debug_file):#如果存在调试文件, #则进入调试模式 ipdb.set_trace() vis.plot('loss',loss_meter.value()[0]) # 诗歌原文 poetrys=[ [ix2word[_word] for _word in data_[:,_iii]] #每一个句子(诗歌)的每一个item(id)要转换成文本 for _iii in range(data_.size(1))][:16]#_iii的取值范围[,127] vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]),win=u'origin_poem') #在visdom中输出这些句子(诗歌)中的前16个 gen_poetries = [] # 分别以这几个字作为诗歌的第一个字,生成8首诗 for word in list(u'春江花月夜凉如水'): gen_poetry = ''.join(generate(model,word,ix2word,word2ix)) gen_poetries.append(gen_poetry) vis.text('</br>'.join([''.join(poetry) for poetry in gen_poetries]),win=u'gen_poem') t.save(model.state_dict(),'%s_%s.pth' %(opt.model_prefix,epoch))
def train_torch_lstm(conf, args=None): pdata = PoemData() pdata.read_data(conf) pdata.get_vocab() if conf.use_gpu: device = torch.device('cuda') else: device = torch.device('cpu') model = PoetryModel(pdata.vocab_size, conf, device) train_data = pdata.train_data test_data = pdata.test_data train_data = torch.from_numpy(np.array(train_data['pad_words'])) dev_data = torch.from_numpy(np.array(test_data['pad_words'])) dataloader = DataLoader(train_data, batch_size=conf.batch_size, shuffle=True, num_workers=conf.num_workers) devloader = DataLoader(dev_data, batch_size=conf.batch_size, shuffle=True, num_workers=conf.num_workers) optimizer = Adam(model.parameters(), lr=conf.learning_rate) criterion = nn.CrossEntropyLoss() loss_meter = meter.AverageValueMeter() if conf.load_best_model: model.load_state_dict(torch.load(conf.beat_model_path)) if conf.use_gpu: model.cuda() criterion.cuda() step = 0 bestppl = 1e9 early_stop_controller = 0 for epoch in range(conf.n_epochs): losses = [] loss_meter.reset() model.train() for i, data in enumerate(dataloader): data = data.long().transpose(1, 0).contiguous() if conf.use_gpu: data = data.cuda() input, target = data[:-1, :], data[1:, :] optimizer.zero_grad() output, _ = model(input) loss = criterion(output, target.contiguous().view(-1)) loss.backward() optimizer.step() losses.append(loss.item()) loss_meter.add(loss.item()) step += 1 if step % 100 == 0: print("epoch_%d_step_%d_loss:%0.4f" % (epoch + 1, step, loss.item())) train_loss = float(loss_meter.value()[0]) model.eval() for i, data in enumerate(devloader): data = data.long().transpose(1, 0).contiguous() if conf.use_gpu: data = data.cuda() input, target = data[:-1, :], data[1:, :] output, _ = model(input) loss = criterion(output, target.view(-1)) loss_meter.add(loss.item()) ppl = math.exp(loss_meter.value()[0]) print("epoch_%d_loss:%0.4f , ppl:%0.4f" % (epoch + 1, train_loss, ppl)) if epoch % conf.save_every == 0: torch.save(model.state_dict(), "{0}_{1}".format(conf.model_prefix, epoch)) fout = open("{0}out_{1}".format(conf.out_path, epoch), 'w', encoding='utf-8') for word in list('日红山夜湖海月'): gen_poetry = generate_poet(model, word, pdata.vocab, conf) fout.write("".join(gen_poetry) + '\n\n') fout.close() if ppl < bestppl: bestppl = ppl early_stop_controller = 0 torch.save(model.state_dict(), "{0}_{1}".format(conf.best_model_path, "best_model")) else: early_stop_controller += 1 if early_stop_controller > conf.patience: print("early stop.") break