def run(args, local_rank):
""" Distributed Synchronous """
torch.manual_seed(1234)
vocab = Vocab(args.vocab, min_occur_cnt=args.min_occur_cnt, specials=[])
if (args.world_size == 1 or dist.get_rank() == 0):
print ("vocab.size = %d"%vocab.size, flush=True)
model = BIGLM(local_rank, vocab, args.embed_dim, args.ff_embed_dim,\
args.num_heads, args.dropout, args.layers, args.smoothing, args.approx)
if args.start_from is not None:
ckpt = torch.load(args.start_from, map_location='cpu')
model.load_state_dict(ckpt['model'])
model = model.cuda(local_rank)
if args.world_size > 1:
torch.manual_seed(1234 + dist.get_rank())
random.seed(5678 + dist.get_rank())
optimizer = Optim(model.embed_dim, args.lr, args.warmup_steps, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.998), eps=1e-9))
if args.start_from is not None:
optimizer.load_state_dict(ckpt['optimizer'])
#train_data = DataLoader(vocab, args.train_data+"0"+str(local_rank), args.batch_size, args.max_len, args.min_len)
train_data = DataLoader(vocab, args.train_data, args.batch_size, args.max_len, args.min_len)
batch_acm = 0
acc_acm, nll_acm, ppl_acm, ntokens_acm, nxs, npairs_acm, loss_acm = 0., 0., 0., 0., 0., 0., 0.
while True:
model.train()
for truth, inp, msk in train_data:
batch_acm += 1
truth = truth.cuda(local_rank)
inp = inp.cuda(local_rank)
msk = msk.cuda(local_rank)
model.zero_grad()
res, loss, acc, nll, ppl, ntokens, npairs = model(truth, inp, msk)
loss_acm += loss.item()
acc_acm += acc
nll_acm += nll
ppl_acm += ppl
ntokens_acm += ntokens
npairs_acm += npairs
nxs += npairs
loss.backward()
if args.world_size > 1:
average_gradients(model)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
if (args.world_size==1 or dist.get_rank() ==0) and batch_acm%args.print_every == -1%args.print_every:
print ('batch_acm %d, loss %.3f, acc %.3f, nll %.3f, ppl %.3f, x_acm %d, lr %.6f'\
%(batch_acm, loss_acm/args.print_every, acc_acm/ntokens_acm, \
nll_acm/nxs, ppl_acm/nxs, npairs_acm, optimizer._rate), flush=True)
acc_acm, nll_acm, ppl_acm, ntokens_acm, loss_acm, nxs = 0., 0., 0., 0., 0., 0.
if (args.world_size==1 or dist.get_rank() ==0) and batch_acm%args.save_every == -1%args.save_every:
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
torch.save({'args':args, 'model':model.state_dict(), 'optimizer':optimizer.state_dict()}, '%s/epoch%d_batch_%d'%(args.save_dir, train_data.epoch_id, batch_acm))
max_len = max([len(single) for single in tokens]) # 最大的句子长度
for j in range(len(tokens)):
padding = [0] * (max_len - len(tokens[j]))
tokens[j] += padding
segments[j] += padding
input_masks[j] += padding
tokens_tensor = torch.tensor(tokens).to(device)
segments_tensors = torch.tensor(segments).to(device)
input_masks_tensors = torch.tensor(input_masks).to(device)
#print(tokens_tensor[0])
#print(segments_tensors.size(0),input_masks_tensors.size())
model.zero_grad()
loss, acc1, acc2 = model(tokens_tensor, segments_tensors,
input_masks_tensors)
loss_acm += loss.item()
acc1_acm += acc1
acc2_acm += acc2
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
if batch_acm % print_every == 0:
print('epoch %d, batch_acm %d, loss %.3f, easy_acc %.3f, hard_acc %.3f'\
%(train_data.epoch_id, batch_acm, loss_acm/print_every, acc2_acm/print_every, acc1_acm/print_every), flush=True)
if acc1_acm / print_every > stoa_acc:
stoa_acc = acc1_acm / print_every
torch.save({'model':model.state_dict(), 'optimizer':optimizer.state_dict()},\
'%s/epoch%d_batch_%dacc_%.3f'%(data_path, train_data.epoch_id, batch_acm, stoa_acc))
loss_acm, acc1_acm, acc2_acm = 0., 0., 0.
batch_acm = 0
acc_acm, loss_acm = 0., 0.
while True:
residual_model.train()
for inp, target in train_data:
# inp=(batch_size, length_number*768), target=(batch_size, 768)
batch_acm += 1
inp = inp.to(device)
target = target.to(device)
residual_model.zero_grad()
res, loss, acc = residual_model(inp, target)
loss_acm += loss.item()
acc_acm += acc
loss.backward()
torch.nn.utils.clip_grad_norm_(residual_model.parameters(), 1.0)
optimizer.step()
if batch_acm % save_every == 0:
torch.save({'model':residual_model.state_dict(), 'optimizer':optimizer.state_dict()},\
'%s/epoch%d_batch_%dacc_%.3f'%(model_path, train_data.epoch, batch_acm, acc_acm/print_every))
if batch_acm % print_every == 0:
print('epoch %d, batch_acm %d, loss %.3f, acc %.3f'\
%(train_data.epoch, batch_acm, loss_acm/print_every, acc_acm/print_every), flush=True)
print('lr: ', optimizer._rate)
loss_acm, acc_acm = 0., 0.
def run(args, local_rank):
""" Distributed Synchronous """
torch.manual_seed(1234)
vocab = Vocab(args.vocab, min_occur_cnt=args.min_occur_cnt, specials=[])
if (args.world_size == 1 or dist.get_rank() == 0):
print("vocab.size = " + str(vocab.size), flush=True)
model = BIGLM(local_rank, vocab, args.embed_dim, args.ff_embed_dim,
args.num_heads, args.dropout, args.layers, args.smoothing)
if args.start_from is not None:
ckpt = torch.load(args.start_from, map_location='cpu')
model.load_state_dict(ckpt['model'])
model = model.cuda(local_rank)
optimizer = Optim(
model.embed_dim, args.lr, args.warmup_steps,
torch.optim.Adam(model.parameters(),
lr=0,
betas=(0.9, 0.998),
eps=1e-9))
if args.start_from is not None:
optimizer.load_state_dict(ckpt['optimizer'])
train_data = DataLoader(vocab, args.train_data, args.batch_size,
args.max_len, args.min_len)
batch_acm = 0
acc_acm, nll_acm, ppl_acm, ntokens_acm, nxs, npairs_acm, loss_acm = 0., 0., 0., 0., 0., 0., 0.
while True:
model.train()
if train_data.epoch_id > args.max_epoch:
break
for xs_tpl, xs_seg, xs_pos, ys_truth, ys_inp, ys_tpl, ys_seg, ys_pos, msk in train_data:
batch_acm += 1
xs_tpl = xs_tpl.cuda(local_rank)
xs_seg = xs_seg.cuda(local_rank)
xs_pos = xs_pos.cuda(local_rank)
ys_truth = ys_truth.cuda(local_rank)
ys_inp = ys_inp.cuda(local_rank)
ys_tpl = ys_tpl.cuda(local_rank)
ys_seg = ys_seg.cuda(local_rank)
ys_pos = ys_pos.cuda(local_rank)
msk = msk.cuda(local_rank)
model.zero_grad()
res, loss, acc, nll, ppl, ntokens, npairs = model(
xs_tpl, xs_seg, xs_pos, ys_truth, ys_inp, ys_tpl, ys_seg,
ys_pos, msk)
# http://www.myzaker.com/article/5f3747a28e9f096c723a65e0/ 资料
# 常用的文本生成评测指标 PPL、Distinct 外，
# 本文还专门设计了衡量格式（Format）准确率、韵律（Rhyme）准确率和句子完整性（integrity）的指标。
# 格式（Format）准确率: Precision p、Recall r 和 F1 得分-> Macro-F1 和 Micro-F1
# 完整性有个奇怪的log值
# 传统的BLEU和ROUGE, 再songnet中完全用不到, 创作要求多样性
loss_acm += loss.item() # 损失
acc_acm += acc # 精确度
nll_acm += nll #
ppl_acm += ppl # -log 和, 其实就是句子出现的概率, 越小, 困惑度越高
# 新指标, 困惑度perplexity, 比较两者再预测样本上的优劣, 困惑都越低越好??, 咋定义的
ntokens_acm += ntokens # 字符数
npairs_acm += npairs # 句子?
nxs += npairs
# 为什么啊, 感觉好难啊gpt2
loss.backward()
if args.world_size > 1:
is_normal = average_gradients(model)
else:
is_normal = True
if is_normal:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
else:
print("gradient: none, gpu: " + str(local_rank), flush=True)
continue
if (args.world_size == 1 or dist.get_rank() == 0
) and batch_acm % args.print_every == -1 % args.print_every:
today = datetime.datetime.now()
print(today)
print(
'batch_acm %d, loss %.3f, acc %.3f, nll %.3f, ppl %.3f, x_acm %d, lr %.6f'
% (batch_acm, loss_acm / args.print_every,
acc_acm / ntokens_acm, nll_acm / nxs, ppl_acm / nxs,
npairs_acm, optimizer._rate),
flush=True)
acc_acm, nll_acm, ppl_acm, ntokens_acm, loss_acm, nxs = 0., 0., 0., 0., 0., 0.
if (args.world_size == 1 or dist.get_rank() == 0
) and batch_acm % args.save_every == -1 % args.save_every:
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
model.eval()
eval_epoch(
args, model, vocab, local_rank, "epoch-" +
str(train_data.epoch_id) + "-acm-" + str(batch_acm))
model.train()
torch.save(
{
'args': args,
'model': model.state_dict(),
'optimizer': optimizer.state_dict()
}, '%s/epoch%d_batch_%d' %
(args.save_dir, train_data.epoch_id, batch_acm))
def run(args, local_rank):
""" Distributed Synchronous """
torch.manual_seed(1234)
vocab = Vocab(args.vocab, min_occur_cnt=args.min_occur_cnt, specials=[])
if (args.world_size == 1 or dist.get_rank() == 0):
print("vocab.size = " + str(vocab.size), flush=True)
model = BIGLM(local_rank, vocab, args.embed_dim, args.ff_embed_dim,\
args.num_heads, args.dropout, args.layers, args.smoothing)
if args.start_from is not None:
ckpt = torch.load(args.start_from, map_location='cpu')
model.load_state_dict(ckpt['model'])
model = model.cuda(local_rank)
optimizer = Optim(
model.embed_dim, args.lr, args.warmup_steps,
torch.optim.Adam(model.parameters(),
lr=0,
betas=(0.9, 0.998),
eps=1e-9))
if args.start_from is not None:
optimizer.load_state_dict(ckpt['optimizer'])
train_data = DataLoader(vocab, args.train_data, args.batch_size,
args.max_len, args.min_len)
batch_acm = 0
acc_acm, nll_acm, ppl_acm, ntokens_acm, nxs, npairs_acm, loss_acm = 0., 0., 0., 0., 0., 0., 0.
while True:
model.train()
if train_data.epoch_id > 30:
break
for xs_tpl, xs_seg, xs_pos, ys_truth, ys_inp, ys_tpl, ys_seg, ys_pos, msk in train_data:
batch_acm += 1
xs_tpl = xs_tpl.cuda(local_rank)
xs_seg = xs_seg.cuda(local_rank)
xs_pos = xs_pos.cuda(local_rank)
ys_truth = ys_truth.cuda(local_rank)
ys_inp = ys_inp.cuda(local_rank)
ys_tpl = ys_tpl.cuda(local_rank)
ys_seg = ys_seg.cuda(local_rank)
ys_pos = ys_pos.cuda(local_rank)
msk = msk.cuda(local_rank)
model.zero_grad()
res, loss, acc, nll, ppl, ntokens, npairs = model(
xs_tpl, xs_seg, xs_pos, ys_truth, ys_inp, ys_tpl, ys_seg,
ys_pos, msk)
loss_acm += loss.item()
acc_acm += acc
nll_acm += nll
ppl_acm += ppl
ntokens_acm += ntokens
npairs_acm += npairs
nxs += npairs
loss.backward()
if args.world_size > 1:
is_normal = average_gradients(model)
else:
is_normal = True
if is_normal:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
else:
print("gradient: none, gpu: " + str(local_rank), flush=True)
continue
if (args.world_size == 1 or dist.get_rank() == 0
) and batch_acm % args.print_every == -1 % args.print_every:
print ('batch_acm %d, loss %.3f, acc %.3f, nll %.3f, ppl %.3f, x_acm %d, lr %.6f'\
%(batch_acm, loss_acm/args.print_every, acc_acm/ntokens_acm, \
nll_acm/nxs, ppl_acm/nxs, npairs_acm, optimizer._rate), flush=True)
acc_acm, nll_acm, ppl_acm, ntokens_acm, loss_acm, nxs = 0., 0., 0., 0., 0., 0.
if (args.world_size == 1 or dist.get_rank() == 0
) and batch_acm % args.save_every == -1 % args.save_every:
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
model.eval()
eval_epoch(
args, model, vocab, local_rank, "epoch-" +
str(train_data.epoch_id) + "-acm-" + str(batch_acm))
model.train()
torch.save(
{
'args': args,
'model': model.state_dict(),
'optimizer': optimizer.state_dict()
}, '%s/epoch%d_batch_%d' %
(args.save_dir, train_data.epoch_id, batch_acm))