def train(epoch, net, train_dt, opt, best):
net.train()
data = shuffle_data(train_dt, 1)
total_loss = 0.0
for num, i in enumerate(range(0, len(data), opts["batch"])):
one = data[i:i + opts["batch"]]
query, _ = padding([x[0] for x in one], max_len=50)
passage, _ = padding([x[1] for x in one], max_len=350)
answer = pad_answer([x[2] for x in one])
query, passage, answer = torch.LongTensor(query), torch.LongTensor(
passage), torch.LongTensor(answer)
if args.cuda:
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
opt.zero_grad()
loss = net([query, passage, answer, True])
loss.backward()
total_loss += loss.item()
opt.step()
if (num + 1) % opts["log_interval"] == 0:
print(
'|------epoch {:d} train error is {:f} eclipse {:.2f}% best {}------|'
.format(epoch, total_loss / opts["log_interval"],
i * 100.0 / len(data), best))
total_loss = 0
def inference():
model.eval()
predictions = []
id_prediction = {}
with torch.no_grad():
for i in range(0, len(data), args.batch_size):
print("{} in {}".format(i, len(data)))
one = data[i:i + args.batch_size]
query, _ = padding([x[0] for x in one], max_len=50)
passage, _ = padding([x[1] for x in one], max_len=300)
answer = pad_answer([x[2] for x in one])
str_words = [x[-1] for x in one]
ids = [x[3] for x in one]
answer = pad_wrong_answer(answer)
query = torch.LongTensor(query)
passage = torch.LongTensor(passage)
#print(np.shape(answer))
answer = torch.LongTensor(answer)
if args.cuda:
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
output = model([query, passage, answer, False])
for q_id, prediction, candidates in zip(ids, output, str_words):
id_prediction[q_id] = int(prediction)
prediction_answer = u''.join(candidates[prediction])
predictions.append(str(q_id) + '\t' + prediction_answer)
outputs = u'\n'.join(predictions)
with codecs.open(args.output, 'w', encoding='utf-8') as f:
f.write(outputs)
with open("pkl_records/dev11.pkl", "wb") as f:
pickle.dump(id_prediction, f)
print('done!')
def train(epoch, net, train_dt, opt, best, best_epoch):
net.train()
data = shuffle_data(train_dt, 1)
total_loss = 0.0
time_sum = 0.0
for num, i in enumerate(range(0, len(data), opts["batch"])):
time_start = time.time()
one = data[i:i + opts["batch"]]
query, _ = padding([x[0] for x in one], max_len=opts["q_len"])
passage, _ = padding([x[1] for x in one], max_len=opts["p_len"])
answer = pad_answer([x[2] for x in one])
ids = [x[3] for x in one]
query, passage, answer, ids = torch.LongTensor(
query), torch.LongTensor(passage), torch.LongTensor(answer), ids
if torch.cuda.is_available():
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
opt.zero_grad()
loss = net([query, passage, answer, ids, True, True])
loss.backward()
total_loss += loss.item()
opt.step()
# 计时
time_end = time.time()
cost = (time_end - time_start)
time_sum += cost
if (num + 1) % opts["log_interval"] == 0:
ts = str('%.2f' % time_sum)
print(
'|---epoch {:d} train error is {:f} eclipse {:.2f}% costing: {} best {} on epoch {}---|'
.format(epoch, total_loss / opts["log_interval"],
i * 100.0 / len(data), ts + " s", best, best_epoch))
time_sum = 0.0
total_loss = 0
def test(pred, session, ans, que, para):
r, a = 0.0, 0.0
id_list = [] # 用于记录ensemble所需的数据
pred_list = []
for i in range(0, len(dev_data), 10):
one = dev_data[i:i + 10]
query, _ = padding([x[0] for x in one], max_len=350)
passage, _ = padding([x[1] for x in one], max_len=50)
answer = pad_answer([x[2] for x in one], max_len=70)
ids = [int(c[3]) for c in one]
# query, passage, answer = np.array(query), np.array(passage), np.array(answer)
fd = {para: passage, que: query, ans: answer}
p = session.run(pred, feed_dict=fd)
# 储存q_id与预测答案下标
p = list(p)
ids = list(ids)
id_list.extend(ids)
pred_list.extend(p)
r = 0
for item in p:
if np.argmax(item) == 0:
r += 1
a += len(one)
return r * 100.0 / a, id_list, pred_list
def inference(pred,query,para,ans,sess,data,store_path="esm_record/test.pkl"):
# model.eval()
predictions = []
exception=[]
id_list = [] # 用于记录ensemble所需的数据
pred_list = []
# with torch.no_grad():
for i in range(0, len(data), args.batch_size):
one = data[i:i + args.batch_size]
q, _ = padding([x[0] for x in one], max_len=50)
p, _ = padding([x[1] for x in one], max_len=350)
a = pad_answer([x[2] for x in one],max_length=70)
str_words = [x[-1] for x in one]
ids = [x[3] for x in one]
if not len(np.shape(a)) == 3:
# print(i)
a = pad_wrong_answer(a)
# query, passage, answer = torch.LongTensor(query), torch.LongTensor(passage), torch.LongTensor(answer)
# if args.cuda:
# query = query.cuda()
# passage = passage.cuda()
# answer = answer.cuda()
output =np.argmax(sess.run(pred,feed_dict={
query:q,
para:p,
ans:a
}),axis=1)
# id_list = id_list.extend(ids)
# pred_list = pred_list.extend(output)
output = list(output)
ids = list(ids)
id_list.extend(ids)
pred_list.extend(output)
for q_id, prediction, candidates in zip(ids, output, str_words):
print(q_id)
print(prediction)
#FIXME
l=len(candidates)
fir=candidates[0]
if l<3:
for _ in range(3-l):
candidates.append(fir)
print(candidates)
prediction_answer = u''.join(candidates[prediction])
predictions.append(str(q_id) + '\t' + prediction_answer)
outputs = u'\n'.join(predictions)
with codecs.open(args.output, 'w',encoding='utf-8') as f:
f.write(outputs)
print ('done!')
esm_record(id_list=id_list, pred_list=pred_list, path=store_path)
def pad_wrong_answer(answer_list):
# 3680
# 7136
# 这两批数据中有alternative answer长度小于3的数据，需要补齐否则无法处理
# 该方法通过复制ans[0]补齐数据
padded_list=[]
for ans in answer_list:
ll=len(ans)
if not ll == 3:
for _ in range(3-ll):
ans+=[ans[0]]
padded_list.append(ans)
padded_list=pad_answer(padded_list,70)
return padded_list
def inference(model, data, md_name, dat_name, opts, is_argmax=True):
pkl_path = "pkl_records/" + md_name + "." + dat_name + ".pkl"
if not os.path.exists(pkl_path):
model.eval()
predictions = []
id_prediction = {}
with torch.no_grad():
for i in range(0, len(data), opts["batch"]):
# print("{} in {}".format(i, len(data)))
one = data[i:i + opts["batch"]]
query, _ = padding([x[0] for x in one], max_len=50)
passage, _ = padding([x[1] for x in one], max_len=300)
answer = pad_answer([x[2] for x in one])
str_words = [x[-1] for x in one]
ids = [x[3] for x in one]
answer = pad_wrong_answer(answer)
query = torch.LongTensor(query)
passage = torch.LongTensor(passage)
# print(np.shape(answer))
answer = torch.LongTensor(answer)
if torch.cuda.is_available():
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
output = model([query, passage, answer, ids, False, is_argmax])
for q_id, prediction, candidates in zip(
ids, output, str_words):
if is_argmax:
id_prediction[q_id] = int(prediction)
else:
prediction = prediction.cpu().numpy()
id_prediction[q_id] = list(prediction)
prediction_answer = u''.join(
candidates[np.argmax(prediction)])
predictions.append(str(q_id) + '\t' + prediction_answer)
outputs = u'\n'.join(predictions)
print("score: {}".format(score_on_dt(id_prediction)))
with codecs.open("submit/" + md_name + "." + dat_name + ".txt",
'w',
encoding='utf-8') as f:
f.write(outputs)
with open("pkl_records/" + md_name + "." + dat_name + ".pkl",
"wb") as f: # TODO: 更换pkl文件名称
pickle.dump(id_prediction, f)
print('done!')
else:
pkkl = pickle.load(open(pkl_path, "rb"))
print(pkl_path + " exist, score:", score_on_dt(pkkl))
def test(net, valid_data):
net.eval()
r, a = 0.0, 0.0
with torch.no_grad():
for i in range(0, len(valid_data), opts["batch"]):
print("{} in {}".format(i, len(valid_data)))
one = valid_data[i:i + opts["batch"]]
query, _ = padding([x[0] for x in one], max_len=50)
passage, _ = padding([x[1] for x in one], max_len=500)
answer = pad_answer([x[2] for x in one])
query, passage, answer = torch.LongTensor(query), torch.LongTensor(
passage), torch.LongTensor(answer)
if args.cuda:
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
output = net([query, passage, answer, False])
r += torch.eq(output, 0).sum().item()
a += len(one)
return r * 100.0 / a
def test(net, valid_data):
net.eval()
r, a = 0.0, 0.0
with torch.no_grad():
for i in range(0, len(valid_data), opts["batch"]):
print("{} in {}".format(i, len(valid_data)))
one = valid_data[i:i + opts["batch"]]
query, _ = padding([x[0] for x in one], max_len=opts["q_len"])
passage, _ = padding([x[1] for x in one], max_len=opts["p_len"])
answer = pad_answer([x[2] for x in one], max_len=opts["alt_len"])
ids = [x[3] for x in one]
query, passage, answer, ids = torch.LongTensor(
query), torch.LongTensor(passage), torch.LongTensor(
answer), ids
if torch.cuda.is_available():
query = query.cuda()
passage = passage.cuda()
answer = answer.cuda()
output = net([query, passage, answer, ids, False, True])
r += torch.eq(output, 0).sum().item()
a += len(one)
return r * 100.0 / a