def Evaluate(data, model, opt):
"""
data : dev_examples
"""
save_dir = opt["save_dir"]
dev_preds = {}
dev_na_probs = {}
batch_size = 32
flag = True
for batch_data in get_batch_data(data, batch_size):
context_tokens = batch_data["context_tokens"]
data = model.get_data(batch_data)
ids = data["ids"]
starts, ends, answer_prob = model.prediction(data, flag)
flag = False
answer_context = []
# print(starts)
# print(ends)
# print(context_tokens)
for idx in range(len(starts)):
start = starts[idx]
end = ends[idx]
context_token = context_tokens[idx]
if (start == 0 and end == 0) or start > end:
answer_context.append("")
else:
answer_context.append(" ".join(context_token[start:end + 1]))
for idx in range(len(ids)):
id = ids[idx]
answer = answer_context[idx]
no_prob = 1.0 - answer_prob[idx].item()
dev_preds[id] = answer
dev_na_probs[id] = no_prob
with open(save_dir + "dev_preds", "w", encoding="utf-8") as f:
json.dump(dev_preds, f)
dev_file_path = "/home/FuDawei/NLP/SQUAD/data/dev-v2.0.json"
ignore_ids = []
with open(save_dir + "devdrop_in_process.json", 'r') as f:
dev_drop_process = json.load(f)
ignore_ids.extend(dev_drop_process["id"])
with open(save_dir + "dev_drop.json", 'r') as f:
dev_drop = json.load(f)
ignore_ids.extend(dev_drop["id"])
out_eval = evaluate(dev_file_path,
dev_preds,
na_prob_thresh=0.3,
ignore_ids=ignore_ids)
exact, f1 = out_eval["exact"], out_eval["f1"]
return exact, f1
def alg(game_board):
"""
The algorithm picks a random but possible move
:param map:
:return:
"""
try:
move = random.choice([
commands[x] for x in commands.keys()
if evaluate.evaluate(game_board, int(x)) >= 0
])
except Exception:
move = "a"
return move
def get_f1(model, mode):
pred_all, pred, recall_all, recall = 0, 0, 0, 0
f_pred_all, f_pred, f_recall_all, f_recall = 0, 0, 0, 0
gold_cross_num = 0
pred_cross_num = 0
if mode == "dev":
batch_zip = zip(dev_token_batches, dev_char_batch, dev_char_len_batch,
dev_pos_batches, dev_label_batches)
elif mode == "test":
batch_zip = zip(test_token_batches, test_char_batch,
test_char_len_batch, test_pos_batches,
test_label_batches)
else:
raise ValueError
for token_batch, char_batch, char_len_batch, pos_batch, label_batch in batch_zip:
token_batch_var = Variable(torch.LongTensor(np.array(token_batch)))
pos_batch_var = Variable(torch.LongTensor(np.array(pos_batch)))
if config.if_gpu:
token_batch_var = token_batch_var.cuda()
pos_batch_var = pos_batch_var.cuda()
model.eval()
pred_entities = model.predict(token_batch_var, pos_batch_var)
p_a, p, r_a, r = evaluate(label_batch, pred_entities)
#gold_cross_num += sum(count_overlap(label_batch))
#pred_cross_num += sum(count_overlap(pred_entities))
gold_cross_num += 0
pred_cross_num += 0
pred_all += p_a
pred += p
recall_all += r_a
recall += r
print(pred_all, pred, recall_all, recall)
p = pred / pred_all if pred_all else 0
r = recall / recall_all if recall_all else 0
f1 = 2 * p * r / (p + r) if p + r else 0
# f1 = 2 / ((pred_all / pred) + (recall_all / recall))
print("Precision {0}, Recall {1}, F1 {2}".format(pred / pred_all,
recall / recall_all, f1))
# print("Prediction Crossing: ", pred_cross_num)
# print("Gold Crossing: ", gold_cross_num)
return f1
def test(dataloader, model, args):
# testing
times = []
errors = []
dir_testInfor = "testInfo"
if (not os.path.exists(dir_testInfor)):
os.makedirs(dir_testInfor)
path_testInfo = os.path.join(dir_testInfor,
"%s_%s.pkl" % (args.dataset, args.flag_model))
if (os.path.exists(path_testInfo)):
data = torch.load(path_testInfo)
times.extend(data['time'])
errors.extend(data['err'])
for i in range(len(times)):
print(
"test: %4d | time: %6.3f, d1: %6.3f, EPE: %6.3f, pixelerror: %6.3f"
% (i, times[i], errors[i][0], errors[i][1], errors[i][2]))
model.eval()
for batch_idx, (img1, img2, dispL_gt) in enumerate(dataloader):
if (batch_idx < len(errors)):
continue
stime = time.time()
dispL = model.estimate(img1, img2)
times.append(time.time() - stime)
error = evaluate(img1, img2, dispL, dispL_gt.numpy())
errors.append(error)
print(
"test: %4d | time: %6.3f, d1: %6.3f, EPE: %6.3f, pixelerror: %6.3f"
% (batch_idx, times[-1], error[0], error[1], error[2]))
if (args.flag_save):
dirpath_save = os.path.join(
dir_testInfor, "%s_%s" % (args.dataset, args.flag_model))
if (not os.path.exists(dirpath_save)):
os.makedirs(dirpath_save)
path_file = os.path.join(dirpath_save,
"{:06d}.png".format(batch_idx))
cv2.imwrite(path_file, dispL[0, 0])
# cv2.imwrite("estimate/{:06d}_gt.png".format(step), dispL_gt[0])
if (batch_idx % 100 == 0):
torch.save({"time": times, 'err': errors}, path_testInfo)
torch.save({"time": times, 'err': errors}, path_testInfo)
print np.mean(times), np.mean(errors, axis=0)
def alg(game_board):
"""
This algorithm attempts to only use left/bottom (a/s | 0/3 ) commands. If nto possible, right/down are also accepted.
:param game_board: game board array
:return: returns the move (char)
"""
if evaluate.evaluate(game_board, 0) == -1 and evaluate.evaluate(
game_board, 3) == -1:
# When the preferred steps are deadlocks
if evaluate.evaluate(game_board, 1) > evaluate.evaluate(game_board, 2):
return "w"
else:
return "d"
else:
if evaluate.evaluate(game_board, 0) > evaluate.evaluate(game_board, 3):
return "a"
else:
return "s"
def get_f1(model: BiRecurrentConvCRF4NestedNER,
mode: str,
file_path: str = None) -> float:
with torch.no_grad():
model.eval()
pred_all, pred, recall_all, recall = 0, 0, 0, 0
gold_cross_num = 0
pred_cross_num = 0
if mode == 'dev':
batch_zip = zip(dev_token_iv_batches, dev_token_ooev_batches,
dev_char_batches, dev_label_batches,
dev_mask_batches)
elif mode == 'test':
batch_zip = zip(test_token_iv_batches, test_token_ooev_batches,
test_char_batches, test_label_batches,
test_mask_batches)
else:
raise ValueError
f = None
if file_path is not None:
f = open(file_path, 'w')
for token_iv_batch, token_ooev_batch, char_batch, label_batch, mask_batch in batch_zip:
token_iv_batch_var = torch.LongTensor(np.array(token_iv_batch))
token_ooev_batch_var = torch.LongTensor(np.array(token_ooev_batch))
char_batch_var = torch.LongTensor(np.array(char_batch))
mask_batch_var = torch.ByteTensor(
np.array(mask_batch, dtype=np.uint8))
if config.if_gpu:
token_iv_batch_var = token_iv_batch_var.cuda()
token_ooev_batch_var = token_ooev_batch_var.cuda()
char_batch_var = char_batch_var.cuda()
mask_batch_var = mask_batch_var.cuda()
pred_sequence_entities = model.predict(token_iv_batch_var,
token_ooev_batch_var,
char_batch_var,
mask_batch_var)
pred_entities = unpack_prediction(model, pred_sequence_entities)
p_a, p, r_a, r = evaluate(label_batch, pred_entities)
gold_cross_num += 0
pred_cross_num += 0
pred_all += p_a
pred += p
recall_all += r_a
recall += r
if file_path is not None:
for token_iv, token_ooev, mask, label, preds \
in zip(token_iv_batch, token_ooev_batch, mask_batch, label_batch, pred_entities):
words = []
for t_iv, t_ooev, m in zip(token_iv, token_ooev, mask):
if not m:
break
if t_iv > 0:
words.append(voc_iv_dict.get_instance(t_iv))
else:
words.append(voc_ooev_dict.get_instance(t_ooev))
f.write(' '.join(words) + '\n')
labels = []
for l in sorted(label, key=lambda x: (x[0], x[1], x[2])):
labels.append("{},{} {}".format(
l[0], l[1], label_dict.get_instance(l[2])))
f.write('|'.join(labels) + '\n')
labels = []
for p in sorted(preds, key=lambda x: (x[0], x[1], x[2])):
labels.append("{},{} {}".format(
p[0], p[1], label_dict.get_instance(p[2])))
f.write('|'.join(labels) + '\n')
f.write('\n')
if file_path is not None:
f.close()
pred = pred / pred_all if pred_all > 0 else 1.
recall = recall / recall_all if recall_all > 0 else 1.
f1 = 2 / ((1. / pred) +
(1. / recall)) if pred > 0. and recall > 0. else 0.
logger.info(
"{} precision: {:.2f}%, recall: {:.2f}%, F1: {:.2f}%".format(
mode, pred * 100., recall * 100., f1 * 100.))
# logger.info("Prediction Crossing: ", pred_cross_num)
# logger.info("Gold Crossing: ", gold_cross_num)
return f1
t.set_postfix(loss=loss.item())
total_loss += loss.item()
loss.backward()
optimizer.step()
model.zero_grad()
t.update()
epoch_end = time.time()
epoch_cost = epoch_end - epoch_start
print(
"Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s"
% (idx, epoch_cost, train_num / epoch_cost, total_loss))
# evaluate
if idx >= 0:
BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4 = evaluate(
data, model, data.result_dir + "/output_" + str(idx))
if BLEU4 > best_BLEU4:
best_epoch = idx
time_above = 0
print(
"best BLEU4 score\n BLEU1 %f, BLEU2 %f BLEU3 %f BLEU4 %f, epoch %d"
% (BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4,
best_epoch),
file=open(data.result_dir + "/BLEU_" + str(idx), "w"))
model_name = data.result_dir + "/modelFinal_{}.model".format(
BLEU4)
torch.save(model.state_dict(), model_name)
best_BLEU4 = BLEU4
else:
time_above = time_above + 1
gc.collect()
y_train = t.from_numpy(y_train).long()
inputs = Variable(x_train)
targes = Variable(y_train)
self.num_epochs = num_epochs
for epoch in range(self.num_epochs):
loss = self.optimize(inputs, targes)
if (epoch + 1) % 20 == 0:
print('Epoch[{}/{}],loss:{:.6f}'.format(
epoch + 1, num_epochs, loss.data.numpy()))
t.save(self.model.state_dict(), './torchsave.pt')
def predict(self, x_test):
self.model.load_state_dict(t.load('./torchsave.pt'))
if isinstance(x_test, np.ndarray):
x_test = t.from_numpy(x_test).float()
predict_y = self.forward(x_test).data.numpy()
return [np.argmax(one_hot) for one_hot in predict_y]
if __name__ == '__main__':
loads = data_loads.Data_Load()
batch_xs, batch_ys = loads.chooceall()
dnn = TorchDemo()
dnn.train(batch_xs, batch_ys)
predict_y = dnn.predict(batch_xs)
evaluate.evaluate(batch_ys, predict_y)
batch_xs, batch_ys = loads.all()
predict_y = dnn.predict(batch_xs)
evaluate.evaluate(batch_ys, predict_y)
def train(dataloader, dataloader_val, model, args):
if (model.updated >= args.updates):
print("Training Finished!")
return
#------------------------------------------prepare----------------------------------------------
mlosses = []
merrors = []
path_loss = os.path.join(model.dirpath, 'loss.pkl')
if (os.path.exists(path_loss)):
data = torch.load(path_loss)
mlosses.extend(data['loss'])
merrors.extend(data['error'])
count = len(dataloader)
epoches = (args.updates + count / 2) // count
tepoch = (model.updated) // count
time_start_full = time.time()
flag_val = False
plt.figure()
for epoch in range(tepoch + 1, epoches + 1):
msg = "This is %d-th epoch, updated: %6d, lr: %f " % (
epoch, model.updated, model.optim.param_groups[0]['lr'])
logging.info(msg)
#----------------------------------Training--------------------------------------------
model.train()
loss_train_total = 0
for batch_idx, batch_stereo in enumerate(dataloader):
time_start = time.time()
try:
if (args.loss_name == "supervised"):
loss = model.update_supervised(*batch_stereo[:3])
else:
loss = model.update_selfsupervised(*batch_stereo[:2])
except Exception as err:
logging.error(err)
logging.error(traceback.format_exc())
continue
if (batch_idx % 20 == 0):
msg = "Iter: %d, training loss: %.3f, time: %.2f" % (
batch_idx, loss, time.time() - time_start)
logging.info(msg)
loss_train_total += loss
if (model.updated % 1000 == 1): flag_val = True
if (model.updated % 100000 == 0):
model.save_weight(flag_postfix=True)
loss_mean = loss_train_total / len(dataloader)
mlosses.append([epoch, loss_mean])
model.save_weight(flag_postfix=False)
time_full_h = (time.time() - time_start_full) / 3600
msg = "**Train phase** Total mean training loss: %.3f, full training time: %.2f HR " % (
loss_mean, time_full_h)
logging.info(msg)
#----------------------------------validate--------------------------------------------
if (flag_val):
flag_val = False
model_val = model
model_val.eval()
terrors = []
for batch_idx, (img1, img2, disp1) in enumerate(dataloader_val):
if (batch_idx > 50): break
# visual check
dispL = model_val.estimate(img1, img2)
error = evaluate(img1, img2, dispL, disp1.numpy())
terrors.append(error)
merror = np.mean(terrors, axis=0)
merrors.append([epoch, merror[0], merror[1], merror[2]])
#----------------------------------visual check--------------------------------------------
torch.save({'loss': mlosses, 'error': merrors}, path_loss)
msg = "**Validate phase** epoch: %d, d1: %.2f, EPE: %.2f, pixelerror: %.2f" % (
epoch, merror[0], merror[1], merror[2])
logging.info(msg)
mlosses_np = np.array(mlosses)
merrors_np = np.array(merrors)
msg = str(mlosses_np.shape) + str(merrors_np.shape)
logging.debug(msg)
plt.subplot(221)
plt.cla()
plt.plot(mlosses_np[:, 0], mlosses_np[:, 1])
plt.xlabel("epoch")
plt.ylabel("mean train loss")
plt.subplot(222)
plt.cla()
plt.plot(merrors_np[:, 0], merrors_np[:, 1])
plt.xlabel("epoch")
plt.ylabel("D1")
plt.subplot(223)
plt.cla()
plt.plot(merrors_np[:, 0], merrors_np[:, 2])
plt.xlabel("epoch")
plt.ylabel("EPE")
plt.subplot(224)
plt.cla()
plt.plot(merrors_np[:, 0], merrors_np[:, 3])
plt.xlabel("epoch")
plt.ylabel("RPE")
plt.savefig("check_%s_%s_%s_%s.png" %
(args.mode, args.dataset, args.net, args.loss_name))
print("Training Finish!")
def train(train_data, val_data, Path):
print("Configuring TensorBoard and Saver...")
# 配置 Tensorboard，重新训练时，请将tensorboard文件夹删除，不然图会覆盖
if not os.path.exists(Path.tensorboard_dir):
os.makedirs(Path.tensorboard_dir)
#结果可视化与存储
tf.summary.scalar("loss", model.loss) #可视化loss
tf.summary.scalar("accuracy", model.acc) #可视化acc
merged_summary = tf.summary.merge_all() #将所有操作合并输出
writer = tf.summary.FileWriter(Path.tensorboard_dir) #将summary data写入磁盘
# 配置 Saver
saver = tf.train.Saver()
if not os.path.exists(Path.save_dir):
os.makedirs(Path.save_dir)
print("Loading training and validation data...")
# 载入训练集与验证集
start_time = time.time()
# train_x1_word, train_x2_word, train_ks, train_x2_label, train_y = train_data
# val_x1_word, val_x2_word, val_ks, val_x2_label, val_y = val_data
train_x1_word, train_x2_word, train_ks, train_y = train_data
val_x1_word, val_x2_word, val_ks, val_y = val_data
print('train len', len(train_x1_word))
print('val_len', len(val_x1_word))
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
# 创建session
session = tf.Session()
session.run(tf.global_variables_initializer())
writer.add_graph(session.graph)
print('Training and evaluating...')
start_time = time.time()
total_batch = 0 # 总批次
best_acc_val = 0.0 # 最佳验证集准确率
last_improved = 0 # 记录上一次提升批次
require_improvement = 1000 # 如果超过1000轮未提升，提前结束训练
flag = False
for epoch in range(param.BaseConfig.num_epochs):
print('Epoch:', epoch + 1)
batch_train = get_batch_data(train_x1_word,
train_x2_word,
train_ks,
train_y,
batch_size=param.BaseConfig.batch_size)
for a_word_batch, b_word_batch, c_word_batch, y_batch in batch_train:
feed_dict = feed_data_fun(model, a_word_batch, b_word_batch,
c_word_batch, y_batch,
model.config.dropout_rate)
if total_batch % param.BaseConfig.save_per_batch == 0:
# 每多少轮次将训练结果写入tensorboard scalar
s = session.run(merged_summary, feed_dict=feed_dict)
writer.add_summary(s, total_batch)
if total_batch % param.BaseConfig.print_per_batch == 0:
# 每多少轮次输出在训练集和验证集上的性能
feed_dict[model.dropout_rate] = 1.0
loss_train, acc_train, pre_y, logit, true_y = session.run(
[
model.loss, model.acc, model.pred_y, model.logit,
model.y
],
feed_dict=feed_dict)
loss_val, acc_val = evaluate(
model, session, val_x1_word, val_x2_word, val_ks, val_y,
feed_data_fun) # 验证当前会话中的模型的loss和acc
if acc_val > best_acc_val:
# 保存最好结果
best_acc_val = acc_val
last_improved = total_batch
saver.save(sess=session, save_path=Path.save_path)
improved_str = '*'
else:
improved_str = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0}, Train Loss: {1}, Train Acc: {2},' \
+ ' Val Loss: {3}, Val Acc: {4}, Time: {5} {6}'
print(
msg.format(total_batch, loss_train, acc_train, loss_val,
acc_val, time_dif, improved_str))
session.run(model.optim, feed_dict=feed_dict) # 运行优化
total_batch += 1
if total_batch - last_improved > require_improvement:
# 验证集正确率长期不提升，提前结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break # 跳出循环
if flag: # 同上
break
def test(test_data, Path):
#载入随机森林模型
print("Loading test data...")
start_time = time.time()
test_x1_word, test_x2_word, test_ks, test_y = test_data
session = tf.Session()
session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess=session, save_path=Path.save_path) # 读取保存的模型
print('Testing...')
loss_test, acc_test = evaluate(model, session, test_x1_word, test_x2_word,
test_ks, test_y, feed_data_fun)
msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}'
print(msg.format(loss_test, acc_test))
batch_size = param.BaseConfig.batch_size
data_len = len(test_x1_word)
num_batch = int((data_len) / batch_size)
y_test_cls = np.argmax(test_y, 1)
y_pred_cls = np.zeros(shape=data_len, dtype=np.int32) # 保存预测结果
for i in range(num_batch): # 逐批次处理
start_id = i * batch_size
end_id = min((i + 1) * batch_size, data_len)
feed_dict = feed_data_fun(model, test_x1_word[start_id:end_id],
test_x2_word[start_id:end_id],
test_ks[start_id:end_id],
test_y[start_id:end_id], 1.0)
y_pred_cls[start_id:end_id] = session.run(
model.pred_y, feed_dict=feed_dict) #将所有批次的预测结果都存放在y_pred_cls中
# pool_1,pool_2,pool_3 = session.run([model.fusion_output_max_1,model.fusion_output_max_2,model.fusion_output_max_3],
# feed_dict=feed_dict)
print("Precision, Recall and F1-Score...")
print(metrics.classification_report(y_test_cls, y_pred_cls,
digits=4)) #直接计算准确率，召回率和f值
# 混淆矩阵
print("Confusion Matrix...")
cm = metrics.confusion_matrix(y_test_cls, y_pred_cls)
print(cm)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
# print("beta value", beta1,beta2,beta3)
#check error prediction
# print(y_pred_cls)
# print('check.......')
# print('maxpooling.....')
# print('pool 1...')
# print(pool_1[0])
# print('pool 2...')
# print(pool_2[0])
# print('pool 3...')
# print(pool_3[0])
return y_test_cls, y_pred_cls
vocab_dir = os.path.join(data.file_dir, data.vocab)
data.gVocab = torch.load(vocab_dir)
# load data
test = os.path.join(data.file_dir, data.test)
data.test_dataset = torch.load(test)
# load pretrain embedding
pretrain_src_path = os.path.join(data.file_dir, data.pretrain_src_embedding)
data.pretrain_src_embedding, data.src_embedding_dim, ukn_src_count = build_pretrain_embedding(
pretrain_src_path, data.gVocab, "src")
src_embedding = torch.tensor(data.pretrain_src_embedding)
print("src unknown words: " + str(ukn_src_count))
pretrain_tgt_path = os.path.join(data.file_dir, data.pretrain_tgt_embedding)
data.pretrain_tgt_embedding, data.tgt_embedding_dim, ukn_tgt_count = build_pretrain_embedding(
pretrain_tgt_path, data.gVocab, "tgt")
tgt_embedding = torch.tensor(data.pretrain_tgt_embedding)
print("tgt unknown words: " + str(ukn_tgt_count))
# build model
model = sama(data)
model.load_state_dict(torch.load(os.path.join(data.model_dir, "trainedmodel")))
param_count = 0 # counting the parameters
for param in model.parameters():
param_count += param.view(-1).size()[0]
print('total number of parameters of complete model: %d\n' % param_count)
BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4 = evaluate(
data, model, data.result_dir + "/" + "generation.txt")
gc.collect()
