def Evaluate(data, model, opt): """ data : dev_examples """ save_dir = opt["save_dir"] dev_preds = {} dev_na_probs = {} batch_size = 32 flag = True for batch_data in get_batch_data(data, batch_size): context_tokens = batch_data["context_tokens"] data = model.get_data(batch_data) ids = data["ids"] starts, ends, answer_prob = model.prediction(data, flag) flag = False answer_context = [] # print(starts) # print(ends) # print(context_tokens) for idx in range(len(starts)): start = starts[idx] end = ends[idx] context_token = context_tokens[idx] if (start == 0 and end == 0) or start > end: answer_context.append("") else: answer_context.append(" ".join(context_token[start:end + 1])) for idx in range(len(ids)): id = ids[idx] answer = answer_context[idx] no_prob = 1.0 - answer_prob[idx].item() dev_preds[id] = answer dev_na_probs[id] = no_prob with open(save_dir + "dev_preds", "w", encoding="utf-8") as f: json.dump(dev_preds, f) dev_file_path = "/home/FuDawei/NLP/SQUAD/data/dev-v2.0.json" ignore_ids = [] with open(save_dir + "devdrop_in_process.json", 'r') as f: dev_drop_process = json.load(f) ignore_ids.extend(dev_drop_process["id"]) with open(save_dir + "dev_drop.json", 'r') as f: dev_drop = json.load(f) ignore_ids.extend(dev_drop["id"]) out_eval = evaluate(dev_file_path, dev_preds, na_prob_thresh=0.3, ignore_ids=ignore_ids) exact, f1 = out_eval["exact"], out_eval["f1"] return exact, f1
def alg(game_board): """ The algorithm picks a random but possible move :param map: :return: """ try: move = random.choice([ commands[x] for x in commands.keys() if evaluate.evaluate(game_board, int(x)) >= 0 ]) except Exception: move = "a" return move
def get_f1(model, mode): pred_all, pred, recall_all, recall = 0, 0, 0, 0 f_pred_all, f_pred, f_recall_all, f_recall = 0, 0, 0, 0 gold_cross_num = 0 pred_cross_num = 0 if mode == "dev": batch_zip = zip(dev_token_batches, dev_char_batch, dev_char_len_batch, dev_pos_batches, dev_label_batches) elif mode == "test": batch_zip = zip(test_token_batches, test_char_batch, test_char_len_batch, test_pos_batches, test_label_batches) else: raise ValueError for token_batch, char_batch, char_len_batch, pos_batch, label_batch in batch_zip: token_batch_var = Variable(torch.LongTensor(np.array(token_batch))) pos_batch_var = Variable(torch.LongTensor(np.array(pos_batch))) if config.if_gpu: token_batch_var = token_batch_var.cuda() pos_batch_var = pos_batch_var.cuda() model.eval() pred_entities = model.predict(token_batch_var, pos_batch_var) p_a, p, r_a, r = evaluate(label_batch, pred_entities) #gold_cross_num += sum(count_overlap(label_batch)) #pred_cross_num += sum(count_overlap(pred_entities)) gold_cross_num += 0 pred_cross_num += 0 pred_all += p_a pred += p recall_all += r_a recall += r print(pred_all, pred, recall_all, recall) p = pred / pred_all if pred_all else 0 r = recall / recall_all if recall_all else 0 f1 = 2 * p * r / (p + r) if p + r else 0 # f1 = 2 / ((pred_all / pred) + (recall_all / recall)) print("Precision {0}, Recall {1}, F1 {2}".format(pred / pred_all, recall / recall_all, f1)) # print("Prediction Crossing: ", pred_cross_num) # print("Gold Crossing: ", gold_cross_num) return f1
def test(dataloader, model, args): # testing times = [] errors = [] dir_testInfor = "testInfo" if (not os.path.exists(dir_testInfor)): os.makedirs(dir_testInfor) path_testInfo = os.path.join(dir_testInfor, "%s_%s.pkl" % (args.dataset, args.flag_model)) if (os.path.exists(path_testInfo)): data = torch.load(path_testInfo) times.extend(data['time']) errors.extend(data['err']) for i in range(len(times)): print( "test: %4d | time: %6.3f, d1: %6.3f, EPE: %6.3f, pixelerror: %6.3f" % (i, times[i], errors[i][0], errors[i][1], errors[i][2])) model.eval() for batch_idx, (img1, img2, dispL_gt) in enumerate(dataloader): if (batch_idx < len(errors)): continue stime = time.time() dispL = model.estimate(img1, img2) times.append(time.time() - stime) error = evaluate(img1, img2, dispL, dispL_gt.numpy()) errors.append(error) print( "test: %4d | time: %6.3f, d1: %6.3f, EPE: %6.3f, pixelerror: %6.3f" % (batch_idx, times[-1], error[0], error[1], error[2])) if (args.flag_save): dirpath_save = os.path.join( dir_testInfor, "%s_%s" % (args.dataset, args.flag_model)) if (not os.path.exists(dirpath_save)): os.makedirs(dirpath_save) path_file = os.path.join(dirpath_save, "{:06d}.png".format(batch_idx)) cv2.imwrite(path_file, dispL[0, 0]) # cv2.imwrite("estimate/{:06d}_gt.png".format(step), dispL_gt[0]) if (batch_idx % 100 == 0): torch.save({"time": times, 'err': errors}, path_testInfo) torch.save({"time": times, 'err': errors}, path_testInfo) print np.mean(times), np.mean(errors, axis=0)
def alg(game_board): """ This algorithm attempts to only use left/bottom (a/s | 0/3 ) commands. If nto possible, right/down are also accepted. :param game_board: game board array :return: returns the move (char) """ if evaluate.evaluate(game_board, 0) == -1 and evaluate.evaluate( game_board, 3) == -1: # When the preferred steps are deadlocks if evaluate.evaluate(game_board, 1) > evaluate.evaluate(game_board, 2): return "w" else: return "d" else: if evaluate.evaluate(game_board, 0) > evaluate.evaluate(game_board, 3): return "a" else: return "s"
def get_f1(model: BiRecurrentConvCRF4NestedNER, mode: str, file_path: str = None) -> float: with torch.no_grad(): model.eval() pred_all, pred, recall_all, recall = 0, 0, 0, 0 gold_cross_num = 0 pred_cross_num = 0 if mode == 'dev': batch_zip = zip(dev_token_iv_batches, dev_token_ooev_batches, dev_char_batches, dev_label_batches, dev_mask_batches) elif mode == 'test': batch_zip = zip(test_token_iv_batches, test_token_ooev_batches, test_char_batches, test_label_batches, test_mask_batches) else: raise ValueError f = None if file_path is not None: f = open(file_path, 'w') for token_iv_batch, token_ooev_batch, char_batch, label_batch, mask_batch in batch_zip: token_iv_batch_var = torch.LongTensor(np.array(token_iv_batch)) token_ooev_batch_var = torch.LongTensor(np.array(token_ooev_batch)) char_batch_var = torch.LongTensor(np.array(char_batch)) mask_batch_var = torch.ByteTensor( np.array(mask_batch, dtype=np.uint8)) if config.if_gpu: token_iv_batch_var = token_iv_batch_var.cuda() token_ooev_batch_var = token_ooev_batch_var.cuda() char_batch_var = char_batch_var.cuda() mask_batch_var = mask_batch_var.cuda() pred_sequence_entities = model.predict(token_iv_batch_var, token_ooev_batch_var, char_batch_var, mask_batch_var) pred_entities = unpack_prediction(model, pred_sequence_entities) p_a, p, r_a, r = evaluate(label_batch, pred_entities) gold_cross_num += 0 pred_cross_num += 0 pred_all += p_a pred += p recall_all += r_a recall += r if file_path is not None: for token_iv, token_ooev, mask, label, preds \ in zip(token_iv_batch, token_ooev_batch, mask_batch, label_batch, pred_entities): words = [] for t_iv, t_ooev, m in zip(token_iv, token_ooev, mask): if not m: break if t_iv > 0: words.append(voc_iv_dict.get_instance(t_iv)) else: words.append(voc_ooev_dict.get_instance(t_ooev)) f.write(' '.join(words) + '\n') labels = [] for l in sorted(label, key=lambda x: (x[0], x[1], x[2])): labels.append("{},{} {}".format( l[0], l[1], label_dict.get_instance(l[2]))) f.write('|'.join(labels) + '\n') labels = [] for p in sorted(preds, key=lambda x: (x[0], x[1], x[2])): labels.append("{},{} {}".format( p[0], p[1], label_dict.get_instance(p[2]))) f.write('|'.join(labels) + '\n') f.write('\n') if file_path is not None: f.close() pred = pred / pred_all if pred_all > 0 else 1. recall = recall / recall_all if recall_all > 0 else 1. f1 = 2 / ((1. / pred) + (1. / recall)) if pred > 0. and recall > 0. else 0. logger.info( "{} precision: {:.2f}%, recall: {:.2f}%, F1: {:.2f}%".format( mode, pred * 100., recall * 100., f1 * 100.)) # logger.info("Prediction Crossing: ", pred_cross_num) # logger.info("Gold Crossing: ", gold_cross_num) return f1
t.set_postfix(loss=loss.item()) total_loss += loss.item() loss.backward() optimizer.step() model.zero_grad() t.update() epoch_end = time.time() epoch_cost = epoch_end - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) # evaluate if idx >= 0: BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4 = evaluate( data, model, data.result_dir + "/output_" + str(idx)) if BLEU4 > best_BLEU4: best_epoch = idx time_above = 0 print( "best BLEU4 score\n BLEU1 %f, BLEU2 %f BLEU3 %f BLEU4 %f, epoch %d" % (BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4, best_epoch), file=open(data.result_dir + "/BLEU_" + str(idx), "w")) model_name = data.result_dir + "/modelFinal_{}.model".format( BLEU4) torch.save(model.state_dict(), model_name) best_BLEU4 = BLEU4 else: time_above = time_above + 1 gc.collect()
y_train = t.from_numpy(y_train).long() inputs = Variable(x_train) targes = Variable(y_train) self.num_epochs = num_epochs for epoch in range(self.num_epochs): loss = self.optimize(inputs, targes) if (epoch + 1) % 20 == 0: print('Epoch[{}/{}],loss:{:.6f}'.format( epoch + 1, num_epochs, loss.data.numpy())) t.save(self.model.state_dict(), './torchsave.pt') def predict(self, x_test): self.model.load_state_dict(t.load('./torchsave.pt')) if isinstance(x_test, np.ndarray): x_test = t.from_numpy(x_test).float() predict_y = self.forward(x_test).data.numpy() return [np.argmax(one_hot) for one_hot in predict_y] if __name__ == '__main__': loads = data_loads.Data_Load() batch_xs, batch_ys = loads.chooceall() dnn = TorchDemo() dnn.train(batch_xs, batch_ys) predict_y = dnn.predict(batch_xs) evaluate.evaluate(batch_ys, predict_y) batch_xs, batch_ys = loads.all() predict_y = dnn.predict(batch_xs) evaluate.evaluate(batch_ys, predict_y)
def train(dataloader, dataloader_val, model, args): if (model.updated >= args.updates): print("Training Finished!") return #------------------------------------------prepare---------------------------------------------- mlosses = [] merrors = [] path_loss = os.path.join(model.dirpath, 'loss.pkl') if (os.path.exists(path_loss)): data = torch.load(path_loss) mlosses.extend(data['loss']) merrors.extend(data['error']) count = len(dataloader) epoches = (args.updates + count / 2) // count tepoch = (model.updated) // count time_start_full = time.time() flag_val = False plt.figure() for epoch in range(tepoch + 1, epoches + 1): msg = "This is %d-th epoch, updated: %6d, lr: %f " % ( epoch, model.updated, model.optim.param_groups[0]['lr']) logging.info(msg) #----------------------------------Training-------------------------------------------- model.train() loss_train_total = 0 for batch_idx, batch_stereo in enumerate(dataloader): time_start = time.time() try: if (args.loss_name == "supervised"): loss = model.update_supervised(*batch_stereo[:3]) else: loss = model.update_selfsupervised(*batch_stereo[:2]) except Exception as err: logging.error(err) logging.error(traceback.format_exc()) continue if (batch_idx % 20 == 0): msg = "Iter: %d, training loss: %.3f, time: %.2f" % ( batch_idx, loss, time.time() - time_start) logging.info(msg) loss_train_total += loss if (model.updated % 1000 == 1): flag_val = True if (model.updated % 100000 == 0): model.save_weight(flag_postfix=True) loss_mean = loss_train_total / len(dataloader) mlosses.append([epoch, loss_mean]) model.save_weight(flag_postfix=False) time_full_h = (time.time() - time_start_full) / 3600 msg = "**Train phase** Total mean training loss: %.3f, full training time: %.2f HR " % ( loss_mean, time_full_h) logging.info(msg) #----------------------------------validate-------------------------------------------- if (flag_val): flag_val = False model_val = model model_val.eval() terrors = [] for batch_idx, (img1, img2, disp1) in enumerate(dataloader_val): if (batch_idx > 50): break # visual check dispL = model_val.estimate(img1, img2) error = evaluate(img1, img2, dispL, disp1.numpy()) terrors.append(error) merror = np.mean(terrors, axis=0) merrors.append([epoch, merror[0], merror[1], merror[2]]) #----------------------------------visual check-------------------------------------------- torch.save({'loss': mlosses, 'error': merrors}, path_loss) msg = "**Validate phase** epoch: %d, d1: %.2f, EPE: %.2f, pixelerror: %.2f" % ( epoch, merror[0], merror[1], merror[2]) logging.info(msg) mlosses_np = np.array(mlosses) merrors_np = np.array(merrors) msg = str(mlosses_np.shape) + str(merrors_np.shape) logging.debug(msg) plt.subplot(221) plt.cla() plt.plot(mlosses_np[:, 0], mlosses_np[:, 1]) plt.xlabel("epoch") plt.ylabel("mean train loss") plt.subplot(222) plt.cla() plt.plot(merrors_np[:, 0], merrors_np[:, 1]) plt.xlabel("epoch") plt.ylabel("D1") plt.subplot(223) plt.cla() plt.plot(merrors_np[:, 0], merrors_np[:, 2]) plt.xlabel("epoch") plt.ylabel("EPE") plt.subplot(224) plt.cla() plt.plot(merrors_np[:, 0], merrors_np[:, 3]) plt.xlabel("epoch") plt.ylabel("RPE") plt.savefig("check_%s_%s_%s_%s.png" % (args.mode, args.dataset, args.net, args.loss_name)) print("Training Finish!")
def train(train_data, val_data, Path): print("Configuring TensorBoard and Saver...") # 配置 Tensorboard,重新训练时,请将tensorboard文件夹删除,不然图会覆盖 if not os.path.exists(Path.tensorboard_dir): os.makedirs(Path.tensorboard_dir) #结果可视化与存储 tf.summary.scalar("loss", model.loss) #可视化loss tf.summary.scalar("accuracy", model.acc) #可视化acc merged_summary = tf.summary.merge_all() #将所有操作合并输出 writer = tf.summary.FileWriter(Path.tensorboard_dir) #将summary data写入磁盘 # 配置 Saver saver = tf.train.Saver() if not os.path.exists(Path.save_dir): os.makedirs(Path.save_dir) print("Loading training and validation data...") # 载入训练集与验证集 start_time = time.time() # train_x1_word, train_x2_word, train_ks, train_x2_label, train_y = train_data # val_x1_word, val_x2_word, val_ks, val_x2_label, val_y = val_data train_x1_word, train_x2_word, train_ks, train_y = train_data val_x1_word, val_x2_word, val_ks, val_y = val_data print('train len', len(train_x1_word)) print('val_len', len(val_x1_word)) time_dif = get_time_dif(start_time) print("Time usage:", time_dif) # 创建session session = tf.Session() session.run(tf.global_variables_initializer()) writer.add_graph(session.graph) print('Training and evaluating...') start_time = time.time() total_batch = 0 # 总批次 best_acc_val = 0.0 # 最佳验证集准确率 last_improved = 0 # 记录上一次提升批次 require_improvement = 1000 # 如果超过1000轮未提升,提前结束训练 flag = False for epoch in range(param.BaseConfig.num_epochs): print('Epoch:', epoch + 1) batch_train = get_batch_data(train_x1_word, train_x2_word, train_ks, train_y, batch_size=param.BaseConfig.batch_size) for a_word_batch, b_word_batch, c_word_batch, y_batch in batch_train: feed_dict = feed_data_fun(model, a_word_batch, b_word_batch, c_word_batch, y_batch, model.config.dropout_rate) if total_batch % param.BaseConfig.save_per_batch == 0: # 每多少轮次将训练结果写入tensorboard scalar s = session.run(merged_summary, feed_dict=feed_dict) writer.add_summary(s, total_batch) if total_batch % param.BaseConfig.print_per_batch == 0: # 每多少轮次输出在训练集和验证集上的性能 feed_dict[model.dropout_rate] = 1.0 loss_train, acc_train, pre_y, logit, true_y = session.run( [ model.loss, model.acc, model.pred_y, model.logit, model.y ], feed_dict=feed_dict) loss_val, acc_val = evaluate( model, session, val_x1_word, val_x2_word, val_ks, val_y, feed_data_fun) # 验证当前会话中的模型的loss和acc if acc_val > best_acc_val: # 保存最好结果 best_acc_val = acc_val last_improved = total_batch saver.save(sess=session, save_path=Path.save_path) improved_str = '*' else: improved_str = '' time_dif = get_time_dif(start_time) msg = 'Iter: {0}, Train Loss: {1}, Train Acc: {2},' \ + ' Val Loss: {3}, Val Acc: {4}, Time: {5} {6}' print( msg.format(total_batch, loss_train, acc_train, loss_val, acc_val, time_dif, improved_str)) session.run(model.optim, feed_dict=feed_dict) # 运行优化 total_batch += 1 if total_batch - last_improved > require_improvement: # 验证集正确率长期不提升,提前结束训练 print("No optimization for a long time, auto-stopping...") flag = True break # 跳出循环 if flag: # 同上 break
def test(test_data, Path): #载入随机森林模型 print("Loading test data...") start_time = time.time() test_x1_word, test_x2_word, test_ks, test_y = test_data session = tf.Session() session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess=session, save_path=Path.save_path) # 读取保存的模型 print('Testing...') loss_test, acc_test = evaluate(model, session, test_x1_word, test_x2_word, test_ks, test_y, feed_data_fun) msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}' print(msg.format(loss_test, acc_test)) batch_size = param.BaseConfig.batch_size data_len = len(test_x1_word) num_batch = int((data_len) / batch_size) y_test_cls = np.argmax(test_y, 1) y_pred_cls = np.zeros(shape=data_len, dtype=np.int32) # 保存预测结果 for i in range(num_batch): # 逐批次处理 start_id = i * batch_size end_id = min((i + 1) * batch_size, data_len) feed_dict = feed_data_fun(model, test_x1_word[start_id:end_id], test_x2_word[start_id:end_id], test_ks[start_id:end_id], test_y[start_id:end_id], 1.0) y_pred_cls[start_id:end_id] = session.run( model.pred_y, feed_dict=feed_dict) #将所有批次的预测结果都存放在y_pred_cls中 # pool_1,pool_2,pool_3 = session.run([model.fusion_output_max_1,model.fusion_output_max_2,model.fusion_output_max_3], # feed_dict=feed_dict) print("Precision, Recall and F1-Score...") print(metrics.classification_report(y_test_cls, y_pred_cls, digits=4)) #直接计算准确率,召回率和f值 # 混淆矩阵 print("Confusion Matrix...") cm = metrics.confusion_matrix(y_test_cls, y_pred_cls) print(cm) time_dif = get_time_dif(start_time) print("Time usage:", time_dif) # print("beta value", beta1,beta2,beta3) #check error prediction # print(y_pred_cls) # print('check.......') # print('maxpooling.....') # print('pool 1...') # print(pool_1[0]) # print('pool 2...') # print(pool_2[0]) # print('pool 3...') # print(pool_3[0]) return y_test_cls, y_pred_cls
vocab_dir = os.path.join(data.file_dir, data.vocab) data.gVocab = torch.load(vocab_dir) # load data test = os.path.join(data.file_dir, data.test) data.test_dataset = torch.load(test) # load pretrain embedding pretrain_src_path = os.path.join(data.file_dir, data.pretrain_src_embedding) data.pretrain_src_embedding, data.src_embedding_dim, ukn_src_count = build_pretrain_embedding( pretrain_src_path, data.gVocab, "src") src_embedding = torch.tensor(data.pretrain_src_embedding) print("src unknown words: " + str(ukn_src_count)) pretrain_tgt_path = os.path.join(data.file_dir, data.pretrain_tgt_embedding) data.pretrain_tgt_embedding, data.tgt_embedding_dim, ukn_tgt_count = build_pretrain_embedding( pretrain_tgt_path, data.gVocab, "tgt") tgt_embedding = torch.tensor(data.pretrain_tgt_embedding) print("tgt unknown words: " + str(ukn_tgt_count)) # build model model = sama(data) model.load_state_dict(torch.load(os.path.join(data.model_dir, "trainedmodel"))) param_count = 0 # counting the parameters for param in model.parameters(): param_count += param.view(-1).size()[0] print('total number of parameters of complete model: %d\n' % param_count) BLUE_score_1, BLUE_score_2, BLUE_score_3, BLEU4 = evaluate( data, model, data.result_dir + "/" + "generation.txt") gc.collect()