def test(**kwargs): import glob pths = glob.glob('checkpoints/%s/*.pth' % (opt.model)) pths.sort(key=os.path.getmtime, reverse=True) print(pths) opt.parse(kwargs) # 模型 opt.load_model_path = pths[0] model = getattr(models, opt.model)().eval() assert os.path.exists(opt.load_model_path) if opt.load_model_path: model.load(opt.load_model_path) if opt.use_gpu: model.cuda() model.train(False) # 数据 #result_name = '../../model/se-resnet/test_se_resnet50' test_data = myData( filelists=opt.test_filelists, image_size=opt.image_size, #transform =data_transforms['val'], transform=None, scale=opt.cropscale, test=True, data_source='none') # test_data = myData(root = opt.test_roo,datatxt='test.txt', # test = True,transform = data_transforms['test']) test_loader = DataLoader(dataset=test_data, batch_size=opt.batch_size // 2, shuffle=False) #test_loader =DataLoader(dataset = test_data,batch_size = opt.batch_size//2,shuffle =True) result_list = [] label_list = [] for step, batch in enumerate(tqdm(test_loader, desc='test', unit='batch')): data, label, image_path = batch with torch.no_grad(): if opt.use_gpu: data = data.cuda() outputs = model(data) outputs = torch.softmax(outputs, dim=-1) preds = outputs.to('cpu').numpy() for i in range(preds.shape[0]): result_list.append(preds[i, 1]) label_list.append(label[i]) metric = roc.cal_metric(label_list, result_list) eer = metric[0] tprs = metric[1] auc = metric[2] xy_dic = metric[3] pickle.dump(xy_dic, open('result/xy.pickle', 'wb')) print('EER: {:.6f} TPR(1.0%): {:.6f} TPR(.5%): {:.6f} AUC: {:.8f}'.format( eer, tprs["TPR(1.%)"], tprs["TPR(.5%)"], auc), file=open('result/test.txt', 'a')) print('EER: {:.6f} TPR(1.0%): {:.6f} TPR(.5%): {:.6f} AUC: {:.8f}'.format( eer, tprs["TPR(1.%)"], tprs["TPR(.5%)"], auc))
def val(model, dataloader, data_len): # 把模型设为验证模式 criterion = FocalLoss(2) model.train(False) running_loss = 0 running_corrects = 0 confusion_matrix = meter.ConfusionMeter(2) result_list = [] label_list = [] for ii, data in enumerate(tqdm(dataloader, desc='Val %s On Anti-spoofing' % (opt.model), unit='batch')): input, label = data with torch.no_grad(): val_input = Variable(input) val_label = Variable(label) if opt.use_gpu: val_input = val_input.cuda() val_label = val_label.cuda() score = model(val_input) _, preds = torch.max(score, 1) loss = criterion(score, val_label) # confusion_matrix.add(score.data.squeeze(), val_label) running_loss += loss.item() * val_input.size(0) running_corrects += torch.sum(preds == val_label.data) outputs = torch.softmax(score, dim=-1) preds = outputs.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds[i_batch, 1]) label_list.append(label[i_batch]) # 把模型恢复为训练模式 model.train(True) metric = roc.cal_metric(label_list, result_list) # cm_value = confusion_matrix.value() val_loss = running_loss / data_len val_accuracy = running_corrects.double() / float(data_len) return val_loss, val_accuracy, metric
FN += 1 elif labels[i] == 0 and preds[i] == 1: FP += 1 outputs = torch.softmax(outputs, dim=-1) preds_prob = outputs.to('cpu').detach().numpy() labels = labels.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds_prob[i_batch, 1]) label_list.append(labels[i_batch]) TP_rate = float(TP / (TP + FN)) TN_rate = float(TN / (TN + FP)) HTER = 1 - (TP_rate + TN_rate) / 2 metric = roc.cal_metric(label_list, result_list, True) print('Test set: Accuracy: {:.4f}, Auc: {:.4f}, HTER: {:.4f}'.format( correct.float() / len(test_loader.dataset), metric[0], HTER)) print() # _, pred = output.topk(1, 1, largest=True, sorted=True) # label = label.view(label.size(0), -1).expand_as(pred) # correct = pred.eq(label).float() # # # #compute top 5 # # correct_5 += correct[:, :5].sum() # # #compute top1 # correct_1 += correct[:, :1].sum()
def validate(val_loader, model, criterion, epoch): global time_stp batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() result_list = [] label_list = [] predicted_list = [] # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, (input, target, depth_dirs) in enumerate(val_loader): with torch.no_grad(): input_var = [] for _input in input: input_var.append(Variable(_input).float().to(device)) target_var = Variable(target).long().to(device) # compute output output = model(input_var) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec2 = accuracy(output.data, target_var, topk=(1, 2)) losses.update(loss.data, input[0].size(0)) top1.update(prec1[0], input[0].size(0)) soft_output = torch.softmax(output, dim=-1) preds = soft_output.to('cpu').detach().numpy() label = target.to('cpu').detach().numpy() _, predicted = torch.max(soft_output.data, 1) predicted = predicted.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds[i_batch, 1]) label_list.append(label[i_batch]) predicted_list.append(predicted[i_batch]) if args.val_save: f = open( 'submission/{}_{}_{}_{}_{}_{}_submission.txt'.format(sub, modal, time_stp, args.arch, epoch, 'test' if args.phase_test else 'dev'), 'a+') depth_dir = depth_dirs[i_batch].replace('D:\\py_workspace\\data\\test_cut\\', '') f.write(depth_dir + ' ' + str(preds[i_batch, 1]) + '\n') # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: line = 'Test: [{0}/{1}]\t' \ 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \ 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format(i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1) with open('logs/{}_{}_{}_{}.log'.format(sub, modal, time_stp, args.arch), 'a+') as flog: flog.write('{}\n'.format(line)) print(line) culc = confusion_matrix(label_list, predicted_list).ravel() matrix_edge = int(math.sqrt(len(culc))) print(culc) tn, fp, fn, tp = np.hstack((culc[:2], culc[matrix_edge:matrix_edge + 2])) apcer = fp / (tn + fp) npcer = fn / (fn + tp) acer = (apcer + npcer) / 2 metric = roc.cal_metric(label_list, result_list) eer = metric[0] tprs = metric[1] auc = metric[2] xy_dic = metric[3] # tpr1 = tprs['TPR@FPR=10E-2'] # logger.info('eer: {}\t' # 'tpr1: {}\t' # 'auc: {}\t' # 'acer: {}\t' # 'accuracy: {top1.avg:.3f} ({top1.avg:.3f})' # .format(eer,tpr1,auc,acer,top1=top1)) # pickle.dump(xy_dic, open('xys/xy_{}_{}_{}.pickle'.format(time_stp, args.arch,epoch),'wb')) with open('logs/val_result_{}_{}_{}.txt'.format(sub, time_stp, args.arch), 'a+') as f_result: result_line = 'epoch: {} EER: {:.6f} TPR@FPR=10E-2: {:.6f} TPR@FPR=10E-3: {:.6f} APCER:{:.6f} NPCER:{:.6f} AUC: {:.8f} Acc:{:.3f} TN:{} FP : {} FN:{} TP:{} ACER:{:.8f} '.format( epoch, eer, tprs["TPR@FPR=10E-2"], tprs["TPR@FPR=10E-3"], apcer, npcer, auc, top1.avg, tn, fp, fn, tp, acer) f_result.write('{}\n'.format(result_line)) print(result_line) return top1.avg
def validate(val_loader, model, criterion, epoch): global time_stp batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() result_list = [] label_list = [] predicted_list = [] subject_name_list = ['start'] subject_preds = [] # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, (input, target, dirs) in enumerate(val_loader): with torch.no_grad(): input_var = Variable(input).float().to(device) target_var = Variable(target).long().to(device) # compute output output = model(input_var) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec2 = accuracy(output.data, target_var, topk=(1, 2)) losses.update(loss.data, input.size(0)) top1.update(prec1[0], input.size(0)) soft_output = torch.softmax(output, dim=-1) preds = soft_output.to('cpu').detach().numpy() label = target.to('cpu').detach().numpy() _, predicted = torch.max(soft_output.data, 1) predicted = predicted.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds[i_batch, 1]) label_list.append(label[i_batch]) predicted_list.append(predicted[i_batch]) subject = dirs[i_batch].split('/')[-3] prefix = dirs[i_batch].split('/')[-4] pre_subject = subject_name_list[-1] if subject not in subject_name_list: subject_name_list.append(subject) if i == len( val_loader) - 1 and i_batch == preds.shape[0] - 1: subject_name_list.append('end') subject_preds.append(preds[i_batch, 1]) if args.val_save and pre_subject != 'start' and pre_subject != subject_name_list[ -1]: mean_pred = np.mean(subject_preds[:-1]) # 将0.4-0.9之间的都认为是假的 if mean_pred < 0.9 and mean_pred > 0.4: mean_pred = mean_pred - 0.4 subject_preds = [preds[i_batch, 1]] if 'phase1' in args.data_root: txt = 'dev' else: txt = 'test' f = open( 'submission/{}_{}_{}@{}_submission_'.format( time_stp, args.arch, args.mode, args.sub_prot_test) + txt + '.txt', 'a+') # dir_ = dirs[i_batch].replace('/home/zp/dataset/CASIA-CeFA/phase1/','').replace(subject_name_list[-1],pre_subject) dir_ = prefix + '/' + pre_subject if 'test' in dir_: img_dirs = glob( os.path.join(args.data_root + dir_, 'profile/*.jpg')) length = len(img_dirs) if length < 10: mean_pred = np.random.uniform(0, 0.3) f.write(dir_ + ' ' + str(mean_pred) + '\n') # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: line = 'Test: [{0}/{1}]\t' \ 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \ 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format(i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1) with open( 'logs/{}_{}_{}_@{}.log'.format( time_stp, args.arch, args.mode, args.sub_prot_val), 'a+') as flog: flog.write('{}\n'.format(line)) print(line) print(len(label_list), len(predicted_list), confusion_matrix(label_list, predicted_list).ravel()) tn, fp, fn, tp = confusion_matrix(label_list, predicted_list).ravel() apcer = fp / (tn + fp) npcer = fn / (fn + tp) acer = (apcer + npcer) / 2 metric = roc.cal_metric(label_list, result_list) eer = metric[0] tprs = metric[1] auc = metric[2] xy_dic = metric[3] # tpr1 = tprs['TPR@FPR=10E-2'] # logger.info('eer: {}\t' # 'tpr1: {}\t' # 'auc: {}\t' # 'acer: {}\t' # 'accuracy: {top1.avg:.3f} ({top1.avg:.3f})' # .format(eer,tpr1,auc,acer,top1=top1)) # pickle.dump(xy_dic, open('xys/xy_{}_{}_{}.pickle'.format(time_stp, args.arch,epoch),'wb')) with open( 'logs/val_result_{}_{}_{}_@{}.txt'.format(time_stp, args.arch, args.mode, args.sub_prot_val), 'a+') as f_result: result_line = 'epoch: {} EER: {:.6f} TPR@FPR=10E-2: {:.6f} TPR@FPR=10E-3: {:.6f} APCER:{:.6f} NPCER:{:.6f} AUC: {:.8f} Acc:{:.3f} TN:{} FP : {} FN:{} TP:{} ACER:{:.8f} '.format( epoch, eer, tprs["TPR@FPR=10E-2"], tprs["TPR@FPR=10E-3"], apcer, npcer, auc, top1.avg, tn, fp, fn, tp, acer) f_result.write('{}\n'.format(result_line)) print(result_line) return top1.avg
def eval_training(epoch): net.eval() test_loss = 0.0 # cost function error correct = 0.0 result_list = [] label_list = [] TP = 0. TN = 0. FP = 0. FN = 0. for (images, labels) in test_loader: images = Variable(images) labels = Variable(labels) images = images.cuda() labels = labels.cuda() outputs = net(images) loss = loss_function(outputs, labels) # loss = CB_loss(labels, outputs, samples_per_cls, 2, 'softmax', 0.9999, 2.0) test_loss += loss.item() _, preds = outputs.max(1) correct += preds.eq(labels).sum() for i in range(len(preds)): if labels[i] == 1 and preds[i] == 1: TP += 1 elif labels[i] == 0 and preds[i] == 0: TN += 1 elif labels[i] == 1 and preds[i] == 0: FN += 1 elif labels[i] == 0 and preds[i] == 1: FP += 1 outputs = torch.softmax(outputs, dim=-1) preds_prob = outputs.to('cpu').detach().numpy() labels = labels.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds_prob[i_batch, 1]) label_list.append(labels[i_batch]) TP_rate = float(TP / (TP + FN)) TN_rate = float(TN / (TN + FP)) HTER = 1 - (TP_rate + TN_rate) / 2 metric = roc.cal_metric(label_list, result_list, False) # print('Test set: Average loss: {:.4f}, Accuracy: {:.4f}, Auc: {:.4f}, HTER: {:.4f}'.format( # test_loss / len(test_loader.dataset), # correct.float() / len(test_loader.dataset), # metric[2], HTER # )) log.write('Test set: Average loss: {:.4f}, Accuracy: {:.4f}, Auc: {:.4f}, HTER: {:.4f}'.format( test_loss / len(test_loader.dataset), correct.float() / len(test_loader.dataset), metric[2], HTER )) print() #add informations to tensorboard writer.add_scalar('Test/Average loss', test_loss / len(test_loader.dataset), epoch) writer.add_scalar('Test/Accuracy', correct.float() / len(test_loader.dataset), epoch) return correct.float() / len(test_loader.dataset)
def validate(val_loader, model, criterion, epoch): global time_stp batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() result_list = [] label_list = [] predicted_list = [] # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, data in enumerate(val_loader): with torch.no_grad(): rgb_img, depth_img, ir_img, hsv_img, YCbCr_img, label, dirs = data[ 0], data[1], data[2], data[3], data[4], data[5], data[6] rgb_var = Variable(rgb_img).float().to(device) depth_var = Variable(depth_img).float().to(device) ir_var = Variable(ir_img).float().to(device) hsv_img_var = Variable(hsv_img).float().to(device) YCbCr_img_var = Variable(YCbCr_img).float().to(device) target_var = Variable(label).long().to(device) # compute output output = model(rgb_var, depth_var, ir_var, hsv_img_var, YCbCr_img_var, args.weight_list) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec2 = accuracy(output.data, target_var, topk=(1, 2)) losses.update(loss.data, rgb_img.size(0)) top1.update(prec1[0], rgb_img.size(0)) soft_output = torch.softmax(output, dim=-1) preds = soft_output.to('cpu').detach().numpy() label = label.to('cpu').detach().numpy() _, predicted = torch.max(soft_output.data, 1) predicted = predicted.to('cpu').detach().numpy() for i_batch in range(preds.shape[0]): result_list.append(preds[i_batch, 1]) label_list.append(label[i_batch]) predicted_list.append(predicted[i_batch]) if args.val_save: f = open( 'submission/{}_{}_{}_submission.txt'.format( time_stp, args.arch, epoch), 'a+') rgb_dir = dirs[i_batch].replace( os.getcwd() + '/data/', '') depth_dir = rgb_dir.replace('profile', 'depth') ir_dir = rgb_dir.replace('profile', 'ir') f.write(rgb_dir + ' ' + depth_dir + ' ' + ir_dir + ' ' + str(preds[i_batch, 1]) + '\n') # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: line = 'Test: [{0}/{1}]\t' \ 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \ 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format(i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1) logger.Print(line) tn, fp, fn, tp = confusion_matrix(label_list, predicted_list).ravel() apcer = fp / (tn + fp) npcer = fn / (fn + tp) acer = (apcer + npcer) / 2 metric = roc.cal_metric(label_list, result_list) eer = metric[0] tprs = metric[1] auc = metric[2] xy_dic = metric[3] with open(log_path + '/val_result_{}_{}.txt'.format(time_stp, args.arch), 'a+') as f_result: result_line = 'epoch: {} EER: {:.6f} TPR@FPR=10E-2: {:.6f} TPR@FPR=10E-3: {:.6f} APCER:{:.6f} NPCER:{:.6f} AUC: {:.8f} Acc:{:.3f} TN:{} FP : {} FN:{} TP:{} ACER:{:.8f} '.format( epoch, eer, tprs["TPR@FPR=10E-2"], tprs["TPR@FPR=10E-3"], apcer, npcer, auc, top1.avg, tn, fp, fn, tp, acer) f_result.write('{}\n'.format(result_line)) logger.Print(result_line) return top1.avg