def test(test_loader, model, epoch): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) # compute output out_a, out_p = model(data_a), model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p)**2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy()) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) # measure accuracy (FPR95) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, distances) print('\33[91mTest set: Accuracy(FPR95): {:.8f}\n\33[0m'.format(fpr95)) logger.log_value('fpr95', fpr95)
def test(test_loader, model, epoch, logger, logger_test_name): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) out_a = model(data_a) out_p = model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p)**2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy().reshape(-1, 1)) ll = label.data.cpu().numpy().reshape(-1, 1) labels.append(ll) if batch_idx % args.log_interval == 0: pbar.set_description(logger_test_name + ' Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, 1.0 / (distances + 1e-8)) #fdr95 = ErrorRateFDRAt95Recall(labels, 1.0 / (distances + 1e-8)) #fpr2 = convertFDR2FPR(fdr95, 0.95, 50000, 50000) #fpr2fdr = convertFPR2FDR(fpr2, 0.95, 50000, 50000) #print('\33[91mTest set: Accuracy(FDR95): {:.8f}\n\33[0m'.format(fdr95)) print('\33[91mTest set: Accuracy(FPR95): {:.8f}\n\33[0m'.format(fpr95)) #print('\33[91mTest set: Accuracy(FDR2FPR): {:.8f}\n\33[0m'.format(fpr2)) #print('\33[91mTest set: Accuracy(FPR2FDR): {:.8f}\n\33[0m'.format(fpr2fdr)) #fpr2 = convertFDR2FPR(round(fdr95,2), 0.95, 50000, 50000) #fpr2fdr = convertFPR2FDR(round(fpr2,2), 0.95, 50000, 50000) #print('\33[91mTest set: Accuracy(FDR2FPR): {:.8f}\n\33[0m'.format(fpr2)) #print('\33[91mTest set: Accuracy(FPR2FDR): {:.8f}\n\33[0m'.format(fpr2fdr)) if (args.enable_logging): logger.log_value(logger_test_name + ' fpr95', fpr95) return
def test(test_loader, model, epoch, logger, logger_test_name): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) out_a = model(data_a) out_p = model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p)**2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy().reshape(-1, 1)) ll = label.data.cpu().numpy().reshape(-1, 1) labels.append(ll) if batch_idx % args.log_interval == 0: pbar.set_description(logger_test_name + ' Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, 1.0 / (distances + 1e-8)) print('\33[91mTest set: Accuracy(FPR95): {:.8f}\n\33[0m'.format(fpr95)) #va_writer.add_scalar("fpr95", fpr95, global_step=epoch) tr_writer.add_scalar("fpr95", fpr95, global_step=epoch) f = open(fpr95_log, "a+") np.savetxt(f, [epoch, fpr95], delimiter=',') f.close() if (args.enable_logging): logger.log_value(logger_test_name + ' fpr95', fpr95) return
def test(test_loader, model, epoch, logger, logger_test_name): # switch to evaluate mode model.eval() labels, distances = [], [] # data_anchor(img_idx), data_positive(img_idx), match_or_not(0/1) pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) out_a, out_p = model(data_a), model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p)**2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy().reshape(-1, 1)) ll = label.data.cpu().numpy().reshape(-1, 1) labels.append(ll) if batch_idx % args.log_interval == 0: pbar.set_description(logger_test_name + ' Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, 1.0 / (distances + 1e-8)) print('Test set: Accuracy(FPR95): {:.8f}\n'.format(fpr95)) if (args.enable_logging): logger.log_string( 'logs', 'Test Epoch {}/{}: Accuracy(FPR95)@{}: {:.8f}'.format( epoch, args.start_epoch + args.epochs, logger_test_name, fpr95)) return
def test(self, test_loader, model, epoch, logger, logger_test_name): print("Testing model") # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if self.args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() with torch.no_grad(): data_a, data_p, label = Variable(data_a), \ Variable(data_p), Variable(label) out_a = model(data_a) out_p = model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy().reshape(-1,1)) ll = label.data.cpu().numpy().reshape(-1, 1) labels.append(ll) if batch_idx % self.args.log_interval == 0: pbar.set_description(logger_test_name+' Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, 1.0 / (distances + 1e-8)) print('\33[91mTest set: Accuracy(FPR95): {:.8f}\n\33[0m'.format(fpr95)) if (self.args.enable_logging): logger.log_value(logger_test_name+' fpr95', fpr95) return
def test(test_loader, model, epoch, logger, logger_test_name): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) out_a, out_p = model(data_a), model(data_p) dists = torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance distances.append(dists.data.cpu().numpy().reshape(-1,1)) ll = label.data.cpu().numpy().reshape(-1, 1) labels.append(ll) num_tests = test_loader.dataset.matches.size(0) labels = np.vstack(labels).reshape(num_tests) distances = np.vstack(distances).reshape(num_tests) fpr95 = ErrorRateAt95Recall(labels, distances) print('\33[91mTest set: Accuracy(FPR95): {:.8f}\n\33[0m'.format(fpr95)) if (args.enable_logging): logger.log_value('fpr95', fpr95) if True: try: os.stat('../histogram_map/'+suffix) except: os.mkdir('../histogram_map/'+suffix) bins = np.linspace(0, 2, 100) cos_dists = distances; pos_dists = cos_dists[labels==1] neg_dists = cos_dists[labels==0] plt.hist(pos_dists, bins, alpha = 0.5, label = 'Matched Pairs') plt.hist(neg_dists, bins, alpha = 0.5, label = 'Non-Matched Pairs') plt.legend(loc='upper left') plt.xlim(0, 2) plt.ylim(0, 2e4) plt.xlabel('l2') plt.ylabel('#Pairs') plt.savefig('../histogram_map/{}/iter_{}.png'.format(suffix,epoch), bbox_inches='tight') plt.clf() good_match_ratio = np.sum((distances<0.3)*labels)/np.sum(labels==1) print('Good match ratio for test: {}'.format(good_match_ratio)) if (args.enable_logging): logger.log_value('good_match_ratio', good_match_ratio) good_mismatch_ratio = np.sum((distances>0.8)*(1-labels))/np.sum(labels==0) print('Good mismatch ratio for test: {}'.format(good_mismatch_ratio)) if (args.enable_logging): logger.log_value('good_mismatch_ratio', good_mismatch_ratio) bad_mismatch_ratio = np.sum((distances<0.4)*(1-labels))/np.sum(labels==0) print('Bad mismatch ratio for test: {}'.format(bad_mismatch_ratio)) if (args.enable_logging): logger.log_value('bad_mismatch_ratio', bad_mismatch_ratio) return