def test(test_loader, model, epoch): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: current_sample = data_a.size(0) data_a = data_a.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) data_p = data_p.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) # compute output out_a, out_p = model(data_a), model(data_p) dists = l2_dist.forward(out_a, out_p) dists = dists.data.cpu().numpy() dists = dists.reshape(current_sample, args.test_input_per_file).mean(axis=1) distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) # if epoch==4: # np.save('Data/4_distance.npy', distances) # np.save('Data/4_label.npy', labels) # err, accuracy= evaluate_eer(distances,labels) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test_Result/eer', eer, epoch) writer.add_scalar('Test_Result/threshold', eer_threshold, epoch) writer.add_scalar('Test_Result/accuracy', accuracy, epoch) # tpr, fpr, accuracy, val, far = evaluate(distances, labels) if args.cos_sim: print( '\33[91mFor cos_distance, Test set ERR is {:.8f} when threshold is {}\tAnd test accuracy could be {:.2f}%.\33[0m' .format(100. * eer, eer_threshold, 100. * accuracy)) else: print( '\33[91mFor l2_distance, Test set ERR: {:.8f}%\tBest ACC:{:.8f} \n\33[0m' .format(100. * eer, accuracy))
def sitw_test(sitw_test_loader, model, epoch): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(sitw_test_loader)) for batch_idx, (data_a, data_p, label) in pbar: vec_shape = data_a.shape # pdb.set_trace() if vec_shape[1] != 1: data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output _, out_a_ = model(data_a) _, out_p_ = model(data_p) out_a = out_a_ out_p = out_p_ dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance if vec_shape[1] != 1: dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(axis=1) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * vec_shape[0], len(sitw_test_loader.dataset), 100. * batch_idx / len(sitw_test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer_t, eer_threshold_t, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) torch.cuda.empty_cache() writer.add_scalars('Test/EER', {'sitw_test': 100. * eer_t}, epoch) writer.add_scalars('Test/Threshold', {'sitw_test': eer_threshold_t}, epoch) print('\33[91mFor Sitw Test ERR: {:.4f}%, Threshold: {}.\n\33[0m'.format( 100. * eer_t, eer_threshold_t))
def test(test_loader, model, epoch): # switch to evaluate mode model.eval() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), \ Variable(data_p), Variable(label) # SiResNet34 out_a = model.pre_forward_norm(data_a) out_p = model.pre_forward_norm(data_p) dists = l2_dist.forward(out_a, out_p) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', eer, epoch - 1) writer.add_scalar('Test/Threshold', eer_threshold, epoch - 1) print( '\33[91mFor {}_distance, Test ERR: {:.8f}, Threshold: {:.8f}.\n\33[0m'. format('cos' if args.cos_sim else 'l2', 100. * eer, eer_threshold))
def verification_test(test_loader, dist_type, log_interval): # switch to evaluate mode labels, distances = [], [] dist_fn = nn.CosineSimilarity( ) if dist_type == 'cos' else nn.PairwiseDistance(2) pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: out_a = torch.tensor(data_a) out_p = torch.tensor(data_p) dists = dist_fn.forward(out_a, out_p).numpy() distances.append(dists) labels.append(label.numpy()) if batch_idx % log_interval == 0: pbar.set_description('Verification: [{}/{} ({:.0f}%)]'.format( batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer( distances, labels, cos=True if dist_type == 'cos' else False, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels))) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m\n'.format( mindcf_01, mindcf_001))
def test(test_loader): # switch to evaluate mode labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: out_a = torch.tensor(data_a) out_p = torch.tensor(data_p) dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance dists = dists.numpy() distances.append(dists) labels.append(label.numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test: [{}/{} ({:.0f}%)]'.format( batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) dist_type = 'cos' if args.cos_sim else 'l2' print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels))) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m'.format( mindcf_01, mindcf_001))
def test(test_loader, valid_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output out, _ = model(data) if args.loss_type == 'asoft': predicted_labels, _ = out else: predicted_labels = out true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = ( predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%' .format(epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc)) valid_accuracy = 100. * correct / total_datasize writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) torch.cuda.empty_cache() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: vec_shape = data_a.shape # pdb.set_trace() if vec_shape[1] != 1: data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output _, out_a_ = model(data_a) _, out_p_ = model(data_p) out_a = out_a_ out_p = out_p_ dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(dim=1) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', 100. * eer, epoch) writer.add_scalar('Test/Threshold', eer_threshold, epoch) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) writer.add_scalar('Test/mindcf-0.01', mindcf_01, epoch) writer.add_scalar('Test/mindcf-0.001', mindcf_001, epoch) dist_type = 'cos' if args.cos_sim else 'l2' print('\nFor %s_distance, ' % dist_type) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f},'.format( mindcf_01, mindcf_001)) print(' Valid Accuracy is %.4f %%.\33[0m' % valid_accuracy) torch.cuda.empty_cache()
def test(valid_loader, test_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: if len(data) != args.batch_size * args.tuple_size: continue if args.cuda: data = data.float().squeeze().cuda() label = label.cuda() data, label = Variable(data), Variable(label) feats, classfier = model(data) predicted_one_labels = softmax(classfier) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = ( predicted_one_labels.cuda() == label.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%' .format(epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc)) valid_accuracy = 100. * correct / total_datasize writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (a, p, label) in pbar: if len(a) != int( args.batch_size * args.tuple_size / args.test_input_per_file): continue vec_shape = a.shape # pdb.set_trace() data_a = a.reshape(vec_shape[0] * vec_shape[1], vec_shape[2], vec_shape[3]) data_p = p.reshape(vec_shape[0] * vec_shape[1], vec_shape[2], vec_shape[3]) if args.cuda: data_a, data_p = data_a.float().cuda(), data_p.float().cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output out_a, _ = model.tuple_forward(data_a) out_p, _ = model.tuple_forward(data_p) dists_a = l2_dist.forward(out_a, out_p) dists_a = dists_a.data.cpu().numpy() dists_a = dists_a.reshape(vec_shape[0], vec_shape[1]).mean(axis=1) distances.append(dists_a) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', 100. * eer, epoch) print( '\33[91mERR: {:.8f}. Threshold: {:.8f}. Valid Accuracy is {:.4f}%.\33[0m\n' .format(100. * eer, eer_threshold, valid_accuracy))
def test(test_loader, valid_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output out, _ = model(data) predicted_labels = out true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = ( predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch for Classification: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%' .format(epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc)) valid_accuracy = 100. * correct / total_datasize labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: current_sample = data_a.size(0) data_a = data_a.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) data_p = data_p.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output _, out_a = model(data_a) _, out_p = model(data_p) dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance dists = dists.data.cpu().numpy() dists = dists.reshape(current_sample, args.test_input_per_file).mean(axis=1) distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) writer.add_scalar('Test/EER', eer, epoch) writer.add_scalar('Test/Threshold', eer_threshold, epoch) if args.cos_sim: print( '\33[91mFor cos_distance, Test set verification ERR is {:.8f}%, when threshold is {}. Valid set classificaton accuracy is {:.2f}%.\n\33[0m' .format(100. * eer, eer_threshold, valid_accuracy)) else: print( '\33[91mFor l2_distance, Test set verification ERR is {:.8f}%, when threshold is {}. Valid set classificaton accuracy is {:.2f}%.\n\33[0m' .format(100. * eer, eer_threshold, valid_accuracy))
def test(test_loader, valid_loader, model, epoch): # net = model.to('cuda:1') model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output out = model.pre_forward(data) cls = model(out) predicted_labels = cls true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = (predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch for Classification: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'.format( epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc )) valid_accuracy = 100. * correct / total_datasize writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) # test_set = testset.TestSet('../all_feature/') # todo: # test_set = [] distances = [] labels = [] # testloader = DataLoader(test_set, batch_size=128, num_workers=16, shuffle=True) # torch.set_num_threads(16) pbar = tqdm(enumerate(test_loader)) # .set_trace() for batch_idx, (data_a, data_p, label) in pbar: current_sample = data_a.size(0) data_a = data_a.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) data_p = data_p.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3)) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a, volatile=True), \ Variable(data_p, volatile=True), Variable(label) # compute output x_vector_a, x_vector_p = model.pre_forward(data_a), model.pre_forward(data_p) dists = l2_dist.forward(x_vector_a, x_vector_p) dists = dists.data.cpu().numpy() dists = dists.reshape(current_sample, args.test_input_per_file).mean(axis=1) distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) # err, accuracy= evaluate_eer(distances,labels) # err, accuracy= evaluate_eer(distances,labels) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', eer, epoch) writer.add_scalar('Test/Threshold', eer_threshold, epoch) # tpr, fpr, accuracy, val, far = evaluate(distances, labels) if args.cos_sim: print( '\33[91mFor cos_distance, Test set ERR is {:.4f}, threshold is {}. Valid accuracy could be {:.2f}%.\n\33[0m'.format( 100. * eer, eer_threshold, valid_accuracy)) else: print( '\33[91mFor l2_distance, Test set ERR: {:.8f}%\tBest ACC:{:.8f}\n\33[0m'.format(100. * eer, valid_accuracy))
def test(test_loader, valid_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output # pdb.set_trace() out = model.pre_forward(data) cls = model(out) predicted_labels = cls true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = ( predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%' .format(epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc)) valid_accuracy = 100. * correct / total_datasize labels, distances = [], [] distances_b = [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output out_a = model.pre_forward(data_a) out_p = model.pre_forward(data_p) out_ab = model.segment7(out_a) out_ab = model.relu(model.batch_norm7(out_ab)) out_pb = model.segment7(out_p) out_pb = model.relu(model.batch_norm7(out_pb)) dists = l2_dist.forward(out_a, out_p) dists = dists.data.cpu().numpy() distances.append(dists) distsb = l2_dist.forward(out_ab, out_pb) distsb = distsb.data.cpu().numpy() distances_b.append(distsb) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) distances_b = np.array( [subdist for dist in distances_b for subdist in dist]) # err, accuracy= evaluate_eer(distances,labels) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) eer_b, eer_threshold_b, accuracy_b = evaluate_kaldi_eer(distances_b, labels, cos=args.cos_sim, re_thre=True) print('\33[91m Epoch {}, Valid Accuracy is {}. For {}_distance:'.format( epoch, valid_accuracy, 'cos' if args.cos_sim else 'l2')) print(' For Embedding a: Test ERR: {:.8f}. Threshold: {:.8f}. \n' ' For Embedding b: Test ERR: {:.8f}. Threshold: {:.8f}.\n\33[0m'. format(100. * eer, eer_threshold, 100. * eer_b, eer_threshold_b))
labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) <<<<<<< HEAD if save != '': np.save('%s/labels.npy' % save, labels) np.save('%s/distance.npy' % save, distances) ======= # this_xvector_dir = "%s/epoch_%s" % (xvector_dir, epoch) with open('%s/scores' % xvector_dir, 'w') as f: for d, l in zip(distances, labels): f.write(str(d) + ' ' + str(l) + '\n') >>>>>>> Server/Server eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=True if dist_type == 'cos' else False, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) return eer, eer_threshold, mindcf_01, mindcf_001 # https://github.com/clovaai/voxceleb_trainer def accuracy(output, target, topk=(1,)): """Computes the precision@k for the specified values of k""" maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred))
def test(test_loader, valid_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output # pdb.set_trace() out = model.att_forward_norm(data) cls = model(out) predicted_labels = cls true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = (predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'.format( epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc )) valid_accuracy = 100. * correct / total_datasize writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: vec_shape = data_a.shape # pdb.set_trace() data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(label) # compute output out_a = model.att_forward_norm(data_a) out_p = model.att_forward_norm(data_p) dists = l2_dist.forward(out_a, out_p) dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(axis=1) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) # err, accuracy= evaluate_eer(distances,labels) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', 100. * eer, epoch) writer.add_scalar('Test/Threshold', eer_threshold, epoch) print('\33[91mFor %s_distance, Test ERR is %.4f %%. Threshold is %.4f . Valid Accuracy is %.4f %%.\33[0m' % ( \ 'cos' if args.cos_sim else 'l2', 100. * eer, eer_threshold, valid_accuracy))