def verification_test(test_loader, dist_type, log_interval): # switch to evaluate mode labels, distances = [], [] dist_fn = nn.CosineSimilarity( ) if dist_type == 'cos' else nn.PairwiseDistance(2) pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: out_a = torch.tensor(data_a) out_p = torch.tensor(data_p) dists = dist_fn.forward(out_a, out_p).numpy() distances.append(dists) labels.append(label.numpy()) if batch_idx % log_interval == 0: pbar.set_description('Verification: [{}/{} ({:.0f}%)]'.format( batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer( distances, labels, cos=True if dist_type == 'cos' else False, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels))) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m\n'.format( mindcf_01, mindcf_001))
def test(test_loader): # switch to evaluate mode labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: out_a = torch.tensor(data_a) out_p = torch.tensor(data_p) dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance dists = dists.numpy() distances.append(dists) labels.append(label.numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test: [{}/{} ({:.0f}%)]'.format( batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) dist_type = 'cos' if args.cos_sim else 'l2' print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels))) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m'.format( mindcf_01, mindcf_001))
def test(test_loader, valid_loader, model, epoch): # switch to evaluate mode model.eval() valid_pbar = tqdm(enumerate(valid_loader)) softmax = nn.Softmax(dim=1) correct = 0. total_datasize = 0. for batch_idx, (data, label) in valid_pbar: data = Variable(data.cuda()) # compute output out, _ = model(data) if args.loss_type == 'asoft': predicted_labels, _ = out else: predicted_labels = out true_labels = Variable(label.cuda()) # pdb.set_trace() predicted_one_labels = softmax(predicted_labels) predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1] batch_correct = ( predicted_one_labels.cuda() == true_labels.cuda()).sum().item() minibatch_acc = float(batch_correct / len(predicted_one_labels)) correct += batch_correct total_datasize += len(predicted_one_labels) if batch_idx % args.log_interval == 0: valid_pbar.set_description( 'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%' .format(epoch, batch_idx * len(data), len(valid_loader.dataset), 100. * batch_idx / len(valid_loader), 100. * minibatch_acc)) valid_accuracy = 100. * correct / total_datasize writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch) torch.cuda.empty_cache() labels, distances = [], [] pbar = tqdm(enumerate(test_loader)) for batch_idx, (data_a, data_p, label) in pbar: vec_shape = data_a.shape # pdb.set_trace() if vec_shape[1] != 1: data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3]) if args.cuda: data_a, data_p = data_a.cuda(), data_p.cuda() data_a, data_p, label = Variable(data_a), Variable(data_p), Variable( label) # compute output _, out_a_ = model(data_a) _, out_p_ = model(data_p) out_a = out_a_ out_p = out_p_ dists = l2_dist.forward( out_a, out_p ) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(dim=1) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) if batch_idx % args.log_interval == 0: pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format( epoch, batch_idx * len(data_a), len(test_loader.dataset), 100. * batch_idx / len(test_loader))) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array([subdist for dist in distances for subdist in dist]) eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True) writer.add_scalar('Test/EER', 100. * eer, epoch) writer.add_scalar('Test/Threshold', eer_threshold, epoch) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) writer.add_scalar('Test/mindcf-0.01', mindcf_01, epoch) writer.add_scalar('Test/mindcf-0.001', mindcf_001, epoch) dist_type = 'cos' if args.cos_sim else 'l2' print('\nFor %s_distance, ' % dist_type) print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format( 100. * eer, eer_threshold)) print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f},'.format( mindcf_01, mindcf_001)) print(' Valid Accuracy is %.4f %%.\33[0m' % valid_accuracy) torch.cuda.empty_cache()
distances = np.array([subdist for dist in distances for subdist in dist]) <<<<<<< HEAD if save != '': np.save('%s/labels.npy' % save, labels) np.save('%s/distance.npy' % save, distances) ======= # this_xvector_dir = "%s/epoch_%s" % (xvector_dir, epoch) with open('%s/scores' % xvector_dir, 'w') as f: for d, l in zip(distances, labels): f.write(str(d) + ' ' + str(l) + '\n') >>>>>>> Server/Server eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=True if dist_type == 'cos' else False, re_thre=True) mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels) return eer, eer_threshold, mindcf_01, mindcf_001 # https://github.com/clovaai/voxceleb_trainer def accuracy(output, target, topk=(1,)): """Computes the precision@k for the specified values of k""" maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) res = []