def verification_test(test_loader, dist_type, log_interval):
# switch to evaluate mode
labels, distances = [], []
dist_fn = nn.CosineSimilarity(
) if dist_type == 'cos' else nn.PairwiseDistance(2)
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
out_a = torch.tensor(data_a)
out_p = torch.tensor(data_p)
dists = dist_fn.forward(out_a, out_p).numpy()
distances.append(dists)
labels.append(label.numpy())
if batch_idx % log_interval == 0:
pbar.set_description('Verification: [{}/{} ({:.0f}%)]'.format(
batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(
distances,
labels,
cos=True if dist_type == 'cos' else False,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels)))
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m\n'.format(
mindcf_01, mindcf_001))
def test(test_loader):
# switch to evaluate mode
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
out_a = torch.tensor(data_a)
out_p = torch.tensor(data_p)
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
dists = dists.numpy()
distances.append(dists)
labels.append(label.numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test: [{}/{} ({:.0f}%)]'.format(
batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
dist_type = 'cos' if args.cos_sim else 'l2'
print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels)))
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m'.format(
mindcf_01, mindcf_001))
def test(test_loader, valid_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
out, _ = model(data)
if args.loss_type == 'asoft':
predicted_labels, _ = out
else:
predicted_labels = out
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (
predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'
.format(epoch, batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc))
valid_accuracy = 100. * correct / total_datasize
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
torch.cuda.empty_cache()
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
vec_shape = data_a.shape
# pdb.set_trace()
if vec_shape[1] != 1:
data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
_, out_a_ = model(data_a)
_, out_p_ = model(data_p)
out_a = out_a_
out_p = out_p_
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(dim=1)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test/EER', 100. * eer, epoch)
writer.add_scalar('Test/Threshold', eer_threshold, epoch)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
writer.add_scalar('Test/mindcf-0.01', mindcf_01, epoch)
writer.add_scalar('Test/mindcf-0.001', mindcf_001, epoch)
dist_type = 'cos' if args.cos_sim else 'l2'
print('\nFor %s_distance, ' % dist_type)
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f},'.format(
mindcf_01, mindcf_001))
print(' Valid Accuracy is %.4f %%.\33[0m' % valid_accuracy)
torch.cuda.empty_cache()
distances = np.array([subdist for dist in distances for subdist in dist])
<<<<<<< HEAD
if save != '':
np.save('%s/labels.npy' % save, labels)
np.save('%s/distance.npy' % save, distances)
=======
# this_xvector_dir = "%s/epoch_%s" % (xvector_dir, epoch)
with open('%s/scores' % xvector_dir, 'w') as f:
for d, l in zip(distances, labels):
f.write(str(d) + ' ' + str(l) + '\n')
>>>>>>> Server/Server
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels,
cos=True if dist_type == 'cos' else False,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
return eer, eer_threshold, mindcf_01, mindcf_001
# https://github.com/clovaai/voxceleb_trainer
def accuracy(output, target, topk=(1,)):
"""Computes the precision@k for the specified values of k"""
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []