def test(test_loader, model, epoch):
# switch to evaluate mode
model.eval()
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
current_sample = data_a.size(0)
data_a = data_a.resize_(args.test_input_per_file * current_sample, 1,
data_a.size(2), data_a.size(3))
data_p = data_p.resize_(args.test_input_per_file * current_sample, 1,
data_a.size(2), data_a.size(3))
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a, volatile=True), \
Variable(data_p, volatile=True), Variable(label)
# compute output
out_a, out_p = model(data_a), model(data_p)
dists = l2_dist.forward(out_a, out_p)
dists = dists.data.cpu().numpy()
dists = dists.reshape(current_sample,
args.test_input_per_file).mean(axis=1)
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
# if epoch==4:
# np.save('Data/4_distance.npy', distances)
# np.save('Data/4_label.npy', labels)
# err, accuracy= evaluate_eer(distances,labels)
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test_Result/eer', eer, epoch)
writer.add_scalar('Test_Result/threshold', eer_threshold, epoch)
writer.add_scalar('Test_Result/accuracy', accuracy, epoch)
# tpr, fpr, accuracy, val, far = evaluate(distances, labels)
if args.cos_sim:
print(
'\33[91mFor cos_distance, Test set ERR is {:.8f} when threshold is {}\tAnd test accuracy could be {:.2f}%.\33[0m'
.format(100. * eer, eer_threshold, 100. * accuracy))
else:
print(
'\33[91mFor l2_distance, Test set ERR: {:.8f}%\tBest ACC:{:.8f} \n\33[0m'
.format(100. * eer, accuracy))
def sitw_test(sitw_test_loader, model, epoch):
# switch to evaluate mode
model.eval()
labels, distances = [], []
pbar = tqdm(enumerate(sitw_test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
vec_shape = data_a.shape
# pdb.set_trace()
if vec_shape[1] != 1:
data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
_, out_a_ = model(data_a)
_, out_p_ = model(data_p)
out_a = out_a_
out_p = out_p_
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
if vec_shape[1] != 1:
dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(axis=1)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * vec_shape[0], len(sitw_test_loader.dataset),
100. * batch_idx / len(sitw_test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer_t, eer_threshold_t, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
torch.cuda.empty_cache()
writer.add_scalars('Test/EER', {'sitw_test': 100. * eer_t}, epoch)
writer.add_scalars('Test/Threshold', {'sitw_test': eer_threshold_t}, epoch)
print('\33[91mFor Sitw Test ERR: {:.4f}%, Threshold: {}.\n\33[0m'.format(
100. * eer_t, eer_threshold_t))
def test(test_loader, model, epoch):
# switch to evaluate mode
model.eval()
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), \
Variable(data_p), Variable(label)
# SiResNet34
out_a = model.pre_forward_norm(data_a)
out_p = model.pre_forward_norm(data_p)
dists = l2_dist.forward(out_a, out_p)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test/EER', eer, epoch - 1)
writer.add_scalar('Test/Threshold', eer_threshold, epoch - 1)
print(
'\33[91mFor {}_distance, Test ERR: {:.8f}, Threshold: {:.8f}.\n\33[0m'.
format('cos' if args.cos_sim else 'l2', 100. * eer, eer_threshold))
def verification_test(test_loader, dist_type, log_interval):
# switch to evaluate mode
labels, distances = [], []
dist_fn = nn.CosineSimilarity(
) if dist_type == 'cos' else nn.PairwiseDistance(2)
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
out_a = torch.tensor(data_a)
out_p = torch.tensor(data_p)
dists = dist_fn.forward(out_a, out_p).numpy()
distances.append(dists)
labels.append(label.numpy())
if batch_idx % log_interval == 0:
pbar.set_description('Verification: [{}/{} ({:.0f}%)]'.format(
batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(
distances,
labels,
cos=True if dist_type == 'cos' else False,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels)))
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m\n'.format(
mindcf_01, mindcf_001))
def test(test_loader):
# switch to evaluate mode
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
out_a = torch.tensor(data_a)
out_p = torch.tensor(data_p)
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
dists = dists.numpy()
distances.append(dists)
labels.append(label.numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test: [{}/{} ({:.0f}%)]'.format(
batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
dist_type = 'cos' if args.cos_sim else 'l2'
print('\nFor %s_distance, %d pairs:' % (dist_type, len(labels)))
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f}.\33[0m'.format(
mindcf_01, mindcf_001))
def test(test_loader, valid_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
out, _ = model(data)
if args.loss_type == 'asoft':
predicted_labels, _ = out
else:
predicted_labels = out
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (
predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'
.format(epoch, batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc))
valid_accuracy = 100. * correct / total_datasize
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
torch.cuda.empty_cache()
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
vec_shape = data_a.shape
# pdb.set_trace()
if vec_shape[1] != 1:
data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1,
vec_shape[2], vec_shape[3])
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
_, out_a_ = model(data_a)
_, out_p_ = model(data_p)
out_a = out_a_
out_p = out_p_
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(dim=1)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test/EER', 100. * eer, epoch)
writer.add_scalar('Test/Threshold', eer_threshold, epoch)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
writer.add_scalar('Test/mindcf-0.01', mindcf_01, epoch)
writer.add_scalar('Test/mindcf-0.001', mindcf_001, epoch)
dist_type = 'cos' if args.cos_sim else 'l2'
print('\nFor %s_distance, ' % dist_type)
print(' \33[91mTest ERR is {:.4f}%, Threshold is {}'.format(
100. * eer, eer_threshold))
print(' mindcf-0.01 {:.4f}, mindcf-0.001 {:.4f},'.format(
mindcf_01, mindcf_001))
print(' Valid Accuracy is %.4f %%.\33[0m' % valid_accuracy)
torch.cuda.empty_cache()
def test(valid_loader, test_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
if len(data) != args.batch_size * args.tuple_size:
continue
if args.cuda:
data = data.float().squeeze().cuda()
label = label.cuda()
data, label = Variable(data), Variable(label)
feats, classfier = model(data)
predicted_one_labels = softmax(classfier)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (
predicted_one_labels.cuda() == label.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'
.format(epoch, batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc))
valid_accuracy = 100. * correct / total_datasize
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (a, p, label) in pbar:
if len(a) != int(
args.batch_size * args.tuple_size / args.test_input_per_file):
continue
vec_shape = a.shape
# pdb.set_trace()
data_a = a.reshape(vec_shape[0] * vec_shape[1], vec_shape[2],
vec_shape[3])
data_p = p.reshape(vec_shape[0] * vec_shape[1], vec_shape[2],
vec_shape[3])
if args.cuda:
data_a, data_p = data_a.float().cuda(), data_p.float().cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
out_a, _ = model.tuple_forward(data_a)
out_p, _ = model.tuple_forward(data_p)
dists_a = l2_dist.forward(out_a, out_p)
dists_a = dists_a.data.cpu().numpy()
dists_a = dists_a.reshape(vec_shape[0], vec_shape[1]).mean(axis=1)
distances.append(dists_a)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch,
batch_idx * len(data_a) / args.test_input_per_file,
len(test_loader.dataset), 100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test/EER', 100. * eer, epoch)
print(
'\33[91mERR: {:.8f}. Threshold: {:.8f}. Valid Accuracy is {:.4f}%.\33[0m\n'
.format(100. * eer, eer_threshold, valid_accuracy))
def test(test_loader, valid_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
out, _ = model(data)
predicted_labels = out
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (
predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch for Classification: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'
.format(epoch, batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc))
valid_accuracy = 100. * correct / total_datasize
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
current_sample = data_a.size(0)
data_a = data_a.resize_(args.test_input_per_file * current_sample, 1,
data_a.size(2), data_a.size(3))
data_p = data_p.resize_(args.test_input_per_file * current_sample, 1,
data_a.size(2), data_a.size(3))
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
_, out_a = model(data_a)
_, out_p = model(data_p)
dists = l2_dist.forward(
out_a, out_p
) # torch.sqrt(torch.sum((out_a - out_p) ** 2, 1)) # euclidean distance
dists = dists.data.cpu().numpy()
dists = dists.reshape(current_sample,
args.test_input_per_file).mean(axis=1)
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a), len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
writer.add_scalar('Test/EER', eer, epoch)
writer.add_scalar('Test/Threshold', eer_threshold, epoch)
if args.cos_sim:
print(
'\33[91mFor cos_distance, Test set verification ERR is {:.8f}%, when threshold is {}. Valid set classificaton accuracy is {:.2f}%.\n\33[0m'
.format(100. * eer, eer_threshold, valid_accuracy))
else:
print(
'\33[91mFor l2_distance, Test set verification ERR is {:.8f}%, when threshold is {}. Valid set classificaton accuracy is {:.2f}%.\n\33[0m'
.format(100. * eer, eer_threshold, valid_accuracy))
def test(test_loader, valid_loader, model, epoch):
# net = model.to('cuda:1')
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
out = model.pre_forward(data)
cls = model(out)
predicted_labels = cls
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch for Classification: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'.format(
epoch,
batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc
))
valid_accuracy = 100. * correct / total_datasize
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
# test_set = testset.TestSet('../all_feature/')
# todo:
# test_set = []
distances = []
labels = []
# testloader = DataLoader(test_set, batch_size=128, num_workers=16, shuffle=True)
# torch.set_num_threads(16)
pbar = tqdm(enumerate(test_loader))
# .set_trace()
for batch_idx, (data_a, data_p, label) in pbar:
current_sample = data_a.size(0)
data_a = data_a.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3))
data_p = data_p.resize_(args.test_input_per_file * current_sample, 1, data_a.size(2), data_a.size(3))
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a, volatile=True), \
Variable(data_p, volatile=True), Variable(label)
# compute output
x_vector_a, x_vector_p = model.pre_forward(data_a), model.pre_forward(data_p)
dists = l2_dist.forward(x_vector_a, x_vector_p)
dists = dists.data.cpu().numpy()
dists = dists.reshape(current_sample, args.test_input_per_file).mean(axis=1)
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
# err, accuracy= evaluate_eer(distances,labels)
# err, accuracy= evaluate_eer(distances,labels)
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True)
writer.add_scalar('Test/EER', eer, epoch)
writer.add_scalar('Test/Threshold', eer_threshold, epoch)
# tpr, fpr, accuracy, val, far = evaluate(distances, labels)
if args.cos_sim:
print(
'\33[91mFor cos_distance, Test set ERR is {:.4f}, threshold is {}. Valid accuracy could be {:.2f}%.\n\33[0m'.format(
100. * eer, eer_threshold, valid_accuracy))
else:
print(
'\33[91mFor l2_distance, Test set ERR: {:.8f}%\tBest ACC:{:.8f}\n\33[0m'.format(100. * eer, valid_accuracy))
def test(test_loader, valid_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
# pdb.set_trace()
out = model.pre_forward(data)
cls = model(out)
predicted_labels = cls
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (
predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'
.format(epoch, batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc))
valid_accuracy = 100. * correct / total_datasize
labels, distances = [], []
distances_b = []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(
label)
# compute output
out_a = model.pre_forward(data_a)
out_p = model.pre_forward(data_p)
out_ab = model.segment7(out_a)
out_ab = model.relu(model.batch_norm7(out_ab))
out_pb = model.segment7(out_p)
out_pb = model.relu(model.batch_norm7(out_pb))
dists = l2_dist.forward(out_a, out_p)
dists = dists.data.cpu().numpy()
distances.append(dists)
distsb = l2_dist.forward(out_ab, out_pb)
distsb = distsb.data.cpu().numpy()
distances_b.append(distsb)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch,
batch_idx * len(data_a) / args.test_input_per_file,
len(test_loader.dataset), 100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
distances_b = np.array(
[subdist for dist in distances_b for subdist in dist])
# err, accuracy= evaluate_eer(distances,labels)
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances,
labels,
cos=args.cos_sim,
re_thre=True)
eer_b, eer_threshold_b, accuracy_b = evaluate_kaldi_eer(distances_b,
labels,
cos=args.cos_sim,
re_thre=True)
print('\33[91m Epoch {}, Valid Accuracy is {}. For {}_distance:'.format(
epoch, valid_accuracy, 'cos' if args.cos_sim else 'l2'))
print(' For Embedding a: Test ERR: {:.8f}. Threshold: {:.8f}. \n'
' For Embedding b: Test ERR: {:.8f}. Threshold: {:.8f}.\n\33[0m'.
format(100. * eer, eer_threshold, 100. * eer_b, eer_threshold_b))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
<<<<<<< HEAD
if save != '':
np.save('%s/labels.npy' % save, labels)
np.save('%s/distance.npy' % save, distances)
=======
# this_xvector_dir = "%s/epoch_%s" % (xvector_dir, epoch)
with open('%s/scores' % xvector_dir, 'w') as f:
for d, l in zip(distances, labels):
f.write(str(d) + ' ' + str(l) + '\n')
>>>>>>> Server/Server
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels,
cos=True if dist_type == 'cos' else False,
re_thre=True)
mindcf_01, mindcf_001 = evaluate_kaldi_mindcf(distances, labels)
return eer, eer_threshold, mindcf_01, mindcf_001
# https://github.com/clovaai/voxceleb_trainer
def accuracy(output, target, topk=(1,)):
"""Computes the precision@k for the specified values of k"""
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
def test(test_loader, valid_loader, model, epoch):
# switch to evaluate mode
model.eval()
valid_pbar = tqdm(enumerate(valid_loader))
softmax = nn.Softmax(dim=1)
correct = 0.
total_datasize = 0.
for batch_idx, (data, label) in valid_pbar:
data = Variable(data.cuda())
# compute output
# pdb.set_trace()
out = model.att_forward_norm(data)
cls = model(out)
predicted_labels = cls
true_labels = Variable(label.cuda())
# pdb.set_trace()
predicted_one_labels = softmax(predicted_labels)
predicted_one_labels = torch.max(predicted_one_labels, dim=1)[1]
batch_correct = (predicted_one_labels.cuda() == true_labels.cuda()).sum().item()
minibatch_acc = float(batch_correct / len(predicted_one_labels))
correct += batch_correct
total_datasize += len(predicted_one_labels)
if batch_idx % args.log_interval == 0:
valid_pbar.set_description(
'Valid Epoch: {:2d} [{:8d}/{:8d} ({:3.0f}%)] Batch Accuracy: {:.4f}%'.format(
epoch,
batch_idx * len(data),
len(valid_loader.dataset),
100. * batch_idx / len(valid_loader),
100. * minibatch_acc
))
valid_accuracy = 100. * correct / total_datasize
writer.add_scalar('Test/Valid_Accuracy', valid_accuracy, epoch)
labels, distances = [], []
pbar = tqdm(enumerate(test_loader))
for batch_idx, (data_a, data_p, label) in pbar:
vec_shape = data_a.shape
# pdb.set_trace()
data_a = data_a.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3])
data_p = data_p.reshape(vec_shape[0] * vec_shape[1], 1, vec_shape[2], vec_shape[3])
if args.cuda:
data_a, data_p = data_a.cuda(), data_p.cuda()
data_a, data_p, label = Variable(data_a), Variable(data_p), Variable(label)
# compute output
out_a = model.att_forward_norm(data_a)
out_p = model.att_forward_norm(data_p)
dists = l2_dist.forward(out_a, out_p)
dists = dists.reshape(vec_shape[0], vec_shape[1]).mean(axis=1)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
if batch_idx % args.log_interval == 0:
pbar.set_description('Test Epoch: {} [{}/{} ({:.0f}%)]'.format(
epoch, batch_idx * len(data_a) / args.test_input_per_file, len(test_loader.dataset),
100. * batch_idx / len(test_loader)))
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array([subdist for dist in distances for subdist in dist])
# err, accuracy= evaluate_eer(distances,labels)
eer, eer_threshold, accuracy = evaluate_kaldi_eer(distances, labels, cos=args.cos_sim, re_thre=True)
writer.add_scalar('Test/EER', 100. * eer, epoch)
writer.add_scalar('Test/Threshold', eer_threshold, epoch)
print('\33[91mFor %s_distance, Test ERR is %.4f %%. Threshold is %.4f . Valid Accuracy is %.4f %%.\33[0m' % ( \
'cos' if args.cos_sim else 'l2', 100. * eer, eer_threshold, valid_accuracy))
