def test_individual_attacks(cm_score_file):
asv_score_file = os.path.join(
'/data/neil/DS_10283_3336',
'LA/ASVspoof2019_LA_asv_scores/ASVspoof2019.LA.asv.eval.gi.trl.scores.txt'
)
# Fix tandem detection cost function (t-DCF) parameters
Pspoof = 0.05
cost_model = {
'Pspoof': Pspoof, # Prior probability of a spoofing attack
'Ptar': (1 - Pspoof) * 0.99, # Prior probability of target speaker
'Pnon': (1 - Pspoof) * 0.01, # Prior probability of nontarget speaker
'Cmiss_asv': 1, # Cost of ASV system falsely rejecting target speaker
'Cfa_asv':
10, # Cost of ASV system falsely accepting nontarget speaker
'Cmiss_cm': 1, # Cost of CM system falsely rejecting target speaker
'Cfa_cm': 10, # Cost of CM system falsely accepting spoof
}
# Load organizers' ASV scores
asv_data = np.genfromtxt(asv_score_file, dtype=str)
asv_sources = asv_data[:, 0]
asv_keys = asv_data[:, 1]
asv_scores = asv_data[:, 2].astype(np.float)
# Load CM scores
cm_data = np.genfromtxt(cm_score_file, dtype=str)
cm_utt_id = cm_data[:, 0]
cm_sources = cm_data[:, 1]
cm_keys = cm_data[:, 2]
cm_scores = cm_data[:, 3].astype(np.float)
other_cm_scores = -cm_scores
eer_cm_lst, min_tDCF_lst = [], []
for attack_idx in range(7, 20):
# Extract target, nontarget, and spoof scores from the ASV scores
tar_asv = asv_scores[asv_keys == 'target']
non_asv = asv_scores[asv_keys == 'nontarget']
spoof_asv = asv_scores[asv_sources == 'A%02d' % attack_idx]
# Extract bona fide (real human) and spoof scores from the CM scores
bona_cm = cm_scores[cm_keys == 'bonafide']
spoof_cm = cm_scores[cm_sources == 'A%02d' % attack_idx]
# EERs of the standalone systems and fix ASV operating point to EER threshold
eer_asv, asv_threshold = em.compute_eer(tar_asv, non_asv)
eer_cm = em.compute_eer(bona_cm, spoof_cm)[0]
other_eer_cm = em.compute_eer(
other_cm_scores[cm_keys == 'bonafide'],
other_cm_scores[cm_sources == 'A%02d' % attack_idx])[0]
[Pfa_asv, Pmiss_asv,
Pmiss_spoof_asv] = em.obtain_asv_error_rates(tar_asv, non_asv,
spoof_asv, asv_threshold)
if eer_cm < other_eer_cm:
# Compute t-DCF
tDCF_curve, CM_thresholds = em.compute_tDCF(
bona_cm, spoof_cm, Pfa_asv, Pmiss_asv, Pmiss_spoof_asv,
cost_model, True)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
else:
tDCF_curve, CM_thresholds = em.compute_tDCF(
other_cm_scores[cm_keys == 'bonafide'],
other_cm_scores[cm_sources == 'A%02d' % attack_idx], Pfa_asv,
Pmiss_asv, Pmiss_spoof_asv, cost_model, True)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
eer_cm_lst.append(min(eer_cm, other_eer_cm))
min_tDCF_lst.append(min_tDCF)
return eer_cm_lst, min_tDCF_lst
cm_utt_id = cm_data[:, 0]
cm_sources = cm_data[:, 1]
cm_keys = cm_data[:, 2]
cm_scores = cm_data[:, 3].astype(np.float)
# Extract target, nontarget, and spoof scores from the ASV scores
tar_asv = asv_scores[asv_keys == 'target']
non_asv = asv_scores[asv_keys == 'nontarget']
spoof_asv = asv_scores[asv_keys == 'spoof']
# Extract bona fide (real human) and spoof scores from the CM scores
bona_cm = cm_scores[cm_keys == 'bonafide']
spoof_cm = cm_scores[cm_keys == 'spoof']
# EERs of the standalone systems and fix ASV operating point to EER threshold
eer_asv, asv_threshold = em.compute_eer(tar_asv, non_asv)
eer_cm = em.compute_eer(bona_cm, spoof_cm)[0]
[Pfa_asv, Pmiss_asv, Pmiss_spoof_asv] = em.obtain_asv_error_rates(tar_asv, non_asv, spoof_asv, asv_threshold)
# Compute t-DCF
tDCF_curve, CM_thresholds = em.compute_tDCF(bona_cm, spoof_cm, Pfa_asv, Pmiss_asv, Pmiss_spoof_asv, cost_model, True)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
print('ASV SYSTEM')
def compute_tDCF(self):
'''
Computes the t-DCF score for the model assigned to self.model.
Assumes that the cm score file is created for the model.
'''
# Fix tandem detection cost function (t-DCF) parameters
cost_model = {
'Pspoof': self.p_spoof, # Prior probability of a spoofing attack
'Ptar':
(1 - self.p_spoof) * 0.99, # Prior probability of target speaker
'Pnon': (1 - self.p_spoof) *
0.01, # Prior probability of nontarget speaker
'Cmiss_asv':
1, # Cost of ASV system falsely rejecting target speaker
'Cfa_asv':
10, # Cost of ASV system falsely accepting nontarget speaker
'Cmiss_cm':
1, # Cost of CM system falsely rejecting target speaker
'Cfa_cm': 10, # Cost of CM system falsely accepting spoof
}
# Load organizers' ASV scores
asv_data = np.genfromtxt(self.asv_score_file, dtype=str)
asv_sources = asv_data[:, 0]
asv_keys = asv_data[:, 1]
asv_scores = asv_data[:, 2].astype(np.float)
# Load CM scores
cm_data = np.genfromtxt(self.cm_score_file, dtype=str)
cm_utt_id = cm_data[:, 0]
cm_sources = cm_data[:, 1]
cm_keys = cm_data[:, 2]
cm_scores = cm_data[:, 3].astype(np.float)
#Modify CM not to contain inf or -inf values. They appear if the model is certain about some result
if (cm_scores[cm_scores != float("inf")] != -float("inf")).any():
real_max = cm_scores[cm_scores != float("inf")].max()
real_min = cm_scores[cm_scores != -float("inf")].min()
max_abs = min(20, max(abs(real_max), abs(real_min)))
else:
max_abs = 20
cm_scores[cm_scores == float("inf")] = max_abs
cm_scores[cm_scores == -float("inf")] = -max_abs
# Extract target, nontarget, and spoof scores from the ASV scores
tar_asv = asv_scores[asv_keys == 'target']
non_asv = asv_scores[asv_keys == 'nontarget']
spoof_asv = asv_scores[asv_keys == 'spoof']
# Extract bona fide (real human) and spoof scores from the CM scores
bona_cm = cm_scores[cm_keys == 'bonafide']
spoof_cm = cm_scores[cm_keys == 'spoof']
# EERs of the standalone systems and fix ASV operating point to EER threshold
eer_asv, asv_threshold = em.compute_eer(tar_asv, non_asv)
eer_cm = em.compute_eer(bona_cm, spoof_cm)[0]
[Pfa_asv, Pmiss_asv,
Pmiss_spoof_asv] = em.obtain_asv_error_rates(tar_asv, non_asv,
spoof_asv, asv_threshold)
# Compute t-DCF
tDCF_curve, CM_thresholds = em.compute_tDCF(bona_cm, spoof_cm, Pfa_asv,
Pmiss_asv, Pmiss_spoof_asv,
cost_model, False)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
return min_tDCF, eer_cm
def train(args):
torch.set_default_tensor_type(torch.FloatTensor)
# initialize model
lfcc_model = ResNet(3, args.enc_dim, resnet_type='18', nclasses=2).to(args.device)
if args.continue_training:
lfcc_model = torch.load(os.path.join(args.out_fold, 'anti-spoofing_lfcc_model.pt')).to(args.device)
lfcc_optimizer = torch.optim.Adam(lfcc_model.parameters(), lr=args.lr,
betas=(args.beta_1, args.beta_2), eps=args.eps, weight_decay=0.0005)
training_set = ASVspoof2019(args.access_type, args.path_to_features, args.path_to_protocol, 'train',
'LFCC', feat_len=args.feat_len, padding=args.padding)
validation_set = ASVspoof2019(args.access_type, args.path_to_features, args.path_to_protocol, 'dev',
'LFCC', feat_len=args.feat_len, padding=args.padding)
trainDataLoader = DataLoader(training_set, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
collate_fn=training_set.collate_fn)
valDataLoader = DataLoader(validation_set, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
collate_fn=validation_set.collate_fn)
feat, _, _, _ = training_set[29]
print("Feature shape", feat.shape)
criterion = nn.CrossEntropyLoss()
if args.add_loss == "amsoftmax":
amsoftmax_loss = AMSoftmax(2, args.enc_dim, s=args.alpha, m=args.r_real).to(args.device)
amsoftmax_loss.train()
amsoftmax_optimzer = torch.optim.SGD(amsoftmax_loss.parameters(), lr=0.01)
if args.add_loss == "ocsoftmax":
ocsoftmax = OCSoftmax(args.enc_dim, r_real=args.r_real, r_fake=args.r_fake, alpha=args.alpha).to(args.device)
ocsoftmax.train()
ocsoftmax_optimzer = torch.optim.SGD(ocsoftmax.parameters(), lr=args.lr)
early_stop_cnt = 0
prev_eer = 1e8
monitor_loss = args.add_loss
for epoch_num in tqdm(range(args.num_epochs)):
lfcc_model.train()
trainlossDict = defaultdict(list)
devlossDict = defaultdict(list)
adjust_learning_rate(args, lfcc_optimizer, epoch_num)
if args.add_loss == "ocsoftmax":
adjust_learning_rate(args, ocsoftmax_optimzer, epoch_num)
elif args.add_loss == "amsoftmax":
adjust_learning_rate(args, amsoftmax_optimzer, epoch_num)
print('\nEpoch: %d ' % (epoch_num + 1))
for i, (lfcc, audio_fn, tags, labels) in enumerate(tqdm(trainDataLoader)):
lfcc = lfcc.unsqueeze(1).float().to(args.device)
labels = labels.to(args.device)
feats, lfcc_outputs = lfcc_model(lfcc)
lfcc_loss = criterion(lfcc_outputs, labels)
if args.add_loss == "softmax":
lfcc_optimizer.zero_grad()
trainlossDict[args.add_loss].append(lfcc_loss.item())
lfcc_loss.backward()
lfcc_optimizer.step()
if args.add_loss == "ocsoftmax":
ocsoftmaxloss, _ = ocsoftmax(feats, labels)
lfcc_loss = ocsoftmaxloss * args.weight_loss
lfcc_optimizer.zero_grad()
ocsoftmax_optimzer.zero_grad()
trainlossDict[args.add_loss].append(ocsoftmaxloss.item())
lfcc_loss.backward()
lfcc_optimizer.step()
ocsoftmax_optimzer.step()
if args.add_loss == "amsoftmax":
outputs, moutputs = amsoftmax_loss(feats, labels)
lfcc_loss = criterion(moutputs, labels)
trainlossDict[args.add_loss].append(lfcc_loss.item())
lfcc_optimizer.zero_grad()
amsoftmax_optimzer.zero_grad()
lfcc_loss.backward()
lfcc_optimizer.step()
amsoftmax_optimzer.step()
with open(os.path.join(args.out_fold, "train_loss.log"), "a") as log:
log.write(str(epoch_num) + "\t" + str(i) + "\t" +
str(np.nanmean(trainlossDict[monitor_loss])) + "\n")
# Val the model
lfcc_model.eval()
with torch.no_grad():
idx_loader, score_loader = [], []
for i, (lfcc, audio_fn, tags, labels) in enumerate(tqdm(valDataLoader)):
lfcc = lfcc.unsqueeze(1).float().to(args.device)
labels = labels.to(args.device)
feats, lfcc_outputs = lfcc_model(lfcc)
lfcc_loss = criterion(lfcc_outputs, labels)
score = F.softmax(lfcc_outputs, dim=1)[:, 0]
if args.add_loss == "softmax":
devlossDict["softmax"].append(lfcc_loss.item())
elif args.add_loss == "amsoftmax":
outputs, moutputs = amsoftmax_loss(feats, labels)
lfcc_loss = criterion(moutputs, labels)
score = F.softmax(outputs, dim=1)[:, 0]
devlossDict[args.add_loss].append(lfcc_loss.item())
elif args.add_loss == "ocsoftmax":
ocsoftmaxloss, score = ocsoftmax(feats, labels)
devlossDict[args.add_loss].append(ocsoftmaxloss.item())
idx_loader.append(labels)
score_loader.append(score)
scores = torch.cat(score_loader, 0).data.cpu().numpy()
labels = torch.cat(idx_loader, 0).data.cpu().numpy()
val_eer = em.compute_eer(scores[labels == 0], scores[labels == 1])[0]
with open(os.path.join(args.out_fold, "dev_loss.log"), "a") as log:
log.write(str(epoch_num) + "\t" + str(np.nanmean(devlossDict[monitor_loss])) + "\t" + str(val_eer) +"\n")
print("Val EER: {}".format(val_eer))
torch.save(lfcc_model, os.path.join(args.out_fold, 'checkpoint',
'anti-spoofing_lfcc_model_%d.pt' % (epoch_num + 1)))
if args.add_loss == "ocsoftmax":
loss_model = ocsoftmax
torch.save(loss_model, os.path.join(args.out_fold, 'checkpoint',
'anti-spoofing_loss_model_%d.pt' % (epoch_num + 1)))
elif args.add_loss == "amsoftmax":
loss_model = amsoftmax_loss
torch.save(loss_model, os.path.join(args.out_fold, 'checkpoint',
'anti-spoofing_loss_model_%d.pt' % (epoch_num + 1)))
else:
loss_model = None
if val_eer < prev_eer:
# Save the model checkpoint
torch.save(lfcc_model, os.path.join(args.out_fold, 'anti-spoofing_lfcc_model.pt'))
if args.add_loss == "ocsoftmax":
loss_model = ocsoftmax
torch.save(loss_model, os.path.join(args.out_fold, 'anti-spoofing_loss_model.pt'))
elif args.add_loss == "amsoftmax":
loss_model = amsoftmax_loss
torch.save(loss_model, os.path.join(args.out_fold, 'anti-spoofing_loss_model.pt'))
else:
loss_model = None
prev_eer = val_eer
early_stop_cnt = 0
else:
early_stop_cnt += 1
if early_stop_cnt == 100:
with open(os.path.join(args.out_fold, 'args.json'), 'a') as res_file:
res_file.write('\nTrained Epochs: %d\n' % (epoch_num - 19))
break
return lfcc_model, loss_model
def compute_eer_and_tdcf(cm_score_file, path_to_database):
asv_score_file = os.path.join(
path_to_database,
'LA/ASVspoof2019_LA_asv_scores/ASVspoof2019.LA.asv.eval.gi.trl.scores.txt'
)
# Fix tandem detection cost function (t-DCF) parameters
Pspoof = 0.05
cost_model = {
'Pspoof': Pspoof, # Prior probability of a spoofing attack
'Ptar': (1 - Pspoof) * 0.99, # Prior probability of target speaker
'Pnon': (1 - Pspoof) * 0.01, # Prior probability of nontarget speaker
'Cmiss_asv': 1, # Cost of ASV system falsely rejecting target speaker
'Cfa_asv':
10, # Cost of ASV system falsely accepting nontarget speaker
'Cmiss_cm': 1, # Cost of CM system falsely rejecting target speaker
'Cfa_cm': 10, # Cost of CM system falsely accepting spoof
}
# Load organizers' ASV scores
asv_data = np.genfromtxt(asv_score_file, dtype=str)
asv_sources = asv_data[:, 0]
asv_keys = asv_data[:, 1]
asv_scores = asv_data[:, 2].astype(np.float)
# Load CM scores
cm_data = np.genfromtxt(cm_score_file, dtype=str)
cm_utt_id = cm_data[:, 0]
cm_sources = cm_data[:, 1]
cm_keys = cm_data[:, 2]
cm_scores = cm_data[:, 3].astype(np.float)
other_cm_scores = -cm_scores
# Extract target, nontarget, and spoof scores from the ASV scores
tar_asv = asv_scores[asv_keys == 'target']
non_asv = asv_scores[asv_keys == 'nontarget']
spoof_asv = asv_scores[asv_keys == 'spoof']
# Extract bona fide (real human) and spoof scores from the CM scores
bona_cm = cm_scores[cm_keys == 'bonafide']
spoof_cm = cm_scores[cm_keys == 'spoof']
# EERs of the standalone systems and fix ASV operating point to EER threshold
eer_asv, asv_threshold = em.compute_eer(tar_asv, non_asv)
eer_cm = em.compute_eer(bona_cm, spoof_cm)[0]
other_eer_cm = em.compute_eer(other_cm_scores[cm_keys == 'bonafide'],
other_cm_scores[cm_keys == 'spoof'])[0]
[Pfa_asv, Pmiss_asv,
Pmiss_spoof_asv] = em.obtain_asv_error_rates(tar_asv, non_asv, spoof_asv,
asv_threshold)
if eer_cm < other_eer_cm:
# Compute t-DCF
tDCF_curve, CM_thresholds = em.compute_tDCF(bona_cm, spoof_cm, Pfa_asv,
Pmiss_asv, Pmiss_spoof_asv,
cost_model, True)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
else:
tDCF_curve, CM_thresholds = em.compute_tDCF(
other_cm_scores[cm_keys == 'bonafide'],
other_cm_scores[cm_keys == 'spoof'], Pfa_asv, Pmiss_asv,
Pmiss_spoof_asv, cost_model, True)
# Minimum t-DCF
min_tDCF_index = np.argmin(tDCF_curve)
min_tDCF = tDCF_curve[min_tDCF_index]
# print('ASV SYSTEM')
# print(' EER = {:8.5f} % (Equal error rate (target vs. nontarget discrimination)'.format(eer_asv * 100))
# print(' Pfa = {:8.5f} % (False acceptance rate of nontargets)'.format(Pfa_asv * 100))
# print(' Pmiss = {:8.5f} % (False rejection rate of targets)'.format(Pmiss_asv * 100))
# print(' 1-Pmiss,spoof = {:8.5f} % (Spoof false acceptance rate)'.format((1 - Pmiss_spoof_asv) * 100))
print('\nCM SYSTEM')
print(
' EER = {:8.5f} % (Equal error rate for countermeasure)'.
format(min(eer_cm, other_eer_cm) * 100))
print('\nTANDEM')
print(' min-tDCF = {:8.5f}'.format(min_tDCF))
# Visualize ASV scores and CM scores
plt.figure()
ax = plt.subplot(121)
plt.hist(tar_asv, histtype='step', density=True, bins=50, label='Target')
plt.hist(non_asv,
histtype='step',
density=True,
bins=50,
label='Nontarget')
plt.hist(spoof_asv, histtype='step', density=True, bins=50, label='Spoof')
plt.plot(asv_threshold,
0,
'o',
markersize=10,
mfc='none',
mew=2,
clip_on=False,
label='EER threshold')
plt.legend()
plt.xlabel('ASV score')
plt.ylabel('Density')
plt.title('ASV score histogram')
ax = plt.subplot(122)
plt.hist(bona_cm,
histtype='step',
density=True,
bins=50,
label='Bona fide')
plt.hist(spoof_cm, histtype='step', density=True, bins=50, label='Spoof')
plt.legend()
plt.xlabel('CM score')
# plt.ylabel('Density')
plt.title('CM score histogram')
plt.savefig(cm_score_file[:-4] + '1.png')
# Plot t-DCF as function of the CM threshold.
plt.figure()
plt.plot(CM_thresholds, tDCF_curve)
plt.plot(CM_thresholds[min_tDCF_index],
min_tDCF,
'o',
markersize=10,
mfc='none',
mew=2)
plt.xlabel('CM threshold index (operating point)')
plt.ylabel('Norm t-DCF')
plt.title('Normalized tandem t-DCF')
plt.plot(
[np.min(CM_thresholds), np.max(CM_thresholds)], [1, 1],
'--',
color='black')
plt.legend(('t-DCF', 'min t-DCF ({:.5f})'.format(min_tDCF),
'Arbitrarily bad CM (Norm t-DCF=1)'))
plt.xlim([np.min(CM_thresholds), np.max(CM_thresholds)])
plt.ylim([0, 1.5])
plt.savefig(cm_score_file[:-4] + '2.png')
plt.show()
return min(eer_cm, other_eer_cm), min_tDCF