def eval_calibration(in_score_file, ndx_file, model_file, out_score_file):
logging.info('load ndx: %s' % ndx_file)
try:
ndx = TrialNdx.load_txt(ndx_file)
except:
ndx = TrialKey.load_txt(ndx_file)
logging.info('load scores: %s' % in_score_file)
scr = TrialScores.load_txt(in_score_file)
scr = scr.align_with_ndx(ndx)
logging.info('load model: %s' % model_file)
lr = LR.load(model_file)
logging.info('apply calibration')
s_cal = lr.predict(scr.scores.ravel())
scr.scores = np.reshape(s_cal, scr.scores.shape)
logging.info('save scores: %s' % out_score_file)
scr.save_txt(out_score_file)
def train_calibration(score_file, key_file, model_file, prior, verbose):
logging.info('load key: %s' % key_file)
key = TrialKey.load_txt(key_file)
logging.info('load scores: %s' % score_file)
scr = TrialScores.load_txt(score_file)
tar, non = scr.get_tar_non(key)
ntar = len(tar)
nnon = len(non)
min_dcf, p_miss, p_fa = compute_min_dcf(tar, non, prior)
n_miss = p_miss * ntar
n_fa = p_fa * nnon
logging.info(
'min_dcf: %.3f p_miss: %.2f p_fa: %.2f n_miss: %.1f n_fa: %.1f' %
(min_dcf, p_miss * 100, p_fa * 100, n_miss, n_fa))
logging.info('train calibration')
x = np.concatenate((tar, non))
y = np.concatenate((np.ones(
(ntar, ), dtype='int32'), np.zeros((nnon, ), dtype='int32')))
lr = LR(prior=prior, verbose=verbose)
lr.fit(x, y)
print(lr.A)
print(lr.b)
logging.info('save calibration at %s' % model_file)
lr.save(model_file)
logging.info('calibrate scores')
tar_cal = lr.predict(tar)
non_cal = lr.predict(non)
print(tar_cal)
act_dcf, p_miss, p_fa = compute_act_dcf(tar_cal, non_cal, prior)
n_miss = p_miss * ntar
n_fa = p_fa * nnon
logging.info(
'act_dcf: %.3f p_miss: %.2f p_fa: %.2f n_miss: %.1f n_fa: %.1f' %
(act_dcf, p_miss * 100, p_fa * 100, n_miss, n_fa))
def score_dcf(key_file, score_file, output_path):
logging.info('Load key: %s' % key_file)
key = TrialKey.load_txt(key_file)
logging.info('Load scores: %s' % score_file)
scr = TrialScores.load_txt(score_file)
tar, non = scr.get_tar_non(key)
priors = np.array([0.001, 0.005, 0.01, 0.05 ])
min_dcf, act_dcf, eer, _ = fast_eval(tar, non, priors)
output_dir = os.path.dirname(output_path)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
output_file = output_path + '_results'
with open(output_file, 'w') as f:
s = 'EER: {0:.2f} DCF5e-2: {1:.3f} / {2:.3f} DCF1e-2: {3:.3f} / {4:.3f} DCF5e-3: {5:.3f} / {6:.3f} DCF1e-3: {7:.3f} / {8:.3f}'.format(
eer * 100, min_dcf[3], act_dcf[3],
min_dcf[2], act_dcf[2],
min_dcf[1], act_dcf[1],
min_dcf[0], act_dcf[0])
f.write(s)
logging.info(s)