def min_cllr(self, ptar, with_pav=False): cllrs = [] if with_pav: if self.pav is None: raise Exception( "To extract min_cllr using PAV you need to call DET with pav=True" ) llrs, ntar, nnon = self.pav.llrs() for p in np.atleast_1d(ptar): logitPost = llrs + logit(p) Ctar, Cnon = softplus(-logitPost), softplus(logitPost) min_cllr = p * (Ctar[ntar != 0] @ ntar[ntar != 0]) / ntar.sum( ) + (1 - p) * (Cnon[nnon != 0] @ nnon[nnon != 0]) / nnon.sum() min_cllr /= -p * np.log(p) - (1 - p) * np.log(1 - p) cllrs.append(min_cllr) else: for p in np.atleast_1d(ptar): cal = logregCal(self.tar, self.non, p) tar_cal, non_cal = cal(self.tar), cal(self.non) cllrs.append(cross_entropy(tar_cal, non_cal, p)) return np.array(cllrs).squeeze()
def init_with_logreg(self, scores, same_spk, valid, ptar, std_for_mats=0): scores = scores[valid] labels = same_spk[valid] tar = scores[labels==1] non = scores[labels==0] a, b = calibration.logregCal(tar, non, ptar, return_params=True) if self.is_durdep: # Dur-dep calibration (might also have an si-dependent stage) self.durdep_alpha.init_with_constant(a, std_for_mats) self.durdep_beta.init_with_constant(b, std_for_mats) if self.is_sidep: self.sidep_alpha.init_with_constant(1.0, std_for_mats) self.sidep_beta.init_with_constant(0.0, std_for_mats) elif not self.is_sidep: # Global calibration utils.replace_state_dict(self, {'alpha': a, 'beta': b}) else: # Only si-dependent cal self.sidep_alpha.init_with_constant(a, std_for_mats) self.sidep_beta.init_with_constant(b, std_for_mats)
def test_model(model, data_dict, model_name, ptar, loss_type, print_min_loss=False): # Compute the loss for each of the devsets. The validation loss will be the average over all devsets av_loss = 0 for name, info in data_dict.items(): mask = info['mask'] scores = evaluate(model, info['dataset'], info['emap'], info['tmap']) # Since the key mask was created to be aligned to the emap and tmap, the score_mat # is already aligned to this mask loss, llrs, labels = compute_loss(scores.score_mat, mask=mask, ptar=ptar, loss_type=loss_type, return_info=True) if print_min_loss: tar = llrs[labels == 1].detach().cpu().numpy() non = llrs[labels == 0].detach().cpu().numpy() cal = logregCal(tar, non, ptar) min_loss = cross_entropy(cal(tar), cal(non), ptar) print("%s, loss on dev set %s = %f min = %f" % (model_name, name, loss, min_loss)) else: print("%s, loss on dev set %s = %f" % (model_name, name, loss)) av_loss += loss av_loss /= len(data_dict) return av_loss