def forward(self, logits, randomized=None, allow_zero_sets=None):
if randomized == None:
randomized = self.randomized
if allow_zero_sets == None:
allow_zero_sets = self.allow_zero_sets
with torch.no_grad():
logits_numpy = logits.detach().cpu().numpy()
scores = softmax(logits_numpy / self.T.item(), axis=1)
if not self.LAC:
I, ordered, cumsum = sort_sum(scores)
S = gcq(scores,
self.Qhat,
I=I,
ordered=ordered,
cumsum=cumsum,
penalties=self.penalties,
randomized=randomized,
allow_zero_sets=allow_zero_sets)
else:
S = [
np.where((1 - scores[i, :]) < self.Qhat)[0]
for i in range(scores.shape[0])
]
return logits, S
def conformal_calibration(cmodel, calib_loader):
print("Conformal calibration")
with torch.no_grad():
E = np.array([])
for x, targets in tqdm(calib_loader):
logits = cmodel.model(x.cuda()).detach().cpu().numpy()
scores = softmax(logits/cmodel.T.item(), axis=1)
I, ordered, cumsum = sort_sum(scores)
E = np.concatenate((E,giq(scores,targets,I=I,ordered=ordered,cumsum=cumsum,penalties=cmodel.penalties,randomized=cmodel.randomized)))
Qhat = np.quantile(E,1-cmodel.alpha,interpolation='higher')
return Qhat
def conformal_calibration_logits(cmodel, calib_loader):
with torch.no_grad():
E = np.array([])
for logits, targets in calib_loader:
logits = logits.detach().cpu().numpy()
scores = softmax(logits/cmodel.T.item(), axis=1)
I, ordered, cumsum = sort_sum(scores)
E = np.concatenate((E,giq(scores,targets,I=I,ordered=ordered,cumsum=cumsum,penalties=cmodel.penalties,randomized=cmodel.randomized,allow_zero_sets=cmodel.allow_zero_sets)))
Qhat = np.quantile(E,1-cmodel.alpha,interpolation='higher')
return Qhat
def forward(self, *args, randomized=None, allow_zero_sets=None, **kwargs):
if randomized == None:
randomized = self.randomized
if allow_zero_sets == None:
allow_zero_sets = self.allow_zero_sets
logits = self.model(*args, **kwargs)
with torch.no_grad():
logits_numpy = logits.detach().cpu().numpy()
scores = softmax(logits_numpy/self.T.item(), axis=1)
I, ordered, cumsum = sort_sum(scores)
S = gcq(scores, self.Qhat, I=I, ordered=ordered, cumsum=cumsum, penalties=self.penalties, randomized=randomized, allow_zero_sets=allow_zero_sets)
return logits, S
def forward(self, logits, randomized=None):
if randomized == None:
randomized = self.randomized
with torch.no_grad():
logits_numpy = logits.detach().cpu().numpy()
scores = softmax(logits_numpy / self.T.item(), axis=1)
I, ordered, cumsum = sort_sum(scores)
S = gcq(scores,
self.Qhat,
I=I,
ordered=ordered,
cumsum=cumsum,
penalties=self.penalties,
randomized=randomized)
return logits, S
def get_violation(cmodel, loader_paramtune, strata, alpha):
df = pd.DataFrame(columns=['size', 'correct'])
for logit, target in loader_paramtune:
# compute output
output, S = cmodel(logit) # This is a 'dummy model' which takes logits, for efficiency.
# measure accuracy and record loss
size = np.array([x.size for x in S])
I, _, _ = sort_sum(logit.numpy())
correct = np.zeros_like(size)
for j in range(correct.shape[0]):
correct[j] = int( target[j] in list(S[j]) )
batch_df = pd.DataFrame({'size': size, 'correct': correct})
df = df.append(batch_df, ignore_index=True)
wc_violation = 0
for stratum in strata:
temp_df = df[ (df['size'] >= stratum[0]) & (df['size'] <= stratum[1]) ]
if len(temp_df) == 0:
continue
stratum_violation = abs(temp_df.correct.mean()-(1-alpha))
wc_violation = max(wc_violation, stratum_violation)
return wc_violation # the violation