def predict(self, X):
n = X.shape[0]
epsilon = np.random.uniform(low=0.0, high=1.0, size=n)
p_hat = self.black_box.predict_proba(X)
grey_box = ProbAccum(p_hat)
S_hat = grey_box.predict_sets(self.alpha_calibrated, epsilon=epsilon)
return S_hat
def __init__(self,
X,
Y,
black_box,
alpha,
random_state=2020,
verbose=False):
# Split data into training/calibration sets
X_train, X_calib, Y_train, Y_calib = train_test_split(
X, Y, test_size=0.5, random_state=random_state)
n2 = X_calib.shape[0]
self.black_box = black_box
# Fit model
self.black_box.fit(X_train, Y_train)
# Form prediction sets on calibration data
p_hat_calib = self.black_box.predict_proba(X_calib)
grey_box = ProbAccum(p_hat_calib)
epsilon = np.random.uniform(low=0.0, high=1.0, size=n2)
alpha_max = grey_box.calibrate_scores(Y_calib, epsilon=epsilon)
scores = alpha - alpha_max
level_adjusted = (1.0 - alpha) * (1.0 + 1.0 / float(n2))
alpha_correction = mquantiles(scores, prob=level_adjusted)
# Store calibrate level
self.alpha_calibrated = alpha - alpha_correction
def predict(self, X):
n = X.shape[0]
S = [[]] * n
n_classes = len(self.classes)
epsilon = np.random.uniform(low=0.0, high=1.0, size=n)
prop_smaller = np.zeros((n, n_classes))
if self.verbose:
print("Computing predictive sets for {} samples:".format(n),
file=sys.stderr)
sys.stderr.flush()
for i in range(self.n):
print("{} of {}...".format(i + 1, self.n), file=sys.stderr)
sys.stderr.flush()
gb = ProbAccum(self.mu_LOO[i].predict_proba(X))
for k in range(n_classes):
y_lab = [self.classes[k]] * n
alpha_max_new = gb.calibrate_scores(y_lab, epsilon=epsilon)
prop_smaller[:, k] += (alpha_max_new < self.alpha_max[i])
else:
for i in range(self.n):
gb = ProbAccum(self.mu_LOO[i].predict_proba(X))
for k in range(n_classes):
y_lab = [self.classes[k]] * n
alpha_max_new = gb.calibrate_scores(y_lab, epsilon=epsilon)
prop_smaller[:, k] += (alpha_max_new < self.alpha_max[i])
for k in range(n_classes):
prop_smaller[:, k] /= float(self.n)
level_adjusted = (1.0 - self.alpha) * (1.0 + 1.0 / float(self.n))
S = [
np.where(prop_smaller[i, :] < level_adjusted)[0] for i in range(n)
]
return S
def __init__(self, dataset, X, Y, black_box, alpha, random_state=2020, verbose=False, gamma=0):
# Split data into training/calibration sets
if dataset=='imagenet':
p_hat_calib = X
Y_calib = Y
self.gamma = gamma
n2 = X.shape[0]
else:
X_train, X_calib, Y_train, Y_calib = train_test_split(X, Y, test_size=0.5, random_state=random_state)
n2 = X_calib.shape[0]
self.gamma = gamma
self.black_box = black_box
# Fit model
self.black_box.fit(X_train, Y_train)
# Form prediction sets on calibration data
p_hat_calib = self.black_box.predict_proba(X_calib)
self.alpha_calibrated = []
for i in range(np.amax(Y)+1):
idxs = np.argwhere(Y_calib == i)[:, 0]
grey_box = ProbAccum(p_hat_calib[idxs])
epsilon = np.random.uniform(low=0.0, high=1.0, size=len(idxs))
alpha_max = grey_box.calibrate_scores(Y_calib[idxs], epsilon=epsilon)
score = alpha - alpha_max
level_adjusted = (1.0-alpha)*(1.0+1.0/float(len(idxs)))
alpha_correction = mquantiles(score, prob=level_adjusted)
self.alpha_calibrated.append(alpha - alpha_correction)
def predict(self, dataset, X):
n = X.shape[0]
epsilon = np.random.uniform(low=0.0, high=1.0, size=n)
if dataset=='imagenet':
p_hat = X
else:
p_hat = self.black_box.predict_proba(X)
grey_box = ProbAccum(p_hat)
idxs = np.argsort(np.array(self.alpha_calibrated)[:, 0])
alpha_calib = self.alpha_calibrated[idxs[self.gamma]][0]
S_hat = grey_box.predict_sets(alpha_calib, epsilon=epsilon)
return S_hat
def predict(self, X, randomize=True):
prob_y = self.data_model.compute_prob(X)
grey_box = ProbAccum(prob_y)
S = grey_box.predict_sets(self.alpha, randomize=randomize)
return S