def main():
dataset = 'cifar100'
num_samples = 1000
datafile = DATAFILE_LIST[dataset]
num_classes = NUM_CLASSES_DICT[dataset]
categories, observations, confidences, idx2category, category2idx, labels = prepare_data(datafile, False)
# accuracy models
accuracy_model = BetaBernoulli(k=num_classes, prior=None)
accuracy_model.update_batch(categories, observations)
# ece models for each class
ece_model = ClasswiseEce(num_classes, num_bins=10, pseudocount=2)
ece_model.update_batch(categories, observations, confidences)
# draw samples from posterior of classwise accuracy
accuracy_samples = accuracy_model.sample(num_samples) # (num_categories, num_samples)
ece_samples = ece_model.sample(num_samples) # (num_categories, num_samples)
accuracy = np.array([np.quantile(accuracy_samples, 0.025, axis=1),
np.quantile(accuracy_samples, 0.5, axis=1),
np.quantile(accuracy_samples, 0.975, axis=1)]).T
ece = np.array([np.quantile(ece_samples, 0.025, axis=1),
np.quantile(ece_samples, 0.5, axis=1),
np.quantile(ece_samples, 0.975, axis=1)]).T
fig, axes = plot_figure_1(accuracy, ece, labels=CIFAR100_CLASSES, limit=10, reverse=False)
fig.tight_layout()
fig.subplots_adjust(bottom=-0.2, wspace=0.35)
fig.set_size_inches(COLUMN_WIDTH * 1.3, 2.0)
fig.savefig(FIGURE_DIR + 'figure1.pdf', bbox_inches="tight", pad_inches=0.05)
def get_bayesian_ground_truth(categories: List[int],
observations: List[bool],
confidences: List[float],
num_classes: int,
metric: str,
mode: str,
topk: int = 1,
pseudocount: int = 1,
prior=None) -> np.ndarray:
"""
Compute ground truth given metric and mode with all data points.
:param categories: List[int]
A list of predicted classes.
:param observations: List[bool]
A list of boolean observations.
:param confidences: List[float]
A list of prediction scores.
:param num_classes: int
The number of classes.
:param metric: str
'accuracy' or 'calibration_error'
:param mode: str
'min' or max'
:param topk: int
The number of top classes to return. Default: 1.
:param pseudocount: int
Strength of prior for ClasswiseEce model. Default: 1.
:param prior: np.ndarray
Prior for BetaBernoulli model. Default: None.
:return: binary np.ndarray of shape (num_classes, ) indicating each class in top k or not.
"""
if metric == 'accuracy':
model = BetaBernoulli(num_classes, prior=prior)
model.update_batch(confidences, observations)
elif metric == 'calibration_error':
model = ClasswiseEce(num_classes, num_bins=10, pseudocount=pseudocount)
model.update_batch(categories, observations, confidences)
metric_val = model.eval
output = np.zeros((num_classes, ), dtype=np.bool_)
if mode == 'max':
indices = metric_val.argsort()[-topk:]
else:
indices = metric_val.argsort()[:topk]
output[indices] = 1
return output
def main() -> None:
with mpl.rc_context(rc=DEFAULT_RC):
fig, axes = plt.subplots(ncols=3, nrows=2, dpi=300, sharey=False)
idx = 0
for dataset in DATASET_NAMES:
datafile = DATAFILE_LIST[dataset]
num_classes = NUM_CLASSES_DICT[dataset]
categories, observations, confidences, idx2category, category2idx, labels = prepare_data(datafile, False)
# accuracy models
accuracy_model = BetaBernoulli(k=num_classes, prior=None)
accuracy_model.update_batch(categories, observations)
# ece models for each class
ece_model = ClasswiseEce(num_classes, num_bins=10, pseudocount=2)
ece_model.update_batch(categories, observations, confidences)
# draw samples from posterior of classwise accuracy
accuracy_samples = accuracy_model.sample(num_samples) # (num_categories, num_samples)
ece_samples = ece_model.sample(num_samples) # (num_categories, num_samples)
plot_kwargs = {}
axes[idx // 3, idx % 3] = plot_scatter(axes[idx // 3, idx % 3], accuracy_samples, ece_samples,
limit=TOPK_DICT[dataset], plot_kwargs=plot_kwargs)
axes[idx // 3, idx % 3].set_title(DATASET_NAMES[dataset])
idx += 1
axes[0, 0].set_ylabel('ECE')
axes[1, 0].set_ylabel('ECE')
fig.set_size_inches(TEXT_WIDTH, 4.0)
fig.subplots_adjust(bottom=0.05, wspace=0.2)
fig.delaxes(axes.flatten()[5])
figname = FIGURE_DIR + 'scatter.pdf'
fig.tight_layout()
fig.savefig(figname, bbox_inches='tight', pad_inches=0)