ncols=len(clf_names),
w_pad=1)
# Iterate through data sets, calibrate and plot latent functions
for (i_clf, (Z, y, info_dict)) in enumerate(benchmark.data_gen()):
# Train, test split
cal_ind, test_ind = next(benchmark.cross_validator.split(Z, y))
Z_cal = Z[cal_ind, :]
y_cal = y[cal_ind]
Z_test = Z[test_ind, :]
y_test = y[test_ind]
hist_data = Z_cal.flatten()
# Calibrate
nocal = calm.NoCalibration(logits=use_logits)
ts = calm.TemperatureScaling()
ts.fit(Z_cal, y_cal)
gpc = calm.GPCalibration(n_classes=n_classes,
maxiter=1000,
n_inducing_points=10,
logits=use_logits,
verbose=True,
random_state=random_state)
gpc.fit(Z_cal, y_cal)
# # Compute calibration error
# ECE_nocal = pycalib.scoring.expected_calibration_error(y_test, nocal.predict_proba(Z_test), n_bins=100)
# ECE_ts = pycalib.scoring.expected_calibration_error(y_test, ts.predict_proba(Z_test), n_bins=100)
###############################
# Initialization
clf_output_dir = os.path.join(file, output_folder)
run_dir = os.path.join(file, "calibration")
# Classifiers
classifier_names = list(clf_dict.keys())
# Calibration models
with gpflow.defer_build():
meanfunc = pycalib.gp_classes.ScalarMult()
meanfunc.alpha.transform = gpflow.transforms.positive
cal_methods = {
"Uncal":
calm.NoCalibration(),
"GPcalib":
calm.GPCalibration(n_classes=2,
maxiter=1000,
n_inducing_points=10,
logits=False,
random_state=random_state),
"GPcalib_lin":
calm.GPCalibration(n_classes=2,
maxiter=1000,
mean_function=meanfunc,
n_inducing_points=10,
logits=False,
random_state=random_state),
"GPcalib_approx":
calm.GPCalibration(n_classes=2,
# Seed and directories
random_state = 1
clf_output_dir = os.path.join(file, output_folder)
run_dir = os.path.join(file, "calibration")
n_classes = 100
train_size = 1000
test_size = 9000
# Calibration methods for logits
with gpflow.defer_build():
meanfunc = pycalib.gp_classes.ScalarMult()
meanfunc.alpha.transform = gpflow.transforms.positive
cal_methods_logits = {
"Uncal": calm.NoCalibration(logits=True),
"GPcalib_lin": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10,
mean_function=meanfunc, logits=True, verbose=False,
random_state=random_state),
"GPcalib": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10,
logits=True, random_state=random_state),
"GPcalib_approx": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10,
logits=True, random_state=random_state, inf_mean_approx=True),
"Temp": calm.TemperatureScaling()
}
# Create benchmark object
cifar_benchmark = pycalib.benchmark.CIFARData(run_dir=run_dir, clf_output_dir=clf_output_dir,
classifier_names=clf_names,
cal_methods=list(cal_methods_logits.values()),
cal_method_names=list(cal_methods_logits.keys()),
# Benchmark
###############################
# Initialization
run_dir = os.path.join(file, "calibration")
clf_output_dir = os.path.join(file, output_folder)
# Classifiers
classifier_names = list(clf_dict.keys())
# Calibration models
with gpflow.defer_build():
meanfunc = pycalib.gp_classes.ScalarMult()
meanfunc.alpha.transform = gpflow.transforms.positive
cal_methods = {
"Uncal": calm.NoCalibration(),
"GPcalib": calm.GPCalibration(n_classes=10, maxiter=1000, n_inducing_points=10, logits=False,
random_state=random_state),
"GPcalib_lin": calm.GPCalibration(n_classes=10, maxiter=1000, mean_function=meanfunc,
n_inducing_points=10, logits=False,
random_state=random_state),
"GPcalib_approx": calm.GPCalibration(n_classes=10, maxiter=1000, n_inducing_points=10,
logits=False, random_state=random_state, inf_mean_approx=True),
"Platt": calm.PlattScaling(random_state=random_state),
"Isotonic": calm.IsotonicRegression(),
"Beta": calm.BetaCalibration(),
"BBQ": calm.BayesianBinningQuantiles(),
"Temp": calm.TemperatureScaling()
}
data_folder=val_folder,
output_folder=output_folder)
###############################
# Benchmark
###############################
# Seed and directories
random_state = 1
clf_output_dir = os.path.join(file, "clf_output")
run_dir = os.path.join(file, "calibration")
# Calibration methods for logits
cal_methods_logits = {
"Uncal":
calm.NoCalibration(logits=True),
"GPcalib":
calm.GPCalibration(n_classes=n_classes,
maxiter=1000,
n_inducing_points=10,
logits=True,
random_state=random_state),
"GPcalib_approx":
calm.GPCalibration(n_classes=n_classes,
maxiter=1000,
n_inducing_points=10,
logits=True,
random_state=random_state,
inf_mean_approx=True),
"Temp":
calm.TemperatureScaling()