ncols=len(clf_names), w_pad=1) # Iterate through data sets, calibrate and plot latent functions for (i_clf, (Z, y, info_dict)) in enumerate(benchmark.data_gen()): # Train, test split cal_ind, test_ind = next(benchmark.cross_validator.split(Z, y)) Z_cal = Z[cal_ind, :] y_cal = y[cal_ind] Z_test = Z[test_ind, :] y_test = y[test_ind] hist_data = Z_cal.flatten() # Calibrate nocal = calm.NoCalibration(logits=use_logits) ts = calm.TemperatureScaling() ts.fit(Z_cal, y_cal) gpc = calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, logits=use_logits, verbose=True, random_state=random_state) gpc.fit(Z_cal, y_cal) # # Compute calibration error # ECE_nocal = pycalib.scoring.expected_calibration_error(y_test, nocal.predict_proba(Z_test), n_bins=100) # ECE_ts = pycalib.scoring.expected_calibration_error(y_test, ts.predict_proba(Z_test), n_bins=100)
############################### # Initialization clf_output_dir = os.path.join(file, output_folder) run_dir = os.path.join(file, "calibration") # Classifiers classifier_names = list(clf_dict.keys()) # Calibration models with gpflow.defer_build(): meanfunc = pycalib.gp_classes.ScalarMult() meanfunc.alpha.transform = gpflow.transforms.positive cal_methods = { "Uncal": calm.NoCalibration(), "GPcalib": calm.GPCalibration(n_classes=2, maxiter=1000, n_inducing_points=10, logits=False, random_state=random_state), "GPcalib_lin": calm.GPCalibration(n_classes=2, maxiter=1000, mean_function=meanfunc, n_inducing_points=10, logits=False, random_state=random_state), "GPcalib_approx": calm.GPCalibration(n_classes=2,
# Seed and directories random_state = 1 clf_output_dir = os.path.join(file, output_folder) run_dir = os.path.join(file, "calibration") n_classes = 100 train_size = 1000 test_size = 9000 # Calibration methods for logits with gpflow.defer_build(): meanfunc = pycalib.gp_classes.ScalarMult() meanfunc.alpha.transform = gpflow.transforms.positive cal_methods_logits = { "Uncal": calm.NoCalibration(logits=True), "GPcalib_lin": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, mean_function=meanfunc, logits=True, verbose=False, random_state=random_state), "GPcalib": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, logits=True, random_state=random_state), "GPcalib_approx": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, logits=True, random_state=random_state, inf_mean_approx=True), "Temp": calm.TemperatureScaling() } # Create benchmark object cifar_benchmark = pycalib.benchmark.CIFARData(run_dir=run_dir, clf_output_dir=clf_output_dir, classifier_names=clf_names, cal_methods=list(cal_methods_logits.values()), cal_method_names=list(cal_methods_logits.keys()),
# Benchmark ############################### # Initialization run_dir = os.path.join(file, "calibration") clf_output_dir = os.path.join(file, output_folder) # Classifiers classifier_names = list(clf_dict.keys()) # Calibration models with gpflow.defer_build(): meanfunc = pycalib.gp_classes.ScalarMult() meanfunc.alpha.transform = gpflow.transforms.positive cal_methods = { "Uncal": calm.NoCalibration(), "GPcalib": calm.GPCalibration(n_classes=10, maxiter=1000, n_inducing_points=10, logits=False, random_state=random_state), "GPcalib_lin": calm.GPCalibration(n_classes=10, maxiter=1000, mean_function=meanfunc, n_inducing_points=10, logits=False, random_state=random_state), "GPcalib_approx": calm.GPCalibration(n_classes=10, maxiter=1000, n_inducing_points=10, logits=False, random_state=random_state, inf_mean_approx=True), "Platt": calm.PlattScaling(random_state=random_state), "Isotonic": calm.IsotonicRegression(), "Beta": calm.BetaCalibration(), "BBQ": calm.BayesianBinningQuantiles(), "Temp": calm.TemperatureScaling() }
data_folder=val_folder, output_folder=output_folder) ############################### # Benchmark ############################### # Seed and directories random_state = 1 clf_output_dir = os.path.join(file, "clf_output") run_dir = os.path.join(file, "calibration") # Calibration methods for logits cal_methods_logits = { "Uncal": calm.NoCalibration(logits=True), "GPcalib": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, logits=True, random_state=random_state), "GPcalib_approx": calm.GPCalibration(n_classes=n_classes, maxiter=1000, n_inducing_points=10, logits=True, random_state=random_state, inf_mean_approx=True), "Temp": calm.TemperatureScaling()