n_inducing_points=10,
logits=False,
random_state=random_state),
"GPcalib_approx":
calm.GPCalibration(n_classes=2,
maxiter=1000,
n_inducing_points=10,
logits=False,
random_state=random_state,
inf_mean_approx=True),
"Platt":
calm.PlattScaling(random_state=random_state),
"Isotonic":
calm.IsotonicRegression(),
"Beta":
calm.BetaCalibration(),
"BBQ":
calm.BayesianBinningQuantiles(),
"Temp":
calm.TemperatureScaling()
}
# Create benchmark object
kitti_benchmark = pycalib.benchmark.KITTIBinaryData(
run_dir=run_dir,
clf_output_dir=clf_output_dir,
classifier_names=classifier_names,
cal_methods=list(cal_methods.values()),
cal_method_names=list(cal_methods.keys()),
n_splits=10,
test_size=8000,
miscal_function_names = {
'identity': f_id,
'root': f_root,
"power": f_power
}
# Define calibration methods
cal_methods_dict = {
"No_Calibration":
cm.NoCalibration(),
"Platt_scaling":
cm.PlattScaling(),
"Isotonic_Regression":
cm.IsotonicRegression(),
"Beta_Calibration":
cm.BetaCalibration(params='abm'),
"Histogram_Binning":
cm.HistogramBinning(mode='equal_freq'),
"Bayesian_Binning_into_Quantiles":
cm.BayesianBinningQuantiles(),
"Temperature_Scaling":
cm.TemperatureScaling(verbose=False),
"GP_calibration":
cm.GPCalibration(n_classes=n_classes,
maxiter=300,
n_inducing_points=100)
}
# Evaluate calibration methods
sb = bm.SyntheticBeta(
run_dir=dir_out,