def imagenet_experiments():
probs, labels = cal.load_test_probs_labels('data/imagenet_probs.dat')
if not os.path.exists('./saved_files'):
os.mkdir('./saved_files')
if not os.path.exists('./saved_files/debiased_estimator/'):
os.mkdir('./saved_files/debiased_estimator/')
save_prefix = './saved_files/debiased_estimator/'
compare_scaling_binning_squared_ce(probs,
labels,
platt_data_size=3000,
bin_data_size=3000,
num_bins=100,
save_prefix=save_prefix +
"imnet_scaling_binning_")
compare_scaling_binning_squared_ce(probs,
labels,
platt_data_size=3000,
bin_data_size=3000,
num_bins=15,
save_prefix=save_prefix +
"imnet_scaling_binning_")
compare_scaling_ce(probs,
labels,
platt_data_size=3000,
bin_data_size=3000,
num_bins=100,
save_prefix=save_prefix + "imnet_scaling_ece_")
compare_scaling_ce(probs,
labels,
platt_data_size=3000,
bin_data_size=3000,
num_bins=15,
save_prefix=save_prefix + "imnet_scaling_ece_")
def cifar10_experiment_marginal(probs_path,
ce_save_path,
mse_ce_save_path,
num_trials=100):
probs, labels = cal.load_test_probs_labels(probs_path)
bins_list = list(range(10, 101, 10))
num_calibration = 1000
l2_ces, l2_stddevs, mses = vary_bin_calibration(
data_sampler=make_calibration_data_sampler(probs, labels,
num_calibration),
num_bins_list=bins_list,
Calibrators=[
cal.HistogramMarginalCalibrator, cal.PlattBinnerMarginalCalibrator
],
calibration_evaluators=[
eval_marginal_calibration, eval_marginal_calibration
],
eval_mse=cal.eval_marginal_mse,
num_trials=num_trials)
plot_mse_ce_curve(bins_list,
l2_ces,
mses,
xlim=(0.0, 0.0006),
ylim=(0.04, 0.08),
save_path=mse_ce_save_path)
plot_ces(bins_list, l2_ces, l2_stddevs, save_path=ce_save_path)
def imagenet_experiment(savefile, binning_func=cal.get_equal_bins, lp=2):
np.random.seed(0)
calibration_data_size = 20000
bin_data_size = 5000
probs, labels = cal.load_test_probs_labels('imagenet_probs.dat')
lower_bound_experiment(probs,
labels,
calibration_data_size,
bin_data_size,
bins_list=[2, 4, 8, 16, 32, 64, 128, 256, 512],
save_name=savefile,
binning_func=binning_func,
lp=lp)
def parse_input():
args = parser.parse_args()
probs, labels = cal.load_test_probs_labels(args.probs_file)
if not os.path.exists('./saved_files'):
os.mkdir('./saved_files')
if not os.path.exists('./saved_files/debiased_estimator/'):
os.mkdir('./saved_files/debiased_estimator/')
save_prefix = './saved_files/debiased_estimator/'
compare_scaling_binning_squared_ce(probs,
labels,
args.platt_data_size,
args.bin_data_size,
args.num_bins,
save_prefix=save_prefix)
def imagenet_experiment_top(probs_path,
ce_save_path,
mse_ce_save_path,
num_trials=100):
probs, labels = cal.load_test_probs_labels(probs_path)
bins_list = list(range(10, 101, 10))
num_calibration = 1000
l2_ces, l2_stddevs, mses = vary_bin_calibration(
data_sampler=make_calibration_data_sampler(probs, labels,
num_calibration),
num_bins_list=bins_list,
Calibrators=[cal.HistogramTopCalibrator, cal.PlattBinnerTopCalibrator],
calibration_evaluators=[eval_top_calibration, eval_top_calibration],
eval_mse=cal.eval_top_mse,
num_trials=num_trials)
plot_mse_ce_curve(bins_list, l2_ces, mses, save_path=mse_ce_save_path)
plot_ces(bins_list, l2_ces, l2_stddevs, save_path=ce_save_path)
bins_list=[2, 4, 8, 16, 32, 64, 128, 256, 512],
save_name=savefile,
binning_func=binning_func,
lp=lp)
if __name__ == "__main__":
if not os.path.exists('./saved_files'):
os.mkdir('./saved_files')
if not os.path.exists('./saved_files/platt_not_calibrated/'):
os.mkdir('./saved_files/platt_not_calibrated/')
prefix = './saved_files/platt_not_calibrated/'
args = parser.parse_args()
if args.binning == 'equal_prob_bins':
binning = cal.get_equal_prob_bins
else:
binning = cal.get_equal_bins
probs, labels = cal.load_test_probs_labels(args.probs_path)
print(args.bins_list)
lower_bound_experiment(probs,
labels,
args.calibration_data_size,
args.bin_data_size,
args.bins_list,
save_name=prefix + args.plot_save_file,
binning_func=binning,
lp=args.lp,
num_samples=args.num_samples)