def train_calibration(self, logits, labels): """Train a calibrator given logits and labels. Args: logits: A sequence of dimension (n, k) where n is the number of data points, and k is the number of classes, representing the output probabilities/confidences of the uncalibrated model. labels: A sequence of length n, where n is the number of data points, representing the ground truth label for each data point. """ assert(len(logits) >= self._num_calibration) logits = np.array(logits) self._k = logits.shape[1] # Number of classes. assert self._k == np.max(labels) - np.min(labels) + 1 labels_one_hot = utils.get_labels_one_hot(np.array(labels), self._k) assert labels_one_hot.shape == logits.shape self._platts = [] self._calibrators = [] for c in range(self._k): # For each class c, get the probabilities the model output for that class, and whether # the data point was actually class c, or not. probs_c = logits[:, c] labels_c = labels_one_hot[:, c] platt_c = utils.get_platt_scaler(probs_c, labels_c) self._platts.append(platt_c) platt_probs_c = platt_c(probs_c) bins = utils.get_equal_bins(platt_probs_c, num_bins=self._num_bins) calibrator_c = utils.get_discrete_calibrator(platt_probs_c, bins) self._calibrators.append(calibrator_c)
def train_calibration(self, logits, labels): assert(len(logits) >= self._num_calibration) predictions = utils.get_top_predictions(logits) probs = utils.get_top_probs(logits) correct = (predictions == labels) self._platt = utils.get_platt_scaler( probs, correct) platt_probs = self._platt(probs) bins = utils.get_equal_bins(platt_probs, num_bins=self._num_bins) self._discrete_calibrator = utils.get_discrete_calibrator( platt_probs, bins)
def lower_bound_experiment(logits, labels, calibration_data_size, bin_data_size, bins_list, save_name='cmp_est', binning_func=utils.get_equal_bins, lp=2): # Shuffle the logits and labels. indices = np.random.choice(list(range(len(logits))), size=len(logits), replace=False) logits = [logits[i] for i in indices] labels = [labels[i] for i in indices] predictions = utils.get_top_predictions(logits) probs = utils.get_top_probs(logits) correct = (predictions == labels) print('num_correct: ', sum(correct)) # Platt scale on first chunk of data platt = utils.get_platt_scaler(probs[:calibration_data_size], correct[:calibration_data_size]) platt_probs = platt(probs) lower, middle, upper = [], [], [] for num_bins in bins_list: bins = binning_func(platt_probs[:calibration_data_size + bin_data_size], num_bins=num_bins) verification_probs = platt_probs[calibration_data_size + bin_data_size:] verification_correct = correct[calibration_data_size + bin_data_size:] verification_data = list(zip(verification_probs, verification_correct)) def estimator(data): binned_data = utils.bin(data, bins) return utils.plugin_ce(binned_data, power=lp) print('estimate: ', estimator(verification_data)) estimate_interval = utils.bootstrap_uncertainty(verification_data, estimator, num_samples=1000) lower.append(estimate_interval[0]) middle.append(estimate_interval[1]) upper.append(estimate_interval[2]) print('interval: ', estimate_interval) lower_errors = np.array(middle) - np.array(lower) upper_errors = np.array(upper) - np.array(middle) plt.clf() font = {'family': 'normal', 'size': 18} rc('font', **font) plt.errorbar(bins_list, middle, yerr=[lower_errors, upper_errors], barsabove=True, fmt='none', color='black', capsize=4) plt.scatter(bins_list, middle, color='black') plt.xlabel(r"No. of bins") if lp == 2: plt.ylabel("Calibration error") else: plt.ylabel("L%d Calibration error" % lp) plt.xscale('log', basex=2) ax = plt.gca() ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) ax.yaxis.set_ticks_position('left') ax.xaxis.set_ticks_position('bottom') plt.tight_layout() plt.savefig(save_name)
def calibrate_marginals_experiment(logits, labels, k): num_calib = 3000 num_bin = 3000 num_cert = 4000 assert (logits.shape[0] == num_calib + num_bin + num_cert) num_bins = 100 bootstrap_samples = 100 # First split by label? To ensure equal class numbers? Do this later. labels = utils.get_labels_one_hot(labels[:, 0], k) mse = np.mean(np.square(labels - logits)) print('original mse is ', mse) calib_logits = logits[:num_calib, :] calib_labels = labels[:num_calib, :] bin_logits = logits[num_calib:num_calib + num_bin, :] bin_labels = labels[num_calib:num_calib + num_bin, :] cert_logits = logits[num_calib + num_bin:, :] cert_labels = labels[num_calib + num_bin:, :] mses = [] unbiased_ces = [] biased_ces = [] std_unbiased_ces = [] std_biased_ces = [] for num_bins in range(10, 21, 10): # Train a platt scaler and binner. platts = [] platt_binners_equal_points = [] for l in range(k): platt_l = utils.get_platt_scaler(calib_logits[:, l], calib_labels[:, l]) platts.append(platt_l) cal_logits_l = platt_l(calib_logits[:, l]) # bins_l = utils.get_equal_bins(cal_logits_l, num_bins=num_bins) # Get # bins_l = utils.get_equal_prob_bins(num_bins=num_bins) bins_l = [0.0012, 0.05, 0.01, 0.95, 0.985, 1.0] cal_bin_logits_l = platt_l(bin_logits[:, l]) platt_binner_l = utils.get_discrete_calibrator( cal_bin_logits_l, bins_l) platt_binners_equal_points.append(platt_binner_l) # Write a function that takes data and outputs the mse, ce def get_mse_ce(logits, labels, ce_est): mses = [] ces = [] logits = np.array(logits) labels = np.array(labels) for l in range(k): cal_logits_l = platt_binners_equal_points[l](platts[l]( logits[:, l])) data = list(zip(cal_logits_l, labels[:, l])) bins_l = utils.get_discrete_bins(cal_logits_l) binned_data = utils.bin(data, bins_l) # probs = platts[l](logits[:, l]) # for p in [1, 5, 10, 20, 50, 85, 88.5, 92, 94, 96, 98, 100]: # print(np.percentile(probs, p)) # import time # time.sleep(100) # print('lengths') # print([len(d) for d in binned_data]) ces.append(ce_est(binned_data)) mses.append( np.mean([(prob - label)**2 for prob, label in data])) return np.mean(mses), np.mean(ces) def plugin_ce_squared(data): logits, labels = zip(*data) return get_mse_ce(logits, labels, lambda x: utils.plugin_ce(x)**2)[1] def mse(data): logits, labels = zip(*data) return get_mse_ce(logits, labels, lambda x: utils.plugin_ce(x)**2)[0] def unbiased_ce_squared(data): logits, labels = zip(*data) return get_mse_ce(logits, labels, utils.improved_unbiased_square_ce)[1] mse, improved_unbiased_ce = get_mse_ce( cert_logits, cert_labels, utils.improved_unbiased_square_ce) mse, biased_ce = get_mse_ce(cert_logits, cert_labels, lambda x: utils.plugin_ce(x)**2) mses.append(mse) unbiased_ces.append(improved_unbiased_ce) biased_ces.append(biased_ce) print('biased ce: ', np.sqrt(biased_ce)) print('mse: ', mse) print('improved ce: ', np.sqrt(improved_unbiased_ce)) data = list(zip(list(cert_logits), list(cert_labels))) std_biased_ces.append( utils.bootstrap_std(data, plugin_ce_squared, num_samples=bootstrap_samples)) std_unbiased_ces.append( utils.bootstrap_std(data, unbiased_ce_squared, num_samples=bootstrap_samples)) std_multiplier = 1.3 # For one sided 90% confidence interval. upper_unbiased_ces = list( map(lambda p: np.sqrt(p[0] + std_multiplier * p[1]), zip(unbiased_ces, std_unbiased_ces))) upper_biased_ces = list( map(lambda p: np.sqrt(p[0] + std_multiplier * p[1]), zip(biased_ces, std_biased_ces))) # Get points on the Pareto curve, and plot them. def get_pareto_points(data): pareto_points = [] def dominated(p1, p2): return p1[0] >= p2[0] and p1[1] >= p2[1] for datum in data: num_dominated = sum(map(lambda x: dominated(datum, x), data)) if num_dominated == 1: pareto_points.append(datum) return pareto_points print( get_pareto_points( list(zip(upper_unbiased_ces, mses, list(range(5, 101, 5)))))) print( get_pareto_points( list(zip(upper_biased_ces, mses, list(range(5, 101, 5)))))) plot_unbiased_ces, plot_unbiased_mses = zip( *get_pareto_points(list(zip(upper_unbiased_ces, mses)))) plot_biased_ces, plot_biased_mses = zip( *get_pareto_points(list(zip(upper_biased_ces, mses)))) plt.title("MSE vs Calibration Error") plt.scatter(plot_unbiased_ces, plot_unbiased_mses, c='red', marker='o', label='Ours') plt.scatter(plot_biased_ces, plot_biased_mses, c='blue', marker='s', label='Plugin') plt.legend(loc='upper left') plt.ylim(0.0, 0.013) plt.xlabel("L2 Calibration Error") plt.ylabel("Mean-Squared Error") plt.show()
plt.yticks([0, 1]) plt.xticks([0, 0.5, 1]) ax = plt.gca() ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) ax.yaxis.set_ticks_position('left') ax.xaxis.set_ticks_position('bottom') ax.set_ylim([-0.05, 1.05]) # Set up points. X = np.arange(1.0 / 18, 1.0, 1.0 / 9) Y = [0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0] plt.scatter(X, Y, marker='x', s=40, c='black') # Add Platt curve. platt_scaler = utils.get_platt_scaler(X, Y) finer_x = np.arange(0.0, 1.0, 0.01) platt_finer_y = platt_scaler(finer_x) if calibrator == _PLATT_SCALING: plt.plot(finer_x, platt_finer_y, c='red') elif calibrator == _VAR_RED_BINNING: plt.plot(finer_x, platt_finer_y, c='gray') # Add bin lines. if calibrator == _VAR_RED_BINNING or calibrator == _HIST_BINNING: plt.axvline(x=1/3.0, linestyle='--', linewidth=2, c='gray') plt.axvline(x=2/3.0, linestyle='--', linewidth=2, c='gray') # Add platt values. platt_y = platt_scaler(X) if calibrator == _VAR_RED_BINNING:
def compare_estimators(logits, labels, platt_data_size, bin_data_size, num_bins, ver_base_size=2000, ver_size_increment=1000, num_resamples=100, save_name='cmp_est'): # Convert logits to prediction, probs. predictions = utils.get_top_predictions(logits) probs = utils.get_top_probs(logits) correct = (predictions == labels) # Platt scale on first chunk of data platt = utils.get_platt_scaler(probs[:platt_data_size], correct[:platt_data_size]) platt_probs = platt(probs) estimator_names = ['biased', 'unbiased'] estimators = [ lambda x: utils.plugin_ce(x)**2, utils.improved_unbiased_square_ce ] bins = utils.get_equal_bins(platt_probs[:platt_data_size + bin_data_size], num_bins=num_bins) binner = utils.get_discrete_calibrator( platt_probs[platt_data_size:platt_data_size + bin_data_size], bins) verification_probs = binner(platt_probs[platt_data_size + bin_data_size:]) verification_correct = correct[platt_data_size + bin_data_size:] verification_data = list(zip(verification_probs, verification_correct)) verification_sizes = list( range(ver_base_size, len(verification_probs) + 1, ver_size_increment)) # We want to compare the two estimators when varying the number of samples. # However, a single point of comparison does not tell us much about the estimators. # So we use resampling - we resample from the test set many times, and run the estimators # on the resamples. We stores these values. This gives us a sense of the range of values # the estimator might output. # So estimates[i][j][k] stores the estimate when using estimator i, with verification_sizes[j] # samples, in the k-th resampling. estimates = np.zeros( (len(estimators), len(verification_sizes), num_resamples)) # We also store the certified estimates. These represent the upper bounds we get using # each estimator. They are computing using the std-dev of the estimator estimated by # Bootstrap. cert_estimates = np.zeros( (len(estimators), len(verification_sizes), num_resamples)) for ver_idx, verification_size in zip(range(len(verification_sizes)), verification_sizes): for k in range(num_resamples): # Resample indices = np.random.choice(list(range(len(verification_data))), size=verification_size, replace=True) cur_verification_data = [verification_data[i] for i in indices] cur_verification_probs = [verification_probs[i] for i in indices] bins = utils.get_discrete_bins(cur_verification_probs) # Compute estimates for each estimator. for i in range(len(estimators)): def estimator(data): binned_data = utils.bin(data, bins) return estimators[i](binned_data) cur_estimate = estimator(cur_verification_data) estimates[i][ver_idx][k] = cur_estimate # cert_resampling_estimates[j].append( # cur_estimate + utils.bootstrap_std(cur_verification_data, estimator, num_samples=20)) estimates = np.sort(estimates, axis=-1) lower_bound = int(0.1 * num_resamples) median = int(0.5 * num_resamples) upper_bound = int(0.9 * num_resamples) lower_estimates = estimates[:, :, lower_bound] upper_estimates = estimates[:, :, upper_bound] median_estimates = estimates[:, :, median] # We can also compute the MSEs of the estimator. bins = utils.get_discrete_bins(verification_probs) true_calibration = utils.plugin_ce(utils.bin(verification_data, bins))**2 print(true_calibration) print(np.sqrt(np.mean(estimates[1, -1, :]))) errors = np.abs(estimates - true_calibration) accumulated_errors = np.mean(errors, axis=-1) error_bars_90 = 1.645 * np.std(errors, axis=-1) / np.sqrt(num_resamples) print(accumulated_errors) plt.errorbar(verification_sizes, accumulated_errors[0], yerr=[error_bars_90[0], error_bars_90[0]], barsabove=True, color='red', capsize=4, label='plugin') plt.errorbar(verification_sizes, accumulated_errors[1], yerr=[error_bars_90[1], error_bars_90[1]], barsabove=True, color='blue', capsize=4, label='debiased') plt.ylabel("Mean-Squared-Error") plt.xlabel("Number of Samples") plt.legend(loc='upper right') plt.show() plt.ylabel("Number of estimates") plt.xlabel("Absolute deviation from ground truth") bins = np.linspace(np.min(errors[:, 0, :]), np.max(errors[:, 0, :]), 40) plt.hist(errors[0][0], bins, alpha=0.5, label='plugin') plt.hist(errors[1][0], bins, alpha=0.5, label='debiased') plt.legend(loc='upper right') plt.gca().yaxis.set_major_formatter(PercentFormatter(xmax=num_resamples)) plt.show()
def train_calibration(self, zs, ys): self._platt = utils.get_platt_scaler(zs, ys)
def train_calibration(self, zs, ys): self._platt = utils.get_platt_scaler(zs, ys) platt_probs = self._platt(zs) bins = utils.get_equal_bins(platt_probs, num_bins=self._num_bins) self._discrete_calibrator = utils.get_discrete_calibrator(platt_probs, bins)