def calibrator_module(zs,ys,n_classes,method): if method == 'Platt': calibrator = cal.PlattBinnerMarginalCalibrator(n_classes, num_bins=200) calibrator.train_calibration(zs, ys) calibrated_zs = calibrator.calibrate(zs) calibration_error = cal.get_calibration_error(calibrated_zs, ys) print("Scaling-binning L2 calibration error with %s is %.2f%%" % (method, (100 * calibration_error))) elif method =='Isotonic': # binarize class labels lb = preprocessing.LabelBinarizer().fit(ys) y_cal_ohe = lb.transform(ys) del lb calibrator = [] # calibrated_prob = np.zeros((ys.shape[0],n_classes)) for i in range(n_classes): calibrator.append(IsotonicRegression(out_of_bounds = 'clip').fit(zs[:,i], y_cal_ohe[:,i])) # calibrated_prob[:,i] = calibrator[i].transform(zs[:,i]) # # Normalize the probabilities # calibrated_prob = normalize_prob(calibrated_prob,n_classes) # calibration_error = cal.get_calibration_error(calibrated_prob, ys) # print("Scaling-binning L2 calibration error with %s is %.2f%%" % (method, (100 * calibration_error))) return calibrator
def main(): # Make synthetic dataset. np.random.seed(0) # Keep results consistent. num_points = 1000 d = 10 (zs, ys) = synthetic_data(num_points=num_points, d=d) # Estimate a lower bound on the calibration error. # Here z_i are the per-class confidences of the uncalibrated model, y_i is the true label. calibration_error = calibration.get_calibration_error(zs, ys) print("Uncalibrated model calibration error is > %.2f%%" % (100 * calibration_error)) # Use Platt binning to train a recalibrator. calibrator = calibration.PlattBinnerMarginalCalibrator(num_points, num_bins=10) calibrator.train_calibration(zs, ys) # Measure the calibration error of recalibrated model. (test_zs, test_ys) = synthetic_data(num_points=num_points, d=d) calibrated_zs = calibrator.calibrate(test_zs) calibration_error = calibration.get_calibration_error(calibrated_zs, test_ys) print("Scaling-binning L2 calibration error is %.2f%%" % (100 * calibration_error)) # Get confidence intervals for the calibration error. [lower, _, upper] = calibration.get_calibration_error_uncertainties(calibrated_zs, test_ys) print(" Confidence interval is [%.2f%%, %.2f%%]" % (100 * lower, 100 * upper))
def on_epoch_end(self, data: Data) -> None: self.y_true = np.squeeze(np.stack(self.y_true)) self.y_pred = np.stack(self.y_pred) calibrator = cal.PlattBinnerMarginalCalibrator(num_calibration=len( self.y_true), num_bins=10) calibrator.train_calibration(probs=self.y_pred, labels=self.y_true) if self.save_path: if not self.save_key or (self.save_key and to_number(data[self.save_key]) == 0): with open(self.save_path, 'wb') as f: dill.dump(calibrator.calibrate, file=f) print( f"FastEstimator-PBMCalibrator: Calibrator written to {self.save_path}" ) data.write_without_log(self.outputs[0], calibrator.calibrate(self.y_pred))
test_file = f'{params.exp_dir}/test_ts_prob/results.pgz' with gzip.open(valid_file, 'rb') as f: valid_dict = pickle.load(f) valid_probs = valid_dict['probs'] valid_label = valid_dict['label'] with gzip.open(test_file, 'rb') as f: test_dict = pickle.load(f) test_probs = test_dict['probs'] test_label = test_dict['label'] T = test_probs.shape[1] test_probs_cal = [] for t in range(T): calibrator = calibration.PlattBinnerMarginalCalibrator(len(valid_label), num_bins=10) calibrator.train_calibration(valid_probs[:, t], valid_label.astype(np.int32)) test_probs_cal.append(calibrator.calibrate(test_probs[:, t])) test_probs_cal = np.stack(test_probs_cal, axis=1) preds = np.argmax(test_probs_cal, axis=-1) probs = np.max(test_probs_cal, axis=-1) acc = (preds == np.expand_dims(test_label, axis=1)).astype(np.float32) res = {'probs': probs, 'preds': preds, 'acc': acc} with gzip.open(f'{save_dir}/results.pgz', 'wb') as f: pickle.dump(res, f) # plot T = test_probs.shape[1] x = range(1, T + 1)