def calibrator_module(zs,ys,n_classes,method):
if method == 'Platt':
calibrator = cal.PlattBinnerMarginalCalibrator(n_classes, num_bins=200)
calibrator.train_calibration(zs, ys)
calibrated_zs = calibrator.calibrate(zs)
calibration_error = cal.get_calibration_error(calibrated_zs, ys)
print("Scaling-binning L2 calibration error with %s is %.2f%%" % (method, (100 * calibration_error)))
elif method =='Isotonic':
# binarize class labels
lb = preprocessing.LabelBinarizer().fit(ys)
y_cal_ohe = lb.transform(ys)
del lb
calibrator = []
# calibrated_prob = np.zeros((ys.shape[0],n_classes))
for i in range(n_classes):
calibrator.append(IsotonicRegression(out_of_bounds = 'clip').fit(zs[:,i], y_cal_ohe[:,i]))
# calibrated_prob[:,i] = calibrator[i].transform(zs[:,i])
# # Normalize the probabilities
# calibrated_prob = normalize_prob(calibrated_prob,n_classes)
# calibration_error = cal.get_calibration_error(calibrated_prob, ys)
# print("Scaling-binning L2 calibration error with %s is %.2f%%" % (method, (100 * calibration_error)))
return calibrator
def main():
# Make synthetic dataset.
np.random.seed(0) # Keep results consistent.
num_points = 1000
d = 10
(zs, ys) = synthetic_data(num_points=num_points, d=d)
# Estimate a lower bound on the calibration error.
# Here z_i are the per-class confidences of the uncalibrated model, y_i is the true label.
calibration_error = calibration.get_calibration_error(zs, ys)
print("Uncalibrated model calibration error is > %.2f%%" % (100 * calibration_error))
# Use Platt binning to train a recalibrator.
calibrator = calibration.PlattBinnerMarginalCalibrator(num_points, num_bins=10)
calibrator.train_calibration(zs, ys)
# Measure the calibration error of recalibrated model.
(test_zs, test_ys) = synthetic_data(num_points=num_points, d=d)
calibrated_zs = calibrator.calibrate(test_zs)
calibration_error = calibration.get_calibration_error(calibrated_zs, test_ys)
print("Scaling-binning L2 calibration error is %.2f%%" % (100 * calibration_error))
# Get confidence intervals for the calibration error.
[lower, _, upper] = calibration.get_calibration_error_uncertainties(calibrated_zs, test_ys)
print(" Confidence interval is [%.2f%%, %.2f%%]" % (100 * lower, 100 * upper))
def on_epoch_end(self, data: Data) -> None:
self.y_true = np.squeeze(np.stack(self.y_true))
self.y_pred = np.stack(self.y_pred)
calibrator = cal.PlattBinnerMarginalCalibrator(num_calibration=len(
self.y_true),
num_bins=10)
calibrator.train_calibration(probs=self.y_pred, labels=self.y_true)
if self.save_path:
if not self.save_key or (self.save_key
and to_number(data[self.save_key]) == 0):
with open(self.save_path, 'wb') as f:
dill.dump(calibrator.calibrate, file=f)
print(
f"FastEstimator-PBMCalibrator: Calibrator written to {self.save_path}"
)
data.write_without_log(self.outputs[0],
calibrator.calibrate(self.y_pred))
test_file = f'{params.exp_dir}/test_ts_prob/results.pgz'
with gzip.open(valid_file, 'rb') as f:
valid_dict = pickle.load(f)
valid_probs = valid_dict['probs']
valid_label = valid_dict['label']
with gzip.open(test_file, 'rb') as f:
test_dict = pickle.load(f)
test_probs = test_dict['probs']
test_label = test_dict['label']
T = test_probs.shape[1]
test_probs_cal = []
for t in range(T):
calibrator = calibration.PlattBinnerMarginalCalibrator(len(valid_label),
num_bins=10)
calibrator.train_calibration(valid_probs[:, t],
valid_label.astype(np.int32))
test_probs_cal.append(calibrator.calibrate(test_probs[:, t]))
test_probs_cal = np.stack(test_probs_cal, axis=1)
preds = np.argmax(test_probs_cal, axis=-1)
probs = np.max(test_probs_cal, axis=-1)
acc = (preds == np.expand_dims(test_label, axis=1)).astype(np.float32)
res = {'probs': probs, 'preds': preds, 'acc': acc}
with gzip.open(f'{save_dir}/results.pgz', 'wb') as f:
pickle.dump(res, f)
# plot
T = test_probs.shape[1]
x = range(1, T + 1)