def __init__(self,
labels=None,
calibrated=None,
method='Isotronic Regression',
independent_probabilities=False):
super().__init__(calibrated=calibrated,
labels=labels,
method=method)
self.calibrated = calibrated
self.labels = labels
self.method = method
self.iso = IsotonicRegression(independent_probabilities)
def example_calibration(datafile: str) -> int:
"""
Example of several calibration methods.
Parameters
----------
datafile : str
Path to datafile which contains two NumPy arrays with keys 'ground_truth' and 'predictions'.
Returns
-------
int
0 at success, -1 otherwise.
"""
bins = 10
# diagram = None
diagram = 'diagram'
# define validation split for test data
validation_split = 0.7
# if True, a Pickle-Object will be written out for each calibration model built
save_models = False
histogram = HistogramBinning(bins)
iso = IsotonicRegression()
bbq = BBQ()
enir = ENIR()
lr_calibration = LogisticCalibration()
temperature = TemperatureScaling()
betacal = BetaCalibration()
models = [("Histogram Binning", histogram), ("Isotonic Regression", iso),
("BBQ", bbq), ("ENIR", enir),
("Logistic Calibration", lr_calibration),
("Temperature Scaling", temperature),
("Beta Calibration", betacal)]
# see ../utils.py for calibration and its measurement
success = single_example(models=models,
datafile=datafile,
bins=bins,
diagram=diagram,
validation_split=validation_split,
save_models=save_models)
return success
def cross_validation(datafile: str) -> int:
"""
5x2 cross validation of several calibration methods.
Parameters
----------
datafile : str
Path to datafile which contains two NumPy arrays with keys 'ground_truth' and 'predictions'.
Returns
-------
int
0 at success, -1 otherwise.
"""
bins = 10
# if True, a Pickle-Object will be written out for each calibration model built
save_models = False
histogram = HistogramBinning(bins)
iso = IsotonicRegression()
bbq = BBQ()
enir = ENIR()
lr_calibration = LogisticCalibration()
temperature = TemperatureScaling()
betacal = BetaCalibration()
models = [("Histogram Binning", histogram), ("Isotonic Regression", iso),
("BBQ", bbq), ("ENIR", enir),
("Logistic Calibration", lr_calibration),
("Temperature Scaling", temperature),
("Beta Calibration", betacal)]
# invoke cross validation function from ../utils.py
# see ../utils.py for calibration and its measurement
success = cross_validation_5_2(models=models,
datafile=datafile,
bins=bins,
save_models=save_models)
return success
def get_calibrator_fn(self, cfg):
if cfg.METHOD.NAME == "HistogramBinning":
bins = cfg.METHOD.HIST.N_BINS
if bins == -1:
bins = cfg.TEST.N_BINS
return HistogramBinning(bins, detection=True)
elif cfg.METHOD.NAME == "IsotonicRegression":
return IsotonicRegression(detection=True)
elif cfg.METHOD.NAME == "BBQ":
return BBQ(score_function=cfg.METHOD.BBQ.SCORE_FN, detection=True)
elif cfg.METHOD.NAME == "ENIR":
return ENIR(score_function=cfg.METHOD.ENIR.SCORE_FN,
detection=True)
elif cfg.METHOD.NAME == "LogisticCalibration":
return LogisticCalibration(detection=True)
elif cfg.METHOD.NAME == "BetaCalibration":
return BetaCalibration(detection=True)
elif cfg.METHOD.NAME == "TemperatureScaling":
return TemperatureScaling(detection=True)
else:
raise NotImplementedError
class Isotronic(Evaluation):
"""
Isotonic Regression is a calibration method to correct any monotonic distortion.
"""
def __init__(self,
labels=None,
calibrated=None,
method='Isotronic Regression',
independent_probabilities=False):
super().__init__(calibrated=calibrated,
labels=labels,
method=method)
self.calibrated = calibrated
self.labels = labels
self.method = method
self.iso = IsotonicRegression(independent_probabilities)
def fit_transform(self, train, y_train):
self.labels = y_train
self.calibrated = self.iso.fit_transform(train, y_train)
return self.calibrated
def cross_validation(datafile: str,
use_cuda: Union[bool, str] = False,
domain: str = ".") -> int:
"""
5x2 cross validation of several calibration methods.
Parameters
----------
datafile : str
Path to datafile which contains two NumPy arrays with keys 'ground_truth' and 'predictions'.
Returns
-------
int
0 at success, -1 otherwise.
"""
# kwargs for uncertainty mode. Those can also be safely set on MLE
uncertainty_kwargs = {
'mcmc_chains': 1,
'mcmc_samples': 300,
'mcmc_warmup_steps': 50,
'vi_samples': 300,
'vi_epochs': 3000
}
hist_bins = 20
bins = 15
if domain == 'examination-mcmc':
method = 'mcmc'
elif domain == 'examination-variational':
method = 'variational'
else:
method = 'mle'
# if True, a Pickle-Object will be written out for each calibration model built
save_models = True
histogram = HistogramBinning(hist_bins)
iso = IsotonicRegression()
bbq = BBQ()
enir = ENIR()
lr_calibration = LogisticCalibration(detection=False,
method=method,
use_cuda=use_cuda,
**uncertainty_kwargs)
temperature = TemperatureScaling(detection=False,
method=method,
use_cuda=use_cuda,
**uncertainty_kwargs)
betacal = BetaCalibration(detection=False,
method=method,
use_cuda=use_cuda,
**uncertainty_kwargs)
models = [("hist", histogram), ("iso", iso), ("bbq", bbq), ("enir", enir),
("lr", lr_calibration), ("temperature", temperature),
("beta", betacal)]
# invoke cross validation function from ../utils.py
# see ../utils.py for calibration and its measurement
success = cross_validation_5_2(models=models,
datafile=datafile,
bins=bins,
save_models=save_models,
domain=domain)
return success