def cv_confidence_intervals(base_classifier, x_train, y_train, x_test,
y_test, cv=2, score_type=None):
intervals = None
skf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=seed)
for i, (train, cali) in enumerate(skf.split(X=x_train, y=y_train)):
if i == 0:
x_t = x_train[train]
y_t = y_train[train]
x_c = x_train[cali]
y_c = y_train[cali]
classifier = clone(base_classifier)
classifier.fit(x_t, y_t)
ccv = calibrate(classifier, x_c, y_c, method=None,
score_type=score_type)
scores = ccv.predict_proba(x_c)[:, 1]
scores_test = ccv.predict_proba(x_test)[:, 1]
ll_before = cross_entropy(scores_test, y_test)
brier_before = brier_score(scores_test, y_test)
calibrator = BetaCalibration(parameters="abm").fit(scores, y_c)
ll_after = cross_entropy(calibrator.predict(scores_test), y_test)
brier_after = brier_score(calibrator.predict(scores_test), y_test)
original_map = calibrator.calibrator_.map_
intervals = beta_test(original_map,
test_type="adev", scores=scores)
intervals["ll_diff"] = ll_after - ll_before
intervals["bs_diff"] = brier_after - brier_before
return intervals
def fit(self, X, y, sample_weight=None):
"""Fit beta calibration only if the classifier is considered
uncalibrated.
Parameters
----------
X : array-like, shape (n_samples,)
Training data.
y : array-like, shape (n_samples,)
Training target.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
self._calibrator = BetaCalibration(parameters="abm").fit(X, y)
test = beta_test(self._calibrator.calibrator_.map_,
test_type="adev",
scores=X)
if test["p-value"] >= 0.05:
self._calibrator = _DummyCalibration().fit(X, y)
return self
def test_betacal_ab(self):
bc = BetaCalibration(parameters="ab")
bc.fit(s, y)
pred = bc.predict(s)
np.testing.assert_allclose(pred, pred_ab)
def test_betacal_am(self):
bc = BetaCalibration(parameters="am")
bc.fit(s, y)
pred = bc.predict(s)
# With smaller tolerance does not pass the tests
np.testing.assert_allclose(pred, pred_am, rtol=1e-5)
def fit(self, X, y, sample_weight=None):
"""Calibrate the fitted model
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data.
y : array-like, shape (n_samples,)
Target values.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
lb = LabelBinarizer()
Y = lb.fit_transform(y)
self.classes_ = lb.classes_
df, idx_pos_class = self._preproc(X)
self.calibrators_ = []
for k, this_df in zip(idx_pos_class, df.T):
if self.method is None:
calibrator = _DummyCalibration()
elif self.method == 'isotonic':
calibrator = IsotonicRegression(out_of_bounds='clip')
elif self.method == 'sigmoid':
calibrator = _SigmoidCalibration()
elif self.method == 'beta':
calibrator = BetaCalibration(parameters='abm')
elif self.method == 'beta_am':
calibrator = BetaCalibration(parameters='am')
elif self.method == 'beta_ab':
calibrator = BetaCalibration(parameters='ab')
else:
raise ValueError('method should be None, "sigmoid", '
'"isotonic", "beta", "beta2" or "beta05". '
'Got %s.' % self.method)
calibrator.fit(this_df, Y[:, k], sample_weight)
self.calibrators_.append(calibrator)
return self
def fit(self, X, y, sample_weight=None):
"""Calibrate the fitted model
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data.
y : array-like, shape (n_samples,)
Target values.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
self.label_encoder_ = LabelEncoder()
if self.classes is None:
self.label_encoder_.fit(y)
else:
self.label_encoder_.fit(self.classes)
self.classes_ = self.label_encoder_.classes_
Y = label_binarize(y, self.classes_)
df, idx_pos_class = self._preproc(X)
self.calibrators_ = []
for k, this_df in zip(idx_pos_class, df.T):
if self.method == 'isotonic':
calibrator = IsotonicRegression(out_of_bounds='clip')
elif self.method == 'sigmoid':
calibrator = _SigmoidCalibration()
elif self.method == 'euler':
calibrator = _EulerSigmoidCalibration()
elif self.method == 'beta':
calibrator = BetaCalibration()
elif self.method in ['rocch', 'convex']:
calibrator = _ROCCHCalibration()
elif isinstance(self.method, BaseEstimator):
calibrator = self.method
else:
raise ValueError('method should be "sigmoid" or '
'"isotonic". Got %s.' % self.method)
calibrator.fit(this_df, Y[:, k], sample_weight)
self.calibrators_.append(calibrator)
return self
class _BetaTestedCalibration(BaseEstimator, RegressorMixin):
"""Dummy regression model. The purpose of this class is to give
the CalibratedClassifierCV class the option to just return the
probabilities of the base classifier.
"""
def fit(self, X, y, sample_weight=None):
"""Fit beta calibration only if the classifier is considered
uncalibrated.
Parameters
----------
X : array-like, shape (n_samples,)
Training data.
y : array-like, shape (n_samples,)
Training target.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
self._calibrator = BetaCalibration(parameters="abm").fit(X, y)
test = beta_test(self._calibrator.calibrator_.map_,
test_type="adev",
scores=X)
if test["p-value"] >= 0.05:
self._calibrator = _DummyCalibration().fit(X, y)
return self
def predict(self, T):
"""Return the probabilities of the base classifier.
Parameters
----------
T : array-like, shape (n_samples,)
Data to predict from.
Returns
-------
T_ : array, shape (n_samples,)
The predicted data.
"""
return self._calibrator.predict(T)
multi_class='ovr'),
'logistic_log': LogisticCalibration(C=C_list,
log_transform=True,
multi_class='multinomial'),
'logistic_logit': LogisticCalibration(C=C_list,
log_transform=False,
multi_class='multinomial'),
'binning_width' :OneVsRestCalibrator(BinningCalibration(strategy='uniform',
n_bins=n_bins)),
'binning_freq' :OneVsRestCalibrator(BinningCalibration(strategy='quantile',
n_bins=n_bins)),
'binning_kmeans' :OneVsRestCalibrator(BinningCalibration(strategy='kmeans')), # Not working yet
'uncalibrated': _DummyCalibration(),
'isotonic': OneVsRestCalibrator(IsotonicCalibration(out_of_bounds='clip')),
'sigmoid': OneVsRestCalibrator(SigmoidCalibration()),
'beta': OneVsRestCalibrator(BetaCalibration(parameters="abm")),
'beta_am': OneVsRestCalibrator(BetaCalibration(parameters="am")),
'beta_ab': OneVsRestCalibrator(BetaCalibration(parameters="ab")),
'ovr_dir_full': OneVsRestCalibrator(DirichletCalibrator(matrix_type='full')),
'ovr_dir_full_l2': OneVsRestCalibrator(DirichletCalibrator(matrix_type='full',
l2=l2_list)),
'ovr_dir_diag': OneVsRestCalibrator(DirichletCalibrator(matrix_type='diagonal')),
'ovr_dir_fixd': OneVsRestCalibrator(DirichletCalibrator(matrix_type='fixed_diagonal')),
'dirichlet_keras': Dirichlet_NN(l2=10, mu=0.0001),
'dirichlet_full': DirichletCalibrator(matrix_type='full'),
'dirichlet_full_gen': GenerativeDirichletCalibrator(),
'dirichlet_full_prefixdiag': DirichletCalibrator(matrix_type='full',
initializer='preFixDiag'),
'dirichlet_full_comp_l2': DirichletCalibrator(matrix_type='full',
comp_l2=True,
l2=l2_list),
def fit(self, X, y, sample_weight=None):
"""Fit the calibrated model
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data.
y : array-like, shape (n_samples,)
Target values.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
X, y = check_X_y(X,
y,
accept_sparse=['csc', 'csr', 'coo'],
force_all_finite=False)
X, y = indexable(X, y)
df = self._preproc(X)
weights = None
if self.platts_trick:
# Bayesian priors (see Platt end of section 2.2)
prior0 = float(np.sum(y <= 0))
prior1 = y.shape[0] - prior0
weights = np.zeros_like(y).astype(float)
weights[y > 0] = (prior1 + 1.) / (prior1 + 2.)
weights[y <= 0] = 1. / (prior0 + 2.)
y = np.append(np.ones_like(y), np.zeros_like(y))
weights = np.append(weights, 1.0 - weights)
df = np.append(df, df)
if self.method is None:
self.calibrator = _DummyCalibration()
elif self.method == 'isotonic':
self.calibrator = IsotonicRegression(out_of_bounds='clip')
elif self.method == 'sksigmoid':
self.calibrator = sk_sigmoid()
elif self.method == 'sksigmoid_notrick':
self.calibrator = sk_sigmoid_notrick()
elif self.method == 'sigmoid':
self.calibrator = _SigmoidCalibration()
elif self.method == 'beta':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_am':
self.calibrator = BetaCalibration(parameters="am")
elif self.method == 'beta_ab':
self.calibrator = BetaCalibration(parameters="ab")
elif self.method == 'beta_test_strict':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_test_relaxed':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_test':
self.calibrator = _BetaTestedCalibration()
else:
raise ValueError('method should be None, "sigmoid", '
'"isotonic", "beta", "beta_am" or "beta_ab". '
'Got %s.' % self.method)
self.calibrator.fit(df, y, weights)
if self.method == 'beta':
df_pos = df[y == 1]
df_neg = df[y == 0]
# alpha_pos_nll, beta_pos_nll = fit_beta_nll(df_pos)
# alpha_neg_nll, beta_neg_nll = fit_beta_nll(df_neg)
#
# a_nll = alpha_pos_nll - alpha_neg_nll
# b_nll = beta_neg_nll - beta_pos_nll
# m_nll = fit_beta_midpoint(alpha_pos_nll, beta_pos_nll,
# alpha_neg_nll, beta_neg_nll)
alpha_pos_mmt, beta_pos_mmt = fit_beta_moments(df_pos)
alpha_neg_mmt, beta_neg_mmt = fit_beta_moments(df_neg)
a_mmt = alpha_pos_mmt - alpha_neg_mmt
if a_mmt < 0 or np.isnan(a_mmt):
a_mmt = 0
b_mmt = beta_neg_mmt - beta_pos_mmt
if b_mmt < 0 or np.isnan(b_mmt):
b_mmt = 0
prior_pos = len(df_pos) / len(df)
prior_neg = len(df_neg) / len(df)
m_mmt = fit_beta_midpoint(prior_pos, alpha_pos_mmt, beta_pos_mmt,
prior_neg, alpha_neg_mmt, beta_neg_mmt)
map = self.calibrator.calibrator_.map_
# if a_mmt > 4 and map[0] < 2:
# print [a_mmt, map[0]]
# print [b_mmt, map[1]]
# print [m_mmt, map[2]]
# exit()
# if b_mmt > 4 and map[1] < 2:
# print [a_mmt, map[0]]
# print [b_mmt, map[1]]
# print [m_mmt, map[2]]
# exit()
self.a = [a_mmt, map[0]]
self.b = [b_mmt, map[1]]
self.m = [m_mmt, map[2]]
self.df_pos = df_pos
self.df_neg = df_neg
return self
class CalibratedModel(BaseEstimator, ClassifierMixin):
def __init__(self, base_estimator=None, method=None, score_type=None):
if method is None:
self.method = method
self.platts_trick = False
else:
temp = method.split('-')
self.platts_trick = (len(temp) == 2)
self.method = temp[0]
self.base_estimator = base_estimator
self.score_type = score_type
def set_base_estimator(self, base_estimator, score_type=None):
self.base_estimator = base_estimator
self.score_type = score_type
def _preproc(self, X):
if self.score_type is None:
if hasattr(self.base_estimator, "decision_function"):
df = self.base_estimator.decision_function(X)
if df.ndim == 1:
df = df[:, np.newaxis]
elif hasattr(self.base_estimator, "predict_proba"):
df = self.base_estimator.predict_proba(X)
df = df[:, 1]
else:
raise RuntimeError('classifier has no decision_function or '
'predict_proba method.')
else:
if self.score_type == "sigmoid":
df = self.base_estimator.decision_function(X)
df = expit(df)
if df.ndim == 1:
df = df[:, np.newaxis]
else:
if hasattr(self.base_estimator, self.score_type):
df = getattr(self.base_estimator, self.score_type)(X)
if self.score_type == "decision_function":
if df.ndim == 1:
df = df[:, np.newaxis]
elif self.score_type == "predict_proba":
df = df[:, 1:]
else:
raise RuntimeError('classifier has no ' + self.score_type +
'method.')
return df.reshape(-1)
def fit(self, X, y, sample_weight=None):
"""Fit the calibrated model
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data.
y : array-like, shape (n_samples,)
Target values.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted.
Returns
-------
self : object
Returns an instance of self.
"""
X, y = check_X_y(X,
y,
accept_sparse=['csc', 'csr', 'coo'],
force_all_finite=False)
X, y = indexable(X, y)
df = self._preproc(X)
weights = None
if self.platts_trick:
# Bayesian priors (see Platt end of section 2.2)
prior0 = float(np.sum(y <= 0))
prior1 = y.shape[0] - prior0
weights = np.zeros_like(y).astype(float)
weights[y > 0] = (prior1 + 1.) / (prior1 + 2.)
weights[y <= 0] = 1. / (prior0 + 2.)
y = np.append(np.ones_like(y), np.zeros_like(y))
weights = np.append(weights, 1.0 - weights)
df = np.append(df, df)
if self.method is None:
self.calibrator = _DummyCalibration()
elif self.method == 'isotonic':
self.calibrator = IsotonicRegression(out_of_bounds='clip')
elif self.method == 'sksigmoid':
self.calibrator = sk_sigmoid()
elif self.method == 'sksigmoid_notrick':
self.calibrator = sk_sigmoid_notrick()
elif self.method == 'sigmoid':
self.calibrator = _SigmoidCalibration()
elif self.method == 'beta':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_am':
self.calibrator = BetaCalibration(parameters="am")
elif self.method == 'beta_ab':
self.calibrator = BetaCalibration(parameters="ab")
elif self.method == 'beta_test_strict':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_test_relaxed':
self.calibrator = BetaCalibration(parameters="abm")
elif self.method == 'beta_test':
self.calibrator = _BetaTestedCalibration()
else:
raise ValueError('method should be None, "sigmoid", '
'"isotonic", "beta", "beta_am" or "beta_ab". '
'Got %s.' % self.method)
self.calibrator.fit(df, y, weights)
if self.method == 'beta':
df_pos = df[y == 1]
df_neg = df[y == 0]
# alpha_pos_nll, beta_pos_nll = fit_beta_nll(df_pos)
# alpha_neg_nll, beta_neg_nll = fit_beta_nll(df_neg)
#
# a_nll = alpha_pos_nll - alpha_neg_nll
# b_nll = beta_neg_nll - beta_pos_nll
# m_nll = fit_beta_midpoint(alpha_pos_nll, beta_pos_nll,
# alpha_neg_nll, beta_neg_nll)
alpha_pos_mmt, beta_pos_mmt = fit_beta_moments(df_pos)
alpha_neg_mmt, beta_neg_mmt = fit_beta_moments(df_neg)
a_mmt = alpha_pos_mmt - alpha_neg_mmt
if a_mmt < 0 or np.isnan(a_mmt):
a_mmt = 0
b_mmt = beta_neg_mmt - beta_pos_mmt
if b_mmt < 0 or np.isnan(b_mmt):
b_mmt = 0
prior_pos = len(df_pos) / len(df)
prior_neg = len(df_neg) / len(df)
m_mmt = fit_beta_midpoint(prior_pos, alpha_pos_mmt, beta_pos_mmt,
prior_neg, alpha_neg_mmt, beta_neg_mmt)
map = self.calibrator.calibrator_.map_
# if a_mmt > 4 and map[0] < 2:
# print [a_mmt, map[0]]
# print [b_mmt, map[1]]
# print [m_mmt, map[2]]
# exit()
# if b_mmt > 4 and map[1] < 2:
# print [a_mmt, map[0]]
# print [b_mmt, map[1]]
# print [m_mmt, map[2]]
# exit()
self.a = [a_mmt, map[0]]
self.b = [b_mmt, map[1]]
self.m = [m_mmt, map[2]]
self.df_pos = df_pos
self.df_neg = df_neg
return self
def predict_proba(self, X):
"""Posterior probabilities of classification
This function returns posterior probabilities of classification
according to each class on an array of test vectors X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
The samples.
Returns
-------
C : array, shape (n_samples, n_classes)
The predicted probas. Can be exact zeros.
"""
proba = np.zeros((X.shape[0], 2))
df = self._preproc(X)
proba[:, 1] = self.calibrator.predict(df)
proba[:, 0] = 1. - proba[:, 1]
# Deal with cases where the predicted probability minimally exceeds 1.0
proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0
return proba
def predict(self, X):
"""Predict the target of new samples. Can be different from the
prediction of the uncalibrated classifier.
Parameters
----------
X : array-like, shape (n_samples, n_features)
The samples.
Returns
-------
C : array, shape (n_samples,)
The predicted class.
"""
check_is_fitted(self, ["calibrator"])
return np.argmax(self.predict_proba(X), axis=1)
def train_calibration_new(self, data, pre_cal_model, size):
# check if the scores on calibration data are different, otherwise it is not reasonable to calibrate
if hasattr(pre_cal_model, "predict_proba"):
preds = pre_cal_model.predict_proba(data.get_cal_train_x(size))
scores = pd.Series(preds[:, 1]).tolist()
elif hasattr(pre_cal_model, "decision_function"):
preds = pre_cal_model.decision_function(data.get_cal_train_x(size))
if self.only_log:
scores = pd.Series(preds).tolist()
else:
scores = pd.Series(preds).apply(lambda x: 1 / (1 + np.e ** (-1 * x))).tolist()
if min(scores) == max(scores):
return None
# generate different corrections
y = data.get_cal_train_y(size)
z = scores
z_repl, y_repl, no_need1, no_need2 = data.group_scores(z, y, method="repl")
z_join, y_join, nr0, nr1 = data.group_scores(z, y, method="join")
z_platt, y_platt, no_need1, no_need2 = data.platts_correction(z, y)
if self.only_beta:
cal10 = BetaCalibration(sklearn_lr=False)
cal10_model = cal10.fit(z_repl, y_repl)
cal11 = BetaCalibration(sklearn_lr=False)
cal11_model = cal11.fit(z_platt, y_platt)
return {"cal10_" + str(size): cal10_model, "cal11_" + str(size): cal11_model}
if self.only_log:
cal1 = _MySigmoidCalibration()
cal1_model = cal1.fit(z_repl, y_repl)
cal2 = _MySigmoidCalibration()
cal2_model = cal2.fit(z_platt, y_platt)
return {"cal1_" + str(size): cal1_model, "cal2_" + str(size): cal2_model}
# Logistic calibration
if not self.only_new:
cal1 = _MySigmoidCalibration()
cal1_model = cal1.fit(z_repl, y_repl)
cal2 = _MySigmoidCalibration()
cal2_model = cal2.fit(z_platt, y_platt)
# cal35_model = ClippingCorrection(cal1_model, 0.95)
# cal36_model = ClippingCorrection(cal1_model, 0.99)
# cal37_model = ClippingCorrection(cal1_model, 0.999)
# Isotonic calibration
cal4 = _MyIsotonicCalibration()
cal4_model = cal4.fit(z_repl, y_repl)
if not self.only_new:
cal5 = _MyIsotonicCalibration()
cal5_model = cal5.fit(z_platt, y_platt)
cal7 = _MyIsotonicCalibration_NEW(distr=8, kind=2)
cal7_model = cal7.fit(z_join, y_join, z_repl, y_repl, nr0=nr0, nr1=nr1)
if self.debug:
cal8 = _MyIsotonicCalibration_NEW(distr=9, kind=2)
cal8_model = cal8.fit(z_join, y_join, z_repl, y_repl, nr0=nr0, nr1=nr1)
else:
cal8 = _MyIsotonicCalibration_NEW(distr=9, kind=2)
cal8_model = cal8.fit(z_join, y_join, z_repl, y_repl, nr0=nr0, nr1=nr1)
#cal65_model = ClippingCorrection(cal4_model, 0.95)
#cal66_model = ClippingCorrection(cal4_model, 0.99)
#cal67_model = ClippingCorrection(cal4_model, 0.999)
# ENIR
if not self.debug:
if not self.only_new:
cal8 = _MyENIRCalibration(self.id + "_" + str(size) + "_1", seed = random.randint(1, 10000))
cal8_model = cal8.fit(z_repl, y_repl)
cal9 = _MyENIRCalibration(self.id + "_" + str(size) + "_2", seed = random.randint(1, 10000))
cal9_model = cal9.fit(z_platt, y_platt)
# cal95_model = ClippingCorrection(cal8_model, 0.95)
# cal96_model = ClippingCorrection(cal8_model, 0.99)
# cal97_model = ClippingCorrection(cal8_model, 0.999)
# Betacal
if not self.debug:
if not self.only_new:
cal10 = BetaCalibration(sklearn_lr=False)
cal10_model = cal10.fit(z_repl, y_repl)
cal11 = BetaCalibration(sklearn_lr=False)
cal11_model = cal11.fit(z_platt, y_platt)
# cal125_model = ClippingCorrection(cal10_model, 0.95)
# cal126_model = ClippingCorrection(cal10_model, 0.99)
# cal127_model = ClippingCorrection(cal10_model, 0.999)
if self.only_new:
return {"cal4_" + str(size): cal4_model, "cal8_" + str(size): cal8_model}
if self.debug:
return {"cal1_" + str(size): cal1_model, "cal2_" + str(size): cal2_model, "cal4_" + str(size): cal4_model,
"cal5_" + str(size): cal5_model, "cal6_" + str(size): cal6_model, "cal7_" + str(size): cal7_model,
"cal8_" + str(size): cal8_model}
return {"cal1_" + str(size): cal1_model, "cal2_" + str(size): cal2_model, "cal4_" + str(size): cal4_model,
"cal5_" + str(size): cal5_model,"cal7_" + str(size): cal7_model,
"cal8_" + str(size): cal8_model, "cal9_" + str(size): cal9_model,
"cal10_" + str(size): cal10_model, "cal11_" + str(size): cal11_model}
def confidence_estimation(X_train, y_train, X_test, y_test, contamination, model):
"""
Estimate the example-wise confidence of different methods, included the model ExCeeD provided in the paper.
First, we compute the outlier probabilities for all the methods but Calibrations. Then, we estimate the
example-wise confidence for ExCeeD, ExCeed_sp (with second prior), ExCeeD with outlier probability computed
through the linear, squashing and unify methods, and calibrated probabilities through Logistic (logcal),
Isotonic (isocal) and Beta (betacal) Calibrations.
Parameters
----------
X_train : list of shape (n_train, n_features) containing the training set with only features.
y_train : list of shape (n_train,) containing the actual labels for the training set. It is needed for Calibration.
X_test : list of shape (n_test, n_features) containing the test set with only features.
y_test : list of shape (n_test,) containing the actual labels for the test set. It is needed for Calibration.
contamination : float representing the expected proportion of anomalies in the training set.
model : string, claiming the model to use for evaluating the confidence. It can be one of: KNN, IForest, OCSVM.
Returns
----------
exceed_conf : example-wise confidence using ExCeeD (outlier probability computed by Bayesian Learning with uniform prior)
exceed_conf_sp : example-wise confidence using ExCeeD (outlier probability computed by Bayesian Learning with other prior)
squash_conf : example-wise confidence using ExCeeD (outlier probability computed by squashing function)
linear_conf : example-wise confidence using ExCeeD (outlier probability computed by linear function)
unify_conf : example-wise confidence using ExCeeD (outlier probability computed by unify method)
logcal_conf : example-wise calibrated probability using Logistic Calibration
isocal_conf : example-wise calibrated probability using Isotonic Calibration
betacal_conf : example-wise calibrated probability using Beta Calibration
prediction : list of class predictions with shape (n_test,)
"""
np.random.seed(331)
n = np.shape(X_train)[0]
clf = train_model(X_train, contamination, model)
col = clf.decision_function(X_train)
prediction = clf.predict(X_test)
test_scores = clf.decision_function(X_test)
train_scores = clf.decision_function(X_train)
n_anom = np.int(n*contamination)
m = 10/contamination
count_instances = np.vectorize(lambda x: np.count_nonzero(train_scores <= x))
n_instances = count_instances(test_scores)
prob_func = np.vectorize(lambda x: (1+x)/(2+n))
exceed_posterior = prob_func(n_instances)
adj_prob_func = np.vectorize(lambda x: (10+x)/(m+n))
exceed_posterior_sp = adj_prob_func(n_instances)
unify_proba_anom = [x[1] for x in clf.predict_proba(X_test, method='unify')]
linear_proba_anom = [x[1] for x in clf.predict_proba(X_test, method='linear')]
tmp_score = sorted(col, reverse = True)
gamma = tmp_score[min(np.int(n*contamination)-1,0)]
squashing_proba_anom = [1-squash_proba(x, gamma) for x in clf.decision_function(X_test)]
mapinto01_train = [1-squash_proba(x, gamma) for x in clf.decision_function(X_train)]
mapinto01_test = [1-squash_proba(x, gamma) for x in clf.decision_function(X_test)]
lr = LogisticRegression(C=99999999999)
lr.fit(np.asarray(mapinto01_train).reshape(-1, 1), y_train)
logistic_calibration = lr.predict_proba(np.asarray(mapinto01_test).reshape(-1, 1))[:,1]
iso = IsotonicRegression()
iso.fit(mapinto01_train, y_train)
isotonic_calibration = np.nan_to_num(iso.predict(mapinto01_test), nan = 1.0)
bc = BetaCalibration(parameters="abm")
bc.fit(np.asarray(mapinto01_train).reshape(-1, 1), y_train)
beta_calibration = bc.predict(np.asarray(mapinto01_test).reshape(-1, 1))
conf_func = np.vectorize(lambda p: 1 - binom.cdf(n - n_anom, n, p))
exceed_conf = conf_func(exceed_posterior)
np.place(exceed_conf, prediction == 0, 1 - exceed_conf[prediction == 0])
exceed_conf_sp = conf_func(exceed_posterior_sp)
np.place(exceed_conf_sp, prediction == 0, 1 - exceed_conf_sp[prediction == 0])
squash_conf = conf_func(squashing_proba_anom)
np.place(squash_conf, prediction == 0, 1 - squash_conf[prediction == 0])
linear_conf = conf_func(linear_proba_anom)
np.place(linear_conf, prediction == 0, 1 - linear_conf[prediction == 0])
unify_conf = conf_func(unify_proba_anom)
np.place(unify_conf, prediction == 0, 1 - unify_conf[prediction == 0])
logcal_conf = np.asarray([round(x,4) if prediction[i] == 1 else round(1-x,4) for i,x in enumerate(logistic_calibration)])
isocal_conf = np.asarray([round(x,4) if prediction[i] == 1 else round(1-x,4) for i,x in enumerate(isotonic_calibration)])
betacal_conf = np.asarray([round(x,4) if prediction[i] == 1 else round(1-x,4) for i,x in enumerate(beta_calibration)])
return exceed_conf, exceed_conf_sp, squash_conf, linear_conf, unify_conf, logcal_conf, isocal_conf, betacal_conf, prediction
np.set_printoptions(precision=2)
predict_val = model1.predict(X_val)
plot_confusion_matrix(y_val,
predict_val,
classes=np.array([True, False]),
title='Confusion matrix',
normalize=False)
# -
# # Beta calibration
#
# +
# Fit three-parameter beta calibration
betaCal = BetaCalibration(parameters="abm")
y_cal_pred = model1.predict_proba(X_cal)[:, 1]
betaCal.fit(y_cal_pred.reshape(-1, 1), y_cal)
y_test_pred_prob = model1.predict_proba(X_test)[:, 1]
y_test_pred_prob = betaCal.predict(y_test_pred_prob.reshape(-1, 1))
y_test_pred = y_test_pred_prob > 0.5
fig, ax = plt.subplots()
sns.distplot(model1.predict_proba(X_test)[y_test['is_converted'] == 1, 1],
kde=False)
sns.distplot(y_test_pred_prob[y_test['is_converted'] == 1], kde=False)
plt.show()
# -