def create_nc(model,
err_func=None,
normalizer_model=None,
oob=False,
fit_params=None,
fit_params_normalizer=None):
if normalizer_model is not None:
normalizer_adapter = RegressorAdapter(normalizer_model,
fit_params_normalizer)
else:
normalizer_adapter = None
if isinstance(model, sklearn.base.ClassifierMixin):
err_func = MarginErrFunc() if err_func is None else err_func
if oob:
c = sklearn.base.clone(model)
c.fit([[0], [1]], [0, 1])
if hasattr(c, 'oob_decision_function_'):
adapter = OobClassifierAdapter(model, fit_params)
else:
raise AttributeError('Cannot use out-of-bag '
'calibration with {}'.format(
model.__class__.__name__))
else:
adapter = ClassifierAdapter(model, fit_params)
if normalizer_adapter is not None:
normalizer = RegressorNormalizer(adapter, normalizer_adapter,
err_func)
return ClassifierNc(adapter, err_func, normalizer)
else:
return ClassifierNc(adapter, err_func)
elif isinstance(model, sklearn.base.RegressorMixin):
err_func = AbsErrorErrFunc() if err_func is None else err_func
if oob:
c = sklearn.base.clone(model)
c.fit([[0], [1]], [0, 1])
if hasattr(c, 'oob_prediction_'):
adapter = OobRegressorAdapter(model, fit_params)
else:
raise AttributeError('Cannot use out-of-bag '
'calibration with {}'.format(
model.__class__.__name__))
else:
adapter = RegressorAdapter(model, fit_params)
if normalizer_adapter is not None:
normalizer = RegressorNormalizer(adapter, normalizer_adapter,
err_func)
return RegressorNc(adapter, err_func, normalizer)
else:
return RegressorNc(adapter, err_func)
def test_oob_calibration(self):
# -----------------------------------------------------------------------------
# Classification
# -----------------------------------------------------------------------------
data = load_iris()
icp = OobCpClassifier(
ClassifierNc(
OobClassifierAdapter(
RandomForestClassifier(n_estimators=100, oob_score=True))))
icp_cv = ClassIcpCvHelper(icp)
scores = cross_val_score(
icp_cv,
data.data,
data.target,
iterations=5,
folds=5,
scoring_funcs=[class_mean_errors, class_avg_c],
significance_levels=[0.05, 0.1, 0.2],
)
print("Classification: iris")
scores = scores.drop(["fold", "iter"], axis=1)
print(scores.groupby(["significance"]).mean())
# -----------------------------------------------------------------------------
# Regression, absolute error
# -----------------------------------------------------------------------------
data = load_diabetes()
icp = OobCpRegressor(
RegressorNc(
OobRegressorAdapter(
RandomForestRegressor(n_estimators=100, oob_score=True))))
icp_cv = RegIcpCvHelper(icp)
scores = cross_val_score(
icp_cv,
data.data,
data.target,
iterations=5,
folds=5,
scoring_funcs=[reg_mean_errors, reg_median_size],
significance_levels=[0.05, 0.1, 0.2],
)
print("Absolute error regression: diabetes")
scores = scores.drop(["fold", "iter"], axis=1)
print(scores.groupby(["significance"]).mean())
from nonconformist.base import OobClassifierAdapter, OobRegressorAdapter
from nonconformist.icp import OobCpClassifier, OobCpRegressor
from nonconformist.nc import ClassifierNc, RegressorNc
from nonconformist.evaluation import cross_val_score
from nonconformist.evaluation import ClassIcpCvHelper, RegIcpCvHelper
from nonconformist.evaluation import class_avg_c, class_mean_errors
from nonconformist.evaluation import reg_mean_errors, reg_median_size
# -----------------------------------------------------------------------------
# Classification
# -----------------------------------------------------------------------------
data = load_iris()
icp = OobCpClassifier(ClassifierNc(OobClassifierAdapter(RandomForestClassifier(n_estimators=100, oob_score=True))))
icp_cv = ClassIcpCvHelper(icp)
scores = cross_val_score(icp_cv,
data.data,
data.target,
iterations=5,
folds=5,
scoring_funcs=[class_mean_errors, class_avg_c],
significance_levels=[0.05, 0.1, 0.2])
print('Classification: iris')
scores = scores.drop(['fold', 'iter'], axis=1)
print(scores.groupby(['significance']).mean())
# -----------------------------------------------------------------------------
# Calibrate the ICP using the calibration set
icp.calibrate(x_val_np, y_val_np)
print('predicting inductive conformal prediction')
# Produce predictions for the test set, with confidence 95%
prediction = icp.predict(x_test_np, significance=0.05)
prediction_conf_cred = pd.DataFrame(
icp.predict_conf(x_test_np),
columns=['Label', 'Confidence', 'Credibility'])
# %%
#Cross validation of the conformal predictor
#icp = IcpClassifier(ClassifierNc(ClassifierAdapter(model),MarginErrFunc()))
icp = OobCpClassifier(
ClassifierNc(
OobClassifierAdapter(
RandomForestClassifier(n_estimators=300, oob_score=True))))
significance = np.arange(0, 1, 0.025)
significance[0] = 0.01
icp_cv = ClassIcpCvHelper(icp)
scores = cross_val_score(icp_cv,
x_train_np,
y_train_np,
iterations=1,
folds=5,
scoring_funcs=[
class_mean_errors, class_one_err, class_avg_c,
class_one_c, class_empty, class_two_c
],
significance_levels=significance,