def __init__(self, model, sklearn_model: bool):
r"""__init__ method
This method is used to adapt the input `model` so it can be used for creating
confidente intervals with conformal prediction.
Parameters
----------
model:
Model we want to use as the underlying model to generate predictions and the
confidence interval. This model can only be a scikit learn model, LGBMRegressor,
LGBMClassifier, XGBRegressor, XGBClassifier, CatBoostRegressor or CatBoostClassifier.
sklearn_model: bool
This variable indicates if the model belongs to scikit learn or not.
Returns
-------
cp: obj: Adapt_to_CP
The class of the adapted model.
Examples
--------
>>> model = lightgbm.LGBMRegressor()
>>> cp = Adapt_to_CP(model)
"""
self.model = model
if sklearn_model:
if is_classifier(model):
self.icp = IcpClassifier(NcFactory.create_nc(model))
elif is_regressor(model):
self.icp = IcpRegressor(NcFactory.create_nc(model))
else:
model_adapter = NonConformistAdapter(model)
if is_classifier(model):
self.icp = IcpClassifier(ClassifierNc(model_adapter))
elif is_regressor(model):
self.icp = IcpRegressor(RegressorNc(model_adapter))
elif model.__class__.__name__ == "Booster":
self.icp = IcpRegressor(RegressorNc(model_adapter))
def run_icp(nc,
X_train,
y_train,
X_test,
idx_train,
idx_cal,
significance,
condition=None):
""" Run split conformal method
Parameters
----------
nc : class of nonconformist object
X_train : numpy array, training features (n1Xp)
y_train : numpy array, training labels (n1)
X_test : numpy array, testing features (n2Xp)
idx_train : numpy array, indices of proper training set examples
idx_cal : numpy array, indices of calibration set examples
significance : float, significance level (e.g. 0.1)
condition : function, mapping feature vector to group id
Returns
-------
y_lower : numpy array, estimated lower bound for the labels (n2)
y_upper : numpy array, estimated upper bound for the labels (n2)
"""
icp = IcpRegressor(nc, condition=condition)
# Fit the ICP using the proper training set
icp.fit(X_train[idx_train, :], y_train[idx_train])
# Calibrate the ICP using the calibration set
icp.calibrate(X_train[idx_cal, :], y_train[idx_cal])
# Produce predictions for the test set, with confidence 90%
predictions = icp.predict(X_test, significance=significance)
y_lower = predictions[:, 0]
y_upper = predictions[:, 1]
return y_lower, y_upper
def run_icp_sep(nc, X_train, y_train, X_test, idx_train, idx_cal, significance,
condition):
""" Run split conformal method, train a seperate regressor for each group
Parameters
----------
nc : class of nonconformist object
X_train : numpy array, training features (n1Xp)
y_train : numpy array, training labels (n1)
X_test : numpy array, testing features (n2Xp)
idx_train : numpy array, indices of proper training set examples
idx_cal : numpy array, indices of calibration set examples
significance : float, significance level (e.g. 0.1)
condition : function, mapping a feature vector to group id
Returns
-------
y_lower : numpy array, estimated lower bound for the labels (n2)
y_upper : numpy array, estimated upper bound for the labels (n2)
"""
X_proper_train = X_train[idx_train, :]
y_proper_train = y_train[idx_train]
X_calibration = X_train[idx_cal, :]
y_calibration = y_train[idx_cal]
category_map_proper_train = np.array([
condition((X_proper_train[i, :], y_proper_train[i]))
for i in range(y_proper_train.size)
])
category_map_calibration = np.array([
condition((X_calibration[i, :], y_calibration[i]))
for i in range(y_calibration.size)
])
category_map_test = np.array(
[condition((X_test[i, :], None)) for i in range(X_test.shape[0])])
categories = np.unique(category_map_proper_train)
y_lower = np.zeros(X_test.shape[0])
y_upper = np.zeros(X_test.shape[0])
cnt = 0
for cond in categories:
icp = IcpRegressor(nc[cnt])
idx_proper_train_group = category_map_proper_train == cond
# Fit the ICP using the proper training set
icp.fit(X_proper_train[idx_proper_train_group, :],
y_proper_train[idx_proper_train_group])
idx_calibration_group = category_map_calibration == cond
# Calibrate the ICP using the calibration set
icp.calibrate(X_calibration[idx_calibration_group, :],
y_calibration[idx_calibration_group])
idx_test_group = category_map_test == cond
# Produce predictions for the test set, with confidence 90%
predictions = icp.predict(X_test[idx_test_group, :],
significance=significance)
y_lower[idx_test_group] = predictions[:, 0]
y_upper[idx_test_group] = predictions[:, 1]
cnt = cnt + 1
return y_lower, y_upper
def run_experiment(cur_test_method, cur_dataset_name, cur_batch_size,
cur_lr_loss, cur_lr_dis, cur_loss_steps, cur_dis_steps,
cur_mu_val, cur_epochs, cur_model_type, cur_regression_type,
cur_random_state, cur_second_scale, num_experiments):
method = cur_test_method
seed = cur_random_state
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
dataset = cur_dataset_name
batch_size = cur_batch_size
# step size to minimize loss
lr_loss = cur_lr_loss
# step size used to fit GAN's classifier
lr_dis = cur_lr_dis
# inner epochs to fit loss
loss_steps = cur_loss_steps
# inner epochs to fit GAN's classifier
dis_steps = cur_dis_steps
# total number of epochs
epochs = cur_epochs
# utility loss
if cur_regression_type == "mreg":
cost_pred = torch.nn.MSELoss()
out_shape = 1
else:
raise
model_type = cur_model_type
metric = "equalized_odds"
print(dataset)
print(method)
sys.stdout.flush()
avg_length_0 = np.zeros(num_experiments)
avg_length_1 = np.zeros(num_experiments)
avg_coverage_0 = np.zeros(num_experiments)
avg_coverage_1 = np.zeros(num_experiments)
avg_p_val = np.zeros(num_experiments)
mse = np.zeros(num_experiments)
for i in range(num_experiments):
# Split into train and test
X, A, Y, X_cal, A_cal, Y_cal, X_test, A_test, Y_test = get_dataset.get_train_test_data(
base_path, dataset, seed + i)
in_shape = X.shape[1]
print("n train = " + str(X.shape[0]) + " p = " + str(X.shape[1]))
print("n calibration = " + str(X_cal.shape[0]))
print("n test = " + str(X_test.shape[0]))
sys.stdout.flush()
if method == "AdversarialDebiasing":
class RegAdapter(RegressorAdapter):
def __init__(self, model=None, fit_params=None, params=None):
super(RegAdapter, self).__init__(model, fit_params)
# Instantiate model
self.learner = adv_debiasing.AdvDebiasingRegLearner(
lr=lr_loss,
N_CLF_EPOCHS=loss_steps,
N_ADV_EPOCHS=dis_steps,
N_EPOCH_COMBINED=epochs,
cost_pred=cost_pred,
in_shape=in_shape,
batch_size=batch_size,
model_type=model_type,
out_shape=out_shape,
lambda_vec=cur_mu_val)
def fit(self, x, y):
self.learner.fit(x, y)
def predict(self, x):
return self.learner.predict(x)
elif method == 'FairDummies':
class RegAdapter(RegressorAdapter):
def __init__(self, model=None, fit_params=None, params=None):
super(RegAdapter, self).__init__(model, fit_params)
# Instantiate model
self.learner = fair_dummies_learning.EquiRegLearner(
lr=lr_loss,
pretrain_pred_epochs=0,
pretrain_dis_epochs=0,
epochs=epochs,
loss_steps=loss_steps,
dis_steps=dis_steps,
cost_pred=cost_pred,
in_shape=in_shape,
batch_size=batch_size,
model_type=model_type,
lambda_vec=cur_mu_val,
second_moment_scaling=cur_second_scale,
out_shape=out_shape)
def fit(self, x, y):
self.learner.fit(x, y)
def predict(self, x):
return self.learner.predict(x)
elif method == 'HGR':
class RegAdapter(RegressorAdapter):
def __init__(self, model=None, fit_params=None, params=None):
super(RegAdapter, self).__init__(model, fit_params)
# Instantiate model
self.learner = continuous_fairness.HGR_Reg_Learner(
lr=lr_loss,
epochs=epochs,
mu=cur_mu_val,
cost_pred=cost_pred,
in_shape=in_shape,
out_shape=out_shape,
batch_size=batch_size,
model_type=model_type)
def fit(self, x, y):
self.learner.fit(x, y)
def predict(self, x):
return self.learner.predict(x)
elif method == 'Baseline':
class RegAdapter(RegressorAdapter):
def __init__(self, model=None, fit_params=None, params=None):
super(RegAdapter, self).__init__(model, fit_params)
# Instantiate model
self.learner = fair_dummies_learning.EquiRegLearner(
lr=lr_loss,
pretrain_pred_epochs=epochs,
pretrain_dis_epochs=0,
epochs=0,
loss_steps=0,
dis_steps=0,
cost_pred=cost_pred,
in_shape=in_shape,
batch_size=batch_size,
model_type=model_type,
lambda_vec=0,
second_moment_scaling=0,
out_shape=out_shape)
def fit(self, x, y):
self.learner.fit(x, y)
def predict(self, x):
return self.learner.predict(x)
fairness_reg = RegAdapter(model=None)
if cur_regression_type == "mreg":
nc = RegressorNc(fairness_reg, AbsErrorErrFunc())
else:
raise
# function that extracts the group identifier
def condition(x, y=None):
return int(x[0][0] > 0)
icp = IcpRegressor(nc, condition=condition)
input_data_train = np.concatenate((A[:, np.newaxis], X), 1)
icp.fit(input_data_train, Y)
input_data_cal = np.concatenate((A_cal[:, np.newaxis], X_cal), 1)
icp.calibrate(input_data_cal, Y_cal)
input_data_test = np.concatenate((A_test[:, np.newaxis], X_test), 1)
Yhat_test = icp.predict(input_data_test, significance=0.1)
# compute and print average coverage and average length
coverage_sample, length_sample = compute_coverage_per_sample(
Y_test, Yhat_test[:, 0], Yhat_test[:, 1], 0.1, method,
input_data_test, condition)
avg_coverage, avg_length = compute_coverage_len(
Y_test, Yhat_test[:, 0], Yhat_test[:, 1])
avg_length_0[i] = np.mean(length_sample[0])
avg_coverage_0[i] = np.mean(coverage_sample[0])
avg_length_1[i] = np.mean(length_sample[1])
avg_coverage_1[i] = np.mean(coverage_sample[1])
Yhat_out_cal = fairness_reg.learner.predict(input_data_cal)
Yhat_out_test = fairness_reg.learner.predict(input_data_test)
if out_shape == 1:
mse[i] = np.mean((Yhat_out_test - Y_test)**2)
MSE_trivial = np.mean((np.mean(Y_test) - Y_test)**2)
print("MSE = " + str(mse[i]) + "MSE Trivial = " + str(MSE_trivial))
p_val = utility_functions.fair_dummies_test_regression(
Yhat_out_cal,
A_cal,
Y_cal,
Yhat_out_test,
A_test,
Y_test,
num_reps=1,
num_p_val_rep=1000,
reg_func_name="Net")
avg_p_val[i] = p_val
print("experiment = " + str(i + 1))
# if out_shape==2:
# init_coverage, init_length = compute_coverage_len(Y_test, Yhat_out_test[:,0], Yhat_out_test[:,1])
# print("Init Coverage = " + str(init_coverage))
# print("Init Length = " + str(init_length))
print("Coverage 0 = " + str(avg_coverage_0[i]))
print("Coverage 1 = " + str(avg_coverage_1[i]))
print("Length 0 = " + str(avg_length_0[i]))
print("Length 1 = " + str(avg_length_1[i]))
print("MSE = " + str(mse[i]))
print("p_val = " + str(p_val))
sys.stdout.flush()
outdir = './results/'
if not os.path.exists(outdir):
os.mkdir(outdir)
out_name = outdir + 'results.csv'
full_name = cur_test_method + "_" + cur_model_type + "_" + cur_regression_type
df = pd.DataFrame({
'method': [cur_test_method],
'dataset': [cur_dataset_name],
'batch_size': [cur_batch_size],
'lr_loss': [cur_lr_loss],
'lr_dis': [cur_lr_dis],
'loss_steps': [cur_loss_steps],
'dis_steps': [cur_dis_steps],
'mu_val': [cur_mu_val],
'epochs': [cur_epochs],
'random_state': [seed + i],
'model_type': [cur_model_type],
'metric': [metric],
'cur_second_scale': [cur_second_scale],
'regression_type': [cur_regression_type],
'avg_length': [avg_length],
'avg_coverage': [avg_coverage],
'avg_length_0': [avg_length_0[i]],
'avg_length_1': [avg_length_1[i]],
'mse': [mse[i]],
'avg_coverage_0': [avg_coverage_0[i]],
'avg_coverage_1': [avg_coverage_1[i]],
'p_val': [p_val],
'full_name': [full_name]
})
if os.path.isfile(out_name):
df2 = pd.read_csv(out_name)
df = pd.concat([df2, df], ignore_index=True)
df.to_csv(out_name, index=False)
print(full_name)
print(
"Num experiments %02d | Avg MSE = %.4f | Avg Length 0 = %.4f | Avg Length 1 = %.4f | Avg Coverage 0 = %.4f | Avg Coverage 1 = %.4f | Avg p_val = %.4f | min p_val = %.4f"
% (i + 1, np.mean(mse[:i + 1]), np.mean(avg_length_0[:i + 1]),
np.mean(avg_length_1[:i + 1]), np.mean(avg_coverage_0[:i + 1]),
np.mean(avg_coverage_1[:i + 1]), np.mean(
avg_p_val[:i + 1]), np.min(avg_p_val[:i + 1])))
print("======== Done =========")
sys.stdout.flush()
def __updatePlot(self):
plotIdx = 0
# Plot sampling over ground truth
if self.groundTruth is not None:
self.ax[plotIdx].clear()
self.ax[plotIdx].set_xlim([0, 1.05])
self.ax[plotIdx].set_ylim([0, 1.05])
self.ax[plotIdx].set_title("ATNE sampling")
self.ax[plotIdx].plot(self.groundTruth[:, 0],
self.groundTruth[:, 1],
'x',
color="0.7",
markeredgewidth=1.8,
markersize=5)
if len(self.hintEliminatedIndexes
) > 0 and self.elimWeights is None:
self.ax[plotIdx].plot(
self.groundTruth[self.hintEliminatedIndexes, 0],
self.groundTruth[self.hintEliminatedIndexes, 1],
'x',
color="indianred",
markeredgewidth=1.8,
markersize=5)
if self.elimWeights is None:
self.ax[plotIdx].plot(self.groundTruth[self.relaxedIndexes, 0],
self.groundTruth[self.relaxedIndexes, 1],
'x',
color="g",
markeredgewidth=1.8,
markersize=5)
else:
#self.ax[plotIdx].plot(self.groundTruth[self.sampledIndexes,0], self.groundTruth[self.sampledIndexes,1], 'o', color="b", markeredgewidth=1.8, markersize=5, alpha=0.6)
weights = np.sum(
np.mean(self.elimWeights, axis=1) /
np.max(np.mean(self.elimWeights, axis=1), axis=0),
axis=1)
# weights = np.mean(self.elimWeights[:,:,0], axis=1)
alpha = 1 - (weights / np.max(weights)) / 2
red = weights / np.max(weights)
for i in range(self.groundTruth.shape[0]):
if i not in self.relaxedIndexes: continue
if np.isnan(red[i]): red[i] = 0
self.ax[plotIdx].plot(self.groundTruth[i, 0],
self.groundTruth[i, 1],
'x',
color=[red[i], 1 - red[i], 0],
markeredgewidth=1.8,
markersize=5)
self.ax[plotIdx].plot(self.groundTruth[self.sampledIndexes, 0],
self.groundTruth[self.sampledIndexes, 1],
'x',
color="b",
markeredgewidth=1.8,
markersize=5)
plotIdx += 1
# Plot predicted design space
if self.predictions is not None and self.doPlotPredictions:
self.ax[plotIdx].clear()
labeledMask = np.in1d(self.predictionsIndexes, self.sampledIndexes)
labeledMaskIdx = np.where(labeledMask)[0]
cmap = self.plt.cm.get_cmap('hsv')
shapes = ['x', '.', '+']
# Plot type 1
#self.ax[plotIdx].set_title("Estimated design spaces by each forest")
#for f in range(self.predictions.shape[0]):
# self.ax[plotIdx].plot(self.predictions[f,:,0], self.predictions[f,:,1], 'x', markeredgewidth=1.8, markersize=5)
# #self.ax[plotIdx].plot(self.predictions[f,labeledMask,0], self.predictions[f,labeledMask,1], 'x', markeredgewidth=1.8, markersize=5)
# Plot type 2
#import matplotlib
#for i,p in enumerate(labeledMaskIdx):
# color = cmap(i/len(labeledMaskIdx))
# predmean = self.predictions[:,p,:].mean(0)
# predmed = np.median(self.predictions[:,p,:], 0)
# predstd = self.predictions[:,p,:].std(0)
# # Plot type 2.1
# #self.ax[plotIdx].plot(self.predictions[:,p,0], self.predictions[:,p,1], shapes[i%len(shapes)], markeredgewidth=1.8, markersize=5, color=color)
# #self.ax[plotIdx].plot(predmean[0], predmean[1], shapes[0], markeredgewidth=1.8, markersize=5, color=color)
# #self.ax[plotIdx].plot(predmed[0], predmed[1], shapes[1], markeredgewidth=1.8, markersize=5, color=color)
# # Plot type 2.2
# circle = matplotlib.patches.Ellipse(predmean[[0,1]], predstd[0], predstd[1])
# self.ax[plotIdx].add_artist(circle)
# Plot type 3 (Mean predictions)
# self.ax[plotIdx].set_title("Average estimated P_relaxed")
# pred_mean = self.predictions.mean(0)
# self.ax[plotIdx].plot(pred_mean[:,0], pred_mean[:,1], 'x', markeredgewidth=1.8, markersize=5)
# Plot type 4 (Mean predictions of the entire space)
self.ax[plotIdx].set_title("Average estimated design space")
if self.estimators is not None:
predictions = np.empty([
self.predictions.shape[0],
self.designs.getNumDesigns(), self.predictions.shape[2]
])
for f in range(self.predictions.shape[0]):
for o in range(self.predictions.shape[2]):
predictions[f, :, o] = self.estimators[f][o].predict(
self.allKnobs)
pred_mean = predictions.mean(0)
self.ax[plotIdx].scatter(pred_mean[:, 0],
pred_mean[:, 1],
marker='x',
c=np.arange(pred_mean.shape[0]) /
pred_mean.shape[0])
# Some tests here, although I can't remember what I was testing exactly...
if False:
from nonconformist.cp import IcpRegressor
from nonconformist.nc import NcFactory
from sklearn.ensemble import RandomForestRegressor
model1 = RandomForestRegressor()
nc1 = NcFactory.create_nc(model1)
icp1 = IcpRegressor(nc1)
model2 = RandomForestRegressor()
nc2 = NcFactory.create_nc(model2)
icp2 = IcpRegressor(nc2)
n = self.sampledIndexes.size
idx = np.random.permutation(n)
idx_train, idx_cal = idx[:int(0.8 * n)], idx[int(0.8 * n):]
icp1.fit(
self.allKnobs[self.sampledIndexes][idx_train, :],
self.groundTruth[self.sampledIndexes, 0][idx_train])
icp2.fit(
self.allKnobs[self.sampledIndexes][idx_train, :],
self.groundTruth[self.sampledIndexes, 1][idx_train])
icp1.calibrate(
self.allKnobs[self.sampledIndexes][idx_cal, :],
self.groundTruth[self.sampledIndexes, 0][idx_cal])
icp2.calibrate(
self.allKnobs[self.sampledIndexes][idx_cal, :],
self.groundTruth[self.sampledIndexes, 1][idx_cal])
prediction1 = icp1.predict(self.allKnobs,
significance=0.05)
prediction2 = icp2.predict(self.allKnobs,
significance=0.05)
print(prediction1)
self.ax[plotIdx].errorbar(pred_mean[:, 0],
pred_mean[:, 1],
xerr=prediction1,
yerr=prediction2,
linestyle="None")
# Keep this
#self.ax[plotIdx].set_xlim(left=0, right=2)
#self.ax[plotIdx].set_ylim(bottom=0, top=2)
plotIdx += 1
# Plot hint space if available
if self.doPlotHintSpace and self.hintSpace is not None:
self.ax[plotIdx].clear()
self.ax[plotIdx].set_xlim([0, 1.05])
self.ax[plotIdx].set_ylim([0, 1.05])
self.ax[plotIdx].set_title("Hint space")
self.ax[plotIdx].plot(self.hintSpace[:, 0],
self.hintSpace[:, 1],
'x',
markeredgewidth=1.8,
markersize=5)
plotIdx += 1
# Plot distances for labeled samples
if self.selectedDistances and self.doPlotDistances:
self.ax[plotIdx].clear()
self.ax[plotIdx].set_title(
"Estimated distances for labeled samples")
for d in self.selectedDistances:
self.ax[plotIdx].hist(d.flatten(), 50, alpha=0.65)
#self.ax[plotIdx].hist(d.mean(0), 50, alpha=0.65)
#for d in self.gtDistances:
# self.ax[plotIdx].hist(d.flatten(), 50, alpha=0.65)
plotIdx += 1
# Plot distances for unlabeled samples
if self.predictedDistances and self.doPlotDistances:
self.ax[plotIdx].clear()
self.ax[plotIdx].set_title(
"Estimated distances for unlabeled samples (within P_relaxed)")
predictedDistances = np.array(self.predictedDistances)
for d in range(predictedDistances.shape[2]):
self.ax[plotIdx].hist(predictedDistances[:, :, d].flatten(),
50,
alpha=0.65)
plotIdx += 1
self.plt.show()
try:
self.plt.pause(0.00001)
except:
pass
self.fig.canvas.draw()
if self.blocking:
self.fig.waitforbuttonpress()
self.selectedDistances = []
self.gtDistances = []
self.predictedDistances = []
params = joblib.load("models/params.pkl")
model = CatBoostRegressor()
model.set_params(**params, loss_function='Quantile:alpha=0.025')
model.fit(trainX, trainY)
lower = model.predict(testX.to_numpy())
model = CatBoostRegressor()
model.set_params(**params, loss_function='Quantile:alpha=0.975')
model.fit(trainX, trainY)
upper = model.predict(testX.to_numpy())
prediction = list(zip(lower, upper))
else:
nc = NcFactory.create_nc(
model, normalizer_model=KNeighborsRegressor(
n_neighbors=11)) # Create a default nonconformity function
icp = IcpRegressor(nc) # Create an inductive conformal regressor
# Fit the ICP using the proper training set
icp.fit(trainX.values, trainY.values)
# Calibrate the ICP using the calibration set
icp.calibrate(calX.values, calY.values)
# Produce predictions for the test set, with confidence 95%
prediction = icp.predict(testX.to_numpy(), significance=0.05)
sub = pd.DataFrame(prediction, columns=['lower_bound', 'upper_bound'])
sub.to_csv(f"models/{PROBLEM_TYPE}_{MODEL}_{DATASET}_intervals.csv",
index=False)
def evaluate(model_filepath, train_filepath, test_filepath,
calibrate_filepath):
"""Evaluate model to estimate power.
Args:
model_filepath (str): Path to model.
train_filepath (str): Path to train set.
test_filepath (str): Path to test set.
calibrate_filepath (str): Path to calibrate set.
"""
METRICS_FILE_PATH.parent.mkdir(parents=True, exist_ok=True)
# Load parameters
params = yaml.safe_load(open("params.yaml"))["evaluate"]
params_train = yaml.safe_load(open("params.yaml"))["train"]
params_split = yaml.safe_load(open("params.yaml"))["split"]
test = np.load(test_filepath)
X_test = test["X"]
y_test = test["y"]
# pandas data frame to store predictions and ground truth.
df_predictions = None
y_pred = None
if params_split["calibrate_split"] == 0:
model = models.load_model(model_filepath)
y_pred = model.predict(X_test)
else:
trained_model = models.load_model(model_filepath)
# mycustommodel = MyCustomModel(model_filepath)
mycustommodel = MyCustomModel(trained_model)
m = cnn(X_test.shape[-2],
X_test.shape[-1],
output_length=1,
kernel_size=params_train["kernel_size"])
nc = RegressorNc(
mycustommodel,
err_func=AbsErrorErrFunc(), # non-conformity function
# normalizer_model=KNeighborsRegressor(n_neighbors=15) # normalizer
# normalizer=m
)
# nc = NcFactory.create_nc(mycustommodel,
# err_func=AbsErrorErrFunc(), # non-conformity function
# # normalizer_model=KNeighborsRegressor(n_neighbors=15) # normalizer
# normalizer_model=m
# )
model = IcpRegressor(nc)
# Fit the normalizer.
train = np.load(train_filepath)
X_train = train["X"]
y_train = train["y"]
y_train = y_train.reshape((y_train.shape[0], ))
model.fit(X_train, y_train)
# Calibrate model.
calibrate = np.load(calibrate_filepath)
X_calibrate = calibrate["X"]
y_calibrate = calibrate["y"]
y_calibrate = y_calibrate.reshape((y_calibrate.shape[0], ))
model.calibrate(X_calibrate, y_calibrate)
print(f"Calibration: {X_calibrate.shape}")
# Set conformal prediction error. This should be a parameter specified by the user.
error = 0.05
# Predictions will contain the intervals. We need to compute the middle
# points to get the actual predictions y.
predictions = model.predict(X_test, significance=error)
# Compute middle points.
y_pred = predictions[:,
0] + (predictions[:, 1] - predictions[:, 0]) / 2
# Reshape to put it in the same format as without calibration set.
y_pred = y_pred.reshape((y_pred.shape[0], 1))
# Build data frame with predictions.
my_results = list(
zip(np.reshape(y_test, (y_test.shape[0], )),
np.reshape(y_pred, (y_pred.shape[0], )), predictions[:, 0],
predictions[:, 1]))
df_predictions = pd.DataFrame(my_results,
columns=[
'ground_truth', 'predicted',
'lower_bound', 'upper_bound'
])
save_predictions(df_predictions)
plot_intervals(df_predictions)
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
print("MSE: {}".format(mse))
print("R2: {}".format(r2))
plot_prediction(y_test, y_pred, inputs=X_test, info="(R2: {})".format(r2))
plot_individual_predictions(y_test, y_pred)
with open(METRICS_FILE_PATH, "w") as f:
json.dump(dict(mse=mse, r2=r2), f)