# nc = NcFactory.create_nc(model=simple_model)
# conformal_model = IcpClassifier(nc)
# conformal_model.fit(x_train_sp, y_train_sp)
# conformal_model.calibrate(x_cal, y_cal)
# ---------------------------------------------------
# CP
# nc = NcFactory.create_nc(model=simple_model)
# conformal_model = IcpClassifier(nc)
# conformal_model.fit(x_train, y_train)
# conformal_model.calibrate(x_train, y_train)
prediction = conformal_model.predict(x_test, significance=None)
table = np.hstack((prediction, truth))
result = [class_mean_errors(prediction, truth, significance=sig),
class_avg_c(prediction, truth, significance=sig)]
if k == 0:
summary = result
else:
summary = np.vstack((summary, result))
# print('\nBCP')
# print('Accuracy: {}'.format(result[0]))
# print('Average count: {}'.format(result[1]))
df_summary = pd.DataFrame(summary, columns=['Accuracy', 'Average_count'])
# temp = [sig, df_summary['Accuracy'].mean()]
temp = [sig, df_summary['Average_count'].mean()]
if sig == 0:
error_summary = temp
print(error_summary)
for train_index, test_index in s_folder.split(X, y):
x_train, x_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
truth = y_test.reshape(-1, 1)
for name, model in models.items():
result = []
model.fit(x_train, y_train)
prediction = model.predict(x_test, significance=None)
table = np.hstack((prediction, truth))
df = pd.DataFrame(table, columns=columns)
print('\n{}'.format(name))
print('Error rate: {}'.format(
class_mean_errors(prediction, truth, significance)))
print('class average count: {}'.format(
class_avg_c(prediction, truth, significance)))
result = [
name,
class_mean_errors(prediction, truth, significance),
class_avg_c(prediction, truth, significance),
class_empty(prediction, truth, significance)
]
if len(summary) == 0:
summary = result
else:
summary = np.vstack((summary, result))
# if name == 'ACP-RandomSubSampler':
# if index == 0:
# ACP_Random = result