print "accuracy - before/after:", acc, "/", acc_calibrated print "AUC - before/after: ", auc, "/", auc_calibrated print "log loss - before/after:", ll, "/", ll_calibrated """ accuracy - before/after: 0.847788697789 / 0.846805896806 AUC - before/after: 0.878139845077 / 0.878139845077 log loss - before/after: 0.630525772871 / 0.364873617584 """ ### print "creating diagrams..." n_bins = 10 # uncalibrated mean_predicted_values, true_fractions = get_diagram_data(y_test, p_test, n_bins) plt.plot(mean_predicted_values, true_fractions) # calibrated mean_predicted_values, true_fractions = get_diagram_data(y_test, p_calibrated, n_bins) plt.plot(mean_predicted_values, true_fractions, "green") # perfect calibration line plt.plot(np.linspace(0, 1), np.linspace(0, 1), "gray") plt.show()
#!/usr/bin/env python
"plot a reliability diagram showing how good a classifier's calibration is"
import numpy as np
import matplotlib.pyplot as plt
from get_diagram_data import get_diagram_data
#from load_data import y, p
from load_data_adult import y, p
print("computing...")
n_bins = 20
mean_predicted_values, true_fractions = get_diagram_data( y, p, n_bins )
plt.plot( mean_predicted_values, true_fractions )
# perfect calibration line
plt.plot( np.linspace( 0, 1 ), np.linspace( 0, 1 ), 'gray' )
plt.show()
print "AUC - before/after: ", auc, "/", auc_calibrated
print "log loss - before/after:", ll, "/", ll_calibrated
"""
accuracy - before/after: 0.847788697789 / 0.846805896806
AUC - before/after: 0.878139845077 / 0.878139845077
log loss - before/after: 0.630525772871 / 0.364873617584
"""
###
print "creating diagrams..."
n_bins = 10
# uncalibrated
mean_predicted_values, true_fractions = get_diagram_data(
y_test, p_test, n_bins)
plt.plot(mean_predicted_values, true_fractions)
# calibrated
mean_predicted_values, true_fractions = get_diagram_data(
y_test, p_calibrated, n_bins)
plt.plot(mean_predicted_values, true_fractions, 'green')
# perfect calibration line
plt.plot(np.linspace(0, 1), np.linspace(0, 1), 'gray')
plt.show()
#!/usr/bin/env python "plot a reliability diagram showing how good a classifier's calibration is" import numpy as np import matplotlib.pyplot as plt from get_diagram_data import get_diagram_data #from load_data import y, p from load_data_adult import y, p print "computing..." n_bins = 20 mean_predicted_values, true_fractions = get_diagram_data( y, p, n_bins ) plt.plot( mean_predicted_values, true_fractions ) # perfect calibration line plt.plot( np.linspace( 0, 1 ), np.linspace( 0, 1 ), 'gray' ) plt.show()
