def rocCurve(results,
titleText,
classIndex=0,
stepSize=0.01,
marker='x',
plotArgs=dict()):
thresholds, confusionMatrices = thresholdConfusionMatrices(
results, classIndex, stepSize)
recalls = [orngStat.recall(cm) for cm in confusionMatrices]
specificities = [1 - orngStat.spec(cm) for cm in confusionMatrices]
#fig = figure()
xlabel("FP", fontsize=25)
ylabel("TP", fontsize=25)
#plot([0, 1], [0, 1])
line = plot(specificities, recalls, marker=marker, **plotArgs)
#scatter(specificities, recalls, picker=True, marker=marker) # to get picking.
def onclick(event):
#point = event.artist.get_data()
indexes = event.ind
str = "\n".join(["%.3f" % thresholds[idx] for idx in indexes])
print str
print
# fig.canvas.mpl_connect('pick_event', onclick)
xlim(0, 1.1)
ylim(0, 1.1)
title(titleText)
return line
def get_stats(results):
result_dict = {}
cm = get_confusion_matrix(results)
result_dict['Accuracy'] = orngStat.CA(results)[0]
result_dict['Sensitivity'] = orngStat.sens(cm)
result_dict['Specificity'] = orngStat.spec(cm)
return result_dict
def get_stats(results):
result_dict = {}
cm = get_confusion_matrix(results)
result_dict['Accuracy'] = orngStat.CA(results)[0]
result_dict['Sensitivity'] = orngStat.sens(cm)
result_dict['Specificity'] = orngStat.spec(cm)
return result_dict
def cforange_confusion_matrix_computations(input_dict):
import orngStat
cm = input_dict['cm']
alpha = float(input_dict['alpha'])
output_dict = {}
output_dict['sens']=orngStat.sens(cm)
output_dict['spec']=orngStat.spec(cm)
output_dict['PPV']=orngStat.PPV(cm)
output_dict['NPV']=orngStat.NPV(cm)
output_dict['precision']=orngStat.precision(cm)
output_dict['recall']=orngStat.recall(cm)
output_dict['F1']=orngStat.F1(cm)
output_dict['Falpha']=orngStat.Falpha(cm,alpha=alpha)
output_dict['MCC']=orngStat.MCC(cm)
return output_dict
# set up the learners
bayes = orange.BayesLearner()
tree = orngTree.TreeLearner(mForPruning=2)
bayes.name = "bayes"
tree.name = "tree"
learners = [bayes, tree]
# compute accuracies on data
data = orange.ExampleTable("voting")
res = orngTest.crossValidation(learners, data, folds=10)
cm = orngStat.computeConfusionMatrices(res, classIndex=data.domain.classVar.values.index("democrat"))
stat = (
("CA", lambda res, cm: orngStat.CA(res)),
("Sens", lambda res, cm: orngStat.sens(cm)),
("Spec", lambda res, cm: orngStat.spec(cm)),
("AUC", lambda res, cm: orngStat.AUC(res)),
("IS", lambda res, cm: orngStat.IS(res)),
("Brier", lambda res, cm: orngStat.BrierScore(res)),
("F1", lambda res, cm: orngStat.F1(cm)),
("F2", lambda res, cm: orngStat.Falpha(cm, alpha=2.0)),
("MCC", lambda res, cm: orngStat.MCC(cm)),
("sPi", lambda res, cm: orngStat.scottsPi(cm)),
)
scores = [s[1](res, cm) for s in stat]
print
print "Learner " + "".join(["%-7s" % s[0] for s in stat])
for (i, l) in enumerate(learners):
print "%-8s " % l.name + "".join(["%5.3f " % s[i] for s in scores])
print "Confusion matrix for naive Bayes for 'opel':"
print "TP: %i, FP: %i, FN: %s, TN: %i" % (cm.TP, cm.FP, cm.FN, cm.TN)
print
cm = orngStat.confusionMatrices(resVeh)[0]
classes = vehicle.domain.classVar.values
print "\t"+"\t".join(classes)
for className, classConfusions in zip(classes, cm):
print ("%s" + ("\t%i" * len(classes))) % ((className, ) + tuple(classConfusions))
cm = orngStat.confusionMatrices(res)
print
print "Sensitivity and specificity for 'voting'"
print "method\tsens\tspec"
for l in range(len(learners)):
print "%s\t%5.3f\t%5.3f" % (learners[l].name, orngStat.sens(cm[l]), orngStat.spec(cm[l]))
cm = orngStat.confusionMatrices(resVeh, vehicle.domain.classVar.values.index("van"))
print
print "Sensitivity and specificity for 'vehicle=van'"
print "method\tsens\tspec"
for l in range(len(learners)):
print "%s\t%5.3f\t%5.3f" % (learners[l].name, orngStat.sens(cm[l]), orngStat.spec(cm[l]))
print
print "AUC (voting)"
AUCs = orngStat.AUC(res)
for l in range(len(learners)):
bayes = orange.BayesLearner()
tree = orngTree.TreeLearner(mForPruning=2)
bayes.name = "bayes"
tree.name = "tree"
learners = [bayes, tree]
# compute accuracies on data
data = orange.ExampleTable("voting")
res = orngTest.crossValidation(learners, data, folds=10)
cm = orngStat.computeConfusionMatrices(
res, classIndex=data.domain.classVar.values.index('democrat'))
stat = (
('CA', lambda res, cm: orngStat.CA(res)),
('Sens', lambda res, cm: orngStat.sens(cm)),
('Spec', lambda res, cm: orngStat.spec(cm)),
('AUC', lambda res, cm: orngStat.AUC(res)),
('IS', lambda res, cm: orngStat.IS(res)),
('Brier', lambda res, cm: orngStat.BrierScore(res)),
('F1', lambda res, cm: orngStat.F1(cm)),
('F2', lambda res, cm: orngStat.Falpha(cm, alpha=2.0)),
('MCC', lambda res, cm: orngStat.MCC(cm)),
('sPi', lambda res, cm: orngStat.scottsPi(cm)),
)
scores = [s[1](res, cm) for s in stat]
print
print "Learner " + "".join(["%-7s" % s[0] for s in stat])
for (i, l) in enumerate(learners):
print "%-8s " % l.name + "".join(["%5.3f " % s[i] for s in scores])
