def evaluation_rocevaluation_modular(ground_truth, predicted): from shogun.Features import Labels from shogun.Evaluation import ROCEvaluation ground_truth_labels = Labels(ground_truth) predicted_labels = Labels(predicted) evaluator = ROCEvaluation() evaluator.evaluate(predicted_labels,ground_truth_labels) return evaluator.get_ROC(), evaluator.get_auROC()
def evaluation_multiclassovrevaluation_modular(traindat, label_traindat, testdat, label_testdat): from shogun.Features import MulticlassLabels from shogun.Evaluation import MulticlassOVREvaluation, ROCEvaluation from modshogun import MulticlassLibLinear, RealFeatures, ContingencyTableEvaluation, ACCURACY from shogun.Mathematics import Math Math.init_random(1) ground_truth_labels = MulticlassLabels(label_traindat) svm = MulticlassLibLinear(1.0, RealFeatures(traindat), MulticlassLabels(label_traindat)) svm.train() predicted_labels = svm.apply() binary_evaluator = ROCEvaluation() evaluator = MulticlassOVREvaluation(binary_evaluator) mean_roc = evaluator.evaluate(predicted_labels, ground_truth_labels) #print mean_roc binary_evaluator = ContingencyTableEvaluation(ACCURACY) evaluator = MulticlassOVREvaluation(binary_evaluator) mean_accuracy = evaluator.evaluate(predicted_labels, ground_truth_labels) #print mean_accuracy return mean_roc, mean_accuracy
gk=GaussianKernel(features, features, 1.0) svm = LibSVM(1000.0, gk, labels) svm.train() lda=LDA(1,features,labels) lda.train() ## plot points subplot(211) plot(pos[0,:], pos[1,:], "r.") plot(neg[0,:], neg[1,:], "b.") grid(True) title('Data',size=10) # plot ROC for SVM subplot(223) ROC_evaluation=ROCEvaluation() ROC_evaluation.evaluate(svm.apply(),labels) roc = ROC_evaluation.get_ROC() print roc plot(roc[0], roc[1]) fill_between(roc[0],roc[1],0,alpha=0.1) text(mean(roc[0])/2,mean(roc[1])/2,'auROC = %.5f' % ROC_evaluation.get_auROC()) grid(True) xlabel('FPR') ylabel('TPR') title('LibSVM (Gaussian kernel, C=%.3f) ROC curve' % svm.get_C1(),size=10) # plot ROC for LDA subplot(224) ROC_evaluation.evaluate(lda.apply(),labels) roc = ROC_evaluation.get_ROC()
model_selection=GridSearchModelSelection(param_tree_root, cross_validation) best_parameters=model_selection.select_model(True) print "Best parameters: ", best_parameters.print_tree() best_parameters.apply_to_machine(classifier) classifier.train() A = test_labels B = BinaryLabels(classifier.apply(feats_test).get_labels()) #print evaluation_contingencytableevaluation_modular((trainlab), (classifier.apply(feats_train).get_labels())) #print evaluation_rocevaluation_modular((trainlab), (classifier.apply(feats_train).get_labels())) # Run the SVM subplot(111) ROC_evaluation=ROCEvaluation() ROC_evaluation.evaluate(A, B) roc = ROC_evaluation.get_ROC() plot(roc[0], roc[1]) fill_between(roc[0],roc[1],0,alpha=0.1) grid(True) xlabel('FPR') ylabel('TPR') title('ROC (Width=%.3f, C1=%.3f, C2=%.3f) ROC curve = %.3f' % (kernel.get_width(), classifier.get_C1(), classifier.get_C2(), ROC_evaluation.get_auROC()),size=10) savefig("unamed.png") """ subplot(222) ROC_evaluation=ROCEvaluation() ROC_evaluation.evaluate(classifier.apply(feats_test),Labels(testlab)) roc = ROC_evaluation.get_ROC()
gk=GaussianKernel(features, features, 1.0) svm = LibSVM(1000.0, gk, labels) svm.train() lda=LDA(1,features,labels) lda.train() ## plot points subplot(211) plot(pos[0,:], pos[1,:], "r.") plot(neg[0,:], neg[1,:], "b.") grid(True) title('Data',size=10) # plot ROC for SVM subplot(223) ROC_evaluation=ROCEvaluation() ROC_evaluation.evaluate(svm.classify(),labels) roc = ROC_evaluation.get_ROC() plot(roc[:,0], roc[:,1]) fill_between(roc[:,0],roc[:,1],0,alpha=0.1) text(mean(roc[:,0])/2,mean(roc[:,1])/2,'auROC = %.5f' % ROC_evaluation.get_auROC()) grid(True) xlabel('FPR') ylabel('TPR') title('LibSVM (Gaussian kernel, C=%.3f) ROC curve' % svm.get_C1(),size=10) # plot ROC for LDA subplot(224) ROC_evaluation.evaluate(lda.classify(),labels) roc = ROC_evaluation.get_ROC() plot(roc[:,0], roc[:,1])