def evaluation_contingencytableevaluation_modular(ground_truth, predicted):
from shogun.Features import Labels
from shogun.Evaluation import ContingencyTableEvaluation
from shogun.Evaluation import AccuracyMeasure, ErrorRateMeasure, BALMeasure
from shogun.Evaluation import WRACCMeasure, F1Measure, CrossCorrelationMeasure
from shogun.Evaluation import RecallMeasure, PrecisionMeasure, SpecificityMeasure
ground_truth_labels = Labels(ground_truth)
predicted_labels = Labels(predicted)
base_evaluator = ContingencyTableEvaluation()
base_evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = AccuracyMeasure()
accuracy = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = ErrorRateMeasure()
errorrate = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = BALMeasure()
bal = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = WRACCMeasure()
wracc = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = F1Measure()
f1 = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = CrossCorrelationMeasure()
crosscorrelation = evaluator.evaluate(predicted_labels,
ground_truth_labels)
evaluator = RecallMeasure()
recall = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = PrecisionMeasure()
precision = evaluator.evaluate(predicted_labels, ground_truth_labels)
evaluator = SpecificityMeasure()
specificity = evaluator.evaluate(predicted_labels, ground_truth_labels)
return accuracy, errorrate, bal, wracc, f1, crosscorrelation, recall, precision, specificity
def evaluation_cross_validation_multiclass_storage(
traindat=traindat, label_traindat=label_traindat):
from shogun.Evaluation import CrossValidation, CrossValidationResult
from shogun.Evaluation import CrossValidationPrintOutput
from shogun.Evaluation import CrossValidationMKLStorage, CrossValidationMulticlassStorage
from shogun.Evaluation import MulticlassAccuracy, F1Measure
from shogun.Evaluation import StratifiedCrossValidationSplitting
from shogun.Features import MulticlassLabels
from shogun.Features import RealFeatures, CombinedFeatures
from shogun.Kernel import GaussianKernel, CombinedKernel
from shogun.Classifier import MKLMulticlass
from shogun.Mathematics import Statistics, MSG_DEBUG
# training data, combined features all on same data
features = RealFeatures(traindat)
comb_features = CombinedFeatures()
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
comb_features.append_feature_obj(features)
labels = MulticlassLabels(label_traindat)
# kernel, different Gaussians combined
kernel = CombinedKernel()
kernel.append_kernel(GaussianKernel(10, 0.1))
kernel.append_kernel(GaussianKernel(10, 1))
kernel.append_kernel(GaussianKernel(10, 2))
# create mkl using libsvm, due to a mem-bug, interleaved is not possible
svm = MKLMulticlass(1.0, kernel, labels)
svm.set_kernel(kernel)
# splitting strategy for 5 fold cross-validation (for classification its better
# to use "StratifiedCrossValidation", but the standard
# "StratifiedCrossValidationSplitting" is also available
splitting_strategy = StratifiedCrossValidationSplitting(labels, 5)
# evaluation method
evaluation_criterium = MulticlassAccuracy()
# cross-validation instance
cross_validation = CrossValidation(svm, comb_features, labels,
splitting_strategy,
evaluation_criterium)
cross_validation.set_autolock(False)
# append cross vlaidation output classes
#cross_validation.add_cross_validation_output(CrossValidationPrintOutput())
#mkl_storage=CrossValidationMKLStorage()
#cross_validation.add_cross_validation_output(mkl_storage)
multiclass_storage = CrossValidationMulticlassStorage()
multiclass_storage.append_binary_evaluation(F1Measure())
cross_validation.add_cross_validation_output(multiclass_storage)
cross_validation.set_num_runs(3)
# perform cross-validation
result = cross_validation.evaluate()
roc_0_0_0 = multiclass_storage.get_fold_ROC(0, 0, 0)
#print roc_0_0_0
auc_0_0_0 = multiclass_storage.get_fold_evaluation_result(0, 0, 0, 0)
#print auc_0_0_0
return roc_0_0_0, auc_0_0_0