def TopVarImportanceTest(data, expectNone=False): resA = [] resB = [] learner = AZorngCvSVM.CvSVMLearner(gamma=1.0, svm_type=103, C=1, coef0=0, degree=3, epsR=0.001, kernel_type=2, nu=0.5, p=0.1, probability=0, shrinking=1) CvSVM = learner(data) for ex in data: resA.append(CvSVM.getTopImportantVars(ex, 1)) scratchdir = miscUtilities.createScratchDir( desc="TopVarImportanceTest") modelPath = os.path.join(scratchdir, "CvSVNModel") CvSVM.write(modelPath) LoadedCvSVM = AZorngCvSVM.CvSVMread(modelPath) miscUtilities.removeDir(scratchdir) for ex in data: resB.append(LoadedCvSVM.getTopImportantVars(ex, 1)) if expectNone: return resA == resB == [None] * len(data) else: return resA == resB and None not in resA and resA.count( resA[0]) != len(resA)
def test_CreateLogicalExpressionConsensusLearner(self): """ Test creation of logical expression consensus learner """ # Arrange # Construct expression learner/classifier learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } discreteExpression = [ "firstLearner == Iris-setosa -> Iris-setosa", "-> Iris-virginica" ] discreteLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=discreteExpression) discreteClassifier = discreteLearner(self.irisData) verifiedLearner = AZorngCvSVM.CvSVMLearner() verifiedClassifier = verifiedLearner(self.irisData) # Act result = [] verifiedResult = [] for ex in self.irisData: result.append(discreteClassifier(ex)) verifiedResult.append(verifiedClassifier(ex)) # Assert for index, item in enumerate(result): if not result[index].value == verifiedResult[index].value: print "Not equal on index: ", index self.assertEqual(result[index].value, verifiedResult[index].value)
def trainSVMOptParam(train, SVMparam): # Optimize parameters #SVMparam = [1.0, 0.05] if not SVMparam: trainDataFile = "/scratch/trainDataTmp.tab" train.save(trainDataFile) learner = AZorngCvSVM.CvSVMLearner() param = paramOptUtilities.getOptParam(learner, trainDataFile, paramList=None, useGrid=False, verbose=1, queueType="NoSGE", runPath=None, nExtFolds=None, nFolds=10, logFile="", getTunedPars=True, fixedParams={}) optC = float(param[1]["C"]) optGamma = float(param[1]["gamma"]) SVMparam = [optC, optGamma] else: optC = SVMparam[0] optGamma = SVMparam[1] #print "Optimal SVM parameters ", optC, optGamma model = AZorngCvSVM.CvSVMLearner(train, C=optC, gamma=optGamma) return model, SVMparam
def test_SVMC(self): # Train a svm svmL = AZorngCvSVM.CvSVMLearner(scaleData=False, svm_type=103, gamma=0.01, C=1, nu=0.5, p=1, eps=0.001, coef0=0, degree=3) svm = svmL(self.inDataC) trainedAcc = evalUtilities.getRMSE(self.inDataC, svm) self.assertEqual(round(trainedAcc, 7), round(2.8525863999999999, 7)) # ver 0.3 # Save model rc = svm.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm) # Assure equal accuracy self.assertEqual(trainedAcc, loadedAcc) svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False) svmLearner.name = "CvSVMLearner" svmLearner.eps = 0.001 svmLearner.p = 1 svmLearner.nu = 0.6 svmLearner.kernel_type = 2 svmLearner.svm_type = 103 svmLearner.gamma = 0.0033 svmLearner.C = 47 svmLearner.scaleData = True svmLearner.scaleClass = False Res = orngTest.crossValidation( [svmLearner], self.inDataC, folds=5, strat=orange.MakeRandomIndices.StratifiedIfPossible) RMSE = evalUtilities.RMSE(Res)[0] self.assertEqual(round(RMSE, 2), round(2.96, 2)) #Ver 0.3 newSVM = svmLearner(self.inDataC) trainedAcc = evalUtilities.getRMSE(self.inDataC, newSVM) # Save model rc = newSVM.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm) # Assure equal accuracy self.assertEqual(round(trainedAcc, 4), round(2.8289, 4)) #Ver 0.3 self.assertEqual(round(trainedAcc, 4), round(loadedAcc, 4))
def test_Priors(self): """Test to assure that priors are set correcly.""" # Create a CvSVM model CvSVMlearner = AZorngCvSVM.CvSVMLearner(C=3, priors={ "Iris-versicolor": 2, "Iris-virginica": 4, "Iris-setosa": 6 }) CvSVMmodel = CvSVMlearner(self.inDataD) # Calculate classification accuracy Acc = evalUtilities.getClassificationAccuracy(self.inDataD, CvSVMmodel) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirTest" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "modelPriors.CvSVM") CvSVMmodel.write(modelPath) # Read in the model newCvSVMmodel = AZorngCvSVM.CvSVMread(modelPath) # Calculate classification accuracy savedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, CvSVMmodel) # Test that the accuracy of the two classifiers is the exact same self.assertEqual(Acc, savedAcc) #Check the priors saved in the model file = open(os.path.join(modelPath, "model.svm"), "r") lines = file.readlines() file.close() priors = [ round(x, 2) for x in eval((lines[18].strip()).replace("data:", "")) ] self.assertEqual(len(priors), 3) self.assertEqual( priors[self.inDataD.domain.classVar.values.index("Iris-setosa")], 18.0) self.assertEqual( priors[self.inDataD.domain.classVar.values.index( "Iris-versicolor")], 6.0) self.assertEqual( priors[self.inDataD.domain.classVar.values.index( "Iris-virginica")], 12.0) # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_CreateDefaultClassifierUsingPreTrainedRegressionClassifiers(self): """ Test the creation of custom Consensus Classifier using pre-trained regression classifiers. """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } classifiers = {} for k, v in learners.items(): classifiers[k] = v(self.getRegressionTrainingData()) expression = "a + b + c" # Act classifier = AZorngConsensus.ConsensusClassifier( classifiers=classifiers, expression=expression) # Assert self.assertNotEqual(classifier, None) self.assertEqual(len(classifier.classifiers), len(learners)) self.assertNotEqual(classifier.basicStat, None) self.assertNotEqual(classifier.classVar, None) self.assertNotEqual(classifier.domain, None) self.assertEqual(classifier.expression, expression) self.assertNotEqual(classifier.imputeData, None) #self.assertEqual(classifier.NTrainEx, len(trainingData)) self.assertEqual(classifier.name, "Consensus classifier") self.assertNotEqual(classifier.varNames, None) self.assertEqual(classifier.verbose, 0) self.assertEqual(classifier.weights, None)
def test_CreateDefaultClassifierUsingPreTrainedRegressionClassifiers(self): """ Test the creation of default Consensus Classifier using pre-trained classification classifiers. """ # Arrange learners = [ AZorngCvSVM.CvSVMLearner(), AZorngCvANN.CvANNLearner(), AZorngRF.RFLearner() ] classifiers = [l(self.getRegressionTrainingData()) for l in learners] # Act classifier = AZorngConsensus.ConsensusClassifier( classifiers=classifiers) # Assert self.assertNotEqual(classifier, None) self.assertEqual(len(classifier.classifiers), len(learners)) self.assertNotEqual(classifier.basicStat, None) self.assertNotEqual(classifier.classVar, None) self.assertNotEqual(classifier.domain, None) self.assertEqual(classifier.expression, None) self.assertNotEqual(classifier.imputeData, None) #self.assertEqual(classifier.NTrainEx, len(trainingData)) self.assertEqual(classifier.name, "Consensus classifier") self.assertNotEqual(classifier.varNames, None) self.assertEqual(classifier.verbose, 0) self.assertEqual(classifier.weights, None)
def test_CreateCustomClassificationClassifierUsingTrainingData(self): """ Test the creation of custom classification Classifier by calling learner with training data. """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } expression = [ "firstLearner == Iris-setosa -> Iris-setosa", "-> Iris-virginica" ] trainingData = self.getClassificationTrainingData() learner = AZorngConsensus.ConsensusLearner(learners=learners, expression=expression) # Act classifier = learner(trainingData) # Assert self.assertNotEqual(classifier, None) self.assertEqual(len(classifier.classifiers), len(learners)) self.assertNotEqual(classifier.basicStat, None) self.assertNotEqual(classifier.classVar, None) self.assertNotEqual(classifier.domain, None) self.assertEqual(classifier.expression, expression) self.assertNotEqual(classifier.imputeData, None) self.assertEqual(classifier.NTrainEx, len(trainingData)) self.assertEqual(classifier.name, "Consensus classifier") self.assertNotEqual(classifier.varNames, None) self.assertEqual(classifier.verbose, 0) self.assertEqual(classifier.weights, None)
def test_PredictionWithIncompatibleDomain(self): """Test prediction with uncompatible domain Test the non-prediction of examples with an incompatible domain """ expectedAcc1 = 0.7 #Ver 0.3 # Create a svm model svm = AZorngCvSVM.CvSVMLearner(self.noBadDataTrain) #using from index 3 o the end of data, because we know that from 0 to 2 the examples are not compatible Acc1 = evalUtilities.getClassificationAccuracy(self.noBadDataTest, svm) self.assertEqual(round(Acc1, 9), round(expectedAcc1, 9), "The Accuracy is not the expected. Got: " + str(Acc1)) self.assertEqual(svm(self.badVarTypeData[0]), 'NEG', "This example could still be predicted. Got: " + str(svm(self.badVarTypeData[0]))) #Ver 0.3 self.assertEqual( svm(self.badVarTypeData[1]), 'NEG', "This example could still be predicted. Got: " + str(svm(self.badVarTypeData[1]))) self.assertEqual( svm(self.badVarNameData[0]), None, "This example should NOT be predicted. Got: " + str(svm(self.badVarNameData[0]))) self.assertEqual( svm(self.badVarCountData[0]), None, "This example should NOT be predicted. Got: " + str(svm(self.badVarCountData[0])))
def test_PredictionWithDiffVarType(self): """Test prediction with diff. VarType Test the prediction of examples with different varType """ expectedAcc = 0.666666666667 # Create a svm model svm = AZorngCvSVM.CvSVMLearner(self.noBadDataTrain) #using from index 3 o the end of data, because we know that from 0 to 2 the examples are not compatible Acc2 = evalUtilities.getClassificationAccuracy(self.noBadDataTest[3:], svm) Acc1 = evalUtilities.getClassificationAccuracy(self.badVarTypeData[3:], svm) self.assertEqual(round(Acc1, 7), round(expectedAcc, 7), "The Accuracy is not the expected. Got: " + str(Acc1)) self.assertEqual(round(Acc2, 7), round(expectedAcc, 7), "The Accuracy is not the expected. Got: " + str(Acc2)) self.assert_( ('Fixed Types of variables' in svm.examplesFixedLog) and (svm.examplesFixedLog['Fixed Types of variables'] == 27), "No report of fixing in classifier class") self.assert_( ('Vars needing type fix' in svm.examplesFixedLog) and (svm.examplesFixedLog['Vars needing type fix']['[Br]([C])'] == "EnumVariable to FloatVariable"), "No report of fixing in classifier class")
def test_CreateDefaultClassifierUsingTrainingData(self): """ Test the creation of default Classifier by calling learner with training data. """ # Arrange learners = [ AZorngCvSVM.CvSVMLearner(), AZorngCvANN.CvANNLearner(), AZorngRF.RFLearner() ] trainingData = self.getRegressionTrainingData() learner = AZorngConsensus.ConsensusLearner(learners=learners) # Act classifier = learner(trainingData) # Assert self.assertNotEqual(classifier, None) self.assertEqual(len(classifier.classifiers), len(learners)) self.assertEqual(classifier.expression, None) self.assertEqual(classifier.name, "Consensus classifier") self.assertEqual(classifier.verbose, 0) self.assertNotEqual(classifier.imputeData, None) self.assertEqual(classifier.NTrainEx, len(trainingData)) self.assertNotEqual(classifier.basicStat, None) self.assertEqual(classifier.weights, None)
def test_CreateModelWithLearnerDictionary(self): """ Test the creation of Consensus Model using dictionary of learners """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } expression = "a + b + c" # Act learner = AZorngConsensus.ConsensusLearner(learners=learners, expression=expression) # Assert for k, v in learner.learners.items(): self.assertEqual(learner.learners[k], learners[k]) self.assertEqual(learner.expression, expression) self.assertEqual(learner.name, "Consensus learner") self.assertEqual(learner.verbose, 0) self.assertEqual(learner.imputeData, None) self.assertEqual(learner.NTrainEx, 0) self.assertEqual(learner.basicStat, None) self.assertEqual(learner.weights, None)
def test_MetaDataHandleForSavingModel(self): """Test the handling of SaveModel for Data with Meta Atributes """ #Test the save of a model created from a train data with meta attributes self.assert_( len(self.WMetaTest.domain.getmetas()) >= 1, "The dataset WMetaTest should have Meta Attributes") svmM = AZorngCvSVM.CvSVMLearner(self.WMetaTest) AccNoMetaBefore = evalUtilities.getClassificationAccuracy( self.NoMetaTrain, svmM) AccWMetaBefore = evalUtilities.getClassificationAccuracy( self.WMetaTest, svmM) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirSVMtest" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "CvSVMModel") svmM.write(modelPath) # Read in the model svmR = AZorngCvSVM.CvSVMread(modelPath) self.assert_( len(svmR.imputer.defaults.domain.getmetas()) == 0, "There shouldn't be any Meta data now!") # Calculate classification accuracy AccNoMetaAfter = evalUtilities.getClassificationAccuracy( self.NoMetaTrain, svmR) AccWMetaAfter = evalUtilities.getClassificationAccuracy( self.WMetaTest, svmR) # Test that the accuracy of the model before and after saved self.assertEqual( AccNoMetaBefore, AccNoMetaAfter, "NoMeta: Predictions after loading saved model were different") self.assertEqual( AccWMetaBefore, AccWMetaAfter, "WMeta: Predictions after loading saved model were different") self.assertEqual(round(AccWMetaAfter, 9), round(0.7, 9), "Accuracy was not the expected value!") self.assertEqual(round(AccNoMetaAfter, 9), round(0.6, 9), "Accuracy was not the expected value!") # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_CustomLogicalExpressionUsingOrAndStatement(self): """ Test logical expression using OR/AND statements """ # Arrange # Construct verification learners a = AZorngCvSVM.CvSVMLearner() a = a(self.irisData) b = AZorngCvANN.CvANNLearner() b = b(self.irisData) c = AZorngRF.RFLearner() c = c(self.irisData) # Construct expression learner/classifier learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } discreteExpression = [ "a == Iris-setosa and c == Iris-virginica or b == Iris-setosa -> Iris-setosa", "-> Iris-virginica" ] discreteLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=discreteExpression) discreteClassifier = discreteLearner(self.irisData) # Act result = [] for ex in self.irisData: result.append(discreteClassifier(ex)) verifiedResult = [] for ex in self.irisData: if a(ex).value == "Iris-setosa" and c( ex).value == "Iris-virginica" or b( ex).value == "Iris-setosa": verifiedResult.append("Iris-setosa") else: verifiedResult.append("Iris-virginica") # Assert for index, item in enumerate(result): if not result[index].value == verifiedResult[index]: print "Not equal on index: ", index, " Predicted: ", result[ index].value, " Real: ", verifiedResult[index] self.assertEqual(result[index].value, verifiedResult[index])
def test_ImputeTrain(self): """ Assure that imputation works for the svm models. Test on data with missing values This test just assures the the model is trained. The correct imputation test is made on testImpute """ svm = AZorngCvSVM.CvSVMLearner(self.missingTrain) Acc = evalUtilities.getClassificationAccuracy(self.missingTest, svm) self.assertEqual(round(0.59999999999999998, 5), round(Acc, 5)) # Ver 0.3
def test_AverageNRegressionExpressionUsingObjMap(self): """ Test regular expression using average N regression with object map """ # Arrange learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } # Construct expression learner/classifier regressionExpression = "(firstLearner + secondLearner + thirdLearner) / 3" expressionLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=regressionExpression) expressionClassifier = expressionLearner(self.DataReg) # Construct default learner/classifier defaultLearners = [ AZorngRF.RFLearner(), AZorngCvANN.CvANNLearner(), AZorngCvSVM.CvSVMLearner() ] defaultLearner = AZorngConsensus.ConsensusLearner( learners=defaultLearners) defaultClassifier = defaultLearner(self.DataReg) # Act expressionPredictions = [] for ex in self.DataReg: expressionPredictions.append(expressionClassifier(ex)) defaultPredictions = [] for ex in self.DataReg: defaultPredictions.append(defaultClassifier(ex)) # Assert for index in range(len(expressionPredictions)): self.assertEqual( True, float_compare(expressionPredictions[index], defaultPredictions[index]))
def test_TwoWays(self): """Test two ways svm creation Test that an svm created in one or two steps give the same results """ # One step svm creation svm = AZorngCvSVM.CvSVMLearner(self.train_data) # Calculate classification accuracy for the classifier trained in one step oneStepAcc = evalUtilities.getClassificationAccuracy( self.test_data, svm) # Two step svm creation learner = AZorngCvSVM.CvSVMLearner() svm = learner(self.train_data) # Calculate classification accuracy for the classifier trained in two steps twoStepAcc = evalUtilities.getClassificationAccuracy( self.test_data, svm) # Test that the accuracy of the classifiers created in different ways is the exact same self.assertEqual(oneStepAcc, twoStepAcc)
def test_CreateLearnerWithObjectMapping(self): """ Test the creation of learners with an object map """ # Arrange learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } # Act learner = AZorngConsensus.ConsensusLearner(learners=learners) # Assert self.assertEqual(len(learner.learners), len(learners))
def test_CustomRegressionExpressionUsingWeights(self): """ Test regression expression using weights """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } weights = {'a': lambda x: 1, 'b': lambda x: 2, 'c': lambda x: 3} regressionExpression = "(a + b + c) / 3" expressionLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=regressionExpression, weights=weights) classifier = expressionLearner(self.DataReg) # Act result = [] for ex in self.DataReg: result.append(classifier(ex)) verifiedResult = [] for ex in self.DataReg: a_value = classifier.classifiers['a'](ex) a_weight_value = weights['a'](a_value) b_value = classifier.classifiers['b'](ex) b_weight_value = weights['b'](b_value) c_value = classifier.classifiers['c'](ex) c_weight_value = weights['c'](c_value) prediction = (a_value * a_weight_value + b_value * b_weight_value + c_value * c_weight_value) / 3 verifiedResult.append(prediction) # Assert for index, item in enumerate(result): if float_compare(result[index].value, verifiedResult[index]) == False: print "Not equal on index: ", index print "Result: ", result[ index].value, " Verified: ", verifiedResult[index] print "Delta: ", abs(result[index].value - verifiedResult[index]) self.assertEqual( float_compare(result[index].value, verifiedResult[index]), True)
def test_CreateLearnerWithObjectMappingWithoutExpression(self): """ Test with name variable mapping defined but not expression given """ # Arrange learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } learner = AZorngConsensus.ConsensusLearner(learners=learners) # Act classifier = learner(self.DataReg) # Assert self.assertEqual(classifier, None)
def test_SVMreg(self): CvSVMmodel = AZorngCvSVM.CvSVMLearner(self.regTrainData) predList = [ 5.482803, 4.889269, 5.188474, 5.528782, 6.224637, 4.679743, 2.062022, 7.878900, 5.603292, 5.905775, ] # Ver. 0.3 for idx, ex in enumerate(self.regTrainData[0:10]): self.assertEqual(round(CvSVMmodel(ex), 4), round(predList[idx], 4))
def test_SaveLoadCustomRegressionExpression(self): """ Test save/load custom expression using average N regression with object map """ # Arrange learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } # Construct expression learner/classifier regressionExpression = "(firstLearner + secondLearner + thirdLearner) / 3" expressionLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=regressionExpression) expressionClassifier = expressionLearner(self.DataReg) # Construct default learner/classifier result = [] for ex in self.DataReg: result.append(expressionClassifier(ex)) # Act scratchdir = miscUtilities.createScratchDir( desc="ConsensusSaveLoadTest") expressionClassifier.write(os.path.join(scratchdir, "./CM.model")) resultLoaded = [] loaded = AZorngConsensus.Consensusread( os.path.join(scratchdir, "./CM.model")) self.assertNotEqual(loaded, None) for ex in self.DataReg: resultLoaded.append(loaded(ex)) # Assert for index, item in enumerate(result): if not float_compare(result[index].value, resultLoaded[index].value): print "Not equal on index: ", index self.assertEqual( float_compare(result[index].value, resultLoaded[index].value), True) self.assertEqual(len(loaded.domain), len(self.DataReg.domain)) self.assertEqual(len(loaded.imputeData), len(loaded.domain)) self.assertEqual(len(loaded.basicStat), len(loaded.domain)) self.assertEqual(loaded.NTrainEx, len(self.DataReg)) miscUtilities.removeDir(scratchdir)
def test_PredictionWithDiffVarOrder(self): """Test Prediction with diff. VarOrder Test the prediction examples with different varOrder """ expectedAcc = 0.7 # 0.59999999999999998 #0.7 # Ver 0.3 # Create a svm model svm = AZorngCvSVM.CvSVMLearner(self.noBadDataTrain) #using from index 3 o the end of data, because we know that from 0 to 2 the examples are not compatible Acc1 = evalUtilities.getClassificationAccuracy(self.noBadDataTest, svm) Acc2 = evalUtilities.getClassificationAccuracy(self.badVarOrderData, svm) self.assertEqual(round(Acc1, 9), round(expectedAcc, 9), "The Accuracy is not the expected. Got: " + str(Acc1)) #Ver 0.3 self.assertEqual(round(Acc2, 9), round(expectedAcc, 9), "The Accuracy is not the expected. Got: " + str(Acc2))
def test_MetaDataHandle(self): """Test the handling of Data with Meta Atributes """ # Create an svm model svm = AZorngCvSVM.CvSVMLearner(self.NoMetaTrain) # Calculate classification accuracy (NoMetaTest and WMeta are the same appart from the meta atribute) AccNoMeta = evalUtilities.getClassificationAccuracy( self.NoMetaTest, svm) AccWMeta = evalUtilities.getClassificationAccuracy(self.WMetaTest, svm) self.assertEqual( AccNoMeta, AccWMeta, "Predictions with and without meta data were different!") self.assertEqual(round(AccNoMeta, 9), round( 0.7, 9), "Accuracy was not the expected value! Got: ") #Ver 0.3
def test_SaveLoadCustomLogicalExpression(self): """ Test save/load functionality with a custom logical expression """ # Arrange # Construct expression learner/classifier learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } discreteExpression = [ "firstLearner == Iris-setosa -> Iris-setosa", "-> Iris-virginica" ] discreteLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=discreteExpression) discreteClassifier = discreteLearner(self.irisData) result = [] for ex in self.irisData: result.append(discreteClassifier(ex)) # Act scratchdir = miscUtilities.createScratchDir( desc="ConsensusSaveLoadTest") discreteClassifier.write(os.path.join(scratchdir, "./CM.model")) resultLoaded = [] loaded = AZorngConsensus.Consensusread( os.path.join(scratchdir, "./CM.model")) self.assertNotEqual(loaded, None) for ex in self.irisData: resultLoaded.append(loaded(ex)) # Assert for index, item in enumerate(result): if not result[index].value == resultLoaded[index].value: print "Not equal on index: ", index self.assertEqual(result[index].value, resultLoaded[index].value) self.assertEqual(len(loaded.domain), len(self.irisData.domain)) self.assertEqual(len(loaded.imputeData), len(loaded.domain)) self.assertEqual(len(loaded.basicStat), len(loaded.domain)) self.assertEqual(loaded.NTrainEx, len(self.irisData)) miscUtilities.removeDir(scratchdir)
def test_InvalidCustomRegressionExpression(self): """ Test invalid custom expression """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } regressionExpression = "(a + b + 3cd45 + c) / 3" expressionLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=regressionExpression) # Act classifier = expressionLearner(self.DataReg) # Assert self.assertEqual(classifier(self.DataReg[0]), None)
def applySettings(self): self.error(0) self.warning(0) if self.priorsGUI: self.priors = str(self.priorsGUI) else: self.priors = None #Update the variables handling the learner specific values self.setSVMTypeLearner() self.setStopCritLearner() self.changeKernel() #Create the Learner self.learner=AZorngCvSVM.CvSVMLearner() #Set the several learner parameters self.learner.priors = self.priors for attr in ("nMin","nMax","nClassMin","nClassMax","scaleClass","scaleData"): setattr(self.learner, attr, getattr(self, attr)) for attr in ("name", "kernel_type", "degree", "svm_type", "stopCrit", "maxIter"): setattr(self.learner, attr, getattr(self, attr)) for attr in ("gamma", "coef0", "C", "p", "eps", "nu"): setattr(self.learner, attr, float(getattr(self, attr))) self.classifier=None self.supportVectors=None #Create the classifier if self.data: self.classifier=self.learner(self.data) if self.learner: self.learner.name=str(self.name) self.send("Learner", self.learner) if self.classifier: self.classifier.name=str(self.name) #self.supportVectors=self.classifier.classifier.supportVectors self.send("Classifier", self.classifier) #if self.supportVectors: # self.send("Support Vectors", self.supportVectors) self.refreshParams()
def test_CanPersistRegressionModelUsingClassifiers(self): """Test the save/load for a regression model - Using average of N classifiers""" # Arrange learners = [ AZorngRF.RFLearner(), AZorngCvSVM.CvSVMLearner(), AZorngCvANN.CvANNLearner() ] learner = AZorngConsensus.ConsensusLearner(learners=learners) classifier = learner(self.DataReg) # Act predictions = [] for ex in self.DataReg: predictions.append(classifier(ex)) scratchdir = miscUtilities.createScratchDir( desc="ConsensusSaveLoadTest") classifier.write(os.path.join(scratchdir, "./CM.model")) # Assert predictionsL = [] Loaded = AZorngConsensus.Consensusread( os.path.join(scratchdir, "./CM.model")) for ex in self.DataReg: predictionsL.append(Loaded(ex)) self.assertEqual( [round(pred.value, 4) for pred in predictions], [round(pred.value, 4) for pred in predictionsL], "Loaded model predictions differ: Pred. 1 (saved/loaded):" + str(predictions[0]) + " / " + str(predictionsL[0])) self.assertEqual(len(Loaded.domain), len(self.DataReg.domain)) self.assertEqual(len(Loaded.imputeData), len(Loaded.domain)) self.assertEqual(len(Loaded.basicStat), len(Loaded.domain)) self.assertEqual(Loaded.NTrainEx, len(self.DataReg)) miscUtilities.removeDir(scratchdir)
def test_CanCreateClassifierUsingObjMapping(self): """ Test with name variable mapping defined but not expression given """ # Arrange learners = { 'firstLearner': AZorngCvSVM.CvSVMLearner(), 'secondLearner': AZorngCvANN.CvANNLearner(), 'thirdLearner': AZorngRF.RFLearner() } discreteExpression = "" regressionExpression = "(firstLearner + secondLearner + thirdLearner) / 2" learner = AZorngConsensus.ConsensusLearner( learners=learners, expression=regressionExpression) # Act classifier = learner(self.DataReg) # Assert self.assertNotEqual(classifier, None) self.assertEqual(len(classifier.classifiers), 3) self.assertEqual(classifier.expression, regressionExpression)
def test_InvalidCustomClassificationExpression(self): """ Test invalid custom expression """ # Arrange learners = { 'a': AZorngCvSVM.CvSVMLearner(), 'b': AZorngCvANN.CvANNLearner(), 'c': AZorngRF.RFLearner() } discreteExpression = [ "a == Iris-setosa and or c == Iris-virginica or b == Iris-setosa -> Iris-setosa", "-> Iris-virginica" ] expressionLearner = AZorngConsensus.ConsensusLearner( learners=learners, expression=discreteExpression) # Act classifier = expressionLearner(self.getClassificationTrainingData()) # Assert self.assertEqual(classifier(self.getClassificationTrainingData()[0]), None)