def test_SVMC(self): # Train a svm svmL = AZorngCvSVM.CvSVMLearner(scaleData=False, svm_type=103, gamma=0.01, C=1, nu=0.5, p=1, eps=0.001, coef0=0, degree=3) svm = svmL(self.inDataC) trainedAcc = evalUtilities.getRMSE(self.inDataC, svm) self.assertEqual(round(trainedAcc, 7), round(2.8525863999999999, 7)) # ver 0.3 # Save model rc = svm.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm) # Assure equal accuracy self.assertEqual(trainedAcc, loadedAcc) svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False) svmLearner.name = "CvSVMLearner" svmLearner.eps = 0.001 svmLearner.p = 1 svmLearner.nu = 0.6 svmLearner.kernel_type = 2 svmLearner.svm_type = 103 svmLearner.gamma = 0.0033 svmLearner.C = 47 svmLearner.scaleData = True svmLearner.scaleClass = False Res = orngTest.crossValidation( [svmLearner], self.inDataC, folds=5, strat=orange.MakeRandomIndices.StratifiedIfPossible) RMSE = evalUtilities.RMSE(Res)[0] self.assertEqual(round(RMSE, 2), round(2.96, 2)) #Ver 0.3 newSVM = svmLearner(self.inDataC) trainedAcc = evalUtilities.getRMSE(self.inDataC, newSVM) # Save model rc = newSVM.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm) # Assure equal accuracy self.assertEqual(round(trainedAcc, 4), round(2.8289, 4)) #Ver 0.3 self.assertEqual(round(trainedAcc, 4), round(loadedAcc, 4))
def TopVarImportanceTest(data, expectNone=False): resA = [] resB = [] learner = AZorngCvSVM.CvSVMLearner(gamma=1.0, svm_type=103, C=1, coef0=0, degree=3, epsR=0.001, kernel_type=2, nu=0.5, p=0.1, probability=0, shrinking=1) CvSVM = learner(data) for ex in data: resA.append(CvSVM.getTopImportantVars(ex, 1)) scratchdir = miscUtilities.createScratchDir( desc="TopVarImportanceTest") modelPath = os.path.join(scratchdir, "CvSVNModel") CvSVM.write(modelPath) LoadedCvSVM = AZorngCvSVM.CvSVMread(modelPath) miscUtilities.removeDir(scratchdir) for ex in data: resB.append(LoadedCvSVM.getTopImportantVars(ex, 1)) if expectNone: return resA == resB == [None] * len(data) else: return resA == resB and None not in resA and resA.count( resA[0]) != len(resA)
def test_Priors(self): """Test to assure that priors are set correcly.""" # Create a CvSVM model CvSVMlearner = AZorngCvSVM.CvSVMLearner(C=3, priors={ "Iris-versicolor": 2, "Iris-virginica": 4, "Iris-setosa": 6 }) CvSVMmodel = CvSVMlearner(self.inDataD) # Calculate classification accuracy Acc = evalUtilities.getClassificationAccuracy(self.inDataD, CvSVMmodel) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirTest" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "modelPriors.CvSVM") CvSVMmodel.write(modelPath) # Read in the model newCvSVMmodel = AZorngCvSVM.CvSVMread(modelPath) # Calculate classification accuracy savedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, CvSVMmodel) # Test that the accuracy of the two classifiers is the exact same self.assertEqual(Acc, savedAcc) #Check the priors saved in the model file = open(os.path.join(modelPath, "model.svm"), "r") lines = file.readlines() file.close() priors = [ round(x, 2) for x in eval((lines[18].strip()).replace("data:", "")) ] self.assertEqual(len(priors), 3) self.assertEqual( priors[self.inDataD.domain.classVar.values.index("Iris-setosa")], 18.0) self.assertEqual( priors[self.inDataD.domain.classVar.values.index( "Iris-versicolor")], 6.0) self.assertEqual( priors[self.inDataD.domain.classVar.values.index( "Iris-virginica")], 12.0) # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_MetaDataHandleForSavingModel(self): """Test the handling of SaveModel for Data with Meta Atributes """ #Test the save of a model created from a train data with meta attributes self.assert_( len(self.WMetaTest.domain.getmetas()) >= 1, "The dataset WMetaTest should have Meta Attributes") svmM = AZorngCvSVM.CvSVMLearner(self.WMetaTest) AccNoMetaBefore = evalUtilities.getClassificationAccuracy( self.NoMetaTrain, svmM) AccWMetaBefore = evalUtilities.getClassificationAccuracy( self.WMetaTest, svmM) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirSVMtest" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "CvSVMModel") svmM.write(modelPath) # Read in the model svmR = AZorngCvSVM.CvSVMread(modelPath) self.assert_( len(svmR.imputer.defaults.domain.getmetas()) == 0, "There shouldn't be any Meta data now!") # Calculate classification accuracy AccNoMetaAfter = evalUtilities.getClassificationAccuracy( self.NoMetaTrain, svmR) AccWMetaAfter = evalUtilities.getClassificationAccuracy( self.WMetaTest, svmR) # Test that the accuracy of the model before and after saved self.assertEqual( AccNoMetaBefore, AccNoMetaAfter, "NoMeta: Predictions after loading saved model were different") self.assertEqual( AccWMetaBefore, AccWMetaAfter, "WMeta: Predictions after loading saved model were different") self.assertEqual(round(AccWMetaAfter, 9), round(0.7, 9), "Accuracy was not the expected value!") self.assertEqual(round(AccNoMetaAfter, 9), round(0.6, 9), "Accuracy was not the expected value!") # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_Impute(self): """Test missing values imputation Assure that imputation works for the svm models. Test on data with missing values """ ex1 = self.contTest[1] ex2 = self.contTest[6] self.assert_(ex1["DiscAttr2"] != "?", "The var DiscAttr2 shouldn't be missing!") self.assert_(ex2["Level"] != "?", "The var Level shouldn't be missing!") imputer = orange.ImputerConstructor_average(self.contTrain) svmL = AZorngCvSVM.CvSVMLearner( p=0.2 ) #Here if p=2 and scaleClass is False, is ok, but with p=2 and also scale calss, the model will have no support vectors. So, with p=0.2 and also scale class, it goes right. svmL.svm_type = 103 svm = svmL(self.contTrain) # Prediction for data as it is P1 = svm(ex1) P2 = svm(ex2) # Predictions changing one continuous and one discrete variable to 0 ex1["DiscAttr2"] = 0 ex2["Level"] = 0 P1_0 = svm(ex1) P2_0 = svm(ex2) # Predictions changing the same continuous and discrete variable to it's correspondent imputation value ex1["DiscAttr2"] = imputer.defaults["DiscAttr2"] ex2["Level"] = imputer.defaults["Level"] P1_imp = svm(ex1) P2_imp = svm(ex2) # Predictions changing the same continuous and discrete variable to '?' wich means that the same imputation # as in the last case will have to be made inside the classifier. So, the predicted value must be the same ex1["DiscAttr2"] = "?" ex2["Level"] = "?" self.assert_(ex1["DiscAttr2"] == "?", "The var DiscAttr2 should be missing now!") self.assert_(ex2["Level"] == "?", "The var Level should be missing now!") P1Miss = svm(ex1) P2Miss = svm(ex2) # Test if the prediction made for the example with mising value is the same as the one # for the example which missing values were substituted using the same method as the classifier does. self.assert_( round(P1_imp, 4) == round(P1Miss, 4), "Imputation was not made correctly inside the classifier") self.assert_( round(P2_imp, 4) == round(P2Miss, 4), "Imputation was not made correctly inside the classifier") # Assure that if other substitutions on those variables were made, the predicted value would be different, # and so, this is a valid method for testing the imputation self.assert_(round(P1.value, 4) != round( P2.value, 4)) # Just to assure that we are not comaring equal examples self.assert_(round(P1.value, 4) != round(P1_imp.value, 4)) self.assert_(round(P1_0.value, 4) != round(P1_imp.value, 4)) self.assert_(round(P2.value, 4) != round(P2_imp.value, 4)) self.assert_(round(P2_0.value, 4) != round(P2_imp.value, 4)) #Test the imputer for saved models # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirSVMtest" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "CvSVMModel") svm.write(modelPath) # Read in the model svmM = AZorngCvSVM.CvSVMread(modelPath) # Predict the ex1 and ex2 which are still the examples with missing values '?' self.assert_(ex1["DiscAttr2"] == "?", "Value of Var DiscAttr2 should be missing!") self.assert_(ex2["Level"] == "?", "Value of Var Level should be missing!") self.assert_( round(svmM(ex1), 4) == round(P1Miss, 4), "Imputation on loaded model is not correct") self.assert_( round(svmM(ex2), 4) == round(P2Miss, 4), "Imputation on loaded model is not correct") # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_SVM_Priors_D(self): """Test SVM with priors """ # Train a svm svm = AZorngCvSVM.CvSVMLearner(self.inDataD, priors={ "Iris-setosa": 0.2, "Iris-versicolor": 0.3, "Iris-virginica": 0.5 }) trainedAcc = evalUtilities.getClassificationAccuracy(self.inDataD, svm) self.assertEqual(round(trainedAcc, 7), round(0.73333329999999997, 7)) # Save model rc = svm.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, loadedsvm) # Assure equal accuracy self.assertEqual(trainedAcc, loadedAcc) svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False, priors={ "Iris-setosa": 0.2, "Iris-versicolor": 0.3, "Iris-virginica": 0.5 }) svmLearner.name = "CvSVMLearner" svmLearner.shrinking = 1 svmLearner.eps = 0.001 svmLearner.p = 0.0 svmLearner.nu = 0.6 svmLearner.kernel_type = 2 svmLearner.svm_type = 103 svmLearner.gamma = 0.0033 svmLearner.C = 47 svmLearner.probability = 1 svmLearner.scaleData = True svmLearner.scaleClass = False #svmLearner.for_nomogram=1 Res = orngTest.crossValidation( [svmLearner], self.inDataD, folds=5, strat=orange.MakeRandomIndices.StratifiedIfPossible) CA = evalUtilities.CA(Res)[0] self.assertEqual(round(CA, 2), round(0.940000000, 2)) # orange1.0: 0.93333333333333335]) svmLearner.priors = None Res = orngTest.crossValidation( [svmLearner], self.inDataD, folds=5, strat=orange.MakeRandomIndices.StratifiedIfPossible) CA = evalUtilities.CA(Res)[0] self.assertEqual(round(CA, 2), round(0.94666666666666666, 2)) newSVM = svmLearner(self.inDataD) trainedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, newSVM) # Save model rc = newSVM.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, loadedsvm) # Assure equal accuracy self.assertEqual(round(trainedAcc, 7), round(0.95999999999999996, 7)) #Before in AZSVM: 0.953333300000 self.assertEqual(round(trainedAcc, 1), round(loadedAcc, 1))
def test_SVMD(self): # Train a svm svm = AZorngCvSVM.CvSVMLearner(self.inDataD, scaleData=False, gamma=4, C=1, nu=0.5, p=0.1, eps=0.001, coef0=0, degree=3) trainedAcc = evalUtilities.getClassificationAccuracy(self.inDataD, svm) self.assertEqual(round(trainedAcc, 7), round(0.986666666667, 7)) # Save model rc = svm.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, loadedsvm) # Assure equal accuracy self.assertEqual(trainedAcc, loadedAcc) svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False) svmLearner.name = "CvSVMLearner" svmLearner.eps = 0.001 svmLearner.p = 0.0 svmLearner.nu = 0.6 svmLearner.kernel_type = 2 svmLearner.svm_type = 101 svmLearner.gamma = 0.0033 svmLearner.C = 47 svmLearner.scaleData = True svmLearner.scaleClass = False Res = orngTest.crossValidation( [svmLearner], self.inDataD, folds=5, strat=orange.MakeRandomIndices.StratifiedIfPossible) CA = evalUtilities.CA(Res)[0] self.assertEqual(round(CA, 2), round(0.96666666666666667, 2)) # Before in AZSVM: 0.95999999999999996 newSVM = svmLearner(self.inDataD) trainedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, newSVM) # Save model rc = newSVM.write(self.modelPath) self.assertEqual(rc, True) # Load the saved model loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath) loadedAcc = evalUtilities.getClassificationAccuracy( self.inDataD, loadedsvm) # Assure equal accuracy self.assertEqual(round(trainedAcc, 7), round(0.96666669999999999, 7)) #Before in AZSVM: 0.953333300000 self.assertEqual(round(trainedAcc, 1), round(loadedAcc, 1))
def modelRead(modelFile=None, verbose=0, retrunClassifier=True): """Get the type of model saved in 'modelPath' and loads the respective model Returns the Classifier saved in the respective model path If called without parameters, it returns a list of known classifier types It can returns the classifier, or just a string with the Type modelRead (modelFile [, verbose = 0] [, retrunClassifier = True] )""" if not modelFile: return ("SignSVM", "CvSVM", "CvANN", "PLS", "CvRF", "CvBoost", "CvBayes", "Consensus") modelType = None loadedModel = None if os.path.isfile(os.path.join(modelFile, "model.svm")): modelType = "CvSVM" if not retrunClassifier: return modelType from trainingMethods import AZorngCvSVM loadedModel = AZorngCvSVM.CvSVMread(modelFile, verbose) elif os.path.isdir(os.path.join(modelFile, "model.SignSvm")): modelType = "SignSVM" if not retrunClassifier: return modelType from trainingMethods import AZorngSignSVM loadedModel = AZorngSignSVM.SignSVMread(modelFile, verbose) elif os.path.isfile(os.path.join(modelFile, "model.ann")): modelType = "CvANN" if not retrunClassifier: return modelType from trainingMethods import AZorngCvANN loadedModel = AZorngCvANN.CvANNread(modelFile, verbose) elif os.path.isfile(os.path.join(modelFile, "Model.pls")): modelType = "PLS" if not retrunClassifier: return modelType from trainingMethods import AZorngPLS loadedModel = AZorngPLS.PLSread(modelFile, verbose) elif os.path.isfile(os.path.join(modelFile, "model.rf")): modelType = "RF" if not retrunClassifier: return modelType from trainingMethods import AZorngRF loadedModel = AZorngRF.RFread(modelFile, verbose) elif os.path.isdir(os.path.join(modelFile, "C0.model")): modelType = "Consensus" if not retrunClassifier: return modelType from trainingMethods import AZorngConsensus loadedModel = AZorngConsensus.Consensusread(modelFile, verbose) elif os.path.isfile(os.path.join(modelFile, "model.boost")): modelType = "CvBoost" if not retrunClassifier: return modelType from trainingMethods import AZorngCvBoost loadedModel = AZorngCvBoost.CvBoostread(modelFile, verbose) elif os.path.isfile(os.path.join(modelFile, "model.bayes")): modelType = "CvBayes" if not retrunClassifier: return modelType from trainingMethods import AZorngCvBayes loadedModel = AZorngCvBayes.CvBayesread(modelFile, verbose) else: # Assuming an RF old format for backcompatibility try: if os.path.isdir(modelFile): modelType = "RF" if not retrunClassifier: return modelType from trainingMethods import AZorngRF loadedModel = AZorngRF.RFread(modelFile, verbose) else: modelType = None loadedModel = None except: modelType = None loadedModel = None return loadedModel