def test_SVMC(self):
# Train a svm
svmL = AZorngCvSVM.CvSVMLearner(scaleData=False,
svm_type=103,
gamma=0.01,
C=1,
nu=0.5,
p=1,
eps=0.001,
coef0=0,
degree=3)
svm = svmL(self.inDataC)
trainedAcc = evalUtilities.getRMSE(self.inDataC, svm)
self.assertEqual(round(trainedAcc, 7), round(2.8525863999999999,
7)) # ver 0.3
# Save model
rc = svm.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm)
# Assure equal accuracy
self.assertEqual(trainedAcc, loadedAcc)
svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False)
svmLearner.name = "CvSVMLearner"
svmLearner.eps = 0.001
svmLearner.p = 1
svmLearner.nu = 0.6
svmLearner.kernel_type = 2
svmLearner.svm_type = 103
svmLearner.gamma = 0.0033
svmLearner.C = 47
svmLearner.scaleData = True
svmLearner.scaleClass = False
Res = orngTest.crossValidation(
[svmLearner],
self.inDataC,
folds=5,
strat=orange.MakeRandomIndices.StratifiedIfPossible)
RMSE = evalUtilities.RMSE(Res)[0]
self.assertEqual(round(RMSE, 2), round(2.96, 2)) #Ver 0.3
newSVM = svmLearner(self.inDataC)
trainedAcc = evalUtilities.getRMSE(self.inDataC, newSVM)
# Save model
rc = newSVM.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getRMSE(self.inDataC, loadedsvm)
# Assure equal accuracy
self.assertEqual(round(trainedAcc, 4), round(2.8289, 4)) #Ver 0.3
self.assertEqual(round(trainedAcc, 4), round(loadedAcc, 4))
def TopVarImportanceTest(data, expectNone=False):
resA = []
resB = []
learner = AZorngCvSVM.CvSVMLearner(gamma=1.0,
svm_type=103,
C=1,
coef0=0,
degree=3,
epsR=0.001,
kernel_type=2,
nu=0.5,
p=0.1,
probability=0,
shrinking=1)
CvSVM = learner(data)
for ex in data:
resA.append(CvSVM.getTopImportantVars(ex, 1))
scratchdir = miscUtilities.createScratchDir(
desc="TopVarImportanceTest")
modelPath = os.path.join(scratchdir, "CvSVNModel")
CvSVM.write(modelPath)
LoadedCvSVM = AZorngCvSVM.CvSVMread(modelPath)
miscUtilities.removeDir(scratchdir)
for ex in data:
resB.append(LoadedCvSVM.getTopImportantVars(ex, 1))
if expectNone:
return resA == resB == [None] * len(data)
else:
return resA == resB and None not in resA and resA.count(
resA[0]) != len(resA)
def test_Priors(self):
"""Test to assure that priors are set correcly."""
# Create a CvSVM model
CvSVMlearner = AZorngCvSVM.CvSVMLearner(C=3,
priors={
"Iris-versicolor": 2,
"Iris-virginica": 4,
"Iris-setosa": 6
})
CvSVMmodel = CvSVMlearner(self.inDataD)
# Calculate classification accuracy
Acc = evalUtilities.getClassificationAccuracy(self.inDataD, CvSVMmodel)
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR,
"scratchdirTest" + str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir, "modelPriors.CvSVM")
CvSVMmodel.write(modelPath)
# Read in the model
newCvSVMmodel = AZorngCvSVM.CvSVMread(modelPath)
# Calculate classification accuracy
savedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, CvSVMmodel)
# Test that the accuracy of the two classifiers is the exact same
self.assertEqual(Acc, savedAcc)
#Check the priors saved in the model
file = open(os.path.join(modelPath, "model.svm"), "r")
lines = file.readlines()
file.close()
priors = [
round(x, 2) for x in eval((lines[18].strip()).replace("data:", ""))
]
self.assertEqual(len(priors), 3)
self.assertEqual(
priors[self.inDataD.domain.classVar.values.index("Iris-setosa")],
18.0)
self.assertEqual(
priors[self.inDataD.domain.classVar.values.index(
"Iris-versicolor")], 6.0)
self.assertEqual(
priors[self.inDataD.domain.classVar.values.index(
"Iris-virginica")], 12.0)
# Remove the scratch directory
os.system("/bin/rm -rf " + scratchdir)
def test_MetaDataHandleForSavingModel(self):
"""Test the handling of SaveModel for Data with Meta Atributes
"""
#Test the save of a model created from a train data with meta attributes
self.assert_(
len(self.WMetaTest.domain.getmetas()) >= 1,
"The dataset WMetaTest should have Meta Attributes")
svmM = AZorngCvSVM.CvSVMLearner(self.WMetaTest)
AccNoMetaBefore = evalUtilities.getClassificationAccuracy(
self.NoMetaTrain, svmM)
AccWMetaBefore = evalUtilities.getClassificationAccuracy(
self.WMetaTest, svmM)
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR,
"scratchdirSVMtest" + str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir, "CvSVMModel")
svmM.write(modelPath)
# Read in the model
svmR = AZorngCvSVM.CvSVMread(modelPath)
self.assert_(
len(svmR.imputer.defaults.domain.getmetas()) == 0,
"There shouldn't be any Meta data now!")
# Calculate classification accuracy
AccNoMetaAfter = evalUtilities.getClassificationAccuracy(
self.NoMetaTrain, svmR)
AccWMetaAfter = evalUtilities.getClassificationAccuracy(
self.WMetaTest, svmR)
# Test that the accuracy of the model before and after saved
self.assertEqual(
AccNoMetaBefore, AccNoMetaAfter,
"NoMeta: Predictions after loading saved model were different")
self.assertEqual(
AccWMetaBefore, AccWMetaAfter,
"WMeta: Predictions after loading saved model were different")
self.assertEqual(round(AccWMetaAfter, 9), round(0.7, 9),
"Accuracy was not the expected value!")
self.assertEqual(round(AccNoMetaAfter, 9), round(0.6, 9),
"Accuracy was not the expected value!")
# Remove the scratch directory
os.system("/bin/rm -rf " + scratchdir)
def test_Impute(self):
"""Test missing values imputation
Assure that imputation works for the svm models. Test on data with missing values
"""
ex1 = self.contTest[1]
ex2 = self.contTest[6]
self.assert_(ex1["DiscAttr2"] != "?",
"The var DiscAttr2 shouldn't be missing!")
self.assert_(ex2["Level"] != "?",
"The var Level shouldn't be missing!")
imputer = orange.ImputerConstructor_average(self.contTrain)
svmL = AZorngCvSVM.CvSVMLearner(
p=0.2
) #Here if p=2 and scaleClass is False, is ok, but with p=2 and also scale calss, the model will have no support vectors. So, with p=0.2 and also scale class, it goes right.
svmL.svm_type = 103
svm = svmL(self.contTrain)
# Prediction for data as it is
P1 = svm(ex1)
P2 = svm(ex2)
# Predictions changing one continuous and one discrete variable to 0
ex1["DiscAttr2"] = 0
ex2["Level"] = 0
P1_0 = svm(ex1)
P2_0 = svm(ex2)
# Predictions changing the same continuous and discrete variable to it's correspondent imputation value
ex1["DiscAttr2"] = imputer.defaults["DiscAttr2"]
ex2["Level"] = imputer.defaults["Level"]
P1_imp = svm(ex1)
P2_imp = svm(ex2)
# Predictions changing the same continuous and discrete variable to '?' wich means that the same imputation
# as in the last case will have to be made inside the classifier. So, the predicted value must be the same
ex1["DiscAttr2"] = "?"
ex2["Level"] = "?"
self.assert_(ex1["DiscAttr2"] == "?",
"The var DiscAttr2 should be missing now!")
self.assert_(ex2["Level"] == "?",
"The var Level should be missing now!")
P1Miss = svm(ex1)
P2Miss = svm(ex2)
# Test if the prediction made for the example with mising value is the same as the one
# for the example which missing values were substituted using the same method as the classifier does.
self.assert_(
round(P1_imp, 4) == round(P1Miss, 4),
"Imputation was not made correctly inside the classifier")
self.assert_(
round(P2_imp, 4) == round(P2Miss, 4),
"Imputation was not made correctly inside the classifier")
# Assure that if other substitutions on those variables were made, the predicted value would be different,
# and so, this is a valid method for testing the imputation
self.assert_(round(P1.value, 4) != round(
P2.value,
4)) # Just to assure that we are not comaring equal examples
self.assert_(round(P1.value, 4) != round(P1_imp.value, 4))
self.assert_(round(P1_0.value, 4) != round(P1_imp.value, 4))
self.assert_(round(P2.value, 4) != round(P2_imp.value, 4))
self.assert_(round(P2_0.value, 4) != round(P2_imp.value, 4))
#Test the imputer for saved models
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR,
"scratchdirSVMtest" + str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir, "CvSVMModel")
svm.write(modelPath)
# Read in the model
svmM = AZorngCvSVM.CvSVMread(modelPath)
# Predict the ex1 and ex2 which are still the examples with missing values '?'
self.assert_(ex1["DiscAttr2"] == "?",
"Value of Var DiscAttr2 should be missing!")
self.assert_(ex2["Level"] == "?",
"Value of Var Level should be missing!")
self.assert_(
round(svmM(ex1), 4) == round(P1Miss, 4),
"Imputation on loaded model is not correct")
self.assert_(
round(svmM(ex2), 4) == round(P2Miss, 4),
"Imputation on loaded model is not correct")
# Remove the scratch directory
os.system("/bin/rm -rf " + scratchdir)
def test_SVM_Priors_D(self):
"""Test SVM with priors """
# Train a svm
svm = AZorngCvSVM.CvSVMLearner(self.inDataD,
priors={
"Iris-setosa": 0.2,
"Iris-versicolor": 0.3,
"Iris-virginica": 0.5
})
trainedAcc = evalUtilities.getClassificationAccuracy(self.inDataD, svm)
self.assertEqual(round(trainedAcc, 7), round(0.73333329999999997, 7))
# Save model
rc = svm.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, loadedsvm)
# Assure equal accuracy
self.assertEqual(trainedAcc, loadedAcc)
svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False,
priors={
"Iris-setosa": 0.2,
"Iris-versicolor": 0.3,
"Iris-virginica": 0.5
})
svmLearner.name = "CvSVMLearner"
svmLearner.shrinking = 1
svmLearner.eps = 0.001
svmLearner.p = 0.0
svmLearner.nu = 0.6
svmLearner.kernel_type = 2
svmLearner.svm_type = 103
svmLearner.gamma = 0.0033
svmLearner.C = 47
svmLearner.probability = 1
svmLearner.scaleData = True
svmLearner.scaleClass = False
#svmLearner.for_nomogram=1
Res = orngTest.crossValidation(
[svmLearner],
self.inDataD,
folds=5,
strat=orange.MakeRandomIndices.StratifiedIfPossible)
CA = evalUtilities.CA(Res)[0]
self.assertEqual(round(CA, 2),
round(0.940000000,
2)) # orange1.0: 0.93333333333333335])
svmLearner.priors = None
Res = orngTest.crossValidation(
[svmLearner],
self.inDataD,
folds=5,
strat=orange.MakeRandomIndices.StratifiedIfPossible)
CA = evalUtilities.CA(Res)[0]
self.assertEqual(round(CA, 2), round(0.94666666666666666, 2))
newSVM = svmLearner(self.inDataD)
trainedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, newSVM)
# Save model
rc = newSVM.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, loadedsvm)
# Assure equal accuracy
self.assertEqual(round(trainedAcc, 7),
round(0.95999999999999996,
7)) #Before in AZSVM: 0.953333300000
self.assertEqual(round(trainedAcc, 1), round(loadedAcc, 1))
def test_SVMD(self):
# Train a svm
svm = AZorngCvSVM.CvSVMLearner(self.inDataD,
scaleData=False,
gamma=4,
C=1,
nu=0.5,
p=0.1,
eps=0.001,
coef0=0,
degree=3)
trainedAcc = evalUtilities.getClassificationAccuracy(self.inDataD, svm)
self.assertEqual(round(trainedAcc, 7), round(0.986666666667, 7))
# Save model
rc = svm.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, loadedsvm)
# Assure equal accuracy
self.assertEqual(trainedAcc, loadedAcc)
svmLearner = AZorngCvSVM.CvSVMLearner(scaleData=False)
svmLearner.name = "CvSVMLearner"
svmLearner.eps = 0.001
svmLearner.p = 0.0
svmLearner.nu = 0.6
svmLearner.kernel_type = 2
svmLearner.svm_type = 101
svmLearner.gamma = 0.0033
svmLearner.C = 47
svmLearner.scaleData = True
svmLearner.scaleClass = False
Res = orngTest.crossValidation(
[svmLearner],
self.inDataD,
folds=5,
strat=orange.MakeRandomIndices.StratifiedIfPossible)
CA = evalUtilities.CA(Res)[0]
self.assertEqual(round(CA, 2),
round(0.96666666666666667,
2)) # Before in AZSVM: 0.95999999999999996
newSVM = svmLearner(self.inDataD)
trainedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, newSVM)
# Save model
rc = newSVM.write(self.modelPath)
self.assertEqual(rc, True)
# Load the saved model
loadedsvm = AZorngCvSVM.CvSVMread(self.modelPath)
loadedAcc = evalUtilities.getClassificationAccuracy(
self.inDataD, loadedsvm)
# Assure equal accuracy
self.assertEqual(round(trainedAcc, 7),
round(0.96666669999999999,
7)) #Before in AZSVM: 0.953333300000
self.assertEqual(round(trainedAcc, 1), round(loadedAcc, 1))
def modelRead(modelFile=None, verbose=0, retrunClassifier=True):
"""Get the type of model saved in 'modelPath' and loads the respective model
Returns the Classifier saved in the respective model path
If called without parameters, it returns a list of known classifier types
It can returns the classifier, or just a string with the Type
modelRead (modelFile [, verbose = 0] [, retrunClassifier = True] )"""
if not modelFile:
return ("SignSVM", "CvSVM", "CvANN", "PLS", "CvRF", "CvBoost",
"CvBayes", "Consensus")
modelType = None
loadedModel = None
if os.path.isfile(os.path.join(modelFile, "model.svm")):
modelType = "CvSVM"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvSVM
loadedModel = AZorngCvSVM.CvSVMread(modelFile, verbose)
elif os.path.isdir(os.path.join(modelFile, "model.SignSvm")):
modelType = "SignSVM"
if not retrunClassifier: return modelType
from trainingMethods import AZorngSignSVM
loadedModel = AZorngSignSVM.SignSVMread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.ann")):
modelType = "CvANN"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvANN
loadedModel = AZorngCvANN.CvANNread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "Model.pls")):
modelType = "PLS"
if not retrunClassifier: return modelType
from trainingMethods import AZorngPLS
loadedModel = AZorngPLS.PLSread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.rf")):
modelType = "RF"
if not retrunClassifier: return modelType
from trainingMethods import AZorngRF
loadedModel = AZorngRF.RFread(modelFile, verbose)
elif os.path.isdir(os.path.join(modelFile, "C0.model")):
modelType = "Consensus"
if not retrunClassifier: return modelType
from trainingMethods import AZorngConsensus
loadedModel = AZorngConsensus.Consensusread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.boost")):
modelType = "CvBoost"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvBoost
loadedModel = AZorngCvBoost.CvBoostread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.bayes")):
modelType = "CvBayes"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvBayes
loadedModel = AZorngCvBayes.CvBayesread(modelFile, verbose)
else: # Assuming an RF old format for backcompatibility
try:
if os.path.isdir(modelFile):
modelType = "RF"
if not retrunClassifier: return modelType
from trainingMethods import AZorngRF
loadedModel = AZorngRF.RFread(modelFile, verbose)
else:
modelType = None
loadedModel = None
except:
modelType = None
loadedModel = None
return loadedModel
