def test_Priors(self):
"""Test to assure that priors are set correcly."""
# Create a CvANN model
CvANNlearner = AZorngCvANN.CvANNLearner(stopUPs = 0, priors = {"Iris-versicolor":0.35, "Iris-virginica":0.13, "Iris-setosa":0.52})
CvANNmodel = CvANNlearner(self.irisData)
#Model with No Priors
CvANNlearnerNoP = AZorngCvANN.CvANNLearner(stopUPs=0)
CvANNmodelNoP = CvANNlearnerNoP(self.irisData)
# Calculate classification accuracy
Acc = evalUtilities.getClassificationAccuracy(self.irisData, CvANNmodel)
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdirTest"+str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir,"modelPriors.CvANN")
CvANNmodel.write(modelPath)
# Read in the model
newCvANNmodel = AZorngCvANN.CvANNread(modelPath)
# Calculate classification accuracy
savedAcc = evalUtilities.getClassificationAccuracy(self.irisData, CvANNmodel)
NoPAcc = evalUtilities.getClassificationAccuracy(self.irisData, CvANNmodelNoP)
# Test that the accuracy of the two classifiers is the exact same
self.assertEqual(Acc, savedAcc)
self.assert_(Acc != NoPAcc)
# Remove the scratch directory
os.system("/bin/rm -rf "+scratchdir)
def test_MetaDataHandleForSavingModel(self):
"""Test the handling of SaveModel for Data with Meta Atributes
"""
expectedAccWMeta = 1.0 # Ver 0.3
expectedAccNoMeta = 0.63333333300000005 # Ver 0.3
#Test the save of a model created from a train data with meta attributes
self.assert_(len(self.WMetaTest.domain.getmetas())>=1,"The dataset WMetaTest should have Meta Attributes")
CvANNlearner = AZorngCvANN.CvANNLearner(randomWeights = False, nHidden = [3], nEpochs = 100,stopUPs=0)
annM = CvANNlearner(self.WMetaTest)
AccNoMetaBefore = evalUtilities.getClassificationAccuracy(self.NoMetaTrain,annM)
AccWMetaBefore = evalUtilities.getClassificationAccuracy(self.WMetaTest,annM)
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdiriTest"+str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir,"CvANNModel.CvANN")
annM.write(modelPath)
# Read in the model
annR = AZorngCvANN.CvANNread(modelPath)
self.assert_(len(annR.imputer.defaults.domain.getmetas())==0,"There shouldn't be any Meta data now!")
# Calculate classification accuracy
AccNoMetaAfter = evalUtilities.getClassificationAccuracy(self.NoMetaTrain, annR)
AccWMetaAfter = evalUtilities.getClassificationAccuracy(self.WMetaTest, annR)
# Test that the accuracy of the model before and after saved
self.assertEqual(AccNoMetaBefore, AccNoMetaAfter,"NoMeta: Predictions after loading saved model were different")
self.assertEqual(AccWMetaBefore, AccWMetaAfter, "WMeta: Predictions after loading saved model were different")
self.assertEqual(round(AccWMetaAfter,9), round(expectedAccWMeta,9))
self.assertEqual(round(AccNoMetaAfter,9), round(expectedAccNoMeta,9))
# Remove the scratch directory
os.system("/bin/rm -rf "+scratchdir)
def test_SavedModel(self):
"""Test to assure that a saved ann model gives the same predictions as before saving."""
# Create an ann model
ann = AZorngCvANN.CvANNLearner(self.train_data,stopUPs=0)
# Calculate classification accuracy
Acc = evalUtilities.getClassificationAccuracy(self.test_data, ann)
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdir"+str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir,"ann.cvann")
ann.write(modelPath)
# Read in the model
ann = AZorngCvANN.CvANNread(modelPath)
# Calculate classification accuracy
savedAcc = evalUtilities.getClassificationAccuracy(self.test_data, ann)
# Test that the accuracy of the two classifiers is the exact same
self.assertEqual(Acc, savedAcc)
# Remove the scratch directory
os.system("/bin/rm -rf "+scratchdir)
def TopVarImportanceTest(data, expectNone = False):
resA = []
resB = []
CvANN = AZorngCvANN.CvANNLearner(data, stopUPs=33)
for ex in data:
resA.append(CvANN.getTopImportantVars(ex,1))
scratchdir = miscUtilities.createScratchDir(desc="TopVarImportanceTest")
modelPath = os.path.join(scratchdir,"CvANNModel")
CvANN.write(modelPath)
LoadedCvANN = AZorngCvANN.CvANNread(modelPath)
miscUtilities.removeDir(scratchdir)
for ex in data:
resB.append(LoadedCvANN.getTopImportantVars(ex,1))
if expectNone:
return resA == resB == [None]*len(data)
else:
return resA == resB and None not in resA and resA.count(resA[0]) != len(resA)
def modelRead(modelFile=None, verbose=0, retrunClassifier=True):
"""Get the type of model saved in 'modelPath' and loads the respective model
Returns the Classifier saved in the respective model path
If called without parameters, it returns a list of known classifier types
It can returns the classifier, or just a string with the Type
modelRead (modelFile [, verbose = 0] [, retrunClassifier = True] )"""
if not modelFile:
return ("SignSVM", "CvSVM", "CvANN", "PLS", "CvRF", "CvBoost",
"CvBayes", "Consensus")
modelType = None
loadedModel = None
if os.path.isfile(os.path.join(modelFile, "model.svm")):
modelType = "CvSVM"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvSVM
loadedModel = AZorngCvSVM.CvSVMread(modelFile, verbose)
elif os.path.isdir(os.path.join(modelFile, "model.SignSvm")):
modelType = "SignSVM"
if not retrunClassifier: return modelType
from trainingMethods import AZorngSignSVM
loadedModel = AZorngSignSVM.SignSVMread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.ann")):
modelType = "CvANN"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvANN
loadedModel = AZorngCvANN.CvANNread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "Model.pls")):
modelType = "PLS"
if not retrunClassifier: return modelType
from trainingMethods import AZorngPLS
loadedModel = AZorngPLS.PLSread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.rf")):
modelType = "RF"
if not retrunClassifier: return modelType
from trainingMethods import AZorngRF
loadedModel = AZorngRF.RFread(modelFile, verbose)
elif os.path.isdir(os.path.join(modelFile, "C0.model")):
modelType = "Consensus"
if not retrunClassifier: return modelType
from trainingMethods import AZorngConsensus
loadedModel = AZorngConsensus.Consensusread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.boost")):
modelType = "CvBoost"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvBoost
loadedModel = AZorngCvBoost.CvBoostread(modelFile, verbose)
elif os.path.isfile(os.path.join(modelFile, "model.bayes")):
modelType = "CvBayes"
if not retrunClassifier: return modelType
from trainingMethods import AZorngCvBayes
loadedModel = AZorngCvBayes.CvBayesread(modelFile, verbose)
else: # Assuming an RF old format for backcompatibility
try:
if os.path.isdir(modelFile):
modelType = "RF"
if not retrunClassifier: return modelType
from trainingMethods import AZorngRF
loadedModel = AZorngRF.RFread(modelFile, verbose)
else:
modelType = None
loadedModel = None
except:
modelType = None
loadedModel = None
return loadedModel
def test_Impute(self):
"""Test missing values imputation
Assure that imputation works for the ann models. Test on data with missing values
"""
#This data is loaded here to speed up the test suite since it is too big
contTestDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/linearTest.tab")
contTrainDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/linearTrain.tab")
contTrain = dataUtilities.DataTable(contTrainDataPath)
contTest = dataUtilities.DataTable(contTestDataPath)
ex1=contTest[5]
ex2=contTest[6]
self.assert_(ex1["Desc 71"]!="?","The var Desc 71 shouldn't be missing!")
self.assert_(ex2["Desc 138"]!="?","The var Desc 138 shouldn't be missing!")
imputer = orange.ImputerConstructor_average(contTrain)
CvANNlearner = AZorngCvANN.CvANNLearner(randomWeights = False, nHidden = [3], nEpochs = 100,stopUPs=0)
ann = CvANNlearner(contTrain)
# Prediction for data as it is
P1=ann(ex1)
P2=ann(ex2)
# Predictions changing one continuous and one discrete variable to 0
ex1["Desc 71"]=0
ex2["Desc 138"]=0
P1_0=ann(ex1)
P2_0=ann(ex2)
# Predictions changing the same continuous and discrete variable to it's correspondent imputation value
ex1["Desc 71"]=imputer.defaults["Desc 71"]
ex2["Desc 138"]=imputer.defaults["Desc 138"]
P1_imp=ann(ex1)
P2_imp=ann(ex2)
# Predictions changing the same continuous and discrete variable to '?' wich means that the same imputation
# as in the last case will have to be made inside the classifier. So, the predicted value must be the same
ex1["Desc 71"]="?"
ex2["Desc 138"]="?"
self.assert_(ex1["Desc 71"]=="?","The var Desc 71 should be missing now!")
self.assert_(ex2["Desc 138"]=="?","The var Desc 138 should be missing now!")
P1Miss=ann(ex1)
P2Miss=ann(ex2)
# Test if the prediction made for the example with mising value is the same as the one
# for the example which missing values were substituted using the same method as the classifier does.
self.assert_(P1_imp==P1Miss,"Imputation was not made correctly inside the classifier")
#self.assert_(P2_imp==P2Miss,"Imputation was not made correctly inside the classifier")
# Assure that if other substitutions on those variables were made, the predicted value would be different,
# and so, this is a valid method for testing the imputation
self.assert_(P1.value!=P2.value) # Just to assure that we are not comaring equal examples
self.assert_(P1.value!=P1_imp.value,"The imputed 1 was the same as the original ... try other example")
self.assert_(P1_0.value!=P1_imp.value,"The imputed 1 was the same as the replaced by 0. The classifier may be replacing missing values by 0")
self.assert_(P2.value!=P2Miss.value, "The missing imputed 2 was the same as the original ... try other example")
self.assert_(P2_0.value!=P2Miss.value,"The missing imputed 2 was the same as the replaced by 0. The classifier may be replacing missing values by 0")
#Test the imputer for saved models
# Save the model
scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdir"+str(time.time()))
os.mkdir(scratchdir)
modelPath = os.path.join(scratchdir,"CvANNModel")
ann.write(modelPath)
# Read in the model
annM = AZorngCvANN.CvANNread(modelPath)
# Predict the ex1 and ex2 which are still the examples with missing values '?'
self.assert_( ex1["Desc 71"]=="?","Value of Var Desc 71 should be missing!")
self.assert_( ex2["Desc 138"]=="?","Value of Var Desc 138 should be missing!")
self.assert_(round(annM(ex1),6)==round(P1Miss,6),"Imputation on loaded model is not correct")
self.assert_(round(annM(ex2),6)==round(P2Miss,6),"Imputation on loaded model is not correct")
# Remove the scratch directory
os.system("/bin/rm -rf "+scratchdir)
