def testSVM_MPI_3(self):
###################################################################
# Test other way of setting appspack
###################################################################
# Classification accuracy:
ExpectedCA = 0.847 #orange1: 0.837619047619
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI_3")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useParameters = pars.getParametersDict(),\
verbose = 0,\
useStd = False,\
advancedMPIoptions = "-v -np 4",\
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check if the MPI version was used
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
self.assertEqual(round(tunedPars[0],3),round(ExpectedCA,3))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig 14) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
def getRangeParsAndDistType(self, parametersOrig, param):
"""Get from the parametersOrig the Distribution type and the Range for the parameter named as the input var 'param'
"""
N_EX = 5 #Just a number to test the avaluation of the expressions on atributes definition
N_ATTR = 5 #Just a number to test the avaluation of the expressions on atributes definition
if parametersOrig[param][1] == "interval":
distType = 0
elif parametersOrig[param][1] == "values":
if "power2Range" in parametersOrig[param][2]:
distType = 1
elif "Range" in parametersOrig[param][2]:
distType = 2
else:
distType = 3
else:
self.setErrors("Invalid keyword in Configuration File")
return [None, None]
if distType in (1, 2):
txt = parametersOrig[param][2]
RangePars = txt[txt.find("Range") +
6:txt.find(")", txt.find("Range"))].split(",")
if len(RangePars) < 3:
RangePars.append("1") #the default step size
try:
if distType == 1 and miscUtilities.power2Range(
eval(RangePars[0]), eval(RangePars[1]),
eval(RangePars[2])) != eval(parametersOrig[param][2]):
return [[], 3]
elif distType == 2 and miscUtilities.Range(
eval(RangePars[0]), eval(RangePars[1]),
eval(RangePars[2])) != eval(parametersOrig[param][2]):
return [[], 3]
except:
return [[], 3]
elif distType == 0:
txt = parametersOrig[param][2]
RangePars = [txt[1:-1].split(",")[0], txt[1:-1].split(",")[-1]]
else: #distType==3
txt = parametersOrig[param][2]
RangePars = [x.strip() for x in txt[1:-1].split(" , ")]
if eval(parametersOrig[param][0]) in types.StringTypes or \
(type(eval(parametersOrig[param][0]))==types.ListType and (type(eval(parametersOrig[param][0])[0]) in types.StringTypes)):
newPars = []
for par in RangePars:
if par[0] == par[-1] and par[0] in ("'", '"'):
newPars.append(par[1:-1])
else:
newPars.append(par)
RangePars = newPars
RangePars = [x.strip() for x in RangePars]
return [RangePars, distType]
def getRangeParsAndDistType(self, parametersOrig ,param):
"""Get from the parametersOrig the Distribution type and the Range for the parameter named as the input var 'param'
"""
N_EX = 5 #Just a number to test the avaluation of the expressions on atributes definition
N_ATTR = 5 #Just a number to test the avaluation of the expressions on atributes definition
if parametersOrig[param][1] == "interval":
distType = 0
elif parametersOrig[param][1] == "values":
if "power2Range" in parametersOrig[param][2]:
distType = 1
elif "Range" in parametersOrig[param][2]:
distType = 2
else:
distType = 3
else:
self.setErrors("Invalid keyword in Configuration File")
return [None,None]
if distType in (1,2):
txt = parametersOrig[param][2]
RangePars = txt[txt.find("Range")+6:txt.find(")",txt.find("Range"))].split(",")
if len(RangePars)<3:
RangePars.append("1") #the default step size
try:
if distType == 1 and miscUtilities.power2Range(eval(RangePars[0]),eval(RangePars[1]),eval(RangePars[2])) != eval(parametersOrig[param][2]):
return [[],3]
elif distType == 2 and miscUtilities.Range(eval(RangePars[0]),eval(RangePars[1]),eval(RangePars[2])) != eval(parametersOrig[param][2]):
return [[],3]
except:
return [[],3]
elif distType == 0:
txt = parametersOrig[param][2]
RangePars = [txt[1:-1].split(",")[0],txt[1:-1].split(",")[-1]]
else:#distType==3
txt = parametersOrig[param][2]
RangePars = [x.strip() for x in txt[1:-1].split(" , ")]
if eval(parametersOrig[param][0]) in types.StringTypes or \
(type(eval(parametersOrig[param][0]))==types.ListType and (type(eval(parametersOrig[param][0])[0]) in types.StringTypes)):
newPars = []
for par in RangePars:
if par[0]==par[-1] and par[0] in ("'",'"'):
newPars.append(par[1:-1])
else:
newPars.append(par)
RangePars = newPars
RangePars=[x.strip() for x in RangePars]
return [RangePars,distType]
def disable_testSVM_MPI_3(self):
###################################################################
# Test other way of setting appspack
###################################################################
# Classification accuracy:
ExpectedCA = 0.6
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter, "optimize", True)
# Change the range
pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1))
trainFile = self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI_3")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useParameters = pars.getParametersDict(),\
verbose = 0,\
useStd = False,\
advancedMPIoptions = "-v -np 4",\
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check if the MPI version was used
self.assertEqual(learner.optimized, True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
self.assertEqual(round(tunedPars[0], 3), round(ExpectedCA, 3))
self.assert_(
len(
dataUtilities.DataTable(
os.path.join(runPath, "optimizationLog.txt"))) >=
12) # (orig 14) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
def disable_testSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.6] #New at orange2.0
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter, "optimize", True)
# Change the range
pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1))
trainFile = self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized, True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(
round(verbTunedPars["bestRes"],
3) in [round(x, 3) for x in ExpectedCA],
"Got:" + str(verbTunedPars["bestRes"]))
self.assert_(
len(
dataUtilities.DataTable(
os.path.join(runPath, "optimizationLog.txt"))) >=
12) # (orig: 14) Must be > 2
miscUtilities.removeDir(runPath)
def testCvSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.585] #Should be the result of the default point
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C"] #, "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter, "optimize", True)
# Change the range
pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1))
pars.setParameter("priors", "default", {
"POS": 50,
"NEG": 4
}) # These priors are to ensure the default point will be the best!
pars.setParameter(
"gamma", "default", 0.001
) # This is a bad value to ensure the default point will be the best!
trainFile = self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:", verbTunedPars["ResIdx"]
print "Optimizer runPath:", runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized, True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(
round(verbTunedPars["bestRes"],
3) in [round(x, 3) for x in ExpectedCA],
"Got:" + str(verbTunedPars["bestRes"]))
self.assert_(
len(
dataUtilities.DataTable(
os.path.join(runPath, "optimizationLog.txt"))) >= 5)
#Check Priors
self.assertEqual(tunedPars[1]["priors"], "None")
learner.priors = {'NEG': 4, 'POS': 2}
classifier = learner(self.discTest)
classifier.write(os.path.join(runPath, "CvSVMModel"))
file = open(os.path.join(runPath, "CvSVMModel/model.svm"), "r")
lines = file.readlines()
file.close()
priors = [
round(x, 2) for x in eval((lines[18].strip()).replace("data:", ""))
]
self.assertEqual(len(priors), 2)
self.assertEqual(
priors[self.discTest.domain.classVar.values.index("POS")],
2.0 * float(tunedPars[1]["C"]))
self.assertEqual(
priors[self.discTest.domain.classVar.values.index("NEG")],
4.0 * float(tunedPars[1]["C"]))
miscUtilities.removeDir(runPath)
def testSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.847, 0.838] #New at orange2.0
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:"+str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig: 14) Must be > 2
miscUtilities.removeDir(runPath)
def testCvSVM(self):
"""
Tests changing the default range of the optimizer.
"""
# Classification accuracy:
ExpectedCA = [0.6] # Ver 0.3
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
pars.setParameter("priors","default",{"POS":2, "NEG":4})
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
useStd = False,\
runPath = runPath,\
verbose = 0)
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:",verbTunedPars["ResIdx"]
print "Optimizer runPath:",runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was not used
self.assertEqual(optimizer.usedMPI, False)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],2) in [round(x,2) for x in ExpectedCA],"Got:"+str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig: 14) Must be > 2
#Check Priors
self.assertEqual(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt"))[1]["priors"].value,"{'NEG':4,'POS':2}")
self.assertEqual( tunedPars[1]["priors"],"{'NEG':4,'POS':2}") # Ver 0.3
#Set the priors since it could be choosing the first row as the best, which would be the default values, without the priors
learner.priors = {"POS":2, "NEG":4}
classifier = learner(self.discTest)
classifier.write(os.path.join(runPath,"CvSVMModel"))
file = open(os.path.join(runPath,"CvSVMModel/model.svm"),"r")
lines = file.readlines()
file.close()
priors = [round(x,2) for x in eval((lines[18].strip()).replace("data:",""))]
self.assertEqual(len(priors),2)
self.assertEqual(priors[self.discTest.domain.classVar.values.index("POS")],2.0*float(tunedPars[1]["C"]))
self.assertEqual(priors[self.discTest.domain.classVar.values.index("NEG")],4.0*float(tunedPars[1]["C"]))
miscUtilities.removeDir(runPath)
def no_test_SVMClassificationRange(self): #Disabled since we do not have the AZSVMLearner anymore
"""
SVM - Tests changing the default range of the optimizer.
"""
optimizer = paramOptUtilities.Appspack()
learner = AZorngSVM.AZSVMLearner()
learnerName = "AZSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
useStd = False,\
runPath = runPath,\
verbose = 0)
self.assertEqual(optimizer.usedMPI,False)
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assertEqual(round(verbTunedPars["bestRes"],2), round(0.837619047619,2))
miscUtilities.removeDir(runPath)
def testCvSVM(self):
"""
Tests changing the default range of the optimizer.
"""
# Classification accuracy:
ExpectedCA = [0.6] # Ver 0.3
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter, "optimize", True)
# Change the range
pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1))
pars.setParameter("priors", "default", {"POS": 2, "NEG": 4})
trainFile = self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
useStd = False,\
runPath = runPath,\
verbose = 0)
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:", verbTunedPars["ResIdx"]
print "Optimizer runPath:", runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized, True)
# Check if the MPI version was not used
self.assertEqual(optimizer.usedMPI, False)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(
round(verbTunedPars["bestRes"],
2) in [round(x, 2) for x in ExpectedCA],
"Got:" + str(verbTunedPars["bestRes"]))
self.assert_(
len(
dataUtilities.DataTable(
os.path.join(runPath, "optimizationLog.txt"))) >=
12) # (orig: 14) Must be > 2
#Check Priors
self.assertEqual(
dataUtilities.DataTable(
os.path.join(runPath,
"optimizationLog.txt"))[1]["priors"].value,
"{'NEG':4,'POS':2}")
self.assertEqual(tunedPars[1]["priors"],
"{'NEG':4,'POS':2}") # Ver 0.3
#Set the priors since it could be choosing the first row as the best, which would be the default values, without the priors
learner.priors = {"POS": 2, "NEG": 4}
classifier = learner(self.discTest)
classifier.write(os.path.join(runPath, "CvSVMModel"))
file = open(os.path.join(runPath, "CvSVMModel/model.svm"), "r")
lines = file.readlines()
file.close()
priors = [
round(x, 2) for x in eval((lines[18].strip()).replace("data:", ""))
]
self.assertEqual(len(priors), 2)
self.assertEqual(
priors[self.discTest.domain.classVar.values.index("POS")],
2.0 * float(tunedPars[1]["C"]))
self.assertEqual(
priors[self.discTest.domain.classVar.values.index("NEG")],
4.0 * float(tunedPars[1]["C"]))
miscUtilities.removeDir(runPath)
def testCvSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.585] #Should be the result of the default point
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C"]#, "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
pars.setParameter("priors","default",{"POS":50, "NEG":4}) # These priors are to ensure the default point will be the best!
pars.setParameter("gamma","default",0.001) # This is a bad value to ensure the default point will be the best!
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:",verbTunedPars["ResIdx"]
print "Optimizer runPath:",runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:"+str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=5)
#Check Priors
self.assertEqual( tunedPars[1]["priors"],"None")
learner.priors = {'NEG':4,'POS':2}
classifier = learner(self.discTest)
classifier.write(os.path.join(runPath,"CvSVMModel"))
file = open(os.path.join(runPath,"CvSVMModel/model.svm"),"r")
lines = file.readlines()
file.close()
priors = [round(x,2) for x in eval((lines[18].strip()).replace("data:",""))]
self.assertEqual(len(priors),2)
self.assertEqual(priors[self.discTest.domain.classVar.values.index("POS")],2.0*float(tunedPars[1]["C"]))
self.assertEqual(priors[self.discTest.domain.classVar.values.index("NEG")],4.0*float(tunedPars[1]["C"]))
miscUtilities.removeDir(runPath)