def test_RFRegression(self): """RF - Test of optimizer with continuous class data """ #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.contTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = True # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.R2" evalM = "AZutilities.evalUtilities.RMSE" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API("RFLearner") # Set the parameters in parameterList to be optimized pars.setParameter("NumThreads", "optimize", False) # Change the default pars.setParameter("NumThreads", "default", "1") # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest") # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useStd = False,\ useParameters = pars.getParametersDict(),\ verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" self.assertEqual(opt.usedMPI, False) self.assertEqual(learner.optimized, True) self.assertEqual(round(tunedPars[0], 2), round(3.1499999999999999, 2)) #The learner is now with its optimized parameters already set, so we can now make a classifier out of it classifier = learner(self.contTrain) RMSE = evalUtilities.getRMSE(self.contTest, classifier) self.assertEqual(round(RMSE, 2), round(2.02, 2)) #Ver 0.3 #Check if the best result was not the one with numThreads different of 1 since that way we can get #different results among runs self.assertEqual(int(tunedPars[1]["NumThreads"]), 1) miscUtilities.removeDir(runPath)
def test_RF_Classification(self): """PLS - Test of optimizer with discrete class data """ expectedAcc = [ 0.57999999999999996, 0.58999999999999997, 0.612 ] #Ver 0.3 - Artifact: The second value can be expected on other Systems #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.discTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = False # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.CA" evalM = "AZutilities.evalUtilities.CA" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest") # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useStd = False,\ verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" self.assertEqual(opt.usedMPI, False) self.assertEqual(learner.optimized, True) self.assert_( round(tunedPars[0], 2) in [round(x, 2) for x in expectedAcc]) #Ver 0.3 #The learner is now with its optimized parameters already set, so we can now make a classifier out of it classifier = learner(self.discTrain) CA = evalUtilities.getClassificationAccuracy(self.discTest, classifier) expectedCA = [0.9655 ] # Artifact: Second value expected in UBUNTU 10.10 self.assert_(round(CA, 2) in [round(ca, 2) for ca in expectedCA]) # Ver 0.3 miscUtilities.removeDir(runPath)
def test_RF_Regression(self): """RF - Test of optimizer with continuous class data """ expectedRes = [ 3.27, 3.2599999999999998, 3.15 ] #Ver 0.3 - Artifact: The second value can be expected on other Systems #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization dataSet = self.contTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = True # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.R2" evalM = "AZutilities.evalUtilities.RMSE" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest") # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useStd = False,\ verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" self.assertEqual(opt.usedMPI, False) self.assertEqual(learner.optimized, True) self.assert_( round(tunedPars[0], 2) in [round(x, 2) for x in expectedRes]) #Ver 0.3 #The learner is now with its optimized parameters already set, so we can now make a classifier out of it classifier = learner(self.contTrain) RMSE = evalUtilities.getRMSE(self.contTest, classifier) expectedRes = [2.89, 2.0158] self.assert_(round(RMSE, 2) in [round(x, 2) for x in expectedRes]) #Ver 0.3 miscUtilities.removeDir(runPath)
def __init__(self, parent=None, signalManager=None, name='ParamOptimizer'): OWWidget.__init__(self, parent, signalManager, name, 1) # Define the input and output channels self.inputs = [("Classified Examples", ExampleTable, self.setData), ("Learner", orange.Learner, self.setLearner)] self.outputs = [("Learner - Tuned", orange.Learner), ("Examples - Optimization Steps", ExampleTable)] self.requiredParamVer = 12 self.name = name self.dataset = None self.learner = None self.optimizer = paramOptUtilities.Appspack() self.verbose = 0 self.tunedPars = None self.intRes = None self.paramsNames = [ "Name", "Optimize", "Lower Limit", "Upper Limit", "Distribution", "Step", "Default", "Actual Learner parameter" ] self.learnerType = None self.parameters = None self.nParameters = 0 self.nFolds = 5 self.SMethod = 1 self.execEnv = 0 # Serial self.CMethod = 0 self.RMethod = 0 self.GUIparams = {} self.OptimizeChBox = {} self.DistCombo = {} self.DefaultCombo = {} self.UseGridSearch = False self.nInnerPoints = 5 #Define Evaluation Methods: ["LabelName", "EvaluateFunction", "True=Best is the Min | False=Best is the Max"] self.CMethods = [("CA", "AZutilities.evalUtilities.CA", False)] self.RMethods = [("RMSE", "AZutilities.evalUtilities.RMSE", True)] # ("R^2", "AZutilities.evalUtilities.Rsqrt", False)] self.SMethods = [("Leave-One-Out", 0), ("Cross Validation", 1)] self.execEnvs = AZOC.OWParamOptExecEnvs self.defineGUI()
def test_PLS_MPI_2(self): ################################################################### # Test other way of setting appspack ###################################################################i # Classification accuracy: ExpectedCA = [0.567049808429, 0.593869731801] ExpectedCAwithTest = [0.6, 0.579] #New at orange2.0 #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngPLS.PLSLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.discTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = False # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.CA" evalM = "AZutilities.evalUtilities.CA" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_PLS_MPI_2") # Load the optimization parameters from the default configuration (AZLearnersParamsConfig.py) parameters = AZLearnersParamsConfig.API("PLSLearner") parameters.setParameter("method", "default", 'pls1') # change the optimization parameters parameters.setParameter( "method", "default", 'pls1') # make the method fixed (do not optimize) to be pls1 parameters.setParameter("method", "optimize", False) parameters.setParameter( "method", "rangeType", "values") # assure that the keyword for the values range type is #set correctly for values instead of interval parameters.setParameter( "k", "range", [1, 3, 5, 6, 10 ]) # make the method fixed (do not optimize) to be pls1 parameters.setParameter("k", "optimize", True) parameters.setParameter( "k", "rangeType", "values") # assure that the keyword for the values range type is # set correctly for values instead of interval #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useParameters = parameters.getParametersDict(),\ verbose = 0, useStd = False,\ advancedMPIoptions = None, np = 4, machinefile = ["localhost:2","localhost:2"]) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" self.assertEqual(learner.optimized, True) # Check if the MPI version was used self.assertEqual(opt.usedMPI, True) self.assert_( round(tunedPars[0], 3) in [round(x, 3) for x in ExpectedCAwithTest], "Got:" + str(tunedPars[0])) #The learner is now with its optimized parameters already set, so we can now make a classifier out of it classifier = learner(self.discTrain) CA = evalUtilities.getClassificationAccuracy(self.discTest, classifier) self.assert_( round(CA, 3) in [round(x, 3) for x in ExpectedCA], "Got: " + str(CA)) resData2 = dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt")) self.assert_(len(resData2) >= 4) # (orig 5) Must be > 2 #print runPath miscUtilities.removeDir(runPath)
def disable_testSVM_MPI_3(self): ################################################################### # Test other way of setting appspack ################################################################### # Classification accuracy: ExpectedCA = 0.6 optimizer = paramOptUtilities.Appspack() learner = AZorngCvSVM.CvSVMLearner() learnerName = "CvSVMLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["C", "gamma"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) # Change the range pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1)) trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI_3") evalM = "AZutilities.evalUtilities.CA" fMin = False #[<minimum of objective function found>, <optimized parameters>] tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useParameters = pars.getParametersDict(),\ verbose = 0,\ useStd = False,\ advancedMPIoptions = "-v -np 4",\ machinefile = ["localhost:2","localhost:2"]) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" # Check if the MPI version was used self.assertEqual(learner.optimized, True) # Check if the MPI version was used self.assertEqual(optimizer.usedMPI, True) self.assertEqual(round(tunedPars[0], 3), round(ExpectedCA, 3)) self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 12) # (orig 14) Must be > 2 #print runPath miscUtilities.removeDir(runPath)
def testRF_MPI(self): """ Tests changing the default range of the optimizer. Use MPI versio0n of appspack """ # Classification accuracy: ExpectedCA = [0.612] #opencv1.1: 0.90480000000000005 optimizer = paramOptUtilities.Appspack() learner = AZorngRF.RFLearner() learnerName = "RFLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["nActVars"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) # Set the NumThreads pars.setParameter("NumThreads", "optimize", False) # Change the default pars.setParameter("NumThreads", "default", "1") trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_RF_MPI") evalM = "AZutilities.evalUtilities.CA" fMin = False # Calculate the optimal parameters. This can take a long period of time! tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ useDefaultPoint = False,\ findMin=fMin,\ runPath = runPath,\ useStd = False,\ verbose = 0,\ #advancedMPIoptions = "-all-local -allcpus") # to use this the # file "<MPICHDIR>/share/machines.LINUX must be properly configured" # Alternatively, we can set machinefile=0 to us also all available cores machinefile =0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" print "Number of cores used: ", optimizer.np verbTunedPars = optimizer.getTunedParameters() # Check that the learner was optimized self.assertEqual(learner.optimized, True) #Check if the number of processors used are all the core available notUsed, out = commands.getstatusoutput( "cat /proc/cpuinfo | grep processor") self.assertEqual(optimizer.np, len(out.split("\n"))) # Check if the MPI version was used self.assertEqual(optimizer.usedMPI, True) # Check the number of optimized parameters self.assert_(len(verbTunedPars["optParam"]) in [8, 9, 10]) # Check the accuracy self.assert_( round(verbTunedPars["bestRes"], 3) in [round(x, 3) for x in ExpectedCA], "Got:" + str(verbTunedPars["bestRes"])) self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 3) # Must be > 2 miscUtilities.removeDir(runPath)
def disable_testSVM_MPI(self): """ Tests changing the default range of the optimizer. Use MPI versio0n of appspack """ # Classification accuracy: ExpectedCA = [0.6] #New at orange2.0 optimizer = paramOptUtilities.Appspack() learner = AZorngCvSVM.CvSVMLearner() learnerName = "CvSVMLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["C", "gamma"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) # Change the range pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1)) trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI") evalM = "AZutilities.evalUtilities.CA" fMin = False # Calculate the optimal parameters. This can take a long period of time! tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ findMin=fMin,\ runPath = runPath,\ verbose = 0,\ useStd = False,\ #advancedMPIoptions = "-all-local -allcpus") # to use this the # file "<MPICHDIR>/share/machines.LINUX must be properly configured" np = 4,\ machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines"))) verbTunedPars = optimizer.getTunedParameters() print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" # Check that the learner was optimized self.assertEqual(learner.optimized, True) # Check if the MPI version was used self.assertEqual(optimizer.usedMPI, True) # Check the number of optimized parameters self.assertEqual(len(verbTunedPars["optParam"]), 12) # Check the accuracy self.assert_( round(verbTunedPars["bestRes"], 3) in [round(x, 3) for x in ExpectedCA], "Got:" + str(verbTunedPars["bestRes"])) self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 12) # (orig: 14) Must be > 2 miscUtilities.removeDir(runPath)
def testCvSVM_MPI(self): """ Tests changing the default range of the optimizer. Use MPI versio0n of appspack """ # Classification accuracy: ExpectedCA = [0.585] #Should be the result of the default point optimizer = paramOptUtilities.Appspack() learner = AZorngCvSVM.CvSVMLearner() learnerName = "CvSVMLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["C"] #, "gamma"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) # Change the range pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1)) pars.setParameter("priors", "default", { "POS": 50, "NEG": 4 }) # These priors are to ensure the default point will be the best! pars.setParameter( "gamma", "default", 0.001 ) # This is a bad value to ensure the default point will be the best! trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM_MPI") evalM = "AZutilities.evalUtilities.CA" fMin = False # Calculate the optimal parameters. This can take a long period of time! tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ findMin=fMin,\ runPath = runPath,\ verbose = 0,\ useStd = False,\ #advancedMPIoptions = "-all-local -allcpus") # to use this the # file "<MPICHDIR>/share/machines.LINUX must be properly configured" np = 4,\ machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines"))) verbTunedPars = optimizer.getTunedParameters() print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "Best result index from intRes file:", verbTunedPars["ResIdx"] print "Optimizer runPath:", runPath print "check the file intRes.txt to see the intermediate results of optimizer!" # Check that the learner was optimized self.assertEqual(learner.optimized, True) # Check if the MPI version was used self.assertEqual(optimizer.usedMPI, True) # Check the number of optimized parameters self.assertEqual(len(verbTunedPars["optParam"]), 12) # Check the accuracy self.assert_( round(verbTunedPars["bestRes"], 3) in [round(x, 3) for x in ExpectedCA], "Got:" + str(verbTunedPars["bestRes"])) self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 5) #Check Priors self.assertEqual(tunedPars[1]["priors"], "None") learner.priors = {'NEG': 4, 'POS': 2} classifier = learner(self.discTest) classifier.write(os.path.join(runPath, "CvSVMModel")) file = open(os.path.join(runPath, "CvSVMModel/model.svm"), "r") lines = file.readlines() file.close() priors = [ round(x, 2) for x in eval((lines[18].strip()).replace("data:", "")) ] self.assertEqual(len(priors), 2) self.assertEqual( priors[self.discTest.domain.classVar.values.index("POS")], 2.0 * float(tunedPars[1]["C"])) self.assertEqual( priors[self.discTest.domain.classVar.values.index("NEG")], 4.0 * float(tunedPars[1]["C"])) miscUtilities.removeDir(runPath)
def testCvSVM(self): """ Tests changing the default range of the optimizer. """ # Classification accuracy: ExpectedCA = [0.6] # Ver 0.3 optimizer = paramOptUtilities.Appspack() learner = AZorngCvSVM.CvSVMLearner() learnerName = "CvSVMLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["C", "gamma"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) # Change the range pars.setParameter("C", "range", miscUtilities.power2Range(-5, 2, 1)) pars.setParameter("priors", "default", {"POS": 2, "NEG": 4}) trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM") evalM = "AZutilities.evalUtilities.CA" fMin = False # Calculate the optimal parameters. This can take a long period of time! tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ findMin=fMin,\ useStd = False,\ runPath = runPath,\ verbose = 0) verbTunedPars = optimizer.getTunedParameters() print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "Best result index from intRes file:", verbTunedPars["ResIdx"] print "Optimizer runPath:", runPath print "check the file intRes.txt to see the intermediate results of optimizer!" # Check that the learner was optimized self.assertEqual(learner.optimized, True) # Check if the MPI version was not used self.assertEqual(optimizer.usedMPI, False) # Check the number of optimized parameters self.assertEqual(len(verbTunedPars["optParam"]), 12) # Check the accuracy self.assert_( round(verbTunedPars["bestRes"], 2) in [round(x, 2) for x in ExpectedCA], "Got:" + str(verbTunedPars["bestRes"])) self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 12) # (orig: 14) Must be > 2 #Check Priors self.assertEqual( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))[1]["priors"].value, "{'NEG':4,'POS':2}") self.assertEqual(tunedPars[1]["priors"], "{'NEG':4,'POS':2}") # Ver 0.3 #Set the priors since it could be choosing the first row as the best, which would be the default values, without the priors learner.priors = {"POS": 2, "NEG": 4} classifier = learner(self.discTest) classifier.write(os.path.join(runPath, "CvSVMModel")) file = open(os.path.join(runPath, "CvSVMModel/model.svm"), "r") lines = file.readlines() file.close() priors = [ round(x, 2) for x in eval((lines[18].strip()).replace("data:", "")) ] self.assertEqual(len(priors), 2) self.assertEqual( priors[self.discTest.domain.classVar.values.index("POS")], 2.0 * float(tunedPars[1]["C"])) self.assertEqual( priors[self.discTest.domain.classVar.values.index("NEG")], 4.0 * float(tunedPars[1]["C"])) miscUtilities.removeDir(runPath)
def test_GridSearch(self): """ Test GridSearch Module """ #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.discTestDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = False # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.CA" evalM = "AZutilities.evalUtilities.CA" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="RFTest") # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API("RFLearner") # Set the parameters in parameterList to be optimized pars.setParameter("NumThreads", "optimize", False) # Change the default pars.setParameter("NumThreads", "default", "1") # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useGridSearchFirst = True,\ gridSearchInnerPoints = 3,\ useDefaultPoint = False,\ useStd = False,\ useParameters = pars.getParametersDict(),\ verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" print "CheckSum:", round(sum(opt.GSRes["results"]), 2) print "Number of results: ", len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) print "Running Path:", runPath # Check that the learner was optimized self.assertEqual(learner.optimized, True) self.log.info("") self.log.info("tunedPars[0]=" + str(tunedPars[0])) # Check the accuracy self.assertEqual(round(tunedPars[0], 2), round(0.621, 2)) # Ver 0.3 #Check if the number of results remain equal self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 5) #Check that all points were evaluated self.assert_(opt.GSRes["nFailedPoints"] == 0) self.assert_(opt.GSRes["nPoints"] == 3) #CheckSum to assure results are the same expectedValues = [ -1.78, # Ver 0.3 -1.79 ] acctualValue = sum(opt.GSRes["results"]) self.assertRoundedToExpectedArray(acctualValue, expectedValues, 2) #Check if the best result was not the one with numThreads different of 1 since that way we can get #different results among runs self.assertEqual(int(tunedPars[1]["NumThreads"]), 1) miscUtilities.removeDir(runPath)
def testCvANN(self): """ Tests changing the default range of the optimizer. """ # Classification accuracy: ExpectedCA = [0.585] #Ver 0.3 optimizer = paramOptUtilities.Appspack() learner = AZorngCvANN.CvANNLearner() learnerName = "CvANNLearner" # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API(learnerName) # Set all parameters to not be optimized pars.setOptimizeAllParameters(False) parameterList = ["maxIter", "nHidden"] # Set the parameters in parameterList to be optimized for parameter in parameterList: pars.setParameter(parameter, "optimize", True) trainFile = self.discTrainDataPath # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvANN") evalM = "AZutilities.evalUtilities.CA" fMin = False # Calculate the optimal parameters. This can take a long period of time! tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ findMin=fMin,\ useStd = False,\ runPath = runPath,\ verbose = 0) verbTunedPars = optimizer.getTunedParameters() print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "Best result index from intRes file:", verbTunedPars["ResIdx"] print "Optimizer runPath:", runPath print "check the file intRes.txt to see the intermediate results of optimizer!" # Check that the learner was optimized self.assertEqual(learner.optimized, True) # Check if the MPI version was not used self.assertEqual(optimizer.usedMPI, False) # Check the number of optimized parameters self.assertEqual(len(verbTunedPars["optParam"]), 14) # Check the accuracy nOptPoints = len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) self.assert_(nOptPoints > 5, "N. of optimization points:" + str(nOptPoints)) # Must be > 2 self.assert_( round(verbTunedPars["bestRes"], 3) in [round(x, 3) for x in ExpectedCA], "Actual result:" + str(verbTunedPars["bestRes"])) miscUtilities.removeDir(runPath)
def test_RFClassification(self): """RF - Test of optimizer with discrete class data """ #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.discTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = False # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.CA" evalM = "AZutilities.evalUtilities.CA" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="RFTest") # Create an interface for setting optimizer parameters pars = AZLearnersParamsConfig.API("RFLearner") # Set the parameters in parameterList to be optimized pars.setParameter("NumThreads", "optimize", False) # Change the default pars.setParameter("NumThreads", "default", "1") # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useDefaultPoint = False,\ useStd = False,\ useParameters = pars.getParametersDict(),\ verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" print "Number of optimization steps: ", len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) print "Number of Threads used: ", learner.NumThreads #The learner is now with its optimized parameters already set, so we can now make a classifier out of it learner.NumThreads = 1 classifier = learner(self.discTrain) CA = evalUtilities.getClassificationAccuracy(self.discTest, classifier) print "CA of optimized Learner: ", CA self.assertEqual(opt.usedMPI, False) self.log.info("") self.log.info("tunedPars[0]=" + str(tunedPars[0])) self.assertEqual(learner.optimized, True) self.assertEqual(round(tunedPars[0], 2), round(0.61, 2)) # Ver 0.3:390 self.log.info("CA=" + str(CA)) self.assertEqual(round(CA, 2), round(0.965517241379, 2)) #Ver 0.3 #Check if the best result was not the one with numThreads different of 1 since that way we can get #different results among runs self.assertEqual(int(tunedPars[1]["NumThreads"]), 1) miscUtilities.removeDir(runPath)
def test_RFAdvanced_Usage(self): """PLS - Test of optimizer with advanced configuration """ #Create the appspack instance opt = paramOptUtilities.Appspack() #Learner to be optimized learner = AZorngRF.RFLearner() #dataset to use in the parameters optimization (Discrete class in this example) dataSet = self.discTrainDataPath # Define the objective function. This requires: # defining the extreme to find (the min or max): findMin=True or findMin=False fMin = False # defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults): # evaluateMethod="AZutilities.evalUtilities.CA" evalM = "AZutilities.evalUtilities.CA" # Create a directory for running the appspack (if not defined it will use the present working directory) runPath = miscUtilities.createScratchDir(desc="ParamOptTest") # Load the optimization parameters from the default configuration (AZLearnersParamsConfig.py) parameters = AZLearnersParamsConfig.API("RFLearner") parameters.setParameter("method", "default", 'rf1') # change the optimization parameters parameters.setParameter( "method", "default", 'rf1') # make the method fixed (do not optimize) to be pls1 parameters.setParameter("method", "optimize", False) parameters.setParameter( "method", "rangeType", "values") # assure that the keyword for the values range type is #set correctly for values instead of interval parameters.setParameter( "k", "range", [1, 3, 5, 6, 10 ]) # make the method fixed (do not optimize) to be pls1 parameters.setParameter("k", "optimize", True) parameters.setParameter( "k", "rangeType", "values") # assure that the keyword for the values range type is #set correctly for values instead of interval # Run the appspack which will configure the input learner and aditionaly return #[<minimum of objective function found>, <optimized parameters>] tunedPars = opt(learner=learner,\ dataSet=dataSet,\ evaluateMethod = evalM,\ findMin=fMin,\ runPath = runPath,\ useStd = False,\ useParameters = parameters.getParametersDict(),\ # The 'useParameters' is mandatory, even placing a file with the new configurations in the # running directory, that we pass to the optimizer the correct parameters to use. # The parameters placed on the running directory are for appspack usage, and the # optimizer needs to know what parameters appspack will use, otherwise, it will # load the default ones verbose = 0) print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized print "Tuned parameters = ", tunedPars[1] print "Best optimization result = ", tunedPars[0] print "check the file intRes.txt to see the intermediate results of optimizer!" self.assertEqual(opt.usedMPI, False) self.assertEqual(learner.optimized, True) self.assertEqual(round(tunedPars[0], 2), round(0.61, 2)) #Ver 0.3 #The learner is now with its optimized parameters already set, so we can now make a classifier out of it classifier = learner(self.discTrain) CA = evalUtilities.getClassificationAccuracy(self.discTest, classifier) self.assertEqual(round(CA, 2), round(0.97, 2)) #Ver 0.3 self.assert_( len( dataUtilities.DataTable( os.path.join(runPath, "optimizationLog.txt"))) >= 5) # Must be > 2 miscUtilities.removeDir(runPath)
] data = dataUtilities.attributeDeselectionData(data, descList) # Deselect descriptors with no variance descList = ["rdk.fr_dihydropyridine", "rdk.fr_nitroso", "rdk.fr_benzodiazepine", "rdk.fr_thiocyan", "rdk.VSA_EState4" ,"rdk.VSA_EState6" \ ,"rdk.VSA_EState7" ,"rdk.VSA_EState1" ,"rdk.VSA_EState2" ,"rdk.VSA_EState3" ,"rdk.SlogP_VSA9" ,"rdk.SMR_VSA8" ,"rdk.fr_diazo" \ ,"rdk.fr_prisulfonamd" ,"rdk.fr_isocyan" ,"rdk.fr_azide" ,"rdk.fr_isothiocyan"] data = dataUtilities.attributeDeselectionData(data, descList) print "Length domain ", len(data.domain) print "Running param opt" data.save(trainFile) learner = AZorngCvSVM.CvSVMLearner() evalM = "AZutilities.evalUtilities.CA" fMin = False optimizer = paramOptUtilities.Appspack() learnerName = "CvSVMLearner" runPath = "/ChemistryData/jgw/projects/OI/Assay1350/ParamOpt" pars = AZLearnersParamsConfig.API(learnerName) tunedPars = optimizer(learner=learner,\ dataSet=trainFile,\ evaluateMethod = evalM,\ useParameters = pars.getParametersDict(),\ findMin=fMin,\ useStd = False,\ runPath = runPath,\ verbose = 0) verbTunedPars = optimizer.getTunedParameters() print "Returned: ", tunedPars print "====================== optimization Done ===========================" print "Learner optimized flag = ", learner.optimized