def test_SavedModel(self): """Test to assure that a saved Bayes model gives the same predictions as before saving.""" # Create an Bayes model Bayes = AZorngCvBayes.CvBayesLearner(self.train_data) # Calculate classification accuracy Acc = evalUtilities.getClassificationAccuracy(self.test_data, Bayes) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdir" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "Bayes.fBayes") Bayes.write(modelPath) # Read in the model Bayes = AZorngCvBayes.CvBayesread(modelPath) # Calculate classification accuracy savedAcc = evalUtilities.getClassificationAccuracy( self.test_data, Bayes) # Test that the accuracy of the two classifiers is the exact same self.assertEqual(Acc, savedAcc) #Test using the global read functionality Bayes2 = AZBaseClasses.modelRead(modelPath) savedAcc2 = evalUtilities.getClassificationAccuracy( self.test_data, Bayes2) self.assertEqual(Acc, savedAcc2) # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def test_SavedModel(self): """Test to assure that a saved Bayes model gives the same predictions as before saving.""" # Create an Bayes model Bayes = AZorngCvBayes.CvBayesLearner(self.train_data) # Calculate classification accuracy Acc = evalUtilities.getClassificationAccuracy(self.test_data, Bayes) # Save the model scratchdir = os.path.join(AZOC.SCRATCHDIR, "scratchdir" + str(time.time())) os.mkdir(scratchdir) modelPath = os.path.join(scratchdir, "Bayes.fBayes") Bayes.write(modelPath) # Read in the model Bayes = AZorngCvBayes.CvBayesread(modelPath) # Calculate classification accuracy savedAcc = evalUtilities.getClassificationAccuracy(self.test_data, Bayes) # Test that the accuracy of the two classifiers is the exact same self.assertEqual(Acc, savedAcc) # Test using the global read functionality Bayes2 = AZBaseClasses.modelRead(modelPath) savedAcc2 = evalUtilities.getClassificationAccuracy(self.test_data, Bayes2) self.assertEqual(Acc, savedAcc2) # Remove the scratch directory os.system("/bin/rm -rf " + scratchdir)
def loadModel(self): self.modelFile = str(self.modelFile) self.classifier = AZBaseClasses.modelRead(self.modelFile) self.modelType = AZBaseClasses.modelRead(self.modelFile, retrunClassifier = False) if not self.classifier: QMessageBox.information(None, "Invalid Model", "There is no valid model in the specifyed location,\nor the model is not compatible with the accepted ones.\nPlease choose the correct model path.\nThis widget can only load CvSVM, CvRF, CvANN, PLS and Consensus models.", QMessageBox.Ok + QMessageBox.Default) self.info.setText("No model Loaded") self.send("Classifier", None) return # Model self.classifier.name = self.modelType + " - " + self.name print self.modelType + " model loaded from " + self.modelFile self.info.setText("Loaded model: "+self.modelType) self.send("Classifier", self.classifier)
def loadModel(self): self.modelFile = str(self.modelFile) self.classifier = AZBaseClasses.modelRead(self.modelFile) self.modelType = AZBaseClasses.modelRead(self.modelFile, retrunClassifier = False) if not self.classifier: QMessageBox.information(None, "Invalid Model", "There is no valid model in the specifyed location,\nor the model is not compatible with the accepted ones.\nPlease choose the correct model path.\nThis widget can only load " + string.join(AZBaseClasses.modelRead()[:-1],", ") + " and " + AZBaseClasses.modelRead()[-1] + " models.", QMessageBox.Ok + QMessageBox.Default) self.info.setText("No model Loaded") self.send("Classifier", None) return # Model self.classifier.name = self.modelType + " - " + self.name print self.modelType + " model loaded from " + self.modelFile self.info.setText("Loaded model: "+self.modelType) self.send("Classifier", self.classifier)
def __init__(self, modelPath): self.model = None self.mountPoint = None self.modelLocation = modelPath self.preDefSignatureFile = self.getDataFile(modelPath) self.model = AZBaseClasses.modelRead(self.modelLocation) if not self.model: print "ERROR: Cannot load model ", modelPath return None
def __init__(self, modelPath): self.model = None self.mountPoint = None self.modelLocation = modelPath self.preDefSignatureFile = self.getDataFile(modelPath) self.model = AZBaseClasses.modelRead(self.modelLocation) if not self.model: print "ERROR: Cannot load model ",modelPath return None
def getStatistics( dataset, runningDir, resultsFile, mlList=[ml for ml in MLMETHODS if AZOC.MLMETHODS[ml]["useByDefault"]], queueType="NoSGE", verbose=0, getAllModels=False, callBack=None): """ runningDir (An existing dir for creating one job dir per fold) | +---- status (The overall status: "started", "finished" or the progress "1/10", "2/10", ...) | +---- fold_1 | +---- fold_2 | . . . The running will be monitorized by this method. Whenever a MLMethod fails the respective fold job is restarted """ if dataset.domain.classVar.varType == orange.VarTypes.Discrete: responseType = "Classification" else: responseType = "Regression" #Create the Train and test sets DataIdxs = dataUtilities.SeedDataSampler(dataset, AZOC.QSARNEXTFOLDS) #Check data in advance so that, by chance, it will not faill at the last fold! #for foldN in range(AZOC.QSARNEXTFOLDS): #trainData = dataset.select(DataIdxs,foldN,negate=1) #checkTrainData(trainData) jobs = {} thisDir = os.getcwd() os.chdir(runningDir) #PID = os.getpid() #print "Started getStatistics in Process with PID: "+str(PID) #os.system('echo "'+str(PID)+'" > '+os.path.join(runningDir,"PID")) os.system('echo "started" > ' + os.path.join(runningDir, "status")) # Start all Fold jobs stepsDone = 0 nTotalSteps = AZOC.QSARNEXTFOLDS for fold in range(AZOC.QSARNEXTFOLDS): job = str(fold) print "Starting job for fold ", job trainData = dataset.select(DataIdxs, fold, negate=1) jobs[job] = { "job": job, "path": os.path.join(runningDir, "fold_" + job), "running": False, "failed": False, "finished": False } # Uncomment next 3 lines for running in finished jobs dirs #st, jID = commands.getstatusoutput("cat "+os.path.join(runningDir, "fold_"+job,"jID")) #jobs[job]["jID"] = jID #continue os.system("rm -rf " + jobs[job]["path"]) os.system("mkdir -p " + jobs[job]["path"]) trainData.save(os.path.join(jobs[job]["path"], "trainData.tab")) file_h = open(os.path.join(jobs[job]["path"], "run.sh"), "w") file_h.write("#!/bin/tcsh\n") file_h.write( "source " + os.path.join(os.environ["AZORANGEHOME"], "templateProfile") + "\n") file_h.write("python " + os.path.join(jobs[job]["path"], "QsubScript.py") + "\n") file_h.close() file_h = open(os.path.join(jobs[job]["path"], "QsubScript.py"), "w") file_h.write("import os\n") file_h.write("from AZutilities import dataUtilities\n") file_h.write("from AZutilities import competitiveWorkflow\n") file_h.write("data = dataUtilities.DataTable('" + os.path.join(jobs[job]["path"], "trainData.tab") + "')\n") file_h.write('os.system(\'echo "running" > ' + os.path.join(jobs[job]["path"], "status") + ' \')\n') file_h.write("models = competitiveWorkflow.getModel(data, mlList=" + str(mlList) + ", savePath = '" + os.path.join(jobs[job]["path"], "results.pkl") + "', queueType = '" + queueType + "', getAllModels = " + str(getAllModels) + ")\n") file_h.write("nModelsSaved = 0\n") file_h.write("for model in models:\n") file_h.write(" if not models[model] is None:\n") file_h.write(" models[model].write('" + os.path.join(jobs[job]["path"], "model") + "'+'_'+model)\n") file_h.write(' nModelsSaved += 1\n') file_h.write( 'if nModelsSaved == len([m for m in models if not models[m] is None ]):\n' ) file_h.write(' os.system(\'echo "finished" > ' + os.path.join(jobs[job]["path"], "status") + ' \')\n') file_h.write('else:\n') file_h.write(' os.system(\'echo "failed" > ' + os.path.join(jobs[job]["path"], "status") + ' \')\n') file_h.close() os.chdir(os.path.join(jobs[job]["path"])) if queueType == "NoSGE": # Serial mode status, out = commands.getstatusoutput( "tcsh " + os.path.join(jobs[job]["path"], "run.sh")) if status: print "ERROR on Job " + str( job) + " (will be restarted latter)" print out else: statusFile = os.path.join(jobs[job]["path"], "status") if os.path.isfile(statusFile): st, status = commands.getstatusoutput("cat " + statusFile) else: print "ERROR: Missing status file" status = None if not status: print "ERROR! job " + job + " has no status!" jobs[job]["failed"] = True elif status == "failed": print "Job " + job + " failed to build all models" jobs[job]["failed"] = True elif status == "finished": jobs[job]["finished"] = True if not isJobProgressingOK(jobs[job]): print "Job " + job + " failed to build one or more models in getMLStatistics" jobs[job]["failed"] = True jobs[job]["finished"] = False if jobs[job]["failed"]: print "Job " + job + " FAILED" else: print "Finished Job " + str(job) + " with success" if callBack: stepsDone += 1 if not callBack((100 * stepsDone) / nTotalSteps): return None else: cmd = "qsub -cwd -q batch.q" + AZOC.SGE_QSUB_ARCH_OPTION_CURRENT + os.path.join( jobs[job]["path"], "run.sh") status, out = commands.getstatusoutput(cmd) if status: print "ERROR on Job " + str(job) + " (will be skipped)" print out #raise Exception("ERROR starting job for folder "+str(job)) # Your job 955801 ("template_run.sh") has been submitted jID = out.strip().split(" ")[2] print " jID: ", jID os.system('echo "' + jID + '" > ' + os.path.join(jobs[job]["path"], "jID")) jobs[job]["running"] = True jobs[job]["jID"] = jID os.chdir(runningDir) os.chdir(thisDir) finished = [] if queueType == "NoSGE": failed = [] #Report failed Jobs for job in jobs: if jobs[job]["finished"]: finished.append(job) for job in jobs: if jobs[job]["failed"]: failed.append(job) print "Successful finished Jobs: ", finished print "Failed Jobs: ", failed else: # Monitor SGE jobs untill all are finished #Monitor Fold jobs updateJobsStatus(jobs) for job in jobs: if jobs[job]["finished"]: finished.append(job) print "Jobs already finished: ", finished os.system(' echo "' + str(len(finished)) + '/' + str(AZOC.QSARNEXTFOLDS) + '" > ' + os.path.join(runningDir, "status")) while len(finished) < AZOC.QSARNEXTFOLDS: print ".", sys.stdout.flush() updateJobsStatus(jobs) for job in jobs: if jobs[job]["finished"] and job not in finished: finished.append(job) if callBack: stepsDone += 1 if not callBack((100 * stepsDone) / nTotalSteps): return None print time.asctime() + ": Finished job " + str(job) os.system(' echo "' + str(len(finished)) + '/' + str(AZOC.QSARNEXTFOLDS) + '" > ' + os.path.join(runningDir, "status")) for job in [j for j in jobs if jobs[j]["failed"]]: jobs[job] = restartJob(jobs[job]) time.sleep(5) print "All fold jobs finished!" # Gather the results print "Gathering results..." #Var for saving each Fols result results = {} exp_pred = {} nTrainEx = {} nTestEx = {} # Var for saving the statistics results statistics = {} mlMethods = [ml for ml in AZOC.MLMETHODS] + ["Consensus"] sortedJobs = [job for job in jobs] sortedJobs.sort(cmp=lambda x, y: int(x) > int(y) and 1 or -1) # Place for storing the selected models results results["selectedML"] = [] exp_pred["selectedML"] = [] nTrainEx["selectedML"] = [] nTestEx["selectedML"] = [] foldSelectedML = [] for ml in mlMethods: # Loop over each MLMethod try: #Var for saving each Fols result results[ml] = [] exp_pred[ml] = [] nTrainEx[ml] = [] nTestEx[ml] = [] logTxt = "" for job in sortedJobs: #loop over each fold modelPath = os.path.join(jobs[job]["path"], "model_" + ml) if not os.path.isdir(modelPath): if getAllModels: print "MLMethod " + ml + " not available in fold " + job continue resFile = os.path.join(jobs[job]["path"], "results.pkl") statFile_h = open(resFile) foldStat = pickle.load(statFile_h) statFile_h.close() #load model model = AZBaseClasses.modelRead(modelPath) #Test the model testData = dataset.select(DataIdxs, int(job)) nTrainEx[ml].append(model.NTrainEx) nTestEx[ml].append(len(testData)) if foldStat[ml]["selected"]: foldSelectedML.append(ml) nTrainEx["selectedML"].append(model.NTrainEx) nTestEx["selectedML"].append(len(testData)) if responseType == "Classification": results[ml].append( (evalUtilities.getClassificationAccuracy( testData, model), evalUtilities.getConfMat(testData, model))) if foldStat[ml]["selected"]: results["selectedML"].append(results[ml][-1]) else: local_exp_pred = [] for ex in testData: local_exp_pred.append((ex.getclass(), model(ex))) results[ml].append( (evalUtilities.calcRMSE(local_exp_pred), evalUtilities.calcRsqrt(local_exp_pred))) #Save the experimental value and correspondent predicted value exp_pred[ml] += local_exp_pred if foldStat[ml]["selected"]: results["selectedML"].append(results[ml][-1]) exp_pred["selectedML"] += local_exp_pred res = createStatObj(results[ml], exp_pred[ml], nTrainEx[ml], nTestEx[ml], responseType, len(sortedJobs), logTxt) if not res: raise Exception("No results available!") if getAllModels: statistics[ml] = copy.deepcopy(res) writeResults(statistics, resultsFile) print " OK", ml except: print "Error on MLmethod " + ml + ". It will be skipped" ml = "selectedML" res = createStatObj(results[ml], exp_pred[ml], nTrainEx[ml], nTestEx[ml], responseType, len(sortedJobs), logTxt, foldSelectedML) if not res: raise Exception("No results available!") statistics[ml] = copy.deepcopy(res) writeResults(statistics, resultsFile) os.system(' echo "finished" > ' + os.path.join(runningDir, "status")) return statistics
if __name__ == "__main__": """ Script to calculate the accuracy on a temporal test set with a saved model. Usage; python getTempAcc.py testDataPath modelPath """ # Full path to temporal test data file (with descriptors) in Orange format #testDataFile = "/home/jonna/projects/M-Lab/scfbmPaper/data/trainData.tab" testDataFile = sys.argv[1] # Read the test data testData = dataUtilities.DataTable(testDataFile) # Full path to the model #modelFile = "/home/jonna/projects/M-Lab/scfbmPaper/data/optRF.model" modelFile = sys.argv[2] # Read the model model = AZBaseClasses.modelRead(modelFile) # Use Orange methods to get the accuracy (Please see Orange doc) results = orngTest.testOnData([model], testData) print "Classification accuracy" print orngStat.CA(results)
def getStatistics(dataset, runningDir, resultsFile, queueType = "NoSGE", verbose = 0, getAllModels = False, callBack = None): """ runningDir (An existing dir for creating one job dir per fold) | +---- status (The overall status: "started", "finished" or the progress "1/10", "2/10", ...) | +---- fold_1 | +---- fold_2 | . . . The running will be monitorized by this method. Whenever a MLMethod fails the respective fold job is restarted """ if dataset.domain.classVar.varType == orange.VarTypes.Discrete: responseType = "Classification" else: responseType = "Regression" #Create the Train and test sets DataIdxs = dataUtilities.SeedDataSampler(dataset, AZOC.QSARNEXTFOLDS ) #Check data in advance so that, by chance, it will not faill at the last fold! #for foldN in range(AZOC.QSARNEXTFOLDS): #trainData = dataset.select(DataIdxs[foldN],negate=1) #checkTrainData(trainData) jobs = {} thisDir = os.getcwd() os.chdir(runningDir) #PID = os.getpid() #print "Started getStatistics in Process with PID: "+str(PID) #os.system('echo "'+str(PID)+'" > '+os.path.join(runningDir,"PID")) os.system('echo "started" > '+os.path.join(runningDir,"status")) # Start all Fold jobs stepsDone = 0 nTotalSteps = AZOC.QSARNEXTFOLDS for fold in range(AZOC.QSARNEXTFOLDS): job = str(fold) print "Starting job for fold ",job trainData = dataset.select(DataIdxs[fold],negate=1) jobs[job] = {"job":job,"path":os.path.join(runningDir, "fold_"+job), "running":False, "failed":False, "finished":False} # Uncomment next 3 lines for running in finished jobs dirs #st, jID = commands.getstatusoutput("cat "+os.path.join(runningDir, "fold_"+job,"jID")) #jobs[job]["jID"] = jID #continue os.system("rm -rf "+jobs[job]["path"]) os.system("mkdir -p "+jobs[job]["path"]) trainData.save(os.path.join(jobs[job]["path"],"trainData.tab")) file_h = open(os.path.join(jobs[job]["path"],"run.sh"),"w") file_h.write("#!/bin/tcsh\n") file_h.write("source "+os.path.join(os.environ["AZORANGEHOME"], "templateProfile") + "\n") file_h.write("python "+os.path.join(jobs[job]["path"],"QsubScript.py")+"\n") file_h.close() file_h = open(os.path.join(jobs[job]["path"],"QsubScript.py"),"w") file_h.write("import os\n") file_h.write("from AZutilities import dataUtilities\n") file_h.write("from AZutilities import competitiveWorkflow\n") file_h.write("data = dataUtilities.DataTable('"+os.path.join(jobs[job]["path"],"trainData.tab")+"')\n") file_h.write('os.system(\'echo "running" > '+os.path.join(jobs[job]["path"],"status")+' \')\n') file_h.write("models = competitiveWorkflow.getModel(data, savePath = '"+os.path.join(jobs[job]["path"],"results.pkl")+"', queueType = '"+queueType+"', getAllModels = "+str(getAllModels)+")\n") file_h.write("nModelsSaved = 0\n") file_h.write("for model in models:\n") file_h.write(" if not models[model] is None:\n") file_h.write(" models[model].write('"+os.path.join(jobs[job]["path"],"model")+"'+'_'+model)\n") file_h.write(' nModelsSaved += 1\n') file_h.write('if nModelsSaved == len([m for m in models if not models[m] is None ]):\n') file_h.write(' os.system(\'echo "finished" > '+os.path.join(jobs[job]["path"],"status")+' \')\n') file_h.write('else:\n') file_h.write(' os.system(\'echo "failed" > '+os.path.join(jobs[job]["path"],"status")+' \')\n') file_h.close() os.chdir(os.path.join(jobs[job]["path"])) if queueType == "NoSGE": # Serial mode status, out = commands.getstatusoutput("tcsh " + os.path.join(jobs[job]["path"],"run.sh")) if status: print "ERROR on Job "+str(job)+" (will be restarted latter)" print out else: statusFile = os.path.join(jobs[job]["path"],"status") if os.path.isfile(statusFile): st, status = commands.getstatusoutput("cat "+statusFile) else: print "ERROR: Missing status file" status = None if not status: print "ERROR! job "+job+" has no status!" jobs[job]["failed"] = True elif status == "failed": print "Job "+job+" failed to build all models" jobs[job]["failed"] = True elif status == "finished": jobs[job]["finished"] = True if not isJobProgressingOK(jobs[job]): print "Job "+job+" failed to build one or more models in getMLStatistics" jobs[job]["failed"] = True jobs[job]["finished"] = False if jobs[job]["failed"]: print "Job "+job+" FAILED" else: print "Finished Job "+str(job)+" with success" if callBack: stepsDone += 1 if not callBack((100*stepsDone)/nTotalSteps): return None else: cmd = "qsub -cwd -q batch.q" + AZOC.SGE_QSUB_ARCH_OPTION_CURRENT + os.path.join(jobs[job]["path"],"run.sh") status, out = commands.getstatusoutput(cmd) if status: print "ERROR on Job "+str(job)+" (will be skipped)" print out #raise Exception("ERROR starting job for folder "+str(job)) # Your job 955801 ("template_run.sh") has been submitted jID = out.strip().split(" ")[2] print " jID: ",jID os.system('echo "'+jID+'" > '+os.path.join(jobs[job]["path"], "jID")) jobs[job]["running"] = True jobs[job]["jID"] = jID os.chdir(runningDir) os.chdir(thisDir) finished = [] if queueType == "NoSGE": failed = [] #Report failed Jobs for job in jobs: if jobs[job]["finished"]: finished.append(job) for job in jobs: if jobs[job]["failed"]: failed.append(job) print "Successful finished Jobs: ",finished print "Failed Jobs: ",failed else: # Monitor SGE jobs untill all are finished #Monitor Fold jobs updateJobsStatus(jobs) for job in jobs: if jobs[job]["finished"]: finished.append(job) print "Jobs already finished: ",finished os.system(' echo "'+str(len(finished))+'/'+str(AZOC.QSARNEXTFOLDS)+'" > '+os.path.join(runningDir,"status")) while len(finished) < AZOC.QSARNEXTFOLDS: print ".", sys.stdout.flush() updateJobsStatus(jobs) for job in jobs: if jobs[job]["finished"] and job not in finished: finished.append(job) if callBack: stepsDone += 1 if not callBack((100*stepsDone)/nTotalSteps): return None print time.asctime()+": Finished job "+str(job) os.system(' echo "'+str(len(finished))+'/'+str(AZOC.QSARNEXTFOLDS)+'" > '+os.path.join(runningDir,"status")) for job in [j for j in jobs if jobs[j]["failed"]]: jobs[job] = restartJob(jobs[job]) time.sleep(5) print "All fold jobs finished!" # Gather the results print "Gathering results..." #Var for saving each Fols result results = {} exp_pred = {} nTrainEx = {} nTestEx = {} # Var for saving the statistics results statistics = {} mlMethods = [ml for ml in AZOC.MLMETHODS] + ["Consensus"] sortedJobs = [job for job in jobs] sortedJobs.sort(cmp = lambda x,y:int(x)>int(y) and 1 or -1) # Place for storing the selected models results results["selectedML"] = [] exp_pred["selectedML"] = [] nTrainEx["selectedML"] = [] nTestEx["selectedML"] = [] foldSelectedML = [] for ml in mlMethods: # Loop over each MLMethod try: #Var for saving each Fols result results[ml] = [] exp_pred[ml] = [] nTrainEx[ml] = [] nTestEx[ml] = [] logTxt = "" for job in sortedJobs: #loop over each fold modelPath = os.path.join(jobs[job]["path"], "model_"+ml) if not os.path.isdir(modelPath): if getAllModels: print "MLMethod "+ml+" not available in fold "+job continue resFile = os.path.join(jobs[job]["path"], "results.pkl") statFile_h = open(resFile) foldStat = pickle.load(statFile_h) statFile_h.close() #load model model = AZBaseClasses.modelRead(modelPath) #Test the model testData = dataset.select(DataIdxs[int(job)]) nTrainEx[ml].append(model.NTrainEx) nTestEx[ml].append(len(testData)) if foldStat[ml]["selected"]: foldSelectedML.append(ml) nTrainEx["selectedML"].append(model.NTrainEx) nTestEx["selectedML"].append(len(testData)) if responseType == "Classification": results[ml].append((evalUtilities.getClassificationAccuracy(testData, model), evalUtilities.getConfMat(testData, model) ) ) if foldStat[ml]["selected"]: results["selectedML"].append(results[ml][-1]) else: local_exp_pred = [] for ex in testData: local_exp_pred.append((ex.getclass(), model(ex))) results[ml].append((evalUtilities.calcRMSE(local_exp_pred), evalUtilities.calcRsqrt(local_exp_pred) ) ) #Save the experimental value and correspondent predicted value exp_pred[ml] += local_exp_pred if foldStat[ml]["selected"]: results["selectedML"].append(results[ml][-1]) exp_pred["selectedML"]+= local_exp_pred res = createStatObj(results[ml], exp_pred[ml], nTrainEx[ml], nTestEx[ml],responseType, len(sortedJobs), logTxt) if not res: raise Exception("No results available!") if getAllModels: statistics[ml] = copy.deepcopy(res) writeResults(statistics, resultsFile) print " OK",ml except: print "Error on MLmethod "+ml+". It will be skipped" ml = "selectedML" res = createStatObj(results[ml], exp_pred[ml], nTrainEx[ml], nTestEx[ml],responseType, len(sortedJobs), logTxt, foldSelectedML) if not res: raise Exception("No results available!") statistics[ml] = copy.deepcopy(res) writeResults(statistics, resultsFile) os.system(' echo "finished" > '+os.path.join(runningDir,"status")) return statistics
from trainingMethods import AZBaseClasses from AZutilities import dataUtilities from AZutilities import paramOptUtilities if __name__ == "__main__": """ Script to calculate the accuracy on a temporal test set with a saved model. Usage; python getTempAcc.py testDataPath modelPath """ # Full path to temporal test data file (with descriptors) in Orange format #testDataFile = "/home/jonna/projects/M-Lab/scfbmPaper/data/trainData.tab" testDataFile = sys.argv[1] # Read the test data testData = dataUtilities.DataTable(testDataFile) # Full path to the model #modelFile = "/home/jonna/projects/M-Lab/scfbmPaper/data/optRF.model" modelFile = sys.argv[2] # Read the model model = AZBaseClasses.modelRead(modelFile) # Use Orange methods to get the accuracy (Please see Orange doc) results = orngTest.testOnData([model], testData) print "Classification accuracy" print orngStat.CA(results)