def runTestonly(self):
""" Run testing dataset evaluation on an existing rule population. """
print("Initializing Evaluation of Testing Dataset...")
#-----------------------------------------------------------------------------------------------------------------------------------------
# CHECK FOR POPULATION REBOOT - Required for running Testing Evaluation only on an existing saved rule population.
#-----------------------------------------------------------------------------------------------------------------------------------------
if not cons.doPopulationReboot:
print(
"Algorithm: Error - Existing population required to run rule compaction alone."
)
return
#----------------------------------------------------------
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
testEval = self.doPopEvaluation(False)
else:
testEval = self.doContPopEvaluation(False)
else: #Online Environment
testEval = None
cons.env.stopEvaluationMode()
cons.timer.stopTimeEvaluation()
#------------------------------------------------------------------------------
cons.timer.returnGlobalTimer()
OutputFileManager().editPopStats(testEval)
OutputFileManager().writePredictions(self.exploreIter,
cons.outFileName,
self.predictionList,
self.realList,
self.predictionSets)
print("Testing Evaluation Complete")
def runExSTraCS(self):
""" Runs the initialized ExSTraCS algorithm. """
print("Beginning ExSTraCS learning iterations.")
print(
"------------------------------------------------------------------------------------------------------------------------------------------------------"
)
#START GP INTEGRATION CODE*************************************************************************************************************************************
#-------------------------------------------------------
# Initial round of rule population initialization
#-------------------------------------------------------
cons.env.startEvaluationMode()
initRuleCount = int((1 - cons.popInitGP) * cons.N)
print('Initializing population with ' + str(initRuleCount) +
' LCS rules.')
if cons.trainFile != 'None':
if cons.env.formatData.discretePhenotype:
instances = cons.env.formatData.numTrainInstances
for i in range(initRuleCount):
if i % instances == 0:
cons.env.resetDataRef(
True) # Go to the first instance in dataset
state_phenotype = cons.env.getTrainInstance()
self.population.addClassifierForInit(
state_phenotype[0], state_phenotype[1])
cons.env.newInstance(True)
else: #ContinuousCode #########################
instances = cons.env.formatData.numTrainInstances
for i in range(initRuleCount):
if i % instances == 0:
cons.env.resetDataRef(
True) # Go to the first instance in dataset
state_phenotype = cons.env.getTrainInstance()
self.population.addClassifierForInit(
state_phenotype[0], state_phenotype[1])
cons.env.newInstance(True)
else: #Online Environment
raise NameError("Online has not been implemented")
cons.env.stopEvaluationMode()
#STOP GP INTEGRATION CODE*************************************************************************************************************************************
#-------------------------------------------------------
# MAJOR LEARNING LOOP
#-------------------------------------------------------
while self.exploreIter < cons.maxLearningIterations and not cons.stop: #Major Learning Loop
#print 'learning iteration = '+str(self.exploreIter)
#-------------------------------------------------------
# GET NEW INSTANCE AND RUN A LEARNING ITERATION
#-------------------------------------------------------
state_phenotype = cons.env.getTrainInstance()
self.runIteration(state_phenotype)
#-------------------------------------------------------------------------------------------------------------------------------
# EVALUATIONS OF ALGORITHM
#-------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeEvaluation()
#-------------------------------------------------------
# TRACK LEARNING ESTIMATES
#-------------------------------------------------------
#Learning Tracking----------------------------------------------------------------------------------------------------------------------------------------
if (self.exploreIter % cons.trackingFrequency) == (
cons.trackingFrequency - 1) and self.exploreIter > 0:
self.population.runPopAveEval(self.exploreIter)
trackedAccuracy = sum(self.correct) / float(
cons.trackingFrequency
) #Accuracy over the last "trackingFrequency" number of iterations.
self.learnTrackOut.write(
self.population.getPopTrack(trackedAccuracy,
self.exploreIter + 1,
cons.trackingFrequency)
) #Report learning progress to standard out and tracking file.
for observer in cons.epochCallbacks:
observer(self.exploreIter, self.population,
trackedAccuracy)
cons.timer.stopTimeEvaluation()
#-------------------------------------------------------
# CHECKPOINT - COMPLETE EVALUTATION OF POPULATION - Evaluation strategy different for discrete vs continuous phenotypes
#-------------------------------------------------------
if (self.exploreIter +
1) in cons.learningCheckpoints or cons.forceCheckpoint:
if (cons.forceCheckpoint):
cons.forceCheckpoint = False
cons.timer.startTimeEvaluation()
print(
"------------------------------------------------------------------------------------------------------------------------------------------------------"
)
print("Running Population Evaluation after " +
str(self.exploreIter + 1) + " iterations.")
self.population.runPopAveEval(self.exploreIter)
self.population.runAttGeneralitySum()
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = self.doPopEvaluation(False)
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = self.doContPopEvaluation(False)
elif cons.trainFile != 'None':
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = None
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = None
else: #Online Environment
trainEval = None
testEval = None
cons.env.stopEvaluationMode(
) #Returns to learning position in training data
cons.timer.stopTimeEvaluation()
#-----------------------------------------------------------------------------------------------------------------------------------------
# WRITE OUTPUT FILES
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeOutFile()
OutputFileManager().writePopStats(cons.outFileName, trainEval,
testEval,
self.exploreIter + 1,
self.population,
self.correct)
OutputFileManager().writePop(cons.outFileName,
self.exploreIter + 1,
self.population)
OutputFileManager().attCo_Occurence(cons.outFileName,
self.exploreIter + 1,
self.population)
OutputFileManager().save_tracking(self.exploreIter,
cons.outFileName)
OutputFileManager().writePredictions(self.exploreIter,
cons.outFileName,
self.predictionList,
self.realList,
self.predictionSets)
cons.timer.stopTimeOutFile()
if self.exploreIter + 1 == cons.maxLearningIterations:
FitnessLandscape(self.population)
#if self.exploreIter + 1 == cons.maxLearningIterations:
#plotPopulation(self.population, self.exploreIter)
#GUI ONLY--------------------------------
for observer in cons.checkpointCallbacks:
observer(trainEval, testEval)
#----------------------------------------
print("Continue Learning...")
print(
"------------------------------------------------------------------------------------------------------------------------------------------------------"
)
#-----------------------------------------------------------------------------------------------------------------------------------------
# RULE COMPACTION
#-----------------------------------------------------------------------------------------------------------------------------------------
if self.exploreIter + 1 == cons.maxLearningIterations and cons.doRuleCompaction:
cons.timer.startTimeRuleCmp()
if testEval == None:
RuleCompaction(self.population, trainEval[0], None,
self.exploreIter)
else:
RuleCompaction(self.population, trainEval[0],
testEval[0], self.exploreIter)
cons.timer.stopTimeRuleCmp()
#-----------------------------------------------------------------------------------------------------------------------------------------
# GLOBAL EVALUATION OF COMPACTED RULE POPULATION
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeEvaluation()
self.population.recalculateNumerositySum()
self.population.runPopAveEval(self.exploreIter)
self.population.runAttGeneralitySum()
#----------------------------------------------------------
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = self.doPopEvaluation(False)
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = self.doContPopEvaluation(False)
else:
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = None
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = None
cons.env.stopEvaluationMode()
cons.timer.stopTimeEvaluation()
#-----------------------------------------------------------------------------------------------------------------------------------------
# WRITE OUTPUT FILES
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeOutFile()
OutputFileManager().writePopStats(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
trainEval, testEval, self.exploreIter + 1,
self.population, self.correct)
OutputFileManager().writePop(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.exploreIter + 1, self.population)
OutputFileManager().attCo_Occurence(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.exploreIter + 1, self.population)
OutputFileManager().writePredictions(
self.exploreIter,
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.predictionList, self.realList,
self.predictionSets)
cons.timer.stopTimeOutFile()
#GUI ONLY--------------------------------
for observer in cons.iterationCallbacks:
observer()
#-------------------------------------------------------
# ADJUST MAJOR VALUES FOR NEXT ITERATION
#-------------------------------------------------------
self.exploreIter += 1
cons.env.newInstance(True) #move to next instance in training set
try: # Once ExSTraCS has reached the last learning iteration, close the tracking file
self.learnTrackOut.close()
except IOError as xxx_todo_changeme2:
(errno, strerror) = xxx_todo_changeme2.args
print(("I/O error(%s): %s" % (errno, strerror)))
raise
print("ExSTraCS Run Complete")
def runRConly(self):
""" Run rule compaction on an existing rule population. """
print("Initializing Rule Compaction...")
#-----------------------------------------------------------------------------------------------------------------------------------------
# CHECK FOR POPULATION REBOOT - Required for running Rule Compaction only on an existing saved rule population.
#-----------------------------------------------------------------------------------------------------------------------------------------
if not cons.doPopulationReboot:
print(
"Algorithm: Error - Existing population required to run rule compaction alone."
)
return
try:
fileObject = open(cons.popRebootPath + "_PopStats.txt",
'rU') # opens each datafile to read.
except:
#break
print("Data-set Not Found!")
#Retrieve last training and testing accuracies from saved file---------
tempLine = None
for i in range(3):
tempLine = fileObject.readline()
tempList = tempLine.strip().split('\t')
trainAcc = float(tempList[0])
if cons.testFile != 'None': #If a testing file is available.
testAcc = float(tempList[1])
else:
testAcc = None
#-----------------------------------------------------------------------------------------------------------------------------------------
# RULE COMPACTION
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeRuleCmp()
RuleCompaction(self.population, trainAcc, testAcc, self.exploreIter)
cons.timer.stopTimeRuleCmp()
#-----------------------------------------------------------------------------------------------------------------------------------------
# GLOBAL EVALUATION OF COMPACTED RULE POPULATION
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeEvaluation()
self.population.recalculateNumerositySum()
self.population.runPopAveEval(self.exploreIter)
self.population.runAttGeneralitySum()
#----------------------------------------------------------
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = self.doPopEvaluation(False)
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = self.doContPopEvaluation(False)
elif cons.trainFile != 'None':
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = None
else: #ContinuousCode #########################
trainEval = self.doContPopEvaluation(True)
testEval = None
else: #Online Environment
trainEval = None
testEval = None
cons.env.stopEvaluationMode()
cons.timer.stopTimeEvaluation()
#------------------------------------------------------------------------------
cons.timer.returnGlobalTimer()
#-----------------------------------------------------------------------------------------------------------------------------------------
# WRITE OUTPUT FILES
#-----------------------------------------------------------------------------------------------------------------------------------------
OutputFileManager().writePopStats(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod, trainEval,
testEval, self.exploreIter + 1, self.population, self.correct)
OutputFileManager().writePop(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.exploreIter + 1, self.population)
OutputFileManager().attCo_Occurence(
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.exploreIter + 1, self.population)
OutputFileManager().writePredictions(
self.exploreIter,
cons.outFileName + "_RC_" + cons.ruleCompactionMethod,
self.predictionList, self.realList, self.predictionSets)
#------------------------------------------------------------------------------------------------------------
print("Rule Compaction Complete")
def runExSTraCS(self):
""" Runs the initialized ExSTraCS algorithm. """
print("Beginning ExSTraCS learning iterations.")
print("------------------------------------------------------------------------------------------------------------------------------------------------------")
#-------------------------------------------------------
# MAJOR LEARNING LOOP
#-------------------------------------------------------
while self.exploreIter < cons.maxLearningIterations and not cons.stop: #Major Learning Loop
#-------------------------------------------------------
# GET NEW INSTANCE AND RUN A LEARNING ITERATION
#-------------------------------------------------------
state_phenotype = cons.env.getTrainInstance()
self.runIteration(state_phenotype, self.exploreIter)
#-------------------------------------------------------------------------------------------------------------------------------
# EVALUATIONS OF ALGORITHM
#-------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeEvaluation()
#-------------------------------------------------------
# TRACK LEARNING ESTIMATES
#-------------------------------------------------------
#Learning Tracking----------------------------------------------------------------------------------------------------------------------------------------
if (self.exploreIter%cons.trackingFrequency) == (cons.trackingFrequency - 1) and self.exploreIter > 0:
self.population.runPopAveEval(self.exploreIter)
trackedAccuracy = sum(self.correct)/float(cons.trackingFrequency) #Accuracy over the last "trackingFrequency" number of iterations.
self.learnTrackOut.write(self.population.getPopTrack(trackedAccuracy, self.exploreIter+1,cons.trackingFrequency)) #Report learning progress to standard out and tracking file.
for observer in cons.epochCallbacks:
observer(self.exploreIter, self.population, trackedAccuracy)
cons.timer.stopTimeEvaluation()
#-------------------------------------------------------
# CHECKPOINT - COMPLETE EVALUTATION OF POPULATION - Evaluation strategy different for discrete vs continuous phenotypes
#-------------------------------------------------------
if (self.exploreIter + 1) in cons.learningCheckpoints or cons.forceCheckpoint:
if(cons.forceCheckpoint):
cons.forceCheckpoint = False
cons.timer.startTimeEvaluation()
print("------------------------------------------------------------------------------------------------------------------------------------------------------")
print("Running Population Evaluation after " + str(self.exploreIter + 1)+ " iterations.")
self.population.runPopAveEval(self.exploreIter)
self.population.runAttGeneralitySum()
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = self.doPopEvaluation(False)
else:
print("Algorithm - Error: ExSTraCS 2.0 can not handle continuous endpoints.")
elif cons.trainFile != 'None':
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = None
else:
print("Algorithm - Error: ExSTraCS 2.0 can not handle continuous endpoints.")
else: #Online Environment
trainEval = None
testEval = None
cons.env.stopEvaluationMode() #Returns to learning position in training data
cons.timer.stopTimeEvaluation()
#-----------------------------------------------------------------------------------------------------------------------------------------
# WRITE OUTPUT FILES
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeOutFile()
OutputFileManager().writePopStats(cons.outFileName, trainEval, testEval, self.exploreIter + 1, self.population, self.correct)
OutputFileManager().writePop(cons.outFileName, self.exploreIter + 1, self.population)
OutputFileManager().attCo_Occurence(cons.outFileName, self.exploreIter + 1, self.population)
OutputFileManager().save_tracking(self.exploreIter, cons.outFileName)
OutputFileManager().writePredictions(self.exploreIter, cons.outFileName, self.predictionList, self.realList, self.predictionSets)
cons.timer.stopTimeOutFile()
#GUI ONLY--------------------------------
for observer in cons.checkpointCallbacks:
observer(trainEval, testEval)
#----------------------------------------
print("Continue Learning...")
print("------------------------------------------------------------------------------------------------------------------------------------------------------")
#-----------------------------------------------------------------------------------------------------------------------------------------
# RULE COMPACTION
#-----------------------------------------------------------------------------------------------------------------------------------------
if self.exploreIter + 1 == cons.maxLearningIterations and cons.doRuleCompaction:
cons.timer.startTimeRuleCmp()
if testEval == None:
RuleCompaction(self.population, trainEval[0], None)
else:
RuleCompaction(self.population, trainEval[0], testEval[0])
cons.timer.stopTimeRuleCmp()
#-----------------------------------------------------------------------------------------------------------------------------------------
# GLOBAL EVALUATION OF COMPACTED RULE POPULATION
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeEvaluation()
self.population.recalculateNumerositySum()
self.population.runPopAveEval(self.exploreIter)
self.population.runAttGeneralitySum()
#----------------------------------------------------------
cons.env.startEvaluationMode()
if cons.testFile != 'None': #If a testing file is available.
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = self.doPopEvaluation(False)
else:
print("Algorithm - Error: ExSTraCS 2.0 can not handle continuous endpoints.")
else:
if cons.env.formatData.discretePhenotype:
trainEval = self.doPopEvaluation(True)
testEval = None
else:
print("Algorithm - Error: ExSTraCS 2.0 can not handle continuous endpoints.")
cons.env.stopEvaluationMode()
cons.timer.stopTimeEvaluation()
#-----------------------------------------------------------------------------------------------------------------------------------------
# WRITE OUTPUT FILES
#-----------------------------------------------------------------------------------------------------------------------------------------
cons.timer.startTimeOutFile()
OutputFileManager().writePopStats(cons.outFileName+"_RC_"+cons.ruleCompactionMethod, trainEval, testEval, self.exploreIter + 1, self.population, self.correct)
OutputFileManager().writePop(cons.outFileName+"_RC_"+cons.ruleCompactionMethod, self.exploreIter + 1, self.population)
OutputFileManager().attCo_Occurence(cons.outFileName+"_RC_"+cons.ruleCompactionMethod, self.exploreIter + 1, self.population)
OutputFileManager().writePredictions(self.exploreIter, cons.outFileName+"_RC_"+cons.ruleCompactionMethod, self.predictionList, self.realList, self.predictionSets)
cons.timer.stopTimeOutFile()
#GUI ONLY--------------------------------
for observer in cons.iterationCallbacks:
observer()
#-------------------------------------------------------
# ADJUST MAJOR VALUES FOR NEXT ITERATION
#-------------------------------------------------------
self.exploreIter += 1
cons.env.newInstance(True) #move to next instance in training set
# Once ExSTraCS has reached the last learning iteration, close the tracking file
self.learnTrackOut.close()
print("ExSTraCS Run Complete")