def __loadAll(self):
for nIndex, sFileRec in enumerate(self.ResultFiles):
_, sEpochNumber, _ = Storage.SplitFileName(sFileRec[0])
sFileNameFull = sFileRec[1]
oMetrics = ClassificationMetrics()
oMetrics.Load(sFileNameFull)
print(
"Accuracy:%f Top%dAccuracy%s" %
(oMetrics.Accuracy, oMetrics.TopKappa, oMetrics.TopKAccuracy))
self.EpochNumber[nIndex] = int(sEpochNumber)
self.Accuracy[nIndex] = oMetrics.Accuracy
self.Recall[nIndex] = oMetrics.AverageRecall
self.Precision[nIndex] = oMetrics.AveragePrecision
self.F1Score[nIndex] = oMetrics.AverageF1Score
if oMetrics.ClassCount == 2:
self.IsBinary = True
# Cross entropy of the F1 scores for binary classification.
self.CrossF1Score[nIndex] = -(
oMetrics.F1Score[0] * np.log10(oMetrics.F1Score[1]) +
oMetrics.F1Score[1] * np.log10(oMetrics.F1Score[0]))
self.ObjectiveF1Score[
nIndex] = self.F1Score[nIndex] / self.CrossF1Score[nIndex]
# Special binary classification, with the class 0 the class positives
self.PositiveF1Score[nIndex] = oMetrics.F1Score[0]
print(sEpochNumber, oMetrics.F1Score[0], oMetrics.F1Score[1],
self.CrossF1Score[nIndex], self.ObjectiveF1Score[nIndex])
def __determineInitialModelUID(self):
sFiles = Storage.GetFilesSorted(self.RunSub.ArchitectureCommonFolder)
sUID = None
for sFile in sFiles:
if sFile.startswith("initial-model_"):
_, sName, _ = Storage.SplitFileName(sFile)
sUID = sName[-12:]
# A standard fold number 1 and the last saved initial experiment in the common folder will be returned
return sUID
def ListSavedResults(self):
if Storage.IsExistingPath(self.ExperimentResultsFolder):
sModelResultFiles = Storage.GetFilesSorted(
self.ExperimentResultsFolder)
oModelResults = []
for sResultFile in sModelResultFiles:
_, sFileName, _ = Storage.SplitFileName(sResultFile)
nEpochNumber = int(sFileName)
oModelResults.append([nEpochNumber, sResultFile, None])
return oModelResults
def GetNextConfigToEvaluate(self):
sFiles = Storage.GetFilesSorted(self.ToEvaluteFolder)
sConfigFiles = []
for sFile in sFiles:
_, _, sExt = Storage.SplitFileName(sFile)
if sExt == ".cfg":
sConfigFiles.append(
Storage.JoinPath(self.ToEvaluteFolder, sFile))
if len(sFiles) > 0:
sResult = sConfigFiles[0]
else:
sResult = None
return sResult
def __sequenceFoldNumber(self, p_sSourceFileName, p_nFoldSequence,
p_nCounter):
print(" -> Sequencing fold numbers from template")
sResult = []
nCounter = p_nCounter
for nFoldNumber in p_nFoldSequence:
oNewConfig = NNLearnConfig()
oNewConfig.LoadFromFile(p_sSourceFileName)
oNewConfig.FoldNumber = nFoldNumber
_, sName, _ = Storage.SplitFileName(p_sSourceFileName)
sDestFileName = Storage.JoinFileName(
self.PendingFolder,
"%.3d-" % p_nCounter + sName + "-fold%d" % nFoldNumber, ".cfg")
sResult.append(sDestFileName)
oNewConfig.SaveToFile(sDestFileName)
nCounter += 1
return nCounter, sResult
def GetNextConfig(self):
# By priority first evaluates models to save disk space and then start training
sResult = self.GetNextConfigToEvaluate()
if sResult is None:
sFiles = Storage.GetFilesSorted(self.PendingFolder)
sConfigFiles = []
for sFile in sFiles:
_, _, sExt = Storage.SplitFileName(sFile)
if sExt == ".cfg":
sConfigFiles.append(
Storage.JoinPath(self.PendingFolder, sFile))
if len(sFiles) > 0:
sResult = sConfigFiles[0]
else:
sResult = None
return sResult
def __sequenceLearningRate(self, p_sSourceFileName,
p_nLearningRateSequence, p_nCounter):
print(" -> Sequencing learning rates from template")
sResult = []
nCounter = p_nCounter
for nLearningRate in p_nLearningRateSequence:
oNewConfig = NNLearnConfig()
oNewConfig.LoadFromFile(p_sSourceFileName)
oNewConfig.Learn.LearningRate = nLearningRate
_, sName, _ = Storage.SplitFileName(p_sSourceFileName)
sDestFileName = Storage.JoinFileName(
self.PendingFolder,
"%.3d-" % p_nCounter + sName + "-lr%.6f" % nLearningRate,
".cfg")
sResult.append(sDestFileName)
oNewConfig.SaveToFile(sDestFileName)
nCounter += 1
return nCounter, sResult
def ArchiveFileName(self, p_nCounter):
_, sName, sExt = Storage.SplitFileName(self.FileName)
sPrefix = None
if (self.IsTraining == True):
# Training run
sPrefix = "r"
elif (self.IsTraining == False) and ((self.IsEvaluating == True) or
(self.IsDeterminingBest == True)):
# Evaluation run
sPrefix = "e"
assert sPrefix is not None
# Prefix is valid for display up to 1000 experiments
assert p_nCounter < 1000
sResult = sPrefix + "%.3d-" % p_nCounter + sName + sExt
return sResult
def AddConfig(self, p_sConfigFileName=None):
if p_sConfigFileName is None:
sSourceFileName = self.TemplateConfigFileName
else:
sSourceFileName = p_sConfigFileName
_, sName, _ = Storage.SplitFileName(sSourceFileName)
# Gets the current run/evaluation counter
nCounter = self.__readCounter()
oConfig = NNLearnConfig()
oConfig.LoadFromFile(sSourceFileName)
sDestFileNames = None
if oConfig.LearningRateSequence is not None:
nCounter, sDestFileNames = self.__sequenceLearningRate(
sSourceFileName, oConfig.LearningRateSequence, nCounter)
elif oConfig.FoldSequence is not None:
nCounter, sDestFileNames = self.__sequenceFoldNumber(
sSourceFileName, oConfig.FoldSequence, nCounter)
# for nFoldNumber in oConfig.FoldSequence:
# oNewConfig = NNLearnConfig()
# oNewConfig.LoadFromFile(sSourceFileName)
# oNewConfig.FoldNumber = nFoldNumber
# sDestFileName = Storage.JoinFileName(self.PendingFolder, "%.3d-" % nCounter + sName, ".cfg")
# oNewConfig.SaveToFile(sDestFileName)
# nCounter += 1
else:
sDestFileNames = [
Storage.JoinFileName(self.PendingFolder,
"%.3d-" % nCounter + sName, ".cfg")
]
Storage.CopyFile(sSourceFileName, sDestFileNames[0])
nCounter += 1
# Saves the current run/evaluation counter
self.__writeCounter()
return sDestFileNames
def ParseUID(self):
if len(self.FileName) >= 12:
_, sName, _ = Storage.SplitFileName(self.FileName)
self.SavedExperimentUID = sName[-12:]