def ListCompressedModels(self):
sResult = []
if Storage.IsExistingPath(self.ExperimentModelFolder):
sModelZipFiles = Storage.GetFilesSorted(self.ExperimentModelFolder)
for sZipFile in sModelZipFiles:
sZipFile = Storage.JoinPath(self.ExperimentModelFolder,
sZipFile)
sResult.append(sZipFile)
return sResult
def ListSavedResults(self):
if Storage.IsExistingPath(self.ExperimentResultsFolder):
sModelResultFiles = Storage.GetFilesSorted(
self.ExperimentResultsFolder)
oModelResults = []
for sResultFile in sModelResultFiles:
_, sFileName, _ = Storage.SplitFileName(sResultFile)
nEpochNumber = int(sFileName)
oModelResults.append([nEpochNumber, sResultFile, None])
return oModelResults
def ListSavedModels(self):
sModelFolders = []
if Storage.IsExistingPath(self.ExperimentModelFolder):
if not Storage.IsFolderEmpty(self.ExperimentModelFolder):
sModelFolders = Storage.GetDirectoriesSorted(
self.ExperimentModelFolder)
oModels = []
for sModel in sModelFolders:
sFolder = Storage.JoinPath(self.ExperimentModelFolder, sModel)
sModelFiles = Storage.GetFilesSorted(sFolder)
nEpochNumber = int(sModel)
oModels.append([nEpochNumber, sFolder, sModelFiles])
return oModels
def GetConfig(cls, p_sFolder):
oResult = None
sFolder = Storage.JoinPath(p_sFolder, "config")
if Storage.IsExistingPath(sFolder):
sFileList = Storage.GetFilesSorted(sFolder)
sFileName = None
for sItem in sFileList:
if sItem.startswith("learn-config-used"):
sFileName = Storage.JoinPath(sFolder, sItem)
break
if sFileName is not None:
oResult = NNLearnConfig()
oResult.LoadFromFile(sFileName)
oResult.ParseUID()
return oResult
def DoPickSamples(self, p_nType=tcc.DS_TRAINING):
if p_nType == tcc.DS_TRAINING:
oSet = self.Train
elif p_nType == tcc.DS_VALIDATION:
oSet = self.Validation
elif p_nType == tcc.DS_TESTING:
oSet = self.Testing
else:
raise Exception("Invalid sample set type %d" % p_nType)
# Decompresses the class folders for the selected classes, creates/loads the class folders collection
bContinue = True
if oSet.ClassFolders is None:
oSet.ClassFolders = []
for nClassIndex, sClassCode in enumerate(self.ClassCodes):
print("%d/%d " % (nClassIndex + 1, len(self.ClassCodes)))
sSourceClassFolder = self.DataSetFolder.GetClassFolder(
sClassCode, p_nType)
if not Storage.IsExistingPath(sSourceClassFolder):
#TODO: Decompression
#bContinue=False
if p_nType == tcc.DS_TRAINING:
sSourceClassFolder = self.DataSetFolder.DecompressSamples(
sClassCode)
elif p_nType == tcc.DS_VALIDATION:
nImageNetClassIndex = None
for nSubIndex, sImageNetCode in enumerate(
self.ImageNetClassCodes):
if sImageNetCode == sClassCode:
nImageNetClassIndex = nSubIndex
break
assert nImageNetClassIndex != None
sSourceClassFolder = self.DataSetFolder.CollectValidationSamples(
nImageNetClassIndex, sClassCode)
oSet.ClassFolders.append(sSourceClassFolder)
oSet.IsActive = bContinue
# Uses the saved flag value
bContinue = oSet.IsActive
if bContinue:
oSet.IsActive = bContinue
# Dumps the class folders to a text file
oSet.DumpClassFolders()
# Creates/loads the samples for each one of the selected classes
if not oSet.LoadSampleIndexFromDisk():
oSet.ClassSamplesAvailable = []
oClassSampleFullFileNamesNotTrimmed = []
oClassSampleFileNamesNotTrimmed = []
for sClassFolder in oSet.ClassFolders:
oSampleFullFilenames, oSampleFileNames, nAvailableSamples = self.PickRandomSamplesFromClassFolder(
sClassFolder, self.TrainSamplesPerClass)
oSet.ClassSamplesAvailable.append(nAvailableSamples)
oClassSampleFullFileNamesNotTrimmed.append(
oSampleFullFilenames)
oClassSampleFileNamesNotTrimmed.append(oSampleFileNames)
oSet.TrimClassSamples(oClassSampleFullFileNamesNotTrimmed,
oClassSampleFileNamesNotTrimmed)
self.__saveClassesToDisk()
oSet.SaveSampleIndexToDisk()
# Dumps the sample IDs to a text file
oSet.DumpSampleIDs()
# Creates/loads the samples for each one of the selected classes
if not oSet.LoadFromDisk():
oSet.StratifySamples()
# The stratified order occurs only before shuffling, hence not dumped at each run
oSet.DumpSampleFullFileNamesToDisk("stratified")
oSet.SaveToDisk()
# Shuffled the samples if not already shuffled
if not oSet.IsShuffled:
oSet.ShufflePages(self.PageSize)
oSet.SaveToDisk()
oSet.DumpSampleFullFileNamesToDisk("shuffled")