def ListCompressedModels(self): sResult = [] if Storage.IsExistingPath(self.ExperimentModelFolder): sModelZipFiles = Storage.GetFilesSorted(self.ExperimentModelFolder) for sZipFile in sModelZipFiles: sZipFile = Storage.JoinPath(self.ExperimentModelFolder, sZipFile) sResult.append(sZipFile) return sResult
def ListSavedResults(self): if Storage.IsExistingPath(self.ExperimentResultsFolder): sModelResultFiles = Storage.GetFilesSorted( self.ExperimentResultsFolder) oModelResults = [] for sResultFile in sModelResultFiles: _, sFileName, _ = Storage.SplitFileName(sResultFile) nEpochNumber = int(sFileName) oModelResults.append([nEpochNumber, sResultFile, None]) return oModelResults
def ListSavedModels(self): sModelFolders = [] if Storage.IsExistingPath(self.ExperimentModelFolder): if not Storage.IsFolderEmpty(self.ExperimentModelFolder): sModelFolders = Storage.GetDirectoriesSorted( self.ExperimentModelFolder) oModels = [] for sModel in sModelFolders: sFolder = Storage.JoinPath(self.ExperimentModelFolder, sModel) sModelFiles = Storage.GetFilesSorted(sFolder) nEpochNumber = int(sModel) oModels.append([nEpochNumber, sFolder, sModelFiles]) return oModels
def GetConfig(cls, p_sFolder): oResult = None sFolder = Storage.JoinPath(p_sFolder, "config") if Storage.IsExistingPath(sFolder): sFileList = Storage.GetFilesSorted(sFolder) sFileName = None for sItem in sFileList: if sItem.startswith("learn-config-used"): sFileName = Storage.JoinPath(sFolder, sItem) break if sFileName is not None: oResult = NNLearnConfig() oResult.LoadFromFile(sFileName) oResult.ParseUID() return oResult
def DoPickSamples(self, p_nType=tcc.DS_TRAINING): if p_nType == tcc.DS_TRAINING: oSet = self.Train elif p_nType == tcc.DS_VALIDATION: oSet = self.Validation elif p_nType == tcc.DS_TESTING: oSet = self.Testing else: raise Exception("Invalid sample set type %d" % p_nType) # Decompresses the class folders for the selected classes, creates/loads the class folders collection bContinue = True if oSet.ClassFolders is None: oSet.ClassFolders = [] for nClassIndex, sClassCode in enumerate(self.ClassCodes): print("%d/%d " % (nClassIndex + 1, len(self.ClassCodes))) sSourceClassFolder = self.DataSetFolder.GetClassFolder( sClassCode, p_nType) if not Storage.IsExistingPath(sSourceClassFolder): #TODO: Decompression #bContinue=False if p_nType == tcc.DS_TRAINING: sSourceClassFolder = self.DataSetFolder.DecompressSamples( sClassCode) elif p_nType == tcc.DS_VALIDATION: nImageNetClassIndex = None for nSubIndex, sImageNetCode in enumerate( self.ImageNetClassCodes): if sImageNetCode == sClassCode: nImageNetClassIndex = nSubIndex break assert nImageNetClassIndex != None sSourceClassFolder = self.DataSetFolder.CollectValidationSamples( nImageNetClassIndex, sClassCode) oSet.ClassFolders.append(sSourceClassFolder) oSet.IsActive = bContinue # Uses the saved flag value bContinue = oSet.IsActive if bContinue: oSet.IsActive = bContinue # Dumps the class folders to a text file oSet.DumpClassFolders() # Creates/loads the samples for each one of the selected classes if not oSet.LoadSampleIndexFromDisk(): oSet.ClassSamplesAvailable = [] oClassSampleFullFileNamesNotTrimmed = [] oClassSampleFileNamesNotTrimmed = [] for sClassFolder in oSet.ClassFolders: oSampleFullFilenames, oSampleFileNames, nAvailableSamples = self.PickRandomSamplesFromClassFolder( sClassFolder, self.TrainSamplesPerClass) oSet.ClassSamplesAvailable.append(nAvailableSamples) oClassSampleFullFileNamesNotTrimmed.append( oSampleFullFilenames) oClassSampleFileNamesNotTrimmed.append(oSampleFileNames) oSet.TrimClassSamples(oClassSampleFullFileNamesNotTrimmed, oClassSampleFileNamesNotTrimmed) self.__saveClassesToDisk() oSet.SaveSampleIndexToDisk() # Dumps the sample IDs to a text file oSet.DumpSampleIDs() # Creates/loads the samples for each one of the selected classes if not oSet.LoadFromDisk(): oSet.StratifySamples() # The stratified order occurs only before shuffling, hence not dumped at each run oSet.DumpSampleFullFileNamesToDisk("stratified") oSet.SaveToDisk() # Shuffled the samples if not already shuffled if not oSet.IsShuffled: oSet.ShufflePages(self.PageSize) oSet.SaveToDisk() oSet.DumpSampleFullFileNamesToDisk("shuffled")