def test_Save_LocationIsNotString(self, notStringLocation): trainingLog = Logger(False) trainingLog._content = "Ford Sierra II 2.0 DOHC 125KM" trainingLog.Save(notStringLocation) pathToLogFile = os.path.join(str(notStringLocation), "%s.log" % trainingLog._fileName) doesLogFileExist = os.path.exists(pathToLogFile) self.assertFalse(doesLogFileExist)
def test_Save_LocationDoesNotExist(self): nonExistentLocation = "TEST_NON_EXISTENT_LOCATION" self.assertFalse(os.path.exists(nonExistentLocation)) trainingLog = Logger(False) trainingLog._content = "Ford Sierra II 2.0 DOHC 125KM" trainingLog.Save(nonExistentLocation) pathToLogFile = os.path.join(nonExistentLocation, "%s.log" % trainingLog._fileName) doesLogFileExist = os.path.exists(pathToLogFile) self.assertFalse(doesLogFileExist)
def test_Save_LocationIsNotDir(self): nonDirectoryLocation = "TEST_NON_DIRECTORY_LOCATION.log" with open(nonDirectoryLocation, "w") as tempFile: tempFile.write("kanapka") self.assertTrue(os.path.exists(nonDirectoryLocation)) self.assertFalse(os.path.isdir(nonDirectoryLocation)) trainingLog = Logger(False) trainingLog._content = "Ford Sierra II 2.0 DOHC 125KM" trainingLog.Save(nonDirectoryLocation) pathToLogFile = os.path.os.path.join(nonDirectoryLocation, "%s.log" % trainingLog._fileName) doesLogFileExist = os.path.exists(pathToLogFile) self.assertFalse(doesLogFileExist) os.remove(nonDirectoryLocation) self.assertFalse(os.path.exists(nonDirectoryLocation))
def test_Save_LogContentIsEmptyOrOnlyWhitespaces(self, logContent): testLocation = "TEST_LOCATION_FOR_LOG_FILE" if os.path.isdir(testLocation): rmtree(testLocation) os.mkdir(testLocation) self.assertTrue(os.path.isdir(testLocation)) trainingLog = Logger(False) trainingLog._content = logContent trainingLog.Save(testLocation) pathToLogFile = os.path.join(testLocation, "{0}.log".format(trainingLog._fileName)) self.assertFalse(os.path.exists(pathToLogFile)) rmtree(testLocation) self.assertFalse(os.path.isdir(testLocation))
def test_Save_OK(self): testLocation = "TEST_LOCATION_FOR_LOG_FILE" if os.path.isdir(testLocation): rmtree(testLocation) os.mkdir(testLocation) self.assertTrue(os.path.isdir(testLocation)) trainingLog = Logger(False) trainingLog._content = "Ford Sierra II 2.0 DOHC 125KM" trainingLog.Save(testLocation) pathToLogFile = os.path.join(testLocation, "{0}.log".format(trainingLog._fileName)) self.assertTrue(os.path.isfile(pathToLogFile)) with open(pathToLogFile, "r") as logFile: expectedLogFileContent = trainingLog._content actualLogFileContent = logFile.read() self.assertEqual(actualLogFileContent, expectedLogFileContent) rmtree(testLocation) self.assertFalse(os.path.isdir(testLocation))
def experiment(options): # --- Check if directory with Unity builds does exist --- # pathToBuildsDir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "env_builds") if not os.path.isdir(pathToBuildsDir): print("Error: 'env_builds' directory doesn't exist! " \ "Run 'make_builds.py' to fix it!") exit() # --- Validation of command line args --- # numberOfTrials = options["--num-of-trials"] errorMessage = "Invalid numberOfTrials value! Should be positive integer! " \ "Actual type of numberOfTrials: '{0}', actual value: '{1}'.".format( type(numberOfTrials), numberOfTrials) if isinstance(numberOfTrials, str) and numberOfTrials.isdigit(): numberOfTrials = int(numberOfTrials) if numberOfTrials <= 0: print(errorMessage) exit() else: print(errorMessage) exit() del errorMessage # --- Create logger object --- # experimentLog = \ Logger(isVerbose = options["--verbose"], fileName = "experiment") experimentLog.Append("Experiment log has been created!") experimentLog.Append("numberOfTrials = {0}".format(numberOfTrials)) # --- Load config data from file --- # pathToConfigFile = options["<config-file-path>"] CONFIG_DATA = loadConfigData(pathToConfigFile) experimentLog.Append( "Config data has been loaded from file: {0}".format(pathToConfigFile)) # --- Determine builds paths --- # buildTarget = CONFIG_DATA["MakeBuilds"]["Target"] buildPaths = CONFIG_DATA["BuildPaths"][buildTarget] # --- Create data collector object --- # dataCollector = ExperimentDataCollector() experimentLog.Append("Data collector object has been created!") # --- Prepare minimal fitness dict for validation purposes --- # minFitnessDict = CONFIG_DATA["TrainingParameters"][ "minimalAcceptableFitness"] # --- Experiment sequence loop --- # for trialCounter in range(numberOfTrials): for trackNumber in range(1, 4): experimentLog.Append("Generating data from 'train_de.py', track: " \ "{0}, trial: {1}".format(trackNumber, trialCounter + 1)) generateDataFromTraining(train_de, "DE", trackNumber, pathToConfigFile, buildPaths, experimentLog, dataCollector, minFitnessDict, isVerbose=options["--verbose"]) experimentLog.Append("Generating data from 'train_pso.py', track: " \ "{0}, trial: {1}".format(trackNumber, trialCounter + 1)) generateDataFromTraining(train_pso, "PSO", trackNumber, pathToConfigFile, buildPaths, experimentLog, dataCollector, minFitnessDict, isVerbose=options["--verbose"]) # --- Create location for charts --- # pathToResultsDir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "experiment_results") if not os.path.isdir(pathToResultsDir): os.mkdir(pathToResultsDir) experimentLog.Append( "'{0}' directory has been created!".format(pathToResultsDir)) pathToCurrentResults = createPathForCurrentResults(pathToResultsDir) if os.path.isdir(pathToCurrentResults): rmtree(pathToCurrentResults) os.mkdir(pathToCurrentResults) experimentLog.Append( "'{0}' directory has been created!".format(pathToCurrentResults)) # --- Generate charts --- # chartsGenerator = ChartsGenerator(dataCollector, pathToCurrentResults) experimentLog.Append("Charts generator objects has been created!") chartsGenerator.CreateAll() experimentLog.Append("All charts have been generated!") # --- Save log file --- # experimentLog.Append( "End of log file, save to '{0}'.".format(pathToCurrentResults)) experimentLog.Save(pathToCurrentResults)
class TrainingResultsRepository: def __init__(self, trainingLog=None): self._trainingLog = trainingLog self._pathToLastSavedModel = None def Save(self, population, bestIndividual, shouldSavePopulation): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.Save() warning: " \ "trainingLog was None! Potentially important details about " \ "training (or run) could haven't been saved!") locationForTrainingResults = self._createLocationForTrainingResults() os.mkdir(locationForTrainingResults) if self._doParametersHaveValidTypes(population, bestIndividual, shouldSavePopulation): if len(population) == 0: self._trainingLog.Append( "TrainingResultsRepository.Save() error: " \ "population is empty!") self._pathToLastSavedModel = None else: self._saveBestModel(locationForTrainingResults, bestIndividual) if shouldSavePopulation: self._saveWholePopulation(locationForTrainingResults, population) self._trainingLog.Append( "TrainingResultsRepository.Save() info: " \ "training results were saved to the location " \ "'{0}'!".format(locationForTrainingResults)) self._pathToLastSavedModel = locationForTrainingResults else: self._trainingLog.Append( "TrainingResultsRepository.Save() error: " \ "some of parameters have wrong type!\n" \ "type(population) == {0}, " \ "type(bestIndividual) == {1}, " \ "type(shouldSavePopulation) == {2}".format( type(population), type(bestIndividual), type(shouldSavePopulation))) self._pathToLastSavedModel = None self._trainingLog.Save(locationForTrainingResults) def LoadBestModel(self, dirNameWithModelToLoad): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() warning: " \ "trainingLog was None! Potentially important details about" \ " training (or run) could haven't been saved!") bestModel = None if type(dirNameWithModelToLoad) == str: basePathToModel = self._createBasePathForResults() fullPathToModel = \ os.path.join( basePathToModel, dirNameWithModelToLoad, "best_model.pth") if os.path.isfile(fullPathToModel): bestModel = torch.load(fullPathToModel) self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() info: " \ "'training_results/{0}/best_model.pth' file has been " "loaded!".format(dirNameWithModelToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() error: " \ "cannot load 'best_model.pth' file - path does not" \ " exist! (dirname = '{0}')".format(dirNameWithModelToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() error: " \ "dirNameWithModelToLoad has wrong type! " \ "(expected: str, actual: {0})".format( type(dirNameWithModelToLoad))) return bestModel def LoadPopulation(self, dirNameWithPopulationToLoad): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() warning: " \ "trainingLog was None! Potentially important details about" \ " training (or run) could haven't been saved!") population = None if type(dirNameWithPopulationToLoad) == str: basePathToPopulation = self._createBasePathForResults() fullPathToPopulation = \ os.path.join( basePathToPopulation, dirNameWithPopulationToLoad, "population") if os.path.isdir(fullPathToPopulation): models = [] listOfModelFileNames = os.listdir(fullPathToPopulation) listOfModelFileNames.sort() for fileName in listOfModelFileNames: if fnmatch(fileName, "model_*.pth"): fullPathToModelFile = \ os.path.join(fullPathToPopulation, fileName) tempModel = torch.load(fullPathToModelFile) models.append(tempModel) if len(models) == 0: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "'training_results/{0}/population' is empty - " \ "has no 'model_<n>.pth' files! " \ "(examples: 'model_1.pth', 'model_2.pth' " \ "etc.)".format(dirNameWithPopulationToLoad)) else: population = models self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() info: " \ "'training_results/{0}/population' has been " \ "loaded!".format(dirNameWithPopulationToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "cannot load population from " \ "'training_results/{0}/population' - path " \ "does not exist!".format(dirNameWithPopulationToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "dirNameWithPopulationToLoad has wrong type! " \ "(expected: str, actual: {0})".format( type(dirNameWithPopulationToLoad))) return population def _doParametersHaveValidTypes(self, population, bestIndividual, shouldSavePopulation): return type(population) == list \ and type(bestIndividual) == AgentNeuralNetwork \ and type(shouldSavePopulation) == bool def _createLocationForTrainingResults(self): basePath = self._createBasePathForResults() if not os.path.isdir(basePath): os.mkdir(basePath) dirName = self._createDirNameForResults() fullPathToLocation = os.path.join(basePath, dirName) return fullPathToLocation def _createBasePathForResults(self): basePath = os.path.dirname(os.path.realpath(__file__)) basePathLastPart = "python_external_process" basePathEnd = \ basePath.index(basePathLastPart) + len(basePathLastPart) basePath = basePath[:basePathEnd] basePath = os.path.join(basePath, "training_results") return basePath def _createDirNameForResults(self): currentDateTime = datetime.now() dirName = "{0}_{1}_{2}_{3}_{4}_{5}".format( str(currentDateTime.year).zfill(2), str(currentDateTime.month).zfill(2), str(currentDateTime.day).zfill(2), str(currentDateTime.hour).zfill(2), str(currentDateTime.minute).zfill(2), str(currentDateTime.second).zfill(2)) return dirName def _saveBestModel(self, location, bestModel): bestModelFileName = "best_model.pth" fullPathForModelFile = os.path.join(location, bestModelFileName) torch.save(bestModel, fullPathForModelFile) def _saveWholePopulation(self, location, population): dirForPopulation = "population" locationForPopulationFiles = os.path.join(location, dirForPopulation) if os.path.isdir(locationForPopulationFiles): rmtree(locationForPopulationFiles) os.mkdir(locationForPopulationFiles) for i in range(len(population)): fileNameForModel = "model_{0}.pth".format(str(i + 1).zfill(3)) fullPathForModel = \ os.path.join(locationForPopulationFiles, fileNameForModel) torch.save(population[i], fullPathForModel)