def test_Append_InvalidParameters(self, isLogVerbose, invalidInfoParameter): trainingLog = Logger(isLogVerbose) logContentBeforeCall = "Ford Sierra II 2.0 DOHC 125KM\n" trainingLog._content = logContentBeforeCall with patch('sys.stdout', new=StringIO()) as fakeOutput: trainingLog.Append(invalidInfoParameter) expectedPrintMessage = "" actualPrintMessage = fakeOutput.getvalue() self.assertEqual(actualPrintMessage, expectedPrintMessage) logContentAfterCall = trainingLog._content self.assertEqual(logContentAfterCall, logContentBeforeCall)
def test_Append(self, isLogVerbose, infoToAppend, expectedPrintMessage, mock_datetime): mock_datetime.now.return_value = datetime(1995, 7, 4, 17, 15, 0) trainingLog = Logger(isLogVerbose) logContentBeforeCall = "Ford Sierra II 2.0 DOHC 125KM\n" trainingLog._content = logContentBeforeCall with patch('sys.stdout', new=StringIO()) as fakeOutput: trainingLog.Append(infoToAppend) actualPrintMessage = fakeOutput.getvalue() self.assertEqual(actualPrintMessage, expectedPrintMessage) expectedLogContentAfterCall = \ logContentBeforeCall + "[ 1995-07-04 17:15:00 ] " + \ infoToAppend + "\n" actualLogContentAfterCall = trainingLog._content self.assertEqual(actualLogContentAfterCall, expectedLogContentAfterCall)
def experiment(options): # --- Check if directory with Unity builds does exist --- # pathToBuildsDir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "env_builds") if not os.path.isdir(pathToBuildsDir): print("Error: 'env_builds' directory doesn't exist! " \ "Run 'make_builds.py' to fix it!") exit() # --- Validation of command line args --- # numberOfTrials = options["--num-of-trials"] errorMessage = "Invalid numberOfTrials value! Should be positive integer! " \ "Actual type of numberOfTrials: '{0}', actual value: '{1}'.".format( type(numberOfTrials), numberOfTrials) if isinstance(numberOfTrials, str) and numberOfTrials.isdigit(): numberOfTrials = int(numberOfTrials) if numberOfTrials <= 0: print(errorMessage) exit() else: print(errorMessage) exit() del errorMessage # --- Create logger object --- # experimentLog = \ Logger(isVerbose = options["--verbose"], fileName = "experiment") experimentLog.Append("Experiment log has been created!") experimentLog.Append("numberOfTrials = {0}".format(numberOfTrials)) # --- Load config data from file --- # pathToConfigFile = options["<config-file-path>"] CONFIG_DATA = loadConfigData(pathToConfigFile) experimentLog.Append( "Config data has been loaded from file: {0}".format(pathToConfigFile)) # --- Determine builds paths --- # buildTarget = CONFIG_DATA["MakeBuilds"]["Target"] buildPaths = CONFIG_DATA["BuildPaths"][buildTarget] # --- Create data collector object --- # dataCollector = ExperimentDataCollector() experimentLog.Append("Data collector object has been created!") # --- Prepare minimal fitness dict for validation purposes --- # minFitnessDict = CONFIG_DATA["TrainingParameters"][ "minimalAcceptableFitness"] # --- Experiment sequence loop --- # for trialCounter in range(numberOfTrials): for trackNumber in range(1, 4): experimentLog.Append("Generating data from 'train_de.py', track: " \ "{0}, trial: {1}".format(trackNumber, trialCounter + 1)) generateDataFromTraining(train_de, "DE", trackNumber, pathToConfigFile, buildPaths, experimentLog, dataCollector, minFitnessDict, isVerbose=options["--verbose"]) experimentLog.Append("Generating data from 'train_pso.py', track: " \ "{0}, trial: {1}".format(trackNumber, trialCounter + 1)) generateDataFromTraining(train_pso, "PSO", trackNumber, pathToConfigFile, buildPaths, experimentLog, dataCollector, minFitnessDict, isVerbose=options["--verbose"]) # --- Create location for charts --- # pathToResultsDir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "experiment_results") if not os.path.isdir(pathToResultsDir): os.mkdir(pathToResultsDir) experimentLog.Append( "'{0}' directory has been created!".format(pathToResultsDir)) pathToCurrentResults = createPathForCurrentResults(pathToResultsDir) if os.path.isdir(pathToCurrentResults): rmtree(pathToCurrentResults) os.mkdir(pathToCurrentResults) experimentLog.Append( "'{0}' directory has been created!".format(pathToCurrentResults)) # --- Generate charts --- # chartsGenerator = ChartsGenerator(dataCollector, pathToCurrentResults) experimentLog.Append("Charts generator objects has been created!") chartsGenerator.CreateAll() experimentLog.Append("All charts have been generated!") # --- Save log file --- # experimentLog.Append( "End of log file, save to '{0}'.".format(pathToCurrentResults)) experimentLog.Save(pathToCurrentResults)
class TrainingResultsRepository: def __init__(self, trainingLog=None): self._trainingLog = trainingLog self._pathToLastSavedModel = None def Save(self, population, bestIndividual, shouldSavePopulation): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.Save() warning: " \ "trainingLog was None! Potentially important details about " \ "training (or run) could haven't been saved!") locationForTrainingResults = self._createLocationForTrainingResults() os.mkdir(locationForTrainingResults) if self._doParametersHaveValidTypes(population, bestIndividual, shouldSavePopulation): if len(population) == 0: self._trainingLog.Append( "TrainingResultsRepository.Save() error: " \ "population is empty!") self._pathToLastSavedModel = None else: self._saveBestModel(locationForTrainingResults, bestIndividual) if shouldSavePopulation: self._saveWholePopulation(locationForTrainingResults, population) self._trainingLog.Append( "TrainingResultsRepository.Save() info: " \ "training results were saved to the location " \ "'{0}'!".format(locationForTrainingResults)) self._pathToLastSavedModel = locationForTrainingResults else: self._trainingLog.Append( "TrainingResultsRepository.Save() error: " \ "some of parameters have wrong type!\n" \ "type(population) == {0}, " \ "type(bestIndividual) == {1}, " \ "type(shouldSavePopulation) == {2}".format( type(population), type(bestIndividual), type(shouldSavePopulation))) self._pathToLastSavedModel = None self._trainingLog.Save(locationForTrainingResults) def LoadBestModel(self, dirNameWithModelToLoad): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() warning: " \ "trainingLog was None! Potentially important details about" \ " training (or run) could haven't been saved!") bestModel = None if type(dirNameWithModelToLoad) == str: basePathToModel = self._createBasePathForResults() fullPathToModel = \ os.path.join( basePathToModel, dirNameWithModelToLoad, "best_model.pth") if os.path.isfile(fullPathToModel): bestModel = torch.load(fullPathToModel) self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() info: " \ "'training_results/{0}/best_model.pth' file has been " "loaded!".format(dirNameWithModelToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() error: " \ "cannot load 'best_model.pth' file - path does not" \ " exist! (dirname = '{0}')".format(dirNameWithModelToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadBestModel() error: " \ "dirNameWithModelToLoad has wrong type! " \ "(expected: str, actual: {0})".format( type(dirNameWithModelToLoad))) return bestModel def LoadPopulation(self, dirNameWithPopulationToLoad): if self._trainingLog is None: self._trainingLog = Logger(isVerbose=False) self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() warning: " \ "trainingLog was None! Potentially important details about" \ " training (or run) could haven't been saved!") population = None if type(dirNameWithPopulationToLoad) == str: basePathToPopulation = self._createBasePathForResults() fullPathToPopulation = \ os.path.join( basePathToPopulation, dirNameWithPopulationToLoad, "population") if os.path.isdir(fullPathToPopulation): models = [] listOfModelFileNames = os.listdir(fullPathToPopulation) listOfModelFileNames.sort() for fileName in listOfModelFileNames: if fnmatch(fileName, "model_*.pth"): fullPathToModelFile = \ os.path.join(fullPathToPopulation, fileName) tempModel = torch.load(fullPathToModelFile) models.append(tempModel) if len(models) == 0: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "'training_results/{0}/population' is empty - " \ "has no 'model_<n>.pth' files! " \ "(examples: 'model_1.pth', 'model_2.pth' " \ "etc.)".format(dirNameWithPopulationToLoad)) else: population = models self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() info: " \ "'training_results/{0}/population' has been " \ "loaded!".format(dirNameWithPopulationToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "cannot load population from " \ "'training_results/{0}/population' - path " \ "does not exist!".format(dirNameWithPopulationToLoad)) else: self._trainingLog.Append( "TrainingResultsRepository.LoadPopulation() error: " \ "dirNameWithPopulationToLoad has wrong type! " \ "(expected: str, actual: {0})".format( type(dirNameWithPopulationToLoad))) return population def _doParametersHaveValidTypes(self, population, bestIndividual, shouldSavePopulation): return type(population) == list \ and type(bestIndividual) == AgentNeuralNetwork \ and type(shouldSavePopulation) == bool def _createLocationForTrainingResults(self): basePath = self._createBasePathForResults() if not os.path.isdir(basePath): os.mkdir(basePath) dirName = self._createDirNameForResults() fullPathToLocation = os.path.join(basePath, dirName) return fullPathToLocation def _createBasePathForResults(self): basePath = os.path.dirname(os.path.realpath(__file__)) basePathLastPart = "python_external_process" basePathEnd = \ basePath.index(basePathLastPart) + len(basePathLastPart) basePath = basePath[:basePathEnd] basePath = os.path.join(basePath, "training_results") return basePath def _createDirNameForResults(self): currentDateTime = datetime.now() dirName = "{0}_{1}_{2}_{3}_{4}_{5}".format( str(currentDateTime.year).zfill(2), str(currentDateTime.month).zfill(2), str(currentDateTime.day).zfill(2), str(currentDateTime.hour).zfill(2), str(currentDateTime.minute).zfill(2), str(currentDateTime.second).zfill(2)) return dirName def _saveBestModel(self, location, bestModel): bestModelFileName = "best_model.pth" fullPathForModelFile = os.path.join(location, bestModelFileName) torch.save(bestModel, fullPathForModelFile) def _saveWholePopulation(self, location, population): dirForPopulation = "population" locationForPopulationFiles = os.path.join(location, dirForPopulation) if os.path.isdir(locationForPopulationFiles): rmtree(locationForPopulationFiles) os.mkdir(locationForPopulationFiles) for i in range(len(population)): fileNameForModel = "model_{0}.pth".format(str(i + 1).zfill(3)) fullPathForModel = \ os.path.join(locationForPopulationFiles, fileNameForModel) torch.save(population[i], fullPathForModel)
message += " Try again to train population!" else: message += " Unfortunately, cannot try again. " \ "Reason: achieved maximum number of repeats!" trainingLog.Append(message) except KeyboardInterrupt: trainingLog.Append( "\nTraining interrupted because of KeyboardInterrupt!") trainingLog.Append("End of training!") # --- Close environment --- # env.close() trainingLog.Append("Closed Unity environment.") # --- Save training results --- # shouldSavePopulation = options["--save-population"] resultsRepository.Save(population, bestAgent, shouldSavePopulation) if isTrainInExperimentMode: dataCollector.PathToLastSavedModel = \ resultsRepository._pathToLastSavedModel if __name__ == "__main__": options = getProgramOptions() trainingLog = Logger(isVerbose=options["--verbose"]) trainingLog.Append("Training log has been created!") train_de(options, trainingLog)