def multiOptimizeNSGA2(restart_=False): """ # for multi objective optimization # clear; clear; time nice -4 python OptimizeAlgorithm.py --ai sfAI --checkAgentInteraction --numScenarios 1 --scenarioSetInitId=0 --scenarioDIR data/scenarios/representativeSet_Intersections/ --paramFile xml/config/sf-param-config-multi.xml --statsOnly -p 1 --benchmark compositePLE --optimizer="NSGA2" -c # 3d multi opjective optimization # clear; clear; time nice -4 optirun python OptimizeAlgorithm.py --ai sfAI --checkAgentInteraction --numScenarios 1 --scenarioSetInitId=0 --scenarioDIR data/scenarios/customSet/bottleneck-hallway/ --paramFile xml/config/sf-param-config-multi.xml --statsOnly -p 1 --benchmark compositePLE --optimizer="NSGA2" --numFrames 2500 --RealDataDir=data/RealWorldData/b140_combined/ -v -c # to restart # clear; clear; time nice -4 optirun python OptimizeAlgorithm.py --ai sfAI --checkAgentInteraction --numScenarios 1 --scenarioSetInitId=0 --scenarioDIR data/scenarios/customSet/bottleneck-hallway/ --paramFile xml/config/sf-param-config-multi.xml --statsOnly -p 6 --benchmark compositePLE --optimizer="NSGA2" --numFrames 2500 --RealDataDir=data/RealWorldData/b140_combined/ -c --multiObjRestartFile=SteerStatsOpt_1.log # clear; clear; time nice -4 optirun python OptimizeAlgorithm.py --ai sfAI --checkAgentInteraction --numScenarios 1 --scenarioSetInitId=0 --scenarioDIR data/scenarios/customSet/bottleneck-hallway/ --paramFile xml/config/sf-param-config-multi.xml --statsOnly -p 6 --benchmark compositePLE --optimizer="NSGA2" --numFrames 2500 --RealDataDir=data/RealWorldData/b140_combined/ -c --multiObjRestartFile=SteerStatsOpt_2.log """ options_ = getOptions() availableProcesses_=int(options_.processes) options_.processes=1 [ai_bounds_, ai_params_list, ai_param_names_list, stddev_for_parameters, ai_step_scaling] = prepareOptimization(options.paramFile, options) steerStats_ = SteerStats(options_) steerStats_.setParamNameDict(ai_param_names_list) cmaFilePrefix=options_.cmaFilePrefix print "Algorithm param names: " + str(ai_param_names_list) print ai_bounds_ print ai_params_list NDIM_ = len(ai_param_names_list) # f_ = (steerStats_.performanceMetric, steerStats_.pleMetricGlobal, steerStats_.entropyMetric) f_ = (steerStats_.performanceMetric, steerStats_.pleMetricGlobal) return multiOptimizeWithNSGA2(NDIM=NDIM_, NGEN=3, MU=8, f=f_, options=options_, ai_bounds=ai_bounds_, availableProcesses=availableProcesses_, steerStats=steerStats_, restart=restart_)
def OptimizeParameters2(self, options): options = getOptions() options.noReturn = True availableProcesses = int(options.processes) options.processes = int(1) steerStats = SteerStats(options) steerStats.set_insert_func(InsertStatistics.InsertOptimizationConfig) param_xml_file = open(options.paramFile) param_config_data = param_xml_file.read() OptimExper = OptimizationExperiment(param_config_data) optimizingParameters = OptimExper.parseExperimentXML() # print "Algorithm param names: " + str(ai_param_names_list) experiements = [] aiParams = [] for param in optimizingParameters: # this will create a lot of directories but I won't have to worry about # syncroniztion and reuse of directories for n in range(int(param.get_descetization())): aiParams_tmp = {} """ for param_ in optimizingParameters: if param_.get_type() == "float": aiParams_tmp[param_.get_name()] = str(float(param_.get_original())) elif param_.get_type() == "integer": aiParams_tmp[param_.get_name()] = str(int(param_.get_original())) """ # calculate param value for this iteration if param.get_type() == "float": paramValue = param.get_min() + ( ((param.get_max() - param.get_min()) / param.get_descetization()) * n) elif param.get_type() == "integer": paramValue = int(param.get_min() + (((param.get_max() - param.get_min()) / param.get_descetization()) * n)) aiParams_tmp[param.get_name()] = paramValue tmp_params = [] tmp_param_names = [] for key, value in aiParams_tmp.items(): tmp_params.append(value) tmp_param_names.append(key) experiements.append(copy.deepcopy(tmp_params)) aiParams.append(copy.deepcopy(tmp_param_names)) #TODO stuff here. print "Number of experiments in-line: " + str(len(experiements)) print "Size of process pool: " + str(availableProcesses) # sys.exit() processes_pool = Pool(availableProcesses) processes_pool.map(steerStats.RunStats, experiements, aiParams) print "Waited for subprocess"
def runScenarioSpace(): options = getOptions() steerstats = SteerStats(options) options.dumpTestcases=True random.seed(10) totalSimulations = 0 samples=10 step = 50 dataLog = open("scenarioSpace.cvs", "w+") for nScenarios in range(step, int(501), step): for i in range(0, samples, 1): options.numScenarios = nScenarios steerstats._options.numScenarios = nScenarios _rand = random.randint(1, 5000) options.randomSeed=_rand results = steerstats.RunStats(None, None, options) cov = steerstats.coverageMetric(None, results, options) dataLog.write(str(nScenarios)+", " + str(cov) + "\n") print "Coverage is: " + str(cov) totalSimulations = totalSimulations + nScenarios dataLog.flush() # this should just save the mean and std after every group... print "totalSimulations: " + str(totalSimulations)
def test_one_scenario(self): # _args="--ai=sfAI" _args = ['--ai', 'pprAI', '--skipInsert', '-c', '--numScenarios', '1'] options = getOptions(_args) steerStats = SteerStats(options) results = steerStats.RunStats() # print "Length of result: " + (str(len(results))) self.assertEqual(1, len(results))
def test_ten_random_scenarios(self): # _args="--ai=sfAI" _args = [ '--ai', self._ai, '--skipInsert', '-c', '--numScenarios', '10' ] options = getOptions(_args) steerStats = SteerStats(options) results = steerStats.RunStats() # print "Length of result: " + (str(len(results))) self.assertEqual(10, len(results))
def test_3_squeeze_scenarios(self): # _args="--ai=sfAI" _args = [ '--ai', self._ai, '--skipInsert', '-c', '--numScenarios', '1', '--scenarioDIR', 'data/scenarios/testcases/' ] options = getOptions(_args) steerStats = SteerStats(options) results = steerStats.RunStats() # print "Length of result: " + (str(len(results))) self.assertEqual(1, len(results))
def InsertOptimizationConfig(parameters, configFilePath, aiConfigParams): insertData = InsertStatistics(parameters, configFilePath) print aiConfigParams print insertData options = getOptions() param_xml_file = open(options.paramFile) param_config_data = param_xml_file.read() OptimExper = OptimizationExperiment(param_config_data) optimizingParameters = OptimExper.parseExperimentXML() for param in optimizingParameters: if not param.get_name() in aiConfigParams: if param.get_type() == "float": aiConfigParams[param.get_name()] = str( float(param.get_original())) elif param.get_type() == "integer": aiConfigParams[param.get_name()] = str( int(param.get_original())) aiConfigParams['ppr_ai_config_id'] = insertData['config_id'] _args = parseArg(parameters) if 'dbName' in _args: conM = ConnectionManager(dbname=_args['dbName']) if options.verbose: print "DataBase used: " + _args['dbName'] else: if options.verbose: print "DataBase used: " + "steersuitedb" conM = ConnectionManager() cur = conM.getCursor() if _args['scenarioAI'] == "pprAI": config_ = PPRConfig(aiConfigParams) aiConfigParams['ppr_ai_config_id'] = insertData['config_id'] elif _args['scenarioAI'] == "rvo2dAI": config_ = RVO2DConfig(aiConfigParams) aiConfigParams['rvo2d_ai_config_id'] = insertData['config_id'] elif _args['scenarioAI'] == "sfAI": config_ = SFConfig(aiConfigParams) aiConfigParams['sf_ai_config_id'] = insertData['config_id'] else: print "Configuration not supported for " + str(_args['scenarioAI']) config_.insertConfig2(cur) conM.dispose() # pprC2 = pprC.getConfigData(cur, 1) return insertData
def OptimizeWithCMA_ES_MixedInt(): print "blah" options = getOptions() cmaFilePrefix=options.cmaFilePrefix result = "blah" param_xml_file = open(options.paramFile) param_config_data = param_xml_file.read() OptimExper = OptimizationExperiment(param_config_data) optimizingParameters = OptimExper.parseExperimentXML() stddev_for_parameters=1 scaling_factor=float(0.13) ai_params_list = [] ai_param_names_list = [] ai_lower_bounds = [] ai_upper_bounds = [] ai_step_scaling = [] for ai_param in optimizingParameters: ai_params_list.append(ai_param.get_original()) ai_param_names_list.append(ai_param.get_name()) ai_lower_bounds.append(ai_param.get_min()) ai_upper_bounds.append(ai_param.get_max()) """ This is magic stuff here Scaling stddev for all the parameters acording the min/max values for the search apparently this should be 0.3 of the search area if your search area is perfectly round """ ai_step_scaling.append( (ai_param.get_max()-ai_param.get_min())*scaling_factor ) print "Algorithm param names: " + str(ai_param_names_list) ai_bounds = [ai_lower_bounds, ai_upper_bounds] steerStats = SteerStats(options) steerStats.setParamNameDict(ai_param_names_list) print ai_lower_bounds print ai_params_list print ai_upper_bounds steerStats.pleMetric(ai_params_list, None)
def runScenarioSpace2(): options = getOptions() steerstats = SteerStats(options) options.dumpTestcases=True random.seed(10) totalSimulations = 0 samples=25 step = 500 p = Pool(int(options.processes)) dataLog = open("scenarioSpace.cvs", "w+") dataLog_data = open("scenarioSpace_data.cvs", "w+") for nScenarios in range(step, int(options.numScenarios), step): # for nScenarios in range(step, int(501), step): _rands = [] _nScenarios = [] for i in range(0, samples, 1): totalSimulations = totalSimulations + nScenarios _rands.append(random.randint(1, 5000)) # args = " -module scenario,scenarioAI=sfAI,useBenchmark,benchmarkTechnique=compositePLE,checkAgentValid,reducedGoals,fixedSpeed,checkAgentRelevant,minAgents=3,maxFrames=1000,checkAgentInteraction,egocentric,scenarioSetInitId=0,numScenarios="+str(nScenarios)+",randomSeed="+str(_rand)+",skipInsert=True,logAIStats -config configs/scenarioSpace/scenarioSpace-config.xml -commandLine" # args = args.split(' ') _nScenarios.append(nScenarios) process_pairs = zip(_nScenarios, _rands, [options] * len(_rands)) options.numScenarios = nScenarios # print process_pairs # print zip(_nScenarios, _rands) # sys.exit(0) results = p.map(runSteerSuite, process_pairs) metrics = [] for result in results: if result is not None: cov = (coverage(result, options)) dataLog_data.write(str(nScenarios)+", " + str(cov) + "\n") dataLog_data.flush() metrics.append(cov) print "Coverage is: " + str(cov) metrics = np.array(metrics) dataLog.write(str(nScenarios)+", " + str(np.mean(metrics)) + ", " + str(np.std(metrics)) + "\n") dataLog.flush() # this should just save the mean and std after every group... print "totalSimulations: " + str(totalSimulations)
def InsertStatistics(params, configFilePath, dataQueue=None): # pr = cProfile.Profile() # pr.enable() __dbSem.acquire() options = getOptions() args = parseArg(params) # the same command passed to the module stats = [] cur = None scenario_id = 0 config_id = 0 if 'recFileFolder' in args: recFileNameSuffix = args['recFile'] # "rec" recFileNamePath = args['recFileFolder'] # "./" # frameBasePath=args['framePath']# "scenarioFrames/" benchmarkLog = args['benchmarkLog'] # "test.log" if 'testCasePath' in args: # check if key is in dict testFilePath = args['testCasePath'] testFilePrefix = "testcase-" elif 'subspace' in args: testFilePath = args['subspace'] testFilePrefix = '' subspaceFile = open(testFilePath, 'r') elif options.randScenarios: testFilePrefix = "testcase-" # dumped testcases testFilePath = args['testCasePath'] else: testFilePath = args['scenarioSetPath'] testFilePrefix = "testcase-" #if "scenarioSetInitId" in args: # scenarioSetInitId = 0 # args['scenarioSetInitId'] #else: # scenarioSetInitId=0 videoType = "mp4" videoBaseName = "scenario" # Need to implement in C++ (scenarioModule) scenario_description = str(params) # items that may change (different ai algorithm or benchmark) benchmarkName = args[ 'benchmarkTechnique'] # "Composite02" or "Composite01" algorithmName = args['scenarioAI'] # "pprAI" or "footstapAI" algorithmLogFileName = args['ailogFileName'] config_id = 1 configFile = open(configFilePath, "r") # print "config file path: " + configFilePath configFileData = configFile.read() config = Configuration(configFileData) configFile.close() test_comments = "AlgorithmName: " + algorithmName test_comments = test_comments + ", BenchmarkName: " + benchmarkName # The point of all of these varaibles is to make scripting easier. # Naming convention will tailor to conforming to steersuite practice # allowing for ai names and benchmark names to match the easiest. try: if not 'skipInsert' in args: if 'dbName' in args: conM = ConnectionManager(dbname=args['dbName']) if options.verbose: print "DataBase used: " + args['dbName'] else: if options.verbose: print "DataBase used: " + "steersuitedb" conM = ConnectionManager() cur = conM.getCursor() # need the benchmark_id benchmark = Benchmark() benchmark_id = benchmark.getBenchmarkIndex(cur, benchmarkName) # also need a algorithm id algorithm = Algorithm() algorithm_id = algorithm.getAlgorithmIndex(cur, algorithmName) # We can load the configuration data for this config_data = {} config_data['name'] = "stats_config" config_data['fps'] = config.getFPS() config_data['grid_size_z'] = config.getGridSizeZ() config_data['grid_size_x'] = config.getGridSizeX() config_data['grid_cells_z'] = config.getGridCellsZ() config_data['grid_cells_x'] = config.getGridCellsX() config_data['config_xml'] = configFileData configuration = Config() configuration.setValuesFromDict(config_data) config_id = configuration.insertConfig2(cur) #then insert a scenario for this scenario scenario = Scenario(algorithm_id, benchmark_id, config_id, scenario_description) scenario_id = scenario.insertScenario2(cur) if options.verbose: print "scenario id: " + str(scenario_id) # add steering algorithms as needed if algorithmName == "pprAI": algorithmAI = PprAI() elif algorithmName == "footstepAI": algorithmAI = FootstepAI() elif algorithmName == "ccAI": algorithmAI = CcAI() elif algorithmName == "rvo2dAI": algorithmAI = RVO2DAI() elif algorithmName == "reactiveAI": algorithmAI = ReactiveAI() elif algorithmName == "egocentricAI": algorithmAI = EgocentricAI() elif algorithmName == "sfAI": algorithmAI = SocialForcesAI() else: print "Unknown AI algorithm" sys.exit(0) try: print '============' + benchmarkLog benchmarkLogFile = open(benchmarkLog, "r") rand_seed = benchmarkLogFile.readline() #save / remove random seed lparser = LogParser() logData = lparser.parseLog(benchmarkLogFile) print '============' + algorithmLogFileName # pr = cProfile.Profile() # pr.enable() algorithmLogFile = open(algorithmLogFileName, "r") algorithmData = lparser.parseLog(algorithmLogFile) except IOError: print "Are you sure you have compiled SteerSuite?" raise IOError("Are you sure you have compiled SteerSuite?") # print logData # sys.exit(0) if options.verbose: print "logdata length: " + str(len(logData)) print "algorithmData length: " + str(len(algorithmData)) # sys.exit(0) if 'skipInsert' in args: # pr = cProfile.Profile() # pr.enable() for ldat, aData in zip(logData, algorithmData): n = ldat['scenario_id'] if 'subspace' in args: testcaseFile = open(testFilePath, 'r') else: testcaseFile = open( testFilePath + "/" + testFilePrefix + n + ".xml", "r") testcase = testcaseFile.read() testcase_ = TestCase(testcase) test_status = ldat['agent_success'] ldat['test_comments'] = test_comments # ldat['benchmark_type']=benchmark_id # update benchmark_id because of naming issues # ldat['test_case']=testcase ldat['test_status'] = test_status # ldat['scenario_group']=scenario_id # __test_case = TestCase(testcase) # // TODO # ldat['num_agents']=__test_case.getNumAgents() # ldat['num_obstacles']=__test_case.getNumObstacles() ldat['num_agents'] = testcase_.getNumAgents() ldat['num_obstacles'] = testcase_.getNumObstacles() # needed for data but not used in calculations ldat['algorithm_data_id'] = 0 ldat['benchmark_type'] = 0 ldat['test_comments'] = test_comments ldat['test_case'] = "" ldat['test_status'] = "" ldat['scenario_group'] = scenario_id if benchmarkName == "compositeEntropy": baseStats = BaseCompositeEntropy() elif benchmarkName == "composite02": baseStats = BaseComposite02() else: baseStats = BaseCompositePLE() baseStats.setMetricValuesFromDict( dict(ldat.items() + aData.items())) stats.append(baseStats) else: for ldat, aData in zip(logData, algorithmData): testcase = "" # n= str(int(ldat['scenario_id']) + int(scenarioSetInitId)) # the numbering for the file n = ldat['scenario_id'] # print "Scenario Id: " + n # then insert test data # read dumped test xml if 'subspace' in args: testcaseFile = open(testFilePath, 'r') else: testcaseFile = open( testFilePath + "/" + testFilePrefix + n + ".xml", "r") testcase = testcaseFile.read() testcase_ = TestCase(testcase) # print testcase # read the record data if 'recFileFolder' in args: recordFile = open( recFileNamePath + "rec" + n + "." + recFileNameSuffix, "r+b") recordbytes = recordFile.read() recordFile.close() recordArray = bytearray() for bite in recordbytes: recordArray.append(bite) """pathToVideoFile=frameBasePath+videoBaseName+n+"."+videoType print "Video file: " + pathToVideoFile videoFile = open(pathToVideoFile, "r+b") videobytes = videoFile.read() videoFile.close() videoArray = bytearray() for vbite in videobytes: videoArray.append(vbite)""" # now that all of the file have been read properly insert a test # need a way to check status # test_status=0 test_status = ldat['agent_success'] algorithmAI.setValuesFromDict(aData) algorithm_data_id = algorithmAI.insertAlgorithmData2(cur) ldat['algorithm_data_id'] = algorithm_data_id ldat['test_comments'] = test_comments ldat[ 'benchmark_type'] = benchmark_id # update benchmark_id because of nameing issues ldat['test_case'] = testcase ldat['test_status'] = test_status ldat['scenario_group'] = scenario_id __test_case = TestCase(testcase) # // TODO # ldat['num_agents']=__test_case.getNumAgents() # ldat['num_obstacles']=__test_case.getNumObstacles() ldat['num_agents'] = testcase_.getNumAgents() ldat['num_obstacles'] = testcase_.getNumObstacles() #print ldat # then other stuff for config. # check is config already exists # save width and heigth stuff if benchmarkName == "composite01": benchmark = Composite1Benchmark() elif benchmarkName == "composite02": benchmark = Composite2Benchmark() elif benchmarkName == "compositeGraph": benchmark = CompositeGraphBenchmark() elif benchmarkName == "compositePLE": benchmark = CompositePLEBenchmark() else: print "Invalid benchmark type: " + benchmarkName sys.exit() # print ldat benchmark.setBenchmarkValuesFromDict(ldat) benchmark_data_id = benchmark.insertBenchmark2(cur) if benchmark_data_id % 50 == 0: if options.verbose: print "Test id: " + str(benchmark_data_id) """video = Video(benchmark_data_id, videoArray) status = video.insertVideo(cur)""" if 'recFileFolder' in args: recording = Recording(benchmark_data_id, recordArray) status = recording.insertRecording(cur) testcaseFile.close() if benchmarkName == "compositeEntropy": baseStats = BaseCompositeEntropy() elif benchmarkName == "composite02": baseStats = BaseComposite02() else: baseStats = BaseCompositePLE() baseStats.setMetricValuesFromDict( dict(ldat.items() + aData.items())) stats.append(baseStats) # last is benchmark # oid = psycopg2.extensions.lobject() # close logFile benchmarkLogFile.close() if options.verbose: print "scenario id: " + str(scenario_id) out = {} out['scenario_id'] = scenario_id out['config_id'] = config_id __dbSem.release() if dataQueue: dataQueue.put(stats) # pr.disable() # f = open('x.prof', 'a') # pstats.Stats(pr, stream=f).sort_stats('time').print_stats() # f.close() return out # videoFileOut.close() except psycopg2.DatabaseError, e: print 'Error pprAI-test: %s' % e
def OptimizeWithBruteForce(): # Only works for 2D for now # clear; clear; time nice -18 python OptimizeAlgorithm.py --ai sfAI --checkAgentInteraction --numScenarios 1 --benchmark compositePLE --statsOnly --scenarioSetInitId 0 --subspace=../subspaces/icra-subspaces/hallway-one-way-100-agents-funnel.xml --dataDir=data/ --numFrames=2000 --optimizeWith=bruteForce --paramFile=xml/config/subspaces/icra-subspaces/hallway-one-way-1pillar-smallOptimizationRegion.xml -c -p 4 --logMetrics import time # aint Python crazy like this # from multiprocessing import Pool as Pool from multiprocessing.pool import ThreadPool import itertools options = getOptions() # options.noReturn=True availableProcesses=int(options.processes) options.processes=int(1) steerStats = SteerStats(options) # steerStats.set_insert_func(InsertStatistics.InsertOptimizationConfig) [ai_bounds, ai_params_list, ai_param_names_list, stddev_for_parameters, ai_step_scaling] = prepareOptimization(options.paramFile, options) # steerStats = SteerStats(options) steerStats.setParamNameDict(ai_param_names_list) cmaFilePrefix=options.cmaFilePrefix param_xml_file = open(options.paramFile) param_config_data = param_xml_file.read() default_params={} for param_name,t_param in zip(ai_param_names_list,ai_params_list): default_params[param_name]=t_param print default_params OptimExper = OptimizationExperiment(param_config_data) optimizingParameters = OptimExper.parseExperimentXML() op = OptimizeAlgorithm(options=options) ppe = PostProcessEvent_TarData(op) pped = PostProcessEvent_DumpOptResults(op) ppeh = PostProcessEvent_CreateHeatMap(op) # op._options=options # op.set_metric_func(steerStats.timeMetricGlobal) op.set_steerstats(steerStats) # op.set_metric_func(steerStats.simulationTimeMetricGlobal) if options.objectiveFunction != "": op.set_metric_func(steerStats.getBoundObjective(options.objectiveFunction)) else: # op.set_metric_func(steerStats.agentFlowMetricGlobal) print 'blah' op.set_penatly_funct(overlapPenalty) # result = OptimizeParamsCMA(op.eval, # ai_params_list, # stddev_for_parameters, ai_bounds, # options.cmaProcesses, # cmaFilePrefix, # ai_step_scaling) #opt_log = open(cmaFilePrefix+"SteerStatsOptResult.txt", "w") #writeCMAResults(opt_log, result) #opt_log.close() # print "Algorithm param names: " + str(ai_param_names_list) # print optimizingParameters experiements = [] experiements_param = [] aiParams = [] for param in optimizingParameters: # this will create a lot of directories but I won't have to worry about # syncroniztion and reuse of directories for n in range(int(param.get_descetization())): # aiParams_tmp = default_params aiParams_tmp = {} """ for param_ in optimizingParameters: if param_.get_type() == "float": aiParams_tmp[param_.get_name()] = str(float(param_.get_original())) elif param_.get_type() == "integer": aiParams_tmp[param_.get_name()] = str(int(param_.get_original())) """ # calculate param value for this iteration if param.get_type() == "float": paramValue = param.get_min() + (((param.get_max()-param.get_min())/param.get_descetization())*n) elif param.get_type() == "integer": paramValue = int(param.get_min() + (((param.get_max()-param.get_min())/param.get_descetization())*n)) aiParams_tmp[param.get_name()] = paramValue tmp_params = [] tmp_param_names = [] for key, value in aiParams_tmp.items() : tmp_params.append(value) tmp_param_names.append(key) # print tmp_params # print tmp_param_names experiements_param.append(copy.deepcopy(paramValue)) aiParams.append(copy.deepcopy(tmp_param_names)) experiements.append(experiements_param) experiements_param = [] #TODO stuff here. print "Number of experiments in-line: " + str(len(experiements)) print "Size of process pool: " + str(availableProcesses) #print "experiements: " + str(experiements) # print "" experiements = list(itertools.product(experiements[0],experiements[1])) # print "Cross product: " + str(experiements) # sys.exit() #try: processes_pool = ThreadPool(availableProcesses) # results = processes_pool.apply(op.eval, experiements) for item in experiements: # this ensures the results come out in the same order the the experiemtns are in this list. processes_pool.apply_async(op.eval, args = (item, ), callback = op.log_result) processes_pool.close() processes_pool.join() # print op._result_list # Does not acctually place an obstacle in the scenario because it comes from options.subspaceParams control_result = steerStats.RunStats((0,0), options=options) control_result = op._metric_func((0,0),results=control_result ,options=options) # print "Control result: " + str(control_result) op._result_list = control_result - numpy.array(op._result_list) # print "Corrected results" + str( op._result_list) # processes_pool = Pool(availableProcesses) # results = processes_pool.map(op.eval, experiements) # except : # print "Multi-processing failed: " op.set_experiements(experiements) # results = map(op.eval, experiements) print "Waited for subprocess" # print "Results: " + str(zip(results,experiements)) # write all of the result to a file. op.add_post_process_event(pped) # create a heatmap from the results op.add_post_process_event(ppeh) # this post processing step should be added last op.add_post_process_event(ppe) op.post_process() print "Done"
def OptimizeWithCMA(): # clear; clear; time python SteerStats.py --ai pprAI --checkAgentInteraction --numScenarios 5000 --benchmark compositeGraph -c --scenarioSetInitId=0 --scenarioDIR data/scenarios/representativeSet --statsOnly -p 8 options = getOptions() [ai_bounds, ai_params_defaults_list, ai_param_names_list, stddev_for_parameters, ai_step_scaling] = prepareOptimization(options.paramFile, options) steerStats = SteerStats(options) steerStats.setParamNameDict(ai_param_names_list) cmaFilePrefix=options.cmaFilePrefix print "Algorithm param names: " + str(ai_param_names_list) print ai_bounds print ai_params_defaults_list # sys.exit() # print results # JUst coverage metric #result = OptimizeParamsCMA(steerStats.coverageMetric, ai_params_list, # stddev_for_parameters, ai_bounds, # cmaFilePrefix) # JUst coverage metric #result = OptimizeParamsCMA(steerStats.performanceMetric, ai_params_list, # stddev_for_parameters, ai_bounds, cmaFilePrefix) # # JUst coverage metric # result = OptimizeParamsCMA(steerStats.distanceMetric, ai_params_list, # stddev_for_parameters, ai_bounds, cmaFilePrefix) # Coverage plus frames metric #result = OptimizeParamsCMA(steerStats.coveragePlusFramesMetric, ai_params_list, # stddev_for_parameters, ai_bounds, cmaFilePrefix) # coverage plus distance metric # result = OptimizeParamsCMA(steerStats.coveragePlusDistanceMetric, ai_params_list, # stddev_for_parameters, ai_bounds, cmaFilePrefix) # coverage + computational_time #result = OptimizeParamsCMA(steerStats.coveragePlusPerformanceMetric, # ai_params_list, #1 stddev_for_parameters, ai_bounds, cmaFilePrefix) # computation time + converage + quality_distance # result = OptimizeParamsCMA(steerStats.coveragePlusDistancePlusComputationMetric, # ai_params_list, # stddev_for_parameters, ai_bounds, # cmaFilePrefix) # computation time + converage + quality_distance op = OptimizeAlgorithm(options=options) op.set_steerstats(steerStats) ppe = PostProcessEvent_TarData(op) pped = PostProcessEvent_DumpOptResults(op) ppev = PostProcessEvent_MakeVideo(op) # op.set_metric_func(steerStats.timeMetricGlobal) if options.objectiveFunction != "": metric = steerStats.getBoundObjective(options.objectiveFunction) if metric is None: print '***** objecive function ' + str(options.objectiveFunction) + ' not found *******' sys.exit(1) print "objective Function: " + str(options.objectiveFunction) + ", found: " + str(metric) op.set_metric_func(steerStats.getBoundObjective(options.objectiveFunction)) else: op.set_metric_func(steerStats.agentFlowMetricGlobal) print 'blah' if options.penaltyFunction != "overlapPenalty": penalty = steerStats.getBoundPenaltyFunc(options.objectiveFunction) op.set_penatly_funct(penalty) else: op.set_penatly_funct(overlapPenalty) # Does not acctually place an obstacle in the scenario because it comes from options.subspaceParams #control_result = steerStats.RunStats((0,0), options=options) # hard coded stuff uggh... #control_result = op._metric_func((0,0),results=control_result ,options=options) # control_result = steerStats.RunStats(ai_params_defaults_list, options=options) # print "control result: " + str(control_result) # print "op: " + str(op) # control_result = op._metric_func(ai_params_defaults_list,results=control_result ,options=options) # op._control_result = control_result result = OptimizeParamsCMA(op.eval, ai_params_defaults_list, stddev_for_parameters, ai_bounds, options.cmaProcesses, cmaFilePrefix, ai_step_scaling, maxfevals_=options.cmaMaxEvals) print "Control result: " + str(op._control_result) opt_log = open(cmaFilePrefix+"SteerStatsOptResult.txt", "w") writeCMAResults(opt_log, result) opt_log.close() # write all of the result to a file. op.add_post_process_event(pped) # create a heatmap from the results # op.add_post_process_event(ppeh) # make video ppev.set_ai_params(result[0]) # Record a video of the result #if options.config.getProcessingEvents()['PostProcessEvent_MakeVideo'] == 'true': # op.add_post_process_event(ppev) # this post processing step should be added last (compressing everything) op.add_post_process_event(ppe) op.post_process() print "Done"
# results = map(op.eval, experiements) print "Waited for subprocess" # print "Results: " + str(zip(results,experiements)) # write all of the result to a file. op.add_post_process_event(pped) # create a heatmap from the results op.add_post_process_event(ppeh) # this post processing step should be added last op.add_post_process_event(ppe) op.post_process() print "Done" if __name__ == "__main__": options = getOptions() if options.optimizationAlgorithm == "CMA-ES": OptimizeWithCMA() elif options.optimizationAlgorithm == "midaco": OptimizeWithMIDACO() elif options.optimizationAlgorithm == "bruteForce": OptimizeWithBruteForce() elif options.optimizationAlgorithm == "CMA-ES-MixInt": OptimizeWithCMA_ES_MixedInt() elif options.optimizationAlgorithm == "NSGA2": if options.multiObjRestartFile == "": pop, stats = multiOptimizeNSGA2() else: pop, stats = multiOptimizeNSGA2(restart_=True) print(stats) pop.sort(key=lambda x: x.fitness.values)
def runScenarioSpace3(): options = getOptions() steerstats = SteerStats(options) options.dumpTestcases=True random.seed(10) totalSimulations = 0 samples=24 step = 500 timelimit=600 dataLog = open("scenarioSpace.cvs", "w+") dataLog_data = open("scenarioSpace_data.cvs", "w+") terminate=False for nScenarios in range(step, int(options.numScenarios), step): # for nScenarios in range(step, int(1501), step): processes_pool = Pool(int(options.processes)) terminate=False results=[] for i in range(0, samples, 1): try: totalSimulations = totalSimulations + nScenarios steerstats._options.numScenarios = nScenarios result = processes_pool.apply_async(runSteerSuite2, args = (nScenarios, random.randint(1, 5000), options, ), callback = handleSteerSuite) results.append(result) except Exception as inst: print "Exception processing result " + str(inst) # terminate=True; continue # result.get(timeout=120) # process_pairs = zip(_nScenarios, _rands, [options] * len(_rands)) # processes_pool.close() # print process_pairs # print zip(_nScenarios, _rands) # sys.exit(0) # results = p.map(runSteerSuite, process_pairs) metrics = [] for result in results: try: # print "Getting Result" _result = result.get(timeout=timelimit) # print "RESULT: " + str(_result) cov = (coverage(_result, options)) _time = (qualityTime(_result, options)) _dist = (qualityPathLength(_result, options)) _effort = (qualityEffort(_result, options)) _result=None _line = str(nScenarios)+", " + str(cov) + ", " + str(_time) + ", " + str(_dist) + ", " + str(_effort) + "\n" print _line dataLog_data.write(_line) dataLog_data.flush() metrics.append([cov, _time, _dist, _effort] ) except Exception as inst: print "Exception getting result " + str(inst) terminate=True; continue if terminate is True: # Using seems to leave a lot of memory behind processes_pool.terminate() # Will this end processes early? else: processes_pool.close() # Will this end processes early? # processes_pool.terminate() # Will this end processes early? processes_pool.join() processes_pool=None metrics = np.array(metrics) print "Metrics: " + str(metrics) # dataLog.write(str(nScenarios)+", " + str(np.mean(metrics, axis=0)) + ", " + str(np.std(metrics, axis=0)) + "\n") dataLog.write(str(nScenarios)) for _item in np.mean(metrics, axis=0): dataLog.write(", " + str(_item)) for _item in np.std(metrics, axis=0): dataLog.write(", " + str(_item)) dataLog.write("\n") dataLog.flush() # this should just save the mean and std after every group... print "totalSimulations: " + str(totalSimulations) dataLog.close() dataLog_data.close()