def execute_seims_model(self): """Run SEIMS for evaluating environmental effectiveness. If execution fails, the `self.economy` and `self.environment` will be set the worst values. """ scoop_log('Scenario ID: %d, running SEIMS model...' % self.ID) self.model.scenario_id = self.ID self.modelout_dir = self.model.OutputDirectory self.model.run() self.modelrun = True return self.model.run_success
def main(): wtsd_name = get_watershed_name( 'Specify watershed name to run scenario analysis.') if wtsd_name not in list(DEMO_MODELS.keys()): print('%s is not one of the available demo watershed: %s' % (wtsd_name, ','.join(list(DEMO_MODELS.keys())))) exit(-1) cur_path = UtilClass.current_path(lambda: 0) SEIMS_path = os.path.abspath(cur_path + '../../..') model_paths = ModelPaths(SEIMS_path, wtsd_name, DEMO_MODELS[wtsd_name]) cf = write_scenario_analysis_config_file(model_paths, 'scenario_analysis.ini') base_cfg = SAConfig(cf) # type: SAConfig if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS cfg = SASlpPosConfig(cf) elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD cfg = SAConnFieldConfig(cf) else: # Common spatial units, e.g., HRU and EXPLICITHRU cfg = SACommUnitConfig(cf) cfg.construct_indexes_units_gene() sce = SUScenario(cfg) scoop_log('### START TO SCENARIOS OPTIMIZING ###') start_t = time.time() fpop, fstats = sa_nsga2.main(sce) fpop.sort(key=lambda x: x.fitness.values) scoop_log(fstats) with open(cfg.opt.logbookfile, 'w', encoding='utf-8') as f: # In case of 'TypeError: write() argument 1 must be unicode, not str' in Python2.7 # when using unicode_literals, please use '%s' to concatenate string! f.write('%s' % fstats.__str__()) end_t = time.time() scoop_log('Running time: %.2fs' % (end_t - start_t))
def main(cfg): """Main workflow of NSGA-II based Scenario analysis.""" random.seed() scoop_log('Population: %d, Generation: %d' % (cfg.opt.npop, cfg.opt.ngens)) # Initial timespan variables stime = time.time() plot_time = 0. allmodels_exect = list() # execute time of all model runs # create reference point for hypervolume ref_pt = numpy.array(worse_objects) * multi_weight * -1 stats = tools.Statistics(lambda sind: sind.fitness.values) stats.register('min', numpy.min, axis=0) stats.register('max', numpy.max, axis=0) stats.register('avg', numpy.mean, axis=0) stats.register('std', numpy.std, axis=0) logbook = tools.Logbook() logbook.header = 'gen', 'evals', 'min', 'max', 'avg', 'std' # read observation data from MongoDB cali_obj = Calibration(cfg) # Read observation data just once model_cfg_dict = cali_obj.model.ConfigDict model_obj = MainSEIMS(args_dict=model_cfg_dict) obs_vars, obs_data_dict = model_obj.ReadOutletObservations(object_vars) # Initialize population param_values = cali_obj.initialize(cfg.opt.npop) pop = list() for i in range(cfg.opt.npop): ind = creator.Individual(param_values[i]) ind.gen = 0 ind.id = i ind.obs.vars = obs_vars[:] ind.obs.data = deepcopy(obs_data_dict) pop.append(ind) param_values = numpy.array(param_values) # Write calibrated values to MongoDB # TODO, extract this function, which is same with `Sensitivity::write_param_values_to_mongodb`. write_param_values_to_mongodb(cfg.model.host, cfg.model.port, cfg.model.db_name, cali_obj.ParamDefs, param_values) # get the low and up bound of calibrated parameters bounds = numpy.array(cali_obj.ParamDefs['bounds']) low = bounds[:, 0] up = bounds[:, 1] low = low.tolist() up = up.tolist() pop_select_num = int(cfg.opt.npop * cfg.opt.rsel) init_time = time.time() - stime def check_validation(fitvalues): """Check the validation of the fitness values of an individual.""" flag = True for condidx, condstr in enumerate(conditions): if condstr is None: continue if not eval('%f%s' % (fitvalues[condidx], condstr)): flag = False return flag def evaluate_parallel(invalid_pops): """Evaluate model by SCOOP or map, and set fitness of individuals according to calibration step.""" popnum = len(invalid_pops) labels = list() try: # parallel on multi-processors or clusters using SCOOP from scoop import futures invalid_pops = list( futures.map(toolbox.evaluate, [cali_obj] * popnum, invalid_pops)) except ImportError or ImportWarning: # Python build-in map (serial) invalid_pops = list( toolbox.map(toolbox.evaluate, [cali_obj] * popnum, invalid_pops)) for tmpind in invalid_pops: tmpfitnessv = list() for k, v in list(multiobj.items()): tmpvalues, tmplabel = tmpind.cali.efficiency_values( k, object_names[k]) tmpfitnessv += tmpvalues[:] labels += tmplabel[:] tmpind.fitness.values = tuple(tmpfitnessv) # Filter for a valid solution if filter_ind: invalid_pops = [ tmpind for tmpind in invalid_pops if check_validation(tmpind.fitness.values) ] if len(invalid_pops) < 2: print( 'The initial population should be greater or equal than 2. ' 'Please check the parameters ranges or change the sampling strategy!' ) exit(2) return invalid_pops, labels # Currently, `invalid_pops` contains evaluated individuals # Record the count and execute timespan of model runs during the optimization modelruns_count = {0: len(pop)} modelruns_time = { 0: 0. } # Total time counted according to evaluate_parallel() modelruns_time_sum = { 0: 0. } # Summarize time of every model runs according to pop # Generation 0 before optimization stime = time.time() pop, plotlables = evaluate_parallel(pop) modelruns_time[0] = time.time() - stime for ind in pop: allmodels_exect.append( [ind.io_time, ind.comp_time, ind.simu_time, ind.runtime]) modelruns_time_sum[0] += ind.runtime # currently, len(pop) may less than pop_select_num pop = toolbox.select(pop, pop_select_num) # Output simulated data to json or pickle files for future use. output_population_details(pop, cfg.opt.simdata_dir, 0) record = stats.compile(pop) logbook.record(gen=0, evals=len(pop), **record) scoop_log(logbook.stream) # Begin the generational process output_str = '### Generation number: %d, Population size: %d ###\n' % ( cfg.opt.ngens, cfg.opt.npop) scoop_log(output_str) UtilClass.writelog(cfg.opt.logfile, output_str, mode='replace') modelsel_count = { 0: len(pop) } # type: Dict[int, int] # newly added Pareto fronts for gen in range(1, cfg.opt.ngens + 1): output_str = '###### Generation: %d ######\n' % gen scoop_log(output_str) offspring = [toolbox.clone(ind) for ind in pop] # method1: use crowding distance (normalized as 0~1) as eta # tools.emo.assignCrowdingDist(offspring) # method2: use the index of individual at the sorted offspring list as eta if len(offspring ) >= 2: # when offspring size greater than 2, mate can be done for i, ind1, ind2 in zip(range(len(offspring) // 2), offspring[::2], offspring[1::2]): if random.random() > cfg.opt.rcross: continue eta = i toolbox.mate(ind1, ind2, eta, low, up) toolbox.mutate(ind1, eta, low, up, cfg.opt.rmut) toolbox.mutate(ind2, eta, low, up, cfg.opt.rmut) del ind1.fitness.values, ind2.fitness.values else: toolbox.mutate(offspring[0], 1., low, up, cfg.opt.rmut) del offspring[0].fitness.values # Evaluate the individuals with an invalid fitness invalid_inds = [ind for ind in offspring if not ind.fitness.valid] valid_inds = [ind for ind in offspring if ind.fitness.valid] if len(invalid_inds) == 0: # No need to continue scoop_log( 'Note: No invalid individuals available, the NSGA2 will be terminated!' ) break # Write new calibrated parameters to MongoDB param_values = list() for idx, ind in enumerate(invalid_inds): ind.gen = gen ind.id = idx param_values.append(ind[:]) param_values = numpy.array(param_values) write_param_values_to_mongodb(cfg.model.host, cfg.model.port, cfg.model.db_name, cali_obj.ParamDefs, param_values) # Count the model runs, and execute models invalid_ind_size = len(invalid_inds) modelruns_count.setdefault(gen, invalid_ind_size) stime = time.time() invalid_inds, plotlables = evaluate_parallel(invalid_inds) curtimespan = time.time() - stime modelruns_time.setdefault(gen, curtimespan) modelruns_time_sum.setdefault(gen, 0.) for ind in invalid_inds: allmodels_exect.append( [ind.io_time, ind.comp_time, ind.simu_time, ind.runtime]) modelruns_time_sum[gen] += ind.runtime # Select the next generation population # Previous version may result in duplications of the same scenario in one Pareto front, # thus, I decided to check and remove the duplications first. # pop = toolbox.select(pop + valid_inds + invalid_inds, pop_select_num) tmppop = pop + valid_inds + invalid_inds pop = list() unique_sces = dict() for tmpind in tmppop: if tmpind.gen in unique_sces and tmpind.id in unique_sces[ tmpind.gen]: continue if tmpind.gen not in unique_sces: unique_sces.setdefault(tmpind.gen, [tmpind.id]) elif tmpind.id not in unique_sces[tmpind.gen]: unique_sces[tmpind.gen].append(tmpind.id) pop.append(tmpind) pop = toolbox.select(pop, pop_select_num) output_population_details(pop, cfg.opt.simdata_dir, gen) hyper_str = 'Gen: %d, New model runs: %d, ' \ 'Execute timespan: %.4f, Sum of model run timespan: %.4f, ' \ 'Hypervolume: %.4f\n' % (gen, invalid_ind_size, curtimespan, modelruns_time_sum[gen], hypervolume(pop, ref_pt)) scoop_log(hyper_str) UtilClass.writelog(cfg.opt.hypervlog, hyper_str, mode='append') record = stats.compile(pop) logbook.record(gen=gen, evals=len(invalid_inds), **record) scoop_log(logbook.stream) # Count the newly generated near Pareto fronts new_count = 0 for ind in pop: if ind.gen == gen: new_count += 1 modelsel_count.setdefault(gen, new_count) # Plot 2D near optimal pareto front graphs, # i.e., (NSE, RSR), (NSE, PBIAS), and (RSR,PBIAS) # And 3D near optimal pareto front graphs, i.e., (NSE, RSR, PBIAS) stime = time.time() front = numpy.array([ind.fitness.values for ind in pop]) plot_pareto_front_single(front, plotlables, cfg.opt.out_dir, gen, 'Near Pareto optimal solutions') plot_time += time.time() - stime # save in file # Header information output_str += 'generation\tcalibrationID\t' for kk, vv in list(object_names.items()): output_str += pop[0].cali.output_header(kk, vv, 'Cali') if cali_obj.cfg.calc_validation: for kkk, vvv in list(object_names.items()): output_str += pop[0].vali.output_header(kkk, vvv, 'Vali') output_str += 'gene_values\n' for ind in pop: output_str += '%d\t%d\t' % (ind.gen, ind.id) for kk, vv in list(object_names.items()): output_str += ind.cali.output_efficiency(kk, vv) if cali_obj.cfg.calc_validation: for kkk, vvv in list(object_names.items()): output_str += ind.vali.output_efficiency(kkk, vvv) output_str += str(ind) output_str += '\n' UtilClass.writelog(cfg.opt.logfile, output_str, mode='append') # TODO: Figure out if we should terminate the evolution # Plot hypervolume and newly executed model count plot_hypervolume_single(cfg.opt.hypervlog, cfg.opt.out_dir) # Save newly added Pareto fronts of each generations new_fronts_count = numpy.array(list(modelsel_count.items())) numpy.savetxt('%s/new_pareto_fronts_count.txt' % cfg.opt.out_dir, new_fronts_count, delimiter=str(','), fmt=str('%d')) # Save and print timespan information allmodels_exect = numpy.array(allmodels_exect) numpy.savetxt('%s/exec_time_allmodelruns.txt' % cfg.opt.out_dir, allmodels_exect, delimiter=str(' '), fmt=str('%.4f')) scoop_log('Running time of all SEIMS models:\n' '\tIO\tCOMP\tSIMU\tRUNTIME\n' 'MAX\t%s\n' 'MIN\t%s\n' 'AVG\t%s\n' 'SUM\t%s\n' % ('\t'.join('%.3f' % t for t in allmodels_exect.max(0)), '\t'.join('%.3f' % t for t in allmodels_exect.min(0)), '\t'.join( '%.3f' % t for t in allmodels_exect.mean(0)), '\t'.join( '%.3f' % t for t in allmodels_exect.sum(0)))) exec_time = 0. for genid, tmptime in list(modelruns_time.items()): exec_time += tmptime exec_time_sum = 0. for genid, tmptime in list(modelruns_time_sum.items()): exec_time_sum += tmptime allcount = 0 for genid, tmpcount in list(modelruns_count.items()): allcount += tmpcount scoop_log('Initialization timespan: %.4f\n' 'Model execution timespan: %.4f\n' 'Sum of model runs timespan: %.4f\n' 'Plot Pareto graphs timespan: %.4f' % (init_time, exec_time, exec_time_sum, plot_time)) return pop, logbook
for genid, tmpcount in list(modelruns_count.items()): allcount += tmpcount scoop_log('Initialization timespan: %.4f\n' 'Model execution timespan: %.4f\n' 'Sum of model runs timespan: %.4f\n' 'Plot Pareto graphs timespan: %.4f' % (init_time, exec_time, exec_time_sum, plot_time)) return pop, logbook if __name__ == "__main__": cf, method = get_optimization_config() cali_cfg = CaliConfig(cf, method=method) scoop_log('### START TO CALIBRATION OPTIMIZING ###') startT = time.time() fpop, fstats = main(cali_cfg) fpop.sort(key=lambda x: x.fitness.values) scoop_log(fstats) with open(cali_cfg.opt.logbookfile, 'w', encoding='utf-8') as f: # In case of 'TypeError: write() argument 1 must be unicode, not str' in Python2.7 # when using unicode_literals, please use '%s' to concatenate string! f.write('%s' % fstats.__str__()) endT = time.time() scoop_log('### END OF CALIBRATION OPTIMIZING ###') scoop_log('Running time: %.2fs' % (endT - startT))
def main(sceobj): # type: (SUScenario) -> () """Main workflow of NSGA-II based Scenario analysis.""" if sceobj.cfg.eval_info['BASE_ENV'] < 0: run_base_scenario(sceobj) print('The environment effectiveness value of the ' 'base scenario is %.2f' % sceobj.cfg.eval_info['BASE_ENV']) random.seed() # Initial timespan variables stime = time.time() plot_time = 0. allmodels_exect = list() # execute time of all model runs pop_size = sceobj.cfg.opt.npop gen_num = sceobj.cfg.opt.ngens cx_rate = sceobj.cfg.opt.rcross mut_perc = sceobj.cfg.opt.pmut mut_rate = sceobj.cfg.opt.rmut sel_rate = sceobj.cfg.opt.rsel pop_select_num = int(pop_size * sel_rate) ws = sceobj.cfg.opt.out_dir cfg_unit = sceobj.cfg.bmps_cfg_unit cfg_method = sceobj.cfg.bmps_cfg_method worst_econ = sceobj.worst_econ worst_env = sceobj.worst_env # available gene value list possible_gene_values = list(sceobj.bmps_params.keys()) if 0 not in possible_gene_values: possible_gene_values.append(0) units_info = sceobj.cfg.units_infos suit_bmps = sceobj.suit_bmps gene_to_unit = sceobj.cfg.gene_to_unit unit_to_gene = sceobj.cfg.unit_to_gene updown_units = sceobj.cfg.updown_units scoop_log('Population: %d, Generation: %d' % (pop_size, gen_num)) scoop_log('BMPs configure unit: %s, configuration method: %s' % (cfg_unit, cfg_method)) # create reference point for hypervolume ref_pt = numpy.array([worst_econ, worst_env]) * multi_weight * -1 stats = tools.Statistics(lambda sind: sind.fitness.values) stats.register('min', numpy.min, axis=0) stats.register('max', numpy.max, axis=0) stats.register('avg', numpy.mean, axis=0) stats.register('std', numpy.std, axis=0) logbook = tools.Logbook() logbook.header = 'gen', 'evals', 'min', 'max', 'avg', 'std' # Initialize population initialize_byinputs = False if sceobj.cfg.initial_byinput and sceobj.cfg.input_pareto_file is not None and \ sceobj.cfg.input_pareto_gen > 0: # Initial by input Pareto solutions inpareto_file = sceobj.modelcfg.model_dir + os.sep + sceobj.cfg.input_pareto_file if os.path.isfile(inpareto_file): inpareto_solutions = read_pareto_solutions_from_txt(inpareto_file, sce_name='scenario', field_name='gene_values') if sceobj.cfg.input_pareto_gen in inpareto_solutions: pareto_solutions = inpareto_solutions[sceobj.cfg.input_pareto_gen] pop = toolbox.population_byinputs(sceobj.cfg, pareto_solutions) # type: List initialize_byinputs = True if not initialize_byinputs: pop = toolbox.population(sceobj.cfg, n=pop_size) # type: List init_time = time.time() - stime def delete_fitness(new_ind): """Delete the fitness and other information of new individual.""" del new_ind.fitness.values new_ind.gen = -1 new_ind.id = -1 new_ind.io_time = 0. new_ind.comp_time = 0. new_ind.simu_time = 0. new_ind.runtime = 0. def check_validation(fitvalues): """Check the validation of the fitness values of an individual.""" flag = True for condidx, condstr in enumerate(conditions): if condstr is None: continue if not eval('%f%s' % (fitvalues[condidx], condstr)): flag = False return flag def evaluate_parallel(invalid_pops): """Evaluate model by SCOOP or map, and get fitness of individuals.""" popnum = len(invalid_pops) try: # parallel on multiprocesor or clusters using SCOOP from scoop import futures invalid_pops = list(futures.map(toolbox.evaluate, [sceobj.cfg] * popnum, invalid_pops)) except ImportError or ImportWarning: # serial invalid_pops = list(map(toolbox.evaluate, [sceobj.cfg] * popnum, invalid_pops)) # Filter for a valid solution if filter_ind: invalid_pops = [tmpind for tmpind in invalid_pops if check_validation(tmpind.fitness.values)] if len(invalid_pops) < 2: print('The initial population should be greater or equal than 2. ' 'Please check the parameters ranges or change the sampling strategy!') exit(2) return invalid_pops # Currently, `invalid_pops` contains evaluated individuals # Record the count and execute timespan of model runs during the optimization modelruns_count = {0: len(pop)} modelruns_time = {0: 0.} # Total time counted according to evaluate_parallel() modelruns_time_sum = {0: 0.} # Summarize time of every model runs according to pop # Generation 0 before optimization stime = time.time() pop = evaluate_parallel(pop) modelruns_time[0] = time.time() - stime for ind in pop: ind.gen = 0 allmodels_exect.append([ind.io_time, ind.comp_time, ind.simu_time, ind.runtime]) modelruns_time_sum[0] += ind.runtime # Currently, len(pop) may less than pop_select_num pop = toolbox.select(pop, pop_select_num) record = stats.compile(pop) logbook.record(gen=0, evals=len(pop), **record) scoop_log(logbook.stream) front = numpy.array([ind.fitness.values for ind in pop]) # save front for further possible use numpy.savetxt(sceobj.scenario_dir + os.sep + 'pareto_front_gen0.txt', front, delimiter=str(' '), fmt=str('%.4f')) # Begin the generational process output_str = '### Generation number: %d, Population size: %d ###\n' % (gen_num, pop_size) scoop_log(output_str) UtilClass.writelog(sceobj.cfg.opt.logfile, output_str, mode='replace') modelsel_count = {0: len(pop)} # type: Dict[int, int] # newly added Pareto fronts for gen in range(1, gen_num + 1): output_str = '###### Generation: %d ######\n' % gen scoop_log(output_str) offspring = [toolbox.clone(ind) for ind in pop] if len(offspring) >= 2: # when offspring size greater than 2, mate can be done for ind1, ind2 in zip(offspring[::2], offspring[1::2]): old_ind1 = toolbox.clone(ind1) old_ind2 = toolbox.clone(ind2) if random.random() <= cx_rate: if cfg_method == BMPS_CFG_METHODS[3]: # SLPPOS method toolbox.mate_slppos(ind1, ind2, sceobj.cfg.hillslp_genes_num) elif cfg_method == BMPS_CFG_METHODS[2]: # UPDOWN method toolbox.mate_updown(updown_units, gene_to_unit, unit_to_gene, ind1, ind2) else: toolbox.mate_rdm(ind1, ind2) if cfg_method == BMPS_CFG_METHODS[0]: toolbox.mutate_rdm(possible_gene_values, ind1, perc=mut_perc, indpb=mut_rate) toolbox.mutate_rdm(possible_gene_values, ind2, perc=mut_perc, indpb=mut_rate) else: tagnames = None if sceobj.cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: tagnames = sceobj.cfg.slppos_tagnames toolbox.mutate_rule(units_info, gene_to_unit, unit_to_gene, suit_bmps, ind1, perc=mut_perc, indpb=mut_rate, unit=cfg_unit, method=cfg_method, tagnames=tagnames, thresholds=sceobj.cfg.boundary_adaptive_threshs) toolbox.mutate_rule(units_info, gene_to_unit, unit_to_gene, suit_bmps, ind2, perc=mut_perc, indpb=mut_rate, unit=cfg_unit, method=cfg_method, tagnames=tagnames, thresholds=sceobj.cfg.boundary_adaptive_threshs) if check_individual_diff(old_ind1, ind1): delete_fitness(ind1) if check_individual_diff(old_ind2, ind2): delete_fitness(ind2) # Evaluate the individuals with an invalid fitness invalid_inds = [ind for ind in offspring if not ind.fitness.valid] valid_inds = [ind for ind in offspring if ind.fitness.valid] invalid_ind_size = len(invalid_inds) if invalid_ind_size == 0: # No need to continue scoop_log('Note: No invalid individuals available, the NSGA2 will be terminated!') break modelruns_count.setdefault(gen, invalid_ind_size) stime = time.time() invalid_inds = evaluate_parallel(invalid_inds) curtimespan = time.time() - stime modelruns_time.setdefault(gen, curtimespan) modelruns_time_sum.setdefault(gen, 0.) for ind in invalid_inds: ind.gen = gen allmodels_exect.append([ind.io_time, ind.comp_time, ind.simu_time, ind.runtime]) modelruns_time_sum[gen] += ind.runtime # Select the next generation population # Previous version may result in duplications of the same scenario in one Pareto front, # thus, I decided to check and remove the duplications first. # pop = toolbox.select(pop + valid_inds + invalid_inds, pop_select_num) tmppop = pop + valid_inds + invalid_inds pop = list() unique_sces = dict() for tmpind in tmppop: if tmpind.gen in unique_sces and tmpind.id in unique_sces[tmpind.gen]: continue if tmpind.gen not in unique_sces: unique_sces.setdefault(tmpind.gen, [tmpind.id]) elif tmpind.id not in unique_sces[tmpind.gen]: unique_sces[tmpind.gen].append(tmpind.id) pop.append(tmpind) pop = toolbox.select(pop, pop_select_num) hyper_str = 'Gen: %d, New model runs: %d, ' \ 'Execute timespan: %.4f, Sum of model run timespan: %.4f, ' \ 'Hypervolume: %.4f\n' % (gen, invalid_ind_size, curtimespan, modelruns_time_sum[gen], hypervolume(pop, ref_pt)) scoop_log(hyper_str) UtilClass.writelog(sceobj.cfg.opt.hypervlog, hyper_str, mode='append') record = stats.compile(pop) logbook.record(gen=gen, evals=len(invalid_inds), **record) scoop_log(logbook.stream) # Count the newly generated near Pareto fronts new_count = 0 for ind in pop: if ind.gen == gen: new_count += 1 modelsel_count.setdefault(gen, new_count) # Plot 2D near optimal pareto front graphs stime = time.time() front = numpy.array([ind.fitness.values for ind in pop]) # save front for further possible use numpy.savetxt(sceobj.scenario_dir + os.sep + 'pareto_front_gen%d.txt' % gen, front, delimiter=str(' '), fmt=str('%.4f')) # Comment out the following plot code if matplotlib does not work. try: from scenario_analysis.visualization import plot_pareto_front_single pareto_title = 'Near Pareto optimal solutions' xlabel = 'Economy' ylabel = 'Environment' if sceobj.cfg.plot_cfg.plot_cn: xlabel = r'经济净投入' ylabel = r'环境效益' pareto_title = r'近似最优Pareto解集' plot_pareto_front_single(front, [xlabel, ylabel], ws, gen, pareto_title, plot_cfg=sceobj.cfg.plot_cfg) except Exception as e: scoop_log('Exception caught: %s' % str(e)) plot_time += time.time() - stime # save in file output_str += 'generation\tscenario\teconomy\tenvironment\tgene_values\n' for indi in pop: output_str += '%d\t%d\t%f\t%f\t%s\n' % (indi.gen, indi.id, indi.fitness.values[0], indi.fitness.values[1], str(indi)) UtilClass.writelog(sceobj.cfg.opt.logfile, output_str, mode='append') # Plot hypervolume and newly executed model count # Comment out the following plot code if matplotlib does not work. try: from scenario_analysis.visualization import plot_hypervolume_single plot_hypervolume_single(sceobj.cfg.opt.hypervlog, ws, plot_cfg=sceobj.cfg.plot_cfg) except Exception as e: scoop_log('Exception caught: %s' % str(e)) # Save newly added Pareto fronts of each generations new_fronts_count = numpy.array(list(modelsel_count.items())) numpy.savetxt('%s/new_pareto_fronts_count.txt' % ws, new_fronts_count, delimiter=str(','), fmt=str('%d')) # Save and print timespan information allmodels_exect = numpy.array(allmodels_exect) numpy.savetxt('%s/exec_time_allmodelruns.txt' % ws, allmodels_exect, delimiter=str(' '), fmt=str('%.4f')) scoop_log('Running time of all SEIMS models:\n' '\tIO\tCOMP\tSIMU\tRUNTIME\n' 'MAX\t%s\n' 'MIN\t%s\n' 'AVG\t%s\n' 'SUM\t%s\n' % ('\t'.join('%.3f' % v for v in allmodels_exect.max(0)), '\t'.join('%.3f' % v for v in allmodels_exect.min(0)), '\t'.join('%.3f' % v for v in allmodels_exect.mean(0)), '\t'.join('%.3f' % v for v in allmodels_exect.sum(0)))) exec_time = 0. for genid, tmptime in list(modelruns_time.items()): exec_time += tmptime exec_time_sum = 0. for genid, tmptime in list(modelruns_time_sum.items()): exec_time_sum += tmptime allcount = 0 for genid, tmpcount in list(modelruns_count.items()): allcount += tmpcount scoop_log('Initialization timespan: %.4f\n' 'Model execution timespan: %.4f\n' 'Sum of model runs timespan: %.4f\n' 'Plot Pareto graphs timespan: %.4f' % (init_time, exec_time, exec_time_sum, plot_time)) return pop, logbook
return pop, logbook if __name__ == "__main__": in_cf = get_config_parser() base_cfg = SAConfig(in_cf) # type: SAConfig if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS sa_cfg = SASlpPosConfig(in_cf) elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD sa_cfg = SAConnFieldConfig(in_cf) else: # Common spatial units, e.g., HRU and EXPLICITHRU sa_cfg = SACommUnitConfig(in_cf) sa_cfg.construct_indexes_units_gene() sce = SUScenario(sa_cfg) scoop_log('### START TO SCENARIOS OPTIMIZING ###') startT = time.time() fpop, fstats = main(sce) fpop.sort(key=lambda x: x.fitness.values) scoop_log(fstats) with open(sa_cfg.opt.logbookfile, 'w', encoding='utf-8') as f: # In case of 'TypeError: write() argument 1 must be unicode, not str' in Python2.7 # when using unicode_literals, please use '%s' to concatenate string! f.write('%s' % fstats.__str__()) endT = time.time() scoop_log('Running time: %.2fs' % (endT - startT))