def read_pareto_solutions_from_txt(txt_file, sce_name='scenario', field_name='gene_values'): # type: (AnyStr, AnyStr, AnyStr) -> (Dict[int, List[List[float]]]) """Read Pareto points from `runtime.log` file. Args: txt_file: Full file path of `runtime.log` output by NSGA2 algorithm. sce_name: Field name followed by `generation`, e.g., 'calibrationID', 'scenario', etc. field_name: Filed name in header for gene values, 'gene_values' by default Returns: pareto_solutions: `OrderedDict`, key is generation ID, value is arrays of Pareto solutions """ with open(txt_file, 'r', encoding='utf-8') as f: lines = f.readlines() pareto_solutions = OrderedDict() found = False cur_gen = -1 field_idx = -1 for lno, line in enumerate(lines): str_line = line for LF in LFs: if LF in line: str_line = line.split(LF)[0] break if str_line == '': continue values = StringClass.extract_numeric_values_from_string(str_line) # Check generation if str_line[0] == '#' and 'Generation' in str_line: if len(values) != 1: continue # e.g., ###### Generation: 23 ###### gen = int(values[0]) found = True cur_gen = gen pareto_solutions[cur_gen] = list() continue if not found: # If the first "###### Generation: 1 ######" has not been found. continue line_list = StringClass.split_string(str_line.upper(), ['\t']) if values is None: # means header line if field_idx >= 0: continue for idx, v in enumerate(line_list): if field_name.upper() in v.upper(): field_idx = idx break continue if field_idx < 0: continue # now append the real Pareto solutions data tmpvalues = StringClass.extract_numeric_values_from_string(line_list[field_idx]) pareto_solutions[cur_gen].append(tmpvalues[:]) return pareto_solutions
def read_optionfuzinf_section(self, _optfuzinf): """Optional parameter-settings for Fuzzy slope position inference.""" if _optfuzinf not in self.cf.sections(): return if self.cf.has_option(_optfuzinf, 'inferparams'): fuzinf_strs = self.cf.get(_optfuzinf, 'inferparams') if StringClass.string_match(fuzinf_strs, 'none'): return self.inferparam = dict() fuzinf_types = StringClass.split_string(fuzinf_strs, ';') if len(fuzinf_types) != len(self.slppostype): raise RuntimeError( "InferParams (%s) MUST be consistent with slope position types" " and separated by ';'!" % fuzinf_strs) for i, slppos in enumerate(self.slppostype): self.inferparam[slppos] = dict() infparams = StringClass.extract_numeric_values_from_string( fuzinf_types[i]) if len(infparams) % 4 != 0: raise RuntimeError( "Each item of InferParams MUST contains four elements," "i.e., Attribute No., FMF No., w1, w2! Please check item: " "%s for %s." % (fuzinf_types[i], slppos)) for j in range(int(len(infparams) / 4)): attridx = int(infparams[j * 4]) - 1 attrname = self.selectedtopolist[attridx] fmf = self._FMFTYPE[int(infparams[j * 4 + 1])] curinfparam = self._FMFPARAM[fmf][:] curinfparam[0] = infparams[j * 4 + 2] # w1 curinfparam[3] = infparams[j * 4 + 3] # w2 self.inferparam[slppos][attrname] = [fmf] + curinfparam
def read_bmp_parameters(self): """Read BMP configuration from MongoDB. Each BMP is stored in Collection as one item identified by 'SUBSCENARIO' field, so the `self.bmps_params` is dict with BMP_ID ('SUBSCENARIO') as key. """ # client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port) # conn = client.get_conn() conn = MongoDBObj.client scenariodb = conn[self.scenario_db] bmpcoll = scenariodb[self.cfg.bmps_coll] findbmps = bmpcoll.find({}, no_cursor_timeout=True) for fb in findbmps: fb = UtilClass.decode_strs_in_dict(fb) if 'SUBSCENARIO' not in fb: continue curid = fb['SUBSCENARIO'] if curid not in self.cfg.bmps_subids: continue if curid not in self.bmps_params: self.bmps_params[curid] = dict() for k, v in fb.items(): if k == 'SUBSCENARIO': continue elif k == 'LANDUSE': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = None else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] elif k == 'SLPPOS': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = list(self.cfg.slppos_tags.keys()) else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] else: self.bmps_params[curid][k] = v
def Precipitation(self, subbsn_id, start_time, end_time): # type: (int, datetime, datetime) -> List[List[Union[datetime, float]]] """ The precipitation is read according to the subbasin ID. Especially when plot a specific subbasin (such as ID 3). For the whole basin, the subbasin ID is 0. Returns: Precipitation data list with the first element as datetime. [[Datetime1, value1], [Datetime2, value2], ..., [Datetimen, valuen]] """ pcp_date_value = list() sitelist_tab = self.maindb[DBTableNames.main_sitelist] findsites = sitelist_tab.find_one({ FieldNames.subbasin_id: subbsn_id, FieldNames.mode: self.Mode }) if findsites is not None: site_liststr = findsites[FieldNames.site_p] else: raise RuntimeError( 'Cannot find precipitation site for subbasin %d.' % subbsn_id) site_list = StringClass.extract_numeric_values_from_string( site_liststr) site_list = [int(v) for v in site_list] if len(site_list) == 0: raise RuntimeError( 'Cannot find precipitation site for subbasin %d.' % subbsn_id) pcp_dict = OrderedDict() for pdata in self.climatedb[DBTableNames.data_values].find({ DataValueFields.utc: { "$gte": start_time, '$lte': end_time }, DataValueFields.type: DataType.p, DataValueFields.id: { "$in": site_list } }).sort([(DataValueFields.utc, 1)]): curt = pdata[DataValueFields.utc] curv = pdata[DataValueFields.value] if curt not in pcp_dict: pcp_dict[curt] = 0. pcp_dict[curt] += curv # average if len(site_list) > 1: for t in pcp_dict: pcp_dict[t] /= len(site_list) for t, v in pcp_dict.items(): # print(str(t), v) pcp_date_value.append([t, v]) print('Read precipitation from %s to %s done.' % (start_time.strftime('%c'), end_time.strftime('%c'))) return pcp_date_value
def read_inf_param_from_file(conf): """Read fuzzy inference parameters from file.""" params_list = list() with open(conf, 'r', encoding='utf-8') as f: for line in f.readlines(): eles = line.split('\n')[0].split('\t') params = StringClass.extract_numeric_values_from_string(line.split('\n')[0]) if StringClass.string_match(eles[0], 'Parameters') and len(params) >= 6: params_list.append([eles[1]] + [eles[3]] + params[-6:]) return params_list
def read_bmp_parameters(self): """Read BMP configuration from MongoDB.""" client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() scenariodb = conn[self.bmp_scenario_db] bmpcoll = scenariodb[self.bmps_coll] findbmps = bmpcoll.find({}, no_cursor_timeout=True) for fb in findbmps: fb = UtilClass.decode_strs_in_dict(fb) if 'SUBSCENARIO' not in fb: continue curid = fb['SUBSCENARIO'] if curid not in self.bmps_subids: continue if curid not in self.bmps_params: self.bmps_params[curid] = dict() for k, v in fb.items(): if k == 'SUBSCENARIO': continue elif k == 'LANDUSE': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = None else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] elif k == 'SLPPOS': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = list(self.slppos_tags.keys()) else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] else: self.bmps_params[curid][k] = v client.close()
def read_pareto_popsize_from_txt(txt_file, sce_name='scenario'): # type: (AnyStr, AnyStr) -> (List[int], List[int]) """Read the population size of each generations.""" with open(txt_file, 'r', encoding='utf-8') as f: lines = f.readlines() pareto_popnum = OrderedDict() found = False cur_gen = -1 iden_idx = -1 for line in lines: str_line = line for LF in LFs: if LF in line: str_line = line.split(LF)[0] break if str_line == '': continue values = StringClass.extract_numeric_values_from_string(str_line) # Check generation if str_line[0] == '#' and 'Generation' in str_line: if len(values) != 1: continue gen = int(values[0]) found = True cur_gen = gen pareto_popnum[cur_gen] = list() continue if not found: continue if values is None: # means header line line_list = StringClass.split_string(str_line, ['\t']) for idx, v in enumerate(line_list): if StringClass.string_match(v, sce_name): iden_idx = idx break continue if iden_idx < 0: continue # now append the real Pareto front point data pareto_popnum[cur_gen].append(int(values[iden_idx])) all_sceids = list() acc_num = list() genids = sorted(list(pareto_popnum.keys())) for idx, genid in enumerate(genids): for _id in pareto_popnum[genid]: if _id not in all_sceids: all_sceids.append(_id) acc_num.append(len(all_sceids)) return genids, acc_num
def get_input_cfgs(): """Get model configuration arguments. Returns: InputArgs object. """ parser = argparse.ArgumentParser( description="Read AutoFuzSlpPos configurations.") parser.add_argument('-ini', help="Full path of configuration file.") parser.add_argument('-bin', help="Path of executable programs, which will override" "exeDir in *.ini file.") parser.add_argument('-proc', help="Number of processor for parallel computing, " "which will override inputProc in *.ini file.") parser.add_argument('-dem', help="DEM of study area.") parser.add_argument('-root', help="Workspace to store results, which will override " "rootDir in *.ini file.") args = parser.parse_args() ini_file = args.ini bin_dir = args.bin input_proc = args.proc rawdem = args.dem root_dir = args.root if input_proc is not None: xx = StringClass.extract_numeric_values_from_string(input_proc) if xx is None or len(xx) != 1: raise RuntimeError("-proc MUST be one integer number!") input_proc = int(xx[0]) else: input_proc = -1 if not FileClass.is_file_exists(ini_file): if FileClass.is_file_exists(rawdem) and os.path.isdir(bin_dir): # In this scenario, the script can be executed by default setting, i.e., the *.ini # file is not required. cf = None if input_proc < 0: input_proc = cpu_count() / 2 else: raise RuntimeError( "*.ini file MUST be provided when '-dem', '-bin', " "and '-root' are not provided!") else: cf = ConfigParser() cf.read(ini_file) return AutoFuzSlpPosConfig(cf, bin_dir, input_proc, rawdem, root_dir)
def ParseTimespan(self, items): # type: (List[AnyStr]) -> Dict[AnyStr, Dict[AnyStr, Union[float, Dict[AnyStr, float]]]] """The format of self.timespan is different for OpenMP version and MPI&OpenMP version. For OpenMP version: {'IO': {'Input': 0.2, 'Output': 0.04 } 'COMP': {'TSD_RD_P': 0.0001, # All modules 'ALL': 12.3 } 'SIMU': {'ALL': 14.1} } For MPI&OpenMP version: {'MAX': {'IO': {'Input': 0.1, 'Output': 0.02, 'ALL': 0.12 } 'COMP': {'Slope': 5, 'Channel': 0.5, 'Barrier': 0.1, 'ALL': 5.6 } 'SIMU': {'ALL': 10.1} } 'MIN': {...} 'AVG': {...} } """ for item in items: if 'TIMESPAN' not in item: continue item = item.split('\n')[0] values = StringClass.extract_numeric_values_from_string(item) if values is None or len(values) != 1: continue time = values[0] titles = item.replace( '[', '').split(']')[:-1] # e.g., 'TIMESPAN', 'COMP', 'ALL' if len(titles) < 3: # e.g., 'TIMESPAN', 'MAX', 'COMP', 'ALL' continue titles = [title.strip() for title in titles] self.timespan.setdefault(titles[1], dict()) if len(titles) > 3: self.timespan[titles[1]].setdefault(titles[2], dict()) self.timespan[titles[1]][titles[2]].setdefault(titles[3], time) else: self.timespan[titles[1]].setdefault(titles[2], time) return self.timespan
def export_landuse_lookup_files_from_mongodb(cfg, maindb): """export landuse lookup tables to txt file from MongoDB.""" lookup_dir = cfg.dirs.lookup property_namelist = ModelParamDataUtils.landuse_fields property_map = dict() property_namelist.append('USLE_P') query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError( 'LanduseLoop Collection is not existed or empty!') count = 0 for row in query_result: # print(row) value_map = dict() for i, p_name in enumerate(property_namelist): if StringClass.string_match(p_name, 'USLE_P'): # Currently, USLE_P is set as 1 for all landuse. value_map[p_name] = 1 else: # I do not know why manning * 10 here. Just uncommented now. lj # if StringClass.string_match(p_name, "Manning"): # value_map[p_name] = row.get(p_name) * 10 # else: v = row.get(p_name) if is_string(v): v = StringClass.extract_numeric_values_from_string( v)[0] value_map[p_name] = v count += 1 property_map[count] = value_map n = len(property_map) UtilClass.rmmkdir(lookup_dir) for propertyName in property_namelist: with open('%s/%s.txt' % ( lookup_dir, propertyName, ), 'w', encoding='utf-8') as f: f.write('%d\n' % n) for prop_id in property_map: s = '%d %f\n' % (int(property_map[prop_id]['LANDUSE_ID']), property_map[prop_id][propertyName]) f.write('%s' % s)
def ParseTimespan(self, items): """The format of self.timespan is different for OpenMP version and MPI&OpenMP version. For OpenMP version: {'IO': {'Input': 0.2, 'Output': 0.04 } 'COMP': {'TSD_RD_P': 0.0001, # All modules 'ALL': 12.3 } 'SIMU': {'ALL': 14.1} } For MPI&OpenMP version: {'MAX': {'IO': {'Input': 0.1, 'Output': 0.02, 'ALL': 0.12 } 'COMP': {'Slope': 5, 'Channel': 0.5, 'Barrier': 0.1, 'ALL': 5.6 } 'SIMU': {'ALL': 10.1} } 'MIN': {...} 'AVG': {...} } """ for item in items: if 'TIMESPAN' not in item: continue item = item.split('\n')[0] values = StringClass.extract_numeric_values_from_string(item) if values is None or len(values) != 1: continue time = values[0] titles = item.replace('[', '').split(']')[:-1] # e.g., 'TIMESPAN', 'COMP', 'ALL' if len(titles) < 3: # e.g., 'TIMESPAN', 'MAX', 'COMP', 'ALL' continue titles = [title.strip() for title in titles] self.timespan.setdefault(titles[1], dict()) if len(titles) > 3: self.timespan[titles[1]].setdefault(titles[2], dict()) self.timespan[titles[1]][titles[2]].setdefault(titles[3], time) else: self.timespan[titles[1]].setdefault(titles[2], time)
def read_field_arrays_from_csv(csvf): data_items = read_data_items_from_txt(csvf) if len(data_items) < 2: return flds = data_items[0] flds_array = dict() for idx, data_item in enumerate(data_items): if idx == 0: continue data_item_values = StringClass.extract_numeric_values_from_string(','.join(data_item)) for fld_idx, fld_name in enumerate(flds): if fld_idx == 0 or StringClass.string_match(fld_name, 'FID'): continue if fld_name not in flds_array: flds_array[fld_name] = list() flds_array[fld_name].append(data_item_values[fld_idx]) # for key, value in list(flds_array.items()): # print('%s: %d' % (key, len(value))) return combine_multi_layers_array(flds_array)
def read_hypervolume(hypervlog): if not os.path.exists(hypervlog): return None, None, None x = list() nmodel = list() hyperv = list() with open(hypervlog, 'r') as f: lines = f.readlines() for line in lines: values = StringClass.extract_numeric_values_from_string(line) if values is None: continue if len(values) < 2: continue x.append(int(values[0])) hyperv.append(values[-1]) if len(values) >= 3: nmodel.append(int(values[1])) return x, hyperv, nmodel
def read_field_arrays_from_csv(csvf): data_items = read_data_items_from_txt(csvf) if len(data_items) < 2: return flds = data_items[0] flds_array = dict() for idx, data_item in enumerate(data_items): if idx == 0: continue data_item_values = StringClass.extract_numeric_values_from_string( ','.join(data_item)) for fld_idx, fld_name in enumerate(flds): if fld_idx == 0 or StringClass.string_match(fld_name, 'FID'): continue if fld_name not in flds_array: flds_array[fld_name] = list() flds_array[fld_name].append(data_item_values[fld_idx]) # for key, value in list(flds_array.items()): # print('%s: %d' % (key, len(value))) return combine_multi_layers_array(flds_array)
def main(): txtInOutPath = r'C:\z_data\zhongTianShe\model_data_swat\TxtInOut' #txtInOutPath = r'D:\tmp\update_mgt' plt_mgt_op = r'C:\z_data\zhongTianShe\model_data_swat\TxtInOut\opSchedules.txt' # In SWAT source readmgt.f, the format of one operation item is: # 5200 format (1x,i2,1x,i2,1x,f8.3,1x,i2,1x,i4,1x,i3,1x,i2,1x,f12.5,1x, # & f6.2,1x,f11.5,1x,f4.2,1x,f6.2,1x,f5.2,i12) # which transformed to Python is (totally 92 characters): line_fmt = ' %2d %2d %8.3f %2d %4d %3d %2d %12.5f %6.2f %11.5f %4.2f %6.2f %5.2f%12d\n' op_schedules = list() op_lines = open(plt_mgt_op, 'r').readlines() for i in range(1, len(op_lines)): v = StringClass.extract_numeric_values_from_string(op_lines[i].strip()) if len(v) < 14: break tmp_line = line_fmt % (v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8], v[9], v[10], v[11], v[12], v[13]) op_schedules.append(tmp_line) mgt_files = FileClass.get_filename_by_suffixes(txtInOutPath, ['.mgt']) for mgt_file in mgt_files: # print mgt_file f = open(txtInOutPath + os.sep + mgt_file, 'r') fcont = f.readlines() f.close() if 'Luse:AGRL' not in fcont[0].rstrip(): continue print('update %s...' % mgt_file) fcont_new = list() for line in fcont: fcont_new.append(line) if 'Operation Schedule' in line.rstrip(): break # Write new Operation Schedule fcont_new += op_schedules f = open(txtInOutPath + os.sep + mgt_file, 'w') for line in fcont_new: f.write(line) f.close()
def read_hypervolume(hypervlog): # type: (AnyStr) -> (List[int], List[float], List[float]) """Read hypervolume data from file.""" if not os.path.exists(hypervlog): print('Error: The hypervolume log file %s is not existed!' % hypervlog) return None, None, None x = list() # Generation No. nmodel = list() # Newly executed models count hyperv = list() # Hypervolume value with open(hypervlog, 'r', encoding='utf-8') as f: lines = f.readlines() for line in lines: values = StringClass.extract_numeric_values_from_string(line) if values is None: continue if len(values) < 2: continue x.append(int(values[0])) hyperv.append(values[-1]) if len(values) >= 3: nmodel.append(int(values[1])) return x, hyperv, nmodel
def main(): """TEST CODE""" lat_station = 31.45 ssd_txt = r'C:\z_data\zhongTianShe\model_data_swat\climate\ssd_LY.txt' sr_txt = r'C:\z_data\zhongTianShe\model_data_swat\climate\sr_LY.txt' sr = list() f = open(ssd_txt, 'r') ssd_items = f.readlines() f.close() st_str = ssd_items[0].strip() st_time = StringClass.get_datetime(st_str) for i in range(1, len(ssd_items)): ssd_tmp = StringClass.extract_numeric_values_from_string(ssd_items[i]) time_tmp = st_time + timedelta(days=i - 1) sr_tmp = ([round(rs(DateClass.day_of_year(time_tmp), v, lat_station * PI / 180.), 1)] for v in ssd_tmp) sr.extend(sr_tmp) f = open(sr_txt, 'w') f.write(st_str + '\n') for sr_tmp in sr: f.write(','.join(str(v) for v in sr_tmp) + '\n') f.close()
def export_landuse_lookup_files_from_mongodb(cfg, maindb): """export landuse lookup tables to txt file from MongoDB.""" lookup_dir = cfg.dirs.lookup property_namelist = ModelParamDataUtils.landuse_fields property_map = dict() property_namelist.append('USLE_P') query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError('LanduseLoop Collection is not existed or empty!') count = 0 for row in query_result: # print(row) value_map = dict() for i, p_name in enumerate(property_namelist): if StringClass.string_match(p_name, "USLE_P"): # Currently, USLE_P is set as 1 for all landuse. value_map[p_name] = 1 else: # I do not know why manning * 10 here. Just uncommented now. lj # if StringClass.string_match(p_name, "Manning"): # value_map[p_name] = row.get(p_name) * 10 # else: v = row.get(p_name) if isinstance(v, unicode) or isinstance(v, str): v = StringClass.extract_numeric_values_from_string(v)[0] value_map[p_name] = v count += 1 property_map[count] = value_map n = len(property_map) UtilClass.rmmkdir(lookup_dir) for propertyName in property_namelist: with open("%s/%s.txt" % (lookup_dir, propertyName,), 'w') as f: f.write("%d\n" % n) for prop_id in property_map: s = "%d %f\n" % (int(property_map[prop_id]['LANDUSE_ID']), property_map[prop_id][propertyName]) f.write(s)
def read_optiontyploc_section(self, _opttyploc): """Optional parameter-settings for Typical Locations selection""" if _opttyploc not in self.cf.sections(): return # handling slope position types and tags if self.cf.has_option(_opttyploc, 'slopepositiontypes'): self.slppostype = list() typstrs = self.cf.get(_opttyploc, 'slopepositiontypes') self.slppostype = StringClass.split_string(typstrs.lower(), ',') else: # five slope position system will be adapted. pass if self.cf.has_option(_opttyploc, 'slopepositiontags'): self.slppostag = list() tagstrs = self.cf.get(_opttyploc, 'slopepositiontags') self.slppostag = StringClass.extract_numeric_values_from_string( tagstrs) if len(self.slppostag) != len(self.slppostype): raise RuntimeError( "The input number of slope position types and " "tags are not the same!") else: self.slppostag = list() for i in range(len(self.slppostype)): self.slppostag.append(pow(2, i)) for typ in self.slppostype: self.singleslpposconf[typ] = SingleSlpPosFiles(self.ws, typ) # handling selected topographic attributes if self.cf.has_option(_opttyploc, 'terrainattrdict'): self.selectedtopolist = list() self.selectedtopo = dict() terrain_attr_dict_str = self.cf.get(_opttyploc, 'terrainattrdict') attrpath_strs = StringClass.split_string(terrain_attr_dict_str, ';') for i, singattr in enumerate(attrpath_strs): ap = StringClass.split_string(singattr, ',') attrname = ap[0].lower() if i == 0 and not StringClass.string_match(attrname, 'rpi'): attrname = 'rpi' self.selectedtopolist.append(attrname) attrpath = self.topoparam.get_attr_file(attrname) if attrpath is not None: self.selectedtopo[attrname] = attrpath else: # this should be user-defined attribute, and should has a valid file path if len(ap) != 2: raise RuntimeError( "User defined topographic attribute (%s) MUST have " "an existed file path!" % singattr) attrp = AutoFuzSlpPosConfig.check_file_available(ap[1]) if attrp is None: raise RuntimeError( "User defined topographic attribute (%s) MUST have " "an existed file path!" % singattr) self.selectedtopo[attrname] = attrp is_regional = False if i == 0: # the first one is regional attribute is_regional = True self.topoparam.add_user_defined_attribute( attrname, attrp, is_regional) # handling several parameters used in extracting typical location if self.cf.has_option(_opttyploc, 'typlocextractparam'): self.param4typloc = dict() base_param_str = self.cf.get(_opttyploc, 'typlocextractparam') base_param_floats = StringClass.extract_numeric_values_from_string( base_param_str) defnum = len(self._DEFAULT_PARAM_TYPLOC) if len(base_param_floats) == defnum: for slppos in self.slppostype: self.param4typloc[slppos] = base_param_floats[:] elif len(base_param_floats) == len(self.slppostype) * defnum: for i, slppos in enumerate(self.slppostype): self.param4typloc[slppos] = base_param_floats[i * defnum:(i + 1) * defnum] else: raise RuntimeError("TyplocExtractParam MUST has the number of " "%d or %d!" % (defnum, len(self.slppostype) * defnum)) else: for slppos in self.slppostype: self.param4typloc[slppos] = self._DEFAULT_PARAM_TYPLOC[:] # handling Pre-defined fuzzy membership function shapes of each terrain attribute # for each slope position if self.cf.has_option(_opttyploc, 'fuzinfdefault'): self.infshape = dict() fuz_inf_shp_strs = self.cf.get(_opttyploc, 'fuzinfdefault') # inference shapes are separated by SIMICOLON bewteen slope positions fuz_inf_shp_types = StringClass.split_string(fuz_inf_shp_strs, ';') if len(fuz_inf_shp_types) != len(self.slppostype): raise RuntimeError( "FuzInfDefault (%s) MUST be consistent with slope position types" " and separated by ';'!" % fuz_inf_shp_strs) for i, slppos in enumerate(self.slppostype): self.infshape[slppos] = dict() # inference shapes are separated by COMMA bewteen topographic attributes infshps = StringClass.split_string(fuz_inf_shp_types[i], ',') if len(infshps) != len(self.selectedtopolist): raise RuntimeError( "FuzInfDefault (%s) for each slope position MUST have " "the same size with TerrainAttrDict" % fuz_inf_shp_types[i]) for j, attrn in enumerate(self.selectedtopolist): self.infshape[slppos][attrn] = infshps[j] else: if len(self.slppostype) != 5: raise RuntimeError( "Only the fuzzy membership function shapes of " "5 slope position system are built-in. For other " "classification system, please set as input!") # handling value ranges of terrain attributes for extracting prototypes if self.cf.has_option(_opttyploc, 'valueranges'): self.extractrange = dict() value_rng_strs = self.cf.get(_opttyploc, 'valueranges') value_rng_types = StringClass.split_string(value_rng_strs, ';') if len(value_rng_types) != len(self.slppostype): raise RuntimeError( "ValueRanges (%s) MUST be consistent with slope position types" " and separated by ';'!" % value_rng_strs) for i, slppos in enumerate(self.slppostype): self.extractrange[slppos] = dict() value_rngs = StringClass.extract_numeric_values_from_string( value_rng_types[i]) if len(value_rngs) == 0 or len(value_rngs) % 3 != 0: raise RuntimeError( "Each item of ValueRanges MUST contains three elements," "i.e., Attributes No., Min, Max! Please check item: " "%s for %s." % (value_rng_types[i], slppos)) for j in range(int(len(value_rngs) / 3)): attridx = int(value_rngs[j * 3]) - 1 attrname = self.selectedtopolist[attridx] min_v = value_rngs[j * 3 + 1] max_v = value_rngs[j * 3 + 2] self.extractrange[slppos][attrname] = [min_v, max_v] else: if len(self.slppostype) != 5: raise RuntimeError( "Only the extract value ranges of " "5 slope position system are built-in. For other " "classification system, please set as input!")
def __init__(self, cf, method='nsga2'): # type: (ConfigParser, str) -> None """Initialization.""" # 1. SEIMS model related self.model = ParseSEIMSConfig(cf) # type: ParseSEIMSConfig # 2. Common settings of BMPs scenario self.eval_stime = None # type: Optional[datetime] self.eval_etime = None # type: Optional[datetime] self.worst_econ = 0. self.worst_env = 0. self.runtime_years = 0. self.export_sce_txt = False self.export_sce_tif = False if 'Scenario_Common' not in cf.sections(): raise ValueError( '[Scenario_Common] section MUST be existed in *.ini file.') self.eval_stime = parse_datetime_from_ini(cf, 'Scenario_Common', 'eval_time_start') self.eval_etime = parse_datetime_from_ini(cf, 'Scenario_Common', 'eval_time_end') self.worst_econ = cf.getfloat('Scenario_Common', 'worst_economy') self.worst_env = cf.getfloat('Scenario_Common', 'worst_environment') self.runtime_years = cf.getfloat('Scenario_Common', 'runtime_years') if cf.has_option('Scenario_Common', 'export_scenario_txt'): self.export_sce_txt = cf.getboolean('Scenario_Common', 'export_scenario_txt') if cf.has_option('Scenario_Common', 'export_scenario_tif'): self.export_sce_tif = cf.getboolean('Scenario_Common', 'export_scenario_tif') # 3. Application specific setting section [BMPs] # Selected BMPs, the key is BMPID, and value is the BMP information dict self.bmps_info = dict( ) # type: Dict[int, Dict[AnyStr, Union[int, float, AnyStr, List[Union[int, float, AnyStr]]]]] # BMPs to be constant for generated scenarios during optimization, same format with bmps_info self.bmps_retain = dict( ) # type: Dict[int, Dict[AnyStr, Union[int, float, AnyStr, List[Union[int, float, AnyStr]]]]] self.eval_info = dict( ) # type: Dict[AnyStr, Union[int, float, AnyStr]] self.bmps_cfg_unit = 'CONNFIELD' # type: AnyStr self.bmps_cfg_method = 'RAND' # type: AnyStr if 'BMPs' not in cf.sections(): raise ValueError( '[BMPs] section MUST be existed for specific scenario analysis.' ) bmpsinfostr = cf.get('BMPs', 'bmps_info') self.bmps_info = UtilClass.decode_strs_in_dict(json.loads(bmpsinfostr)) if cf.has_option('BMPs', 'bmps_retain'): bmpsretainstr = cf.get('BMPs', 'bmps_retain') self.bmps_retain = json.loads(bmpsretainstr) self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain) evalinfostr = cf.get('BMPs', 'eval_info') self.eval_info = UtilClass.decode_strs_in_dict(json.loads(evalinfostr)) bmpscfgunitstr = cf.get('BMPs', 'bmps_cfg_units') bmpscfgunitdict = UtilClass.decode_strs_in_dict( json.loads(bmpscfgunitstr)) for unitname, unitcfg in viewitems(bmpscfgunitdict): self.bmps_cfg_unit = unitname if self.bmps_cfg_unit not in BMPS_CFG_UNITS: raise ValueError('BMPs configuration unit MUST be ' 'one of %s' % BMPS_CFG_UNITS.__str__()) if not isinstance(unitcfg, dict): raise ValueError( 'The value of BMPs configuration unit MUST be dict value!') for cfgname, cfgvalue in viewitems(unitcfg): for bmpid, bmpdict in viewitems(self.bmps_info): if cfgname in bmpdict: continue self.bmps_info[bmpid][cfgname] = cfgvalue break if cf.has_option('BMPs', 'bmps_cfg_method'): self.bmps_cfg_method = cf.get('BMPs', 'bmps_cfg_method') if self.bmps_cfg_method not in BMPS_CFG_METHODS: print('BMPs configuration method MUST be one of %s' % BMPS_CFG_METHODS.__str__()) self.bmps_cfg_method = 'RAND' # Check the validation of configuration unit and method if self.bmps_cfg_method not in BMPS_CFG_PAIR.get(self.bmps_cfg_unit): raise ValueError('BMPs configuration method %s ' 'is not supported on unit %s' % (self.bmps_cfg_method, self.bmps_cfg_unit)) # Optimize boundary of BMP configuration unit self.boundary_adaptive = False self.boundary_adaptive_threshs = None if cf.has_option('BMPs', 'bmps_cfg_units_opt'): self.boundary_adaptive = cf.getboolean('BMPs', 'bmps_cfg_units_opt') if cf.has_option('BMPs', 'boundary_adaptive_threshold'): tstr = cf.get('BMPs', 'boundary_adaptive_threshold') self.boundary_adaptive_threshs = StringClass.extract_numeric_values_from_string( tstr) if 0 not in self.boundary_adaptive_threshs: self.boundary_adaptive_threshs.append( 0) # 0 means no adjustment of boundary for tmp_thresh in self.boundary_adaptive_threshs: if -1 * tmp_thresh not in self.boundary_adaptive_threshs: self.boundary_adaptive_threshs.append(-1 * tmp_thresh) # 4. Parameters settings for specific optimization algorithm self.opt_mtd = method self.opt = None # type: Union[ParseNSGA2Config, None] if self.opt_mtd == 'nsga2': self.opt = ParseNSGA2Config( cf, self.model.model_dir, 'SA_NSGA2_%s_%s' % (self.bmps_cfg_unit, self.bmps_cfg_method)) # Using the existed population derived from previous scenario optimization self.initial_byinput = cf.getboolean(self.opt_mtd.upper(), 'inputpopulation') if \ cf.has_option(self.opt_mtd.upper(), 'inputpopulation') else False self.input_pareto_file = None self.input_pareto_gen = -1 if cf.has_option(self.opt_mtd.upper(), 'paretofrontsfile'): self.input_pareto_file = cf.get(self.opt_mtd.upper(), 'paretofrontsfile') if cf.has_option(self.opt_mtd.upper(), 'generationselected'): self.input_pareto_gen = cf.getint(self.opt_mtd.upper(), 'generationselected') self.scenario_dir = self.opt.out_dir + os.path.sep + 'Scenarios' UtilClass.rmmkdir(self.scenario_dir) # 5. (Optional) Plot settings for matplotlib self.plot_cfg = PlotConfig(cf)
def __init__(self, cf): """Initialization.""" # 1. Directories self.base_dir = None self.clim_dir = None self.spatial_dir = None self.observe_dir = None self.scenario_dir = None self.model_dir = None self.txt_db_dir = None self.preproc_script_dir = None self.seims_bin = None self.mpi_bin = None self.workspace = None # 1.1. Directory determined flags self.use_observed = True self.use_scernario = True # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' # 3. Climate inputs self.hydro_climate_vars = None self.prec_sites = None self.prec_data = None self.Meteo_sites = None self.Meteo_data = None self.thiessen_field = 'ID' # 4. Spatial inputs self.prec_sites_thiessen = None self.meteo_sites_thiessen = None self.dem = None self.outlet_file = None self.landuse = None self.landcover_init_param = None self.soil = None self.soil_property = None self.fields_partition = False self.fields_partition_thresh = list() self.additional_rs = dict() # 5. Option parameters self.d8acc_threshold = 0 self.np = 4 self.d8down_method = 's' self.dorm_hr = -1. self.temp_base = 0. self.imper_perc_in_urban = 0. self.default_landuse = -1 self.default_soil = -1 # 1. Directories if 'PATH' in cf.sections(): self.base_dir = cf.get('PATH', 'base_data_dir') self.clim_dir = cf.get('PATH', 'climate_data_dir') self.spatial_dir = cf.get('PATH', 'spatial_data_dir') self.observe_dir = cf.get('PATH', 'measurement_data_dir') self.scenario_dir = cf.get('PATH', 'bmp_data_dir') self.model_dir = cf.get('PATH', 'model_dir') self.txt_db_dir = cf.get('PATH', 'txt_db_dir') self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir') self.seims_bin = cf.get('PATH', 'cpp_program_dir') self.mpi_bin = cf.get('PATH', 'mpiexec_dir') self.workspace = cf.get('PATH', 'working_dir') else: raise ValueError('[PATH] section MUST be existed in *.ini file.') if not (FileClass.is_dir_exists(self.base_dir) and FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.txt_db_dir) and FileClass.is_dir_exists(self.preproc_script_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError( 'Please Check Directories defined in [PATH]. ' 'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, ' 'and CPP_PROGRAM_DIR are required!') if not FileClass.is_dir_exists(self.mpi_bin): self.mpi_bin = None if not FileClass.is_dir_exists(self.workspace): try: # first try to make dirs UtilClass.mkdir(self.workspace) # os.mkdir(self.workspace) except OSError as exc: self.workspace = self.model_dir + os.path.sep + 'preprocess_output' print('WARNING: Make WORKING_DIR failed! Use the default: %s' % self.workspace) if not os.path.exists(self.workspace): UtilClass.mkdir(self.workspace) self.dirs = DirNameUtils(self.workspace) self.logs = LogNameUtils(self.dirs.log) self.vecs = VectorNameUtils(self.dirs.geoshp) self.taudems = TauDEMFilesUtils(self.dirs.taudem) self.spatials = SpatialNamesUtils(self.dirs.geodata2db) self.modelcfgs = ModelCfgUtils(self.model_dir) self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database') if not FileClass.is_dir_exists(self.clim_dir): print( 'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".' ) self.clim_dir = self.base_dir + os.path.sep + 'climate' if not FileClass.is_dir_exists(self.clim_dir): raise IOError( 'Directories named "climate" MUST BE located in [base_dir]!' ) if not FileClass.is_dir_exists(self.spatial_dir): print( 'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".' ) self.spatial_dir = self.base_dir + os.path.sep + 'spatial' raise IOError( 'Directories named "spatial" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.observe_dir): self.observe_dir = None self.use_observed = False if not FileClass.is_dir_exists(self.scenario_dir): self.scenario_dir = None self.use_scernario = False # 2. MongoDB related if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError( '[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Climate Input if 'CLIMATE' in cf.sections(): self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'hydroclimatevarfile') self.prec_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precsitefile') self.prec_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precdatafile') self.Meteo_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteositefile') self.Meteo_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteodatafile') self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield') else: raise ValueError( 'Climate input file names MUST be provided in [CLIMATE]!') # 4. Spatial Input if 'SPATIAL' in cf.sections(): self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'precsitesthiessen') self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'meteositesthiessen') self.dem = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'dem') self.outlet_file = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'outlet_file') if not os.path.exists(self.outlet_file): self.outlet_file = None self.landuse = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'landusefile') self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'landcoverinitfile') self.soil = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqnfile') self.soil_property = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqntext') if cf.has_option('SPATIAL', 'additionalfile'): additional_dict_str = cf.get('SPATIAL', 'additionalfile') tmpdict = json.loads(additional_dict_str) tmpdict = { str(k): (str(v) if is_string(v) else v) for k, v in list(tmpdict.items()) } for k, v in list(tmpdict.items()): # Existence check has been moved to mask_origin_delineated_data() # in sp_delineation.py self.additional_rs[k] = v # Field partition if cf.has_option('SPATIAL', 'field_partition_thresh'): ths = cf.get('SPATIAL', 'field_partition_thresh') thsv = StringClass.extract_numeric_values_from_string(ths) if thsv is not None: self.fields_partition_thresh = [int(v) for v in thsv] self.fields_partition = True else: raise ValueError( 'Spatial input file names MUST be provided in [SPATIAL]!') # 5. Optional parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold') self.np = cf.getint('OPTIONAL_PARAMETERS', 'np') self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod') if StringClass.string_match(self.d8down_method, 'surface'): self.d8down_method = 's' elif StringClass.string_match(self.d8down_method, 'horizontal'): self.d8down_method = 'h' elif StringClass.string_match(self.d8down_method, 'pythagoras'): self.d8down_method = 'p' elif StringClass.string_match(self.d8down_method, 'vertical'): self.d8down_method = 'v' else: self.d8down_method = self.d8down_method.lower() if self.d8down_method not in ['s', 'h', 'p', 'v']: self.d8down_method = 's' self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr') self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base') self.imper_perc_in_urban = cf.getfloat( 'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell') self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse') self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf): """Initialization.""" # 1. Directories self.base_dir = None self.clim_dir = None self.spatial_dir = None self.observe_dir = None self.scenario_dir = None self.model_dir = None self.txt_db_dir = None self.preproc_script_dir = None self.seims_bin = None self.mpi_bin = None self.workspace = None # 1.1. Directory determined flags self.use_observed = True self.use_scernario = True # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' # 3. Switch for building SEIMS. These switches should be removed! By lj. # self.gen_cn = True # self.gen_runoff_coef = True # self.gen_crop = True # self.gen_iuh = True # 4. Climate inputs self.hydro_climate_vars = None self.prec_sites = None self.prec_data = None self.Meteo_sites = None self.Meteo_data = None self.thiessen_field = 'ID' # 5. Spatial inputs self.prec_sites_thiessen = None self.meteo_sites_thiessen = None self.dem = None self.outlet_file = None self.landuse = None self.landcover_init_param = None self.soil = None self.soil_property = None self.fields_partition = False self.fields_partition_thresh = list() self.additional_rs = dict() # 6. Option parameters self.d8acc_threshold = 0 self.np = 4 self.d8down_method = 's' self.dorm_hr = -1. self.temp_base = 0. self.imper_perc_in_urban = 0. self.default_landuse = -1 self.default_soil = -1 # 1. Directories if 'PATH' in cf.sections(): self.base_dir = cf.get('PATH', 'base_data_dir') self.clim_dir = cf.get('PATH', 'climate_data_dir') self.spatial_dir = cf.get('PATH', 'spatial_data_dir') self.observe_dir = cf.get('PATH', 'measurement_data_dir') self.scenario_dir = cf.get('PATH', 'bmp_data_dir') self.model_dir = cf.get('PATH', 'model_dir') self.txt_db_dir = cf.get('PATH', 'txt_db_dir') self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir') self.seims_bin = cf.get('PATH', 'cpp_program_dir') self.mpi_bin = cf.get('PATH', 'mpiexec_dir') self.workspace = cf.get('PATH', 'working_dir') else: raise ValueError('[PATH] section MUST be existed in *.ini file.') if not (FileClass.is_dir_exists(self.base_dir) and FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.txt_db_dir) and FileClass.is_dir_exists(self.preproc_script_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError('Please Check Directories defined in [PATH]. ' 'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, ' 'and CPP_PROGRAM_DIR are required!') if not FileClass.is_dir_exists(self.mpi_bin): self.mpi_bin = None if not FileClass.is_dir_exists(self.workspace): try: # first try to make dirs UtilClass.mkdir(self.workspace) # os.mkdir(self.workspace) except OSError as exc: self.workspace = self.model_dir + os.path.sep + 'preprocess_output' print('WARNING: Make WORKING_DIR failed: %s. ' 'Use the default: %s' % (exc.message, self.workspace)) if not os.path.exists(self.workspace): UtilClass.mkdir(self.workspace) self.dirs = DirNameUtils(self.workspace) self.logs = LogNameUtils(self.dirs.log) self.vecs = VectorNameUtils(self.dirs.geoshp) self.taudems = TauDEMFilesUtils(self.dirs.taudem) self.spatials = SpatialNamesUtils(self.dirs.geodata2db) self.modelcfgs = ModelCfgUtils(self.model_dir) self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database') if not FileClass.is_dir_exists(self.clim_dir): print('The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".') self.clim_dir = self.base_dir + os.path.sep + 'climate' if not FileClass.is_dir_exists(self.clim_dir): raise IOError('Directories named "climate" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.spatial_dir): print('The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".') self.spatial_dir = self.base_dir + os.path.sep + 'spatial' raise IOError('Directories named "spatial" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.observe_dir): self.observe_dir = None self.use_observed = False if not FileClass.is_dir_exists(self.scenario_dir): self.scenario_dir = None self.use_scernario = False # 2. MongoDB related if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError('[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Model related switch. The SWITCH section should be removed! By lj. # by default, OpenMP version and daily (longterm) mode will be built # if 'SWITCH' in cf.sections(): # self.gen_cn = cf.getboolean('SWITCH', 'gencn') # self.gen_runoff_coef = cf.getboolean('SWITCH', 'genrunoffcoef') # self.gen_crop = cf.getboolean('SWITCH', 'gencrop') # # if self.storm_mode: # self.gen_iuh = False # self.climate_db = ModelNameUtils.standardize_climate_dbname(self.climate_db) # 4. Climate Input if 'CLIMATE' in cf.sections(): self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'hydroclimatevarfile') self.prec_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precsitefile') self.prec_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precdatafile') self.Meteo_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteositefile') self.Meteo_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteodatafile') self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield') else: raise ValueError('Climate input file names MUST be provided in [CLIMATE]!') # 5. Spatial Input if 'SPATIAL' in cf.sections(): self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'precsitesthiessen') self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'meteositesthiessen') self.dem = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'dem') self.outlet_file = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'outlet_file') if not os.path.exists(self.outlet_file): self.outlet_file = None self.landuse = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'landusefile') self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get('SPATIAL', 'landcoverinitfile') self.soil = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'soilseqnfile') self.soil_property = self.txt_db_dir + os.path.sep + cf.get('SPATIAL', 'soilseqntext') if cf.has_option('SPATIAL', 'additionalfile'): additional_dict_str = cf.get('SPATIAL', 'additionalfile') tmpdict = json.loads(additional_dict_str) tmpdict = {str(k): (str(v) if isinstance(v, str) else v) for k, v in list(tmpdict.items())} for k, v in list(tmpdict.items()): # Existence check has been moved to mask_origin_delineated_data() # in sp_delineation.py self.additional_rs[k] = v # Field partition if cf.has_option('SPATIAL', 'field_partition_thresh'): ths = cf.get('SPATIAL', 'field_partition_thresh') thsv = StringClass.extract_numeric_values_from_string(ths) if thsv is not None: self.fields_partition_thresh = [int(v) for v in thsv] self.fields_partition = True else: raise ValueError('Spatial input file names MUST be provided in [SPATIAL]!') # 6. Option parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold') self.np = cf.getint('OPTIONAL_PARAMETERS', 'np') self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod') if StringClass.string_match(self.d8down_method, 'surface'): self.d8down_method = 's' elif StringClass.string_match(self.d8down_method, 'horizontal'): self.d8down_method = 'h' elif StringClass.string_match(self.d8down_method, 'pythagoras'): self.d8down_method = 'p' elif StringClass.string_match(self.d8down_method, 'vertical'): self.d8down_method = 'v' else: self.d8down_method = self.d8down_method.lower() if self.d8down_method not in ['s', 'h', 'p', 'v']: self.d8down_method = 'h' self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr') self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base') self.imper_perc_in_urban = cf.getfloat('OPTIONAL_PARAMETERS', 'imperviouspercinurbancell') self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse') self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def read_pareto_points_from_txt(txt_file, sce_name, headers, labels=None): # type: (AnyStr, AnyStr, List[AnyStr], Optional[List[AnyStr]]) -> (Dict[int, Union[List, numpy.ndarray]], Dict[int, int]) """Read Pareto points from `runtime.log` file. Args: txt_file: Full file path of `runtime.log` output by NSGA2 algorithm. sce_name: Field name followed by `generation`, e.g., 'calibrationID', 'scenarioID', etc. headers: Filed names in header for each dimension of Pareto front labels: (Optional) Labels corresponding to `headers` for Pareto graphs Returns: pareto_points: `OrderedDict`, key is generation ID, value is Pareto front array pareto_popnum: `OrderedDict`, key is generation ID, value is newly model runs number """ with open(txt_file, 'r', encoding='utf-8') as f: lines = f.readlines() pareto_points = OrderedDict() pareto_popnum = OrderedDict() found = False cur_gen = -1 iden_idx = -1 new_headers = headers[:] for i, hd in enumerate(new_headers): new_headers[i] = hd.upper() if labels is None: labels = headers[:] headers_idx = list() new_labels = list() for lno, line in enumerate(lines): str_line = line for LF in LFs: if LF in line: str_line = line.split(LF)[0] break if str_line == '': continue values = StringClass.extract_numeric_values_from_string(str_line) # Check generation if str_line[0] == '#' and 'Generation' in str_line: if len(values) != 1: continue # e.g., ###### Generation: 23 ###### gen = int(values[0]) found = True cur_gen = gen pareto_popnum[cur_gen] = list() pareto_points[cur_gen] = list() continue if not found: # If the first "###### Generation: 1 ######" has not been found. continue line_list = StringClass.split_string(str_line.upper(), ['\t']) if values is None: # means header line if headers_idx and new_labels: continue for idx, v in enumerate(line_list): if sce_name.upper() in v.upper(): iden_idx = idx break for fldno, fld in enumerate(new_headers): if fld in line_list: tmpidx = line_list.index(fld) headers_idx.append(tmpidx) new_labels.append(labels[fldno]) continue if iden_idx < 0: continue # now append the real Pareto front point data tmpvalues = list() for tmpidx in headers_idx: tmpvalues.append(StringClass.extract_numeric_values_from_string(line_list[tmpidx])[0]) pareto_points[cur_gen].append(tmpvalues[:]) iden_str = line_list[iden_idx] # e.g., 1-44 iden_strs = iden_str.split('-') if len(iden_strs) == 1: pareto_popnum[cur_gen].append(int(iden_strs[0])) if len(iden_strs) == 2: pareto_popnum.setdefault(int(iden_strs[0]), list()) pareto_popnum[int(iden_strs[0])].append(int(iden_strs[1])) return pareto_points, pareto_popnum
def plot_hypervolume(method_paths, ws, cn=False): """Plot hypervolume of multiple optimization methods Args: method_paths: Dict, key is method name and value is full path of the directory ws: Full path of the destination directory cn: (Optional) Use Chinese """ hyperv = OrderedDict() for k, v in list(method_paths.items()): v = v + os.path.sep + 'hypervolume.txt' x = list() y = list() with open(v, 'r') as f: lines = f.readlines() for line in lines: values = StringClass.extract_numeric_values_from_string(line) if values is None: continue if len(values) < 2: continue x.append(int(values[0])) y.append(values[-1]) if len(x) == len(y) > 0: hyperv[k] = [x[:], y[:]] plt.rcParams['xtick.direction'] = 'out' plt.rcParams['ytick.direction'] = 'out' plt.rcParams['font.family'] = 'Times New Roman' generation_str = 'Generation' hyperv_str = 'Hypervolume index' if cn: plt.rcParams['font.family'] = 'SimSun' # 宋体 generation_str = u'进化代数' hyperv_str = u'Hypervolume 指数' linestyles = ['-', '--', '-.', ':'] # plot accumulate pop size fig, ax = plt.subplots(figsize=(10, 8)) mark_idx = 0 for method, gen_hyperv in hyperv.items(): xdata = gen_hyperv[0] ydata = gen_hyperv[1] plt.plot(xdata, ydata, linestyle=linestyles[mark_idx], color='black', label=method, linewidth=2) mark_idx += 1 plt.legend(fontsize=24, loc=2) xaxis = plt.gca().xaxis yaxis = plt.gca().yaxis for xlebal in xaxis.get_ticklabels(): xlebal.set_fontsize(20) for ylebal in yaxis.get_ticklabels(): ylebal.set_fontsize(20) plt.xlabel(generation_str, fontsize=20) plt.ylabel(hyperv_str, fontsize=20) ax.set_xlim(left=0, right=ax.get_xlim()[1] + 2) plt.tight_layout() save_png_eps(plt, ws, 'hypervolume') # close current plot in case of 'figure.max_open_warning' plt.cla() plt.clf() plt.close()