def __init__(self, cf): """Initialization.""" # 1. Directories self.model_dir = None self.scenario_id = -1 if 'PATH' in cf.sections(): self.model_dir = cf.get('PATH', 'model_dir') self.scenario_id = cf.getint('PATH', 'scenarioid') if self.scenario_id < 0: self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' else: self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str( self.scenario_id) else: raise ValueError("[PATH] section MUST be existed in *.ini file.") if not FileClass.is_dir_exists(self.model_dir): raise ValueError("Please Check Directories defined in [PATH]") # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError( '[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Parameters self.plt_subbsnid = -1 self.plt_vars = list() if 'PARAMETERS' in cf.sections(): self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid') plt_vars_str = cf.get('PARAMETERS', 'plot_variables') else: raise ValueError( "[PARAMETERS] section MUST be existed in *.ini file.") if self.plt_subbsnid < 0: raise ValueError( "PLOT_SUBBASINID must be greater or equal than 0.") if plt_vars_str != '': self.plt_vars = StringClass.split_string(plt_vars_str) else: raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!") # 4. Optional_Parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start') tend = cf.get('OPTIONAL_PARAMETERS', 'time_end') else: raise ValueError( "[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.") try: # UTCTIME self.time_start = StringClass.get_datetime(tstart) self.time_end = StringClass.get_datetime(tend) if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \ cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'): tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start') tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end') self.vali_stime = StringClass.get_datetime(tstart) self.vali_etime = StringClass.get_datetime(tend) else: self.vali_stime = None self.vali_etime = None except ValueError: raise ValueError( 'The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".' ) if self.time_start >= self.time_end: raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!") # 5. Switches self.lang_cn = False if 'SWITCH' in cf.sections(): self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def __init__(self, cf=None): # type: (Optional[ConfigParser]) -> None self.host = '127.0.0.1' # localhost by default self.port = 27017 self.bin_dir = '' self.model_dir = '' self.db_name = '' self.version = 'OMP' self.mpi_bin = None self.hosts_opt = None self.hostfile = None self.nprocess = 1 self.nthread = 1 self.lyrmtd = 1 self.scenario_id = 0 self.calibration_id = -1 self.subbasin_id = 0 self.simu_stime = None self.simu_etime = None self.out_stime = None self.out_etime = None self.config_dict = dict() # Running time counted by time.time() of Python, in case of GetTimespan() function failed self.runtime = 0. if cf is None: return sec_name = 'SEIMS_Model' if sec_name not in cf.sections(): raise ValueError('[%s] section MUST be existed in *.ini file.' % sec_name) if cf.has_option(sec_name, 'hostname'): self.host = cf.get(sec_name, 'hostname') if cf.has_option(sec_name, 'port'): self.port = cf.getint(sec_name, 'port') if not StringClass.is_valid_ip_addr(self.host): raise ValueError('HOSTNAME defined in [%s] is illegal!' % sec_name) self.bin_dir = cf.get(sec_name, 'bin_dir') self.model_dir = cf.get(sec_name, 'model_dir') if not (FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.bin_dir)): raise IOError('Please Check Directories defined in [%s]. ' 'BIN_DIR and MODEL_DIR are required!' % sec_name) self.db_name = os.path.split( self.model_dir)[1] # Defaultly, spatial dbname equals dirname if cf.has_option(sec_name, 'spatial_dbname'): self.db_name = cf.get(sec_name, 'spatial_dbname') if cf.has_option(sec_name, 'version'): self.version = cf.get(sec_name, 'version') if cf.has_option(sec_name, 'mpi_bin'): # full path of the executable MPI program self.mpi_bin = cf.get(sec_name, 'mpi_bin') if cf.has_option(sec_name, 'hostopt'): self.hosts_opt = cf.get(sec_name, 'hostopt') if cf.has_option(sec_name, 'hostfile'): self.hostfile = cf.get(sec_name, 'hostfile') if cf.has_option(sec_name, 'processnum'): self.nprocess = cf.getint(sec_name, 'processnum') if cf.has_option(sec_name, 'threadsnum'): self.nthread = cf.getint(sec_name, 'threadsnum') if cf.has_option(sec_name, 'layeringmethod'): self.lyrmtd = cf.getint(sec_name, 'layeringmethod') if cf.has_option(sec_name, 'scenarioid'): self.scenario_id = cf.getint(sec_name, 'scenarioid') if cf.has_option(sec_name, 'calibrationid'): self.calibration_id = cf.getint(sec_name, 'calibrationid') if cf.has_option(sec_name, 'subbasinid'): self.subbasin_id = cf.getint(sec_name, 'subbasinid') self.simu_stime = parse_datetime_from_ini(cf, sec_name, 'sim_time_start') self.simu_etime = parse_datetime_from_ini(cf, sec_name, 'sim_time_end') if self.simu_stime and self.simu_etime and self.simu_stime >= self.simu_etime: raise ValueError('Wrong simulation time settings in [%s]!' % sec_name) self.out_stime = parse_datetime_from_ini(cf, sec_name, 'output_time_start') self.out_etime = parse_datetime_from_ini(cf, sec_name, 'output_time_end') if self.out_stime and self.out_etime and self.out_stime >= self.out_etime: raise ValueError('Wrong output time settings in [%s]!' % sec_name)
def __init__(self, cf): """Initialization.""" # 1. Directories self.base_dir = None self.clim_dir = None self.spatial_dir = None self.observe_dir = None self.scenario_dir = None self.model_dir = None self.txt_db_dir = None self.preproc_script_dir = None self.seims_bin = None self.mpi_bin = None self.workspace = None # 1.1. Directory determined flags self.use_observed = True self.use_scernario = True # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' # 3. Climate inputs self.hydro_climate_vars = None self.prec_sites = None self.prec_data = None self.Meteo_sites = None self.Meteo_data = None self.thiessen_field = 'ID' # 4. Spatial inputs self.prec_sites_thiessen = None self.meteo_sites_thiessen = None self.dem = None self.outlet_file = None self.landuse = None self.landcover_init_param = None self.soil = None self.soil_property = None self.fields_partition = False self.fields_partition_thresh = list() self.additional_rs = dict() # 5. Option parameters self.d8acc_threshold = 0 self.np = 4 self.d8down_method = 's' self.dorm_hr = -1. self.temp_base = 0. self.imper_perc_in_urban = 0. self.default_landuse = -1 self.default_soil = -1 # 1. Directories if 'PATH' in cf.sections(): self.base_dir = cf.get('PATH', 'base_data_dir') self.clim_dir = cf.get('PATH', 'climate_data_dir') self.spatial_dir = cf.get('PATH', 'spatial_data_dir') self.observe_dir = cf.get('PATH', 'measurement_data_dir') self.scenario_dir = cf.get('PATH', 'bmp_data_dir') self.model_dir = cf.get('PATH', 'model_dir') self.txt_db_dir = cf.get('PATH', 'txt_db_dir') self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir') self.seims_bin = cf.get('PATH', 'cpp_program_dir') self.mpi_bin = cf.get('PATH', 'mpiexec_dir') self.workspace = cf.get('PATH', 'working_dir') else: raise ValueError('[PATH] section MUST be existed in *.ini file.') if not (FileClass.is_dir_exists(self.base_dir) and FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.txt_db_dir) and FileClass.is_dir_exists(self.preproc_script_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError( 'Please Check Directories defined in [PATH]. ' 'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, ' 'and CPP_PROGRAM_DIR are required!') if not FileClass.is_dir_exists(self.mpi_bin): self.mpi_bin = None if not FileClass.is_dir_exists(self.workspace): try: # first try to make dirs UtilClass.mkdir(self.workspace) # os.mkdir(self.workspace) except OSError as exc: self.workspace = self.model_dir + os.path.sep + 'preprocess_output' print('WARNING: Make WORKING_DIR failed! Use the default: %s' % self.workspace) if not os.path.exists(self.workspace): UtilClass.mkdir(self.workspace) self.dirs = DirNameUtils(self.workspace) self.logs = LogNameUtils(self.dirs.log) self.vecs = VectorNameUtils(self.dirs.geoshp) self.taudems = TauDEMFilesUtils(self.dirs.taudem) self.spatials = SpatialNamesUtils(self.dirs.geodata2db) self.modelcfgs = ModelCfgUtils(self.model_dir) self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database') if not FileClass.is_dir_exists(self.clim_dir): print( 'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".' ) self.clim_dir = self.base_dir + os.path.sep + 'climate' if not FileClass.is_dir_exists(self.clim_dir): raise IOError( 'Directories named "climate" MUST BE located in [base_dir]!' ) if not FileClass.is_dir_exists(self.spatial_dir): print( 'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".' ) self.spatial_dir = self.base_dir + os.path.sep + 'spatial' raise IOError( 'Directories named "spatial" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.observe_dir): self.observe_dir = None self.use_observed = False if not FileClass.is_dir_exists(self.scenario_dir): self.scenario_dir = None self.use_scernario = False # 2. MongoDB related if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError( '[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Climate Input if 'CLIMATE' in cf.sections(): self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'hydroclimatevarfile') self.prec_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precsitefile') self.prec_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precdatafile') self.Meteo_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteositefile') self.Meteo_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteodatafile') self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield') else: raise ValueError( 'Climate input file names MUST be provided in [CLIMATE]!') # 4. Spatial Input if 'SPATIAL' in cf.sections(): self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'precsitesthiessen') self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'meteositesthiessen') self.dem = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'dem') self.outlet_file = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'outlet_file') if not os.path.exists(self.outlet_file): self.outlet_file = None self.landuse = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'landusefile') self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'landcoverinitfile') self.soil = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqnfile') self.soil_property = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqntext') if cf.has_option('SPATIAL', 'additionalfile'): additional_dict_str = cf.get('SPATIAL', 'additionalfile') tmpdict = json.loads(additional_dict_str) tmpdict = { str(k): (str(v) if is_string(v) else v) for k, v in list(tmpdict.items()) } for k, v in list(tmpdict.items()): # Existence check has been moved to mask_origin_delineated_data() # in sp_delineation.py self.additional_rs[k] = v # Field partition if cf.has_option('SPATIAL', 'field_partition_thresh'): ths = cf.get('SPATIAL', 'field_partition_thresh') thsv = StringClass.extract_numeric_values_from_string(ths) if thsv is not None: self.fields_partition_thresh = [int(v) for v in thsv] self.fields_partition = True else: raise ValueError( 'Spatial input file names MUST be provided in [SPATIAL]!') # 5. Optional parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold') self.np = cf.getint('OPTIONAL_PARAMETERS', 'np') self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod') if StringClass.string_match(self.d8down_method, 'surface'): self.d8down_method = 's' elif StringClass.string_match(self.d8down_method, 'horizontal'): self.d8down_method = 'h' elif StringClass.string_match(self.d8down_method, 'pythagoras'): self.d8down_method = 'p' elif StringClass.string_match(self.d8down_method, 'vertical'): self.d8down_method = 'v' else: self.d8down_method = self.d8down_method.lower() if self.d8down_method not in ['s', 'h', 'p', 'v']: self.d8down_method = 's' self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr') self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base') self.imper_perc_in_urban = cf.getfloat( 'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell') self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse') self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf): """Initialization.""" # 1. Directories self.model_dir = None self.scenario_id = -1 if 'PATH' in cf.sections(): self.model_dir = cf.get('PATH', 'model_dir') self.scenario_id = cf.getint('PATH', 'scenarioid') if self.scenario_id < 0: self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' else: self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str(self.scenario_id) else: raise ValueError("[PATH] section MUST be existed in *.ini file.") if not FileClass.is_dir_exists(self.model_dir): raise ValueError("Please Check Directories defined in [PATH]") # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError('[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Parameters self.plt_subbsnid = -1 self.plt_vars = list() if 'PARAMETERS' in cf.sections(): self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid') plt_vars_str = cf.get('PARAMETERS', 'plot_variables') else: raise ValueError("[PARAMETERS] section MUST be existed in *.ini file.") if self.plt_subbsnid < 0: raise ValueError("PLOT_SUBBASINID must be greater or equal than 0.") if plt_vars_str != '': self.plt_vars = StringClass.split_string(plt_vars_str) else: raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!") # 4. Optional_Parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start') tend = cf.get('OPTIONAL_PARAMETERS', 'time_end') else: raise ValueError("[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.") try: # UTCTIME self.time_start = StringClass.get_datetime(tstart) self.time_end = StringClass.get_datetime(tend) if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \ cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'): tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start') tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end') self.vali_stime = StringClass.get_datetime(tstart) self.vali_etime = StringClass.get_datetime(tend) else: self.vali_stime = None self.vali_etime = None except ValueError: raise ValueError('The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".') if self.time_start >= self.time_end: raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!") # 5. Switches self.lang_cn = False if 'SWITCH' in cf.sections(): self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def __init__(self, cfg): # type: (PostConfig) -> None """Constructor""" self.model = MainSEIMS(args_dict=cfg.model_cfg.ConfigDict) self.ws = self.model.OutputDirectory if not FileClass.is_dir_exists(self.ws): raise ValueError('The output directory %s is not existed!' % self.ws) self.plot_vars = cfg.plot_vars self.plot_cfg = cfg.plot_cfg # type: PlotConfig # UTCTIME, calibration period self.stime = cfg.cali_stime self.etime = cfg.cali_etime self.subbsnID = cfg.plt_subbsnid # validation period self.vali_stime = cfg.vali_stime self.vali_etime = cfg.vali_etime # Read model data from MongoDB, the time period of simulation is read from FILE_IN. mongoclient = ConnectMongoDB(self.model.host, self.model.port).get_conn() self.readData = ReadModelData(mongoclient, self.model.db_name) self.mode = self.readData.Mode self.interval = self.readData.Interval # check start and end time of calibration st, et = self.readData.SimulationPeriod self.plot_validation = True if st > self.stime: self.stime = st if et < self.etime: self.etime = et if st > self.etime > self.stime: self.stime = st self.etime = et # in this circumstance, no validation should be calculated. self.vali_stime = None self.vali_etime = None self.plot_validation = False # check validation time period if self.vali_stime and self.vali_etime: if self.vali_stime >= self.vali_etime or st > self.vali_etime > self.vali_stime \ or self.vali_stime >= et: self.vali_stime = None self.vali_etime = None self.plot_validation = False elif st > self.vali_stime: self.vali_stime = st elif et < self.vali_etime: self.vali_etime = et else: self.plot_validation = False # Set start time and end time of both calibration and validation periods start = self.stime end = self.etime if self.plot_validation: start = self.stime if self.stime < self.vali_stime else self.vali_stime end = self.etime if self.etime > self.vali_etime else self.vali_etime self.outletid = self.readData.OutletID # read precipitation self.pcp_date_value = self.readData.Precipitation( self.subbsnID, start, end) # read simulated data and update the available variables self.plot_vars, self.sim_data_dict = read_simulation_from_txt( self.ws, self.plot_vars, self.outletid, start, end) self.sim_data_value = list( ) # type: List[List[Union[datetime, float]]] for d, vs in self.sim_data_dict.items(): self.sim_data_value.append([d] + vs[:]) # reset start time and end time if len(self.sim_data_value) == 0: raise RuntimeError( 'No available simulate data, please check the start and end time!' ) # read observation data from MongoDB self.obs_vars, self.obs_data_dict = self.readData.Observation( self.subbsnID, self.plot_vars, start, end) # Calibration period self.sim_obs_dict = match_simulation_observation(self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.stime, end_time=self.etime) calculate_statistics(self.sim_obs_dict) # Validation period if existed self.vali_sim_obs_dict = dict() if self.plot_validation: self.vali_sim_obs_dict = match_simulation_observation( self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.vali_stime, end_time=self.vali_etime) calculate_statistics(self.vali_sim_obs_dict)
def __init__(self, cf): """Initialization.""" # 1. Directories self.base_dir = None self.clim_dir = None self.spatial_dir = None self.observe_dir = None self.scenario_dir = None self.model_dir = None self.txt_db_dir = None self.preproc_script_dir = None self.seims_bin = None self.mpi_bin = None self.workspace = None # 1.1. Directory determined flags self.use_observed = True self.use_scernario = True # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' # 3. Switch for building SEIMS. These switches should be removed! By lj. # self.gen_cn = True # self.gen_runoff_coef = True # self.gen_crop = True # self.gen_iuh = True # 4. Climate inputs self.hydro_climate_vars = None self.prec_sites = None self.prec_data = None self.Meteo_sites = None self.Meteo_data = None self.thiessen_field = 'ID' # 5. Spatial inputs self.prec_sites_thiessen = None self.meteo_sites_thiessen = None self.dem = None self.outlet_file = None self.landuse = None self.landcover_init_param = None self.soil = None self.soil_property = None self.fields_partition = False self.fields_partition_thresh = list() self.additional_rs = dict() # 6. Option parameters self.d8acc_threshold = 0 self.np = 4 self.d8down_method = 's' self.dorm_hr = -1. self.temp_base = 0. self.imper_perc_in_urban = 0. self.default_landuse = -1 self.default_soil = -1 # 1. Directories if 'PATH' in cf.sections(): self.base_dir = cf.get('PATH', 'base_data_dir') self.clim_dir = cf.get('PATH', 'climate_data_dir') self.spatial_dir = cf.get('PATH', 'spatial_data_dir') self.observe_dir = cf.get('PATH', 'measurement_data_dir') self.scenario_dir = cf.get('PATH', 'bmp_data_dir') self.model_dir = cf.get('PATH', 'model_dir') self.txt_db_dir = cf.get('PATH', 'txt_db_dir') self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir') self.seims_bin = cf.get('PATH', 'cpp_program_dir') self.mpi_bin = cf.get('PATH', 'mpiexec_dir') self.workspace = cf.get('PATH', 'working_dir') else: raise ValueError('[PATH] section MUST be existed in *.ini file.') if not (FileClass.is_dir_exists(self.base_dir) and FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.txt_db_dir) and FileClass.is_dir_exists(self.preproc_script_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError('Please Check Directories defined in [PATH]. ' 'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, ' 'and CPP_PROGRAM_DIR are required!') if not FileClass.is_dir_exists(self.mpi_bin): self.mpi_bin = None if not FileClass.is_dir_exists(self.workspace): try: # first try to make dirs UtilClass.mkdir(self.workspace) # os.mkdir(self.workspace) except OSError as exc: self.workspace = self.model_dir + os.path.sep + 'preprocess_output' print('WARNING: Make WORKING_DIR failed: %s. ' 'Use the default: %s' % (exc.message, self.workspace)) if not os.path.exists(self.workspace): UtilClass.mkdir(self.workspace) self.dirs = DirNameUtils(self.workspace) self.logs = LogNameUtils(self.dirs.log) self.vecs = VectorNameUtils(self.dirs.geoshp) self.taudems = TauDEMFilesUtils(self.dirs.taudem) self.spatials = SpatialNamesUtils(self.dirs.geodata2db) self.modelcfgs = ModelCfgUtils(self.model_dir) self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database') if not FileClass.is_dir_exists(self.clim_dir): print('The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".') self.clim_dir = self.base_dir + os.path.sep + 'climate' if not FileClass.is_dir_exists(self.clim_dir): raise IOError('Directories named "climate" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.spatial_dir): print('The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".') self.spatial_dir = self.base_dir + os.path.sep + 'spatial' raise IOError('Directories named "spatial" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.observe_dir): self.observe_dir = None self.use_observed = False if not FileClass.is_dir_exists(self.scenario_dir): self.scenario_dir = None self.use_scernario = False # 2. MongoDB related if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError('[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Model related switch. The SWITCH section should be removed! By lj. # by default, OpenMP version and daily (longterm) mode will be built # if 'SWITCH' in cf.sections(): # self.gen_cn = cf.getboolean('SWITCH', 'gencn') # self.gen_runoff_coef = cf.getboolean('SWITCH', 'genrunoffcoef') # self.gen_crop = cf.getboolean('SWITCH', 'gencrop') # # if self.storm_mode: # self.gen_iuh = False # self.climate_db = ModelNameUtils.standardize_climate_dbname(self.climate_db) # 4. Climate Input if 'CLIMATE' in cf.sections(): self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'hydroclimatevarfile') self.prec_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precsitefile') self.prec_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precdatafile') self.Meteo_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteositefile') self.Meteo_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteodatafile') self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield') else: raise ValueError('Climate input file names MUST be provided in [CLIMATE]!') # 5. Spatial Input if 'SPATIAL' in cf.sections(): self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'precsitesthiessen') self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'meteositesthiessen') self.dem = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'dem') self.outlet_file = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'outlet_file') if not os.path.exists(self.outlet_file): self.outlet_file = None self.landuse = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'landusefile') self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get('SPATIAL', 'landcoverinitfile') self.soil = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'soilseqnfile') self.soil_property = self.txt_db_dir + os.path.sep + cf.get('SPATIAL', 'soilseqntext') if cf.has_option('SPATIAL', 'additionalfile'): additional_dict_str = cf.get('SPATIAL', 'additionalfile') tmpdict = json.loads(additional_dict_str) tmpdict = {str(k): (str(v) if isinstance(v, str) else v) for k, v in list(tmpdict.items())} for k, v in list(tmpdict.items()): # Existence check has been moved to mask_origin_delineated_data() # in sp_delineation.py self.additional_rs[k] = v # Field partition if cf.has_option('SPATIAL', 'field_partition_thresh'): ths = cf.get('SPATIAL', 'field_partition_thresh') thsv = StringClass.extract_numeric_values_from_string(ths) if thsv is not None: self.fields_partition_thresh = [int(v) for v in thsv] self.fields_partition = True else: raise ValueError('Spatial input file names MUST be provided in [SPATIAL]!') # 6. Option parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold') self.np = cf.getint('OPTIONAL_PARAMETERS', 'np') self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod') if StringClass.string_match(self.d8down_method, 'surface'): self.d8down_method = 's' elif StringClass.string_match(self.d8down_method, 'horizontal'): self.d8down_method = 'h' elif StringClass.string_match(self.d8down_method, 'pythagoras'): self.d8down_method = 'p' elif StringClass.string_match(self.d8down_method, 'vertical'): self.d8down_method = 'v' else: self.d8down_method = self.d8down_method.lower() if self.d8down_method not in ['s', 'h', 'p', 'v']: self.d8down_method = 'h' self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr') self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base') self.imper_perc_in_urban = cf.getfloat('OPTIONAL_PARAMETERS', 'imperviouspercinurbancell') self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse') self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf): # Default arguments self.host = '127.0.0.1' # localhost by default self.port = 27017 self.bin_dir = '' self.model_dir = '' self.db_name = '' self.version = 'OMP' self.mpi_bin = None self.hosts_opt = None self.hostfile = None self.nprocess = 1 self.nthread = 1 self.lyrmtd = 1 self.scenario_id = 0 self.calibration_id = -1 self.config_dict = dict() if 'SEIMS_Model' not in cf.sections(): raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.") self.host = cf.get('SEIMS_Model', 'hostname') self.port = cf.getint('SEIMS_Model', 'port') if not StringClass.is_valid_ip_addr(self.host): raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!') self.bin_dir = cf.get('SEIMS_Model', 'bin_dir') self.model_dir = cf.get('SEIMS_Model', 'model_dir') if not (FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.bin_dir)): raise IOError('Please Check Directories defined in [SEIMS_Model]. ' 'BIN_DIR and MODEL_DIR are required!') self.db_name = os.path.split(self.model_dir)[1] if cf.has_option('SEIMS_Model', 'version'): self.version = cf.get('SEIMS_Model', 'version') if cf.has_option('SEIMS_Model', 'mpi_bin'): # full path of the executable MPI program self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin') if cf.has_option('SEIMS_Model', 'hostopt'): self.hosts_opt = cf.get('SEIMS_Model', 'hostopt') if cf.has_option('SEIMS_Model', 'hostfile'): self.hostfile = cf.get('SEIMS_Model', 'hostfile') if cf.has_option('SEIMS_Model', 'processnum'): self.nprocess = cf.getint('SEIMS_Model', 'processnum') if cf.has_option('SEIMS_Model', 'threadsnum'): self.nthread = cf.getint('SEIMS_Model', 'threadsnum') if cf.has_option('SEIMS_Model', 'layeringmethod'): self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod') if cf.has_option('SEIMS_Model', 'scenarioid'): self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid') if cf.has_option('SEIMS_Model', 'calibrationid'): self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid') if not (cf.has_option('SEIMS_Model', 'sim_time_start') and cf.has_option('SEIMS_Model', 'sim_time_end')): raise ValueError("Start and end time MUST be specified in [SEIMS_Model].") try: # UTCTIME tstart = cf.get('SEIMS_Model', 'sim_time_start') tend = cf.get('SEIMS_Model', 'sim_time_end') self.time_start = StringClass.get_datetime(tstart) self.time_end = StringClass.get_datetime(tend) except ValueError: raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".') if self.time_start >= self.time_end: raise ValueError("Wrong time settings in [SEIMS_Model]!") # Running time counted by time.time() of Python, in case of failed of GetTimespan() self.runtime = 0.
def __init__(self, cf): # Default arguments self.host = '127.0.0.1' # localhost by default self.port = 27017 self.bin_dir = '' self.model_dir = '' self.db_name = '' self.version = 'OMP' self.mpi_bin = None self.hosts_opt = None self.hostfile = None self.nprocess = 1 self.nthread = 1 self.lyrmtd = 1 self.scenario_id = 0 self.calibration_id = -1 self.config_dict = dict() if 'SEIMS_Model' not in cf.sections(): raise ValueError( "[SEIMS_Model] section MUST be existed in *.ini file.") self.host = cf.get('SEIMS_Model', 'hostname') self.port = cf.getint('SEIMS_Model', 'port') if not StringClass.is_valid_ip_addr(self.host): raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!') self.bin_dir = cf.get('SEIMS_Model', 'bin_dir') self.model_dir = cf.get('SEIMS_Model', 'model_dir') if not (FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.bin_dir)): raise IOError('Please Check Directories defined in [SEIMS_Model]. ' 'BIN_DIR and MODEL_DIR are required!') self.db_name = os.path.split(self.model_dir)[1] if cf.has_option('SEIMS_Model', 'version'): self.version = cf.get('SEIMS_Model', 'version') if cf.has_option('SEIMS_Model', 'mpi_bin'): # full path of the executable MPI program self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin') if cf.has_option('SEIMS_Model', 'hostopt'): self.hosts_opt = cf.get('SEIMS_Model', 'hostopt') if cf.has_option('SEIMS_Model', 'hostfile'): self.hostfile = cf.get('SEIMS_Model', 'hostfile') if cf.has_option('SEIMS_Model', 'processnum'): self.nprocess = cf.getint('SEIMS_Model', 'processnum') if cf.has_option('SEIMS_Model', 'threadsnum'): self.nthread = cf.getint('SEIMS_Model', 'threadsnum') if cf.has_option('SEIMS_Model', 'layeringmethod'): self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod') if cf.has_option('SEIMS_Model', 'scenarioid'): self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid') if cf.has_option('SEIMS_Model', 'calibrationid'): self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid') if not (cf.has_option('SEIMS_Model', 'sim_time_start') and cf.has_option('SEIMS_Model', 'sim_time_end')): raise ValueError( "Start and end time MUST be specified in [SEIMS_Model].") try: # UTCTIME tstart = cf.get('SEIMS_Model', 'sim_time_start') tend = cf.get('SEIMS_Model', 'sim_time_end') self.time_start = StringClass.get_datetime(tstart) self.time_end = StringClass.get_datetime(tend) except ValueError: raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".') if self.time_start >= self.time_end: raise ValueError("Wrong time settings in [SEIMS_Model]!") # Running time counted by time.time() of Python, in case of failed of GetTimespan() self.runtime = 0.
def __init__(self, cf): """Initialization.""" # 1. NSGA-II related parameters self.nsga2_ngens = 1 self.nsga2_npop = 4 self.nsga2_rcross = 0.75 self.nsga2_pmut = 0.05 self.nsga2_rmut = 0.1 self.nsga2_rsel = 0.8 if 'NSGA2' in cf.sections(): self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum') self.nsga2_npop = cf.getint('NSGA2', 'populationsize') self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate') self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc') self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate') self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate') else: raise ValueError('[NSGA2] section MUST be existed in *.ini file.') if self.nsga2_npop % 4 != 0: raise ValueError('PopulationSize must be a multiple of 4.') # 2. MongoDB self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.spatial_db = '' self.bmp_scenario_db = '' if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.spatial_db = cf.get('MONGODB', 'spatialdbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') else: raise ValueError( '[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. SEIMS_Model self.model_dir = '' self.seims_bin = '' self.seims_nthread = 1 self.seims_lyrmethod = 0 if 'SEIMS_Model' in cf.sections(): self.model_dir = cf.get('SEIMS_Model', 'model_dir') self.seims_bin = cf.get('SEIMS_Model', 'bin_dir') self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum') self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod') else: raise ValueError( "[SEIMS_Model] section MUST be existed in *.ini file.") if not (FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError('Please Check Directories defined in [PATH]. ' 'BIN_DIR and MODEL_DIR are required!') # 4. Application specific setting section [BMPs] self.bmps_info = dict() self.bmps_rule = False self.rule_method = 1 self.bmps_retain = dict() self.export_sce_txt = False self.export_sce_tif = False if 'BMPs' in cf.sections(): bmpsinfostr = cf.get('BMPs', 'bmps_info') self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule') if cf.has_option('BMPs', 'rule_method'): self.rule_method = cf.getint('BMPs', 'rule_method') if cf.has_option('BMPs', 'bmps_retain'): bmpsretainstr = cf.get('BMPs', 'bmps_retain') self.bmps_retain = json.loads(bmpsretainstr) self.bmps_retain = UtilClass.decode_strs_in_dict( self.bmps_retain) if cf.has_option('BMPs', 'export_scenario_txt'): self.export_sce_txt = cf.getboolean('BMPs', 'export_scenario_txt') if cf.has_option('BMPs', 'export_scenario_tif'): self.export_sce_tif = cf.getboolean('BMPs', 'export_scenario_tif') else: raise ValueError("[BMPs] section MUST be existed for specific SA.") self.bmps_info = json.loads(bmpsinfostr) self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info) # 5. Application specific setting section [Effectiveness] self.worst_econ = 0 self.worst_env = 0 self.runtime_years = 0 if 'Effectiveness' in cf.sections(): self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy') self.worst_env = cf.getfloat('Effectiveness', 'worst_environment') self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years') self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years') # 6. define gene_values fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop) fn += '_rule' if self.bmps_rule else '_random' self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios' UtilClass.rmmkdir(self.nsga2_dir) UtilClass.rmmkdir(self.scenario_dir) self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt' self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt' self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log' self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'
def __init__(self, cf): """Initialization.""" # 1. NSGA-II related parameters self.nsga2_ngens = 1 self.nsga2_npop = 4 self.nsga2_rcross = 0.75 self.nsga2_pmut = 0.05 self.nsga2_rmut = 0.1 self.nsga2_rsel = 0.8 if 'NSGA2' in cf.sections(): self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum') self.nsga2_npop = cf.getint('NSGA2', 'populationsize') self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate') self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc') self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate') self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate') else: raise ValueError('[NSGA2] section MUST be existed in *.ini file.') if self.nsga2_npop % 4 != 0: raise ValueError('PopulationSize must be a multiple of 4.') # 2. MongoDB self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.spatial_db = '' self.bmp_scenario_db = '' if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.spatial_db = cf.get('MONGODB', 'spatialdbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') else: raise ValueError('[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. SEIMS_Model self.model_dir = '' self.seims_bin = '' self.seims_nthread = 1 self.seims_lyrmethod = 0 if 'SEIMS_Model' in cf.sections(): self.model_dir = cf.get('SEIMS_Model', 'model_dir') self.seims_bin = cf.get('SEIMS_Model', 'bin_dir') self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum') self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod') else: raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.") if not (FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError('Please Check Directories defined in [PATH]. ' 'BIN_DIR and MODEL_DIR are required!') # 4. Application specific setting section [BMPs] self.bmps_info = dict() self.bmps_rule = False self.rule_method = 1 self.bmps_retain = dict() self.export_sce_txt = False self.export_sce_tif = False if 'BMPs' in cf.sections(): bmpsinfostr = cf.get('BMPs', 'bmps_info') self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule') if cf.has_option('BMPs', 'rule_method'): self.rule_method = cf.getint('BMPs', 'rule_method') if cf.has_option('BMPs', 'bmps_retain'): bmpsretainstr = cf.get('BMPs', 'bmps_retain') self.bmps_retain = json.loads(bmpsretainstr) self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain) if cf.has_option('BMPs', 'export_scenario_txt'): self.export_sce_txt = cf.getboolean('BMPs', 'export_scenario_txt') if cf.has_option('BMPs', 'export_scenario_tif'): self.export_sce_tif = cf.getboolean('BMPs', 'export_scenario_tif') else: raise ValueError("[BMPs] section MUST be existed for specific SA.") self.bmps_info = json.loads(bmpsinfostr) self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info) # 5. Application specific setting section [Effectiveness] self.worst_econ = 0 self.worst_env = 0 self.runtime_years = 0 if 'Effectiveness' in cf.sections(): self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy') self.worst_env = cf.getfloat('Effectiveness', 'worst_environment') self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years') self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years') # 6. define gene_values fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop) fn += '_rule' if self.bmps_rule else '_random' self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios' UtilClass.rmmkdir(self.nsga2_dir) UtilClass.rmmkdir(self.scenario_dir) self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt' self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt' self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log' self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'