def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.base_dir = None
self.clim_dir = None
self.spatial_dir = None
self.observe_dir = None
self.scenario_dir = None
self.model_dir = None
self.txt_db_dir = None
self.preproc_script_dir = None
self.seims_bin = None
self.mpi_bin = None
self.workspace = None
# 1.1. Directory determined flags
self.use_observed = True
self.use_scernario = True
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
# 3. Climate inputs
self.hydro_climate_vars = None
self.prec_sites = None
self.prec_data = None
self.Meteo_sites = None
self.Meteo_data = None
self.thiessen_field = 'ID'
# 4. Spatial inputs
self.prec_sites_thiessen = None
self.meteo_sites_thiessen = None
self.dem = None
self.outlet_file = None
self.landuse = None
self.landcover_init_param = None
self.soil = None
self.soil_property = None
self.fields_partition = False
self.fields_partition_thresh = list()
self.additional_rs = dict()
# 5. Option parameters
self.d8acc_threshold = 0
self.np = 4
self.d8down_method = 's'
self.dorm_hr = -1.
self.temp_base = 0.
self.imper_perc_in_urban = 0.
self.default_landuse = -1
self.default_soil = -1
# 1. Directories
if 'PATH' in cf.sections():
self.base_dir = cf.get('PATH', 'base_data_dir')
self.clim_dir = cf.get('PATH', 'climate_data_dir')
self.spatial_dir = cf.get('PATH', 'spatial_data_dir')
self.observe_dir = cf.get('PATH', 'measurement_data_dir')
self.scenario_dir = cf.get('PATH', 'bmp_data_dir')
self.model_dir = cf.get('PATH', 'model_dir')
self.txt_db_dir = cf.get('PATH', 'txt_db_dir')
self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir')
self.seims_bin = cf.get('PATH', 'cpp_program_dir')
self.mpi_bin = cf.get('PATH', 'mpiexec_dir')
self.workspace = cf.get('PATH', 'working_dir')
else:
raise ValueError('[PATH] section MUST be existed in *.ini file.')
if not (FileClass.is_dir_exists(self.base_dir)
and FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.txt_db_dir)
and FileClass.is_dir_exists(self.preproc_script_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError(
'Please Check Directories defined in [PATH]. '
'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, '
'and CPP_PROGRAM_DIR are required!')
if not FileClass.is_dir_exists(self.mpi_bin):
self.mpi_bin = None
if not FileClass.is_dir_exists(self.workspace):
try: # first try to make dirs
UtilClass.mkdir(self.workspace)
# os.mkdir(self.workspace)
except OSError as exc:
self.workspace = self.model_dir + os.path.sep + 'preprocess_output'
print('WARNING: Make WORKING_DIR failed! Use the default: %s' %
self.workspace)
if not os.path.exists(self.workspace):
UtilClass.mkdir(self.workspace)
self.dirs = DirNameUtils(self.workspace)
self.logs = LogNameUtils(self.dirs.log)
self.vecs = VectorNameUtils(self.dirs.geoshp)
self.taudems = TauDEMFilesUtils(self.dirs.taudem)
self.spatials = SpatialNamesUtils(self.dirs.geodata2db)
self.modelcfgs = ModelCfgUtils(self.model_dir)
self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir +
os.path.sep + 'database')
if not FileClass.is_dir_exists(self.clim_dir):
print(
'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".'
)
self.clim_dir = self.base_dir + os.path.sep + 'climate'
if not FileClass.is_dir_exists(self.clim_dir):
raise IOError(
'Directories named "climate" MUST BE located in [base_dir]!'
)
if not FileClass.is_dir_exists(self.spatial_dir):
print(
'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".'
)
self.spatial_dir = self.base_dir + os.path.sep + 'spatial'
raise IOError(
'Directories named "spatial" MUST BE located in [base_dir]!')
if not FileClass.is_dir_exists(self.observe_dir):
self.observe_dir = None
self.use_observed = False
if not FileClass.is_dir_exists(self.scenario_dir):
self.scenario_dir = None
self.use_scernario = False
# 2. MongoDB related
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Climate Input
if 'CLIMATE' in cf.sections():
self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'hydroclimatevarfile')
self.prec_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precsitefile')
self.prec_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precdatafile')
self.Meteo_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteositefile')
self.Meteo_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteodatafile')
self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield')
else:
raise ValueError(
'Climate input file names MUST be provided in [CLIMATE]!')
# 4. Spatial Input
if 'SPATIAL' in cf.sections():
self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'precsitesthiessen')
self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'meteositesthiessen')
self.dem = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'dem')
self.outlet_file = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'outlet_file')
if not os.path.exists(self.outlet_file):
self.outlet_file = None
self.landuse = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'landusefile')
self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'landcoverinitfile')
self.soil = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqnfile')
self.soil_property = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqntext')
if cf.has_option('SPATIAL', 'additionalfile'):
additional_dict_str = cf.get('SPATIAL', 'additionalfile')
tmpdict = json.loads(additional_dict_str)
tmpdict = {
str(k): (str(v) if is_string(v) else v)
for k, v in list(tmpdict.items())
}
for k, v in list(tmpdict.items()):
# Existence check has been moved to mask_origin_delineated_data()
# in sp_delineation.py
self.additional_rs[k] = v
# Field partition
if cf.has_option('SPATIAL', 'field_partition_thresh'):
ths = cf.get('SPATIAL', 'field_partition_thresh')
thsv = StringClass.extract_numeric_values_from_string(ths)
if thsv is not None:
self.fields_partition_thresh = [int(v) for v in thsv]
self.fields_partition = True
else:
raise ValueError(
'Spatial input file names MUST be provided in [SPATIAL]!')
# 5. Optional parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS',
'd8accthreshold')
self.np = cf.getint('OPTIONAL_PARAMETERS', 'np')
self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod')
if StringClass.string_match(self.d8down_method, 'surface'):
self.d8down_method = 's'
elif StringClass.string_match(self.d8down_method, 'horizontal'):
self.d8down_method = 'h'
elif StringClass.string_match(self.d8down_method, 'pythagoras'):
self.d8down_method = 'p'
elif StringClass.string_match(self.d8down_method, 'vertical'):
self.d8down_method = 'v'
else:
self.d8down_method = self.d8down_method.lower()
if self.d8down_method not in ['s', 'h', 'p', 'v']:
self.d8down_method = 's'
self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr')
self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base')
self.imper_perc_in_urban = cf.getfloat(
'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell')
self.default_landuse = cf.getint('OPTIONAL_PARAMETERS',
'defaultlanduse')
self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.model_dir = None
self.scenario_id = -1
if 'PATH' in cf.sections():
self.model_dir = cf.get('PATH', 'model_dir')
self.scenario_id = cf.getint('PATH', 'scenarioid')
if self.scenario_id < 0:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT'
else:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str(
self.scenario_id)
else:
raise ValueError("[PATH] section MUST be existed in *.ini file.")
if not FileClass.is_dir_exists(self.model_dir):
raise ValueError("Please Check Directories defined in [PATH]")
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Parameters
self.plt_subbsnid = -1
self.plt_vars = list()
if 'PARAMETERS' in cf.sections():
self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid')
plt_vars_str = cf.get('PARAMETERS', 'plot_variables')
else:
raise ValueError(
"[PARAMETERS] section MUST be existed in *.ini file.")
if self.plt_subbsnid < 0:
raise ValueError(
"PLOT_SUBBASINID must be greater or equal than 0.")
if plt_vars_str != '':
self.plt_vars = StringClass.split_string(plt_vars_str)
else:
raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!")
# 4. Optional_Parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'time_end')
else:
raise ValueError(
"[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.")
try:
# UTCTIME
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \
cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'):
tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
else:
self.vali_stime = None
self.vali_etime = None
except ValueError:
raise ValueError(
'The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".'
)
if self.time_start >= self.time_end:
raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!")
# 5. Switches
self.lang_cn = False
if 'SWITCH' in cf.sections():
self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.model_dir = None
self.scenario_id = -1
if 'PATH' in cf.sections():
self.model_dir = cf.get('PATH', 'model_dir')
self.scenario_id = cf.getint('PATH', 'scenarioid')
if self.scenario_id < 0:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT'
else:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str(self.scenario_id)
else:
raise ValueError("[PATH] section MUST be existed in *.ini file.")
if not FileClass.is_dir_exists(self.model_dir):
raise ValueError("Please Check Directories defined in [PATH]")
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError('[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Parameters
self.plt_subbsnid = -1
self.plt_vars = list()
if 'PARAMETERS' in cf.sections():
self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid')
plt_vars_str = cf.get('PARAMETERS', 'plot_variables')
else:
raise ValueError("[PARAMETERS] section MUST be existed in *.ini file.")
if self.plt_subbsnid < 0:
raise ValueError("PLOT_SUBBASINID must be greater or equal than 0.")
if plt_vars_str != '':
self.plt_vars = StringClass.split_string(plt_vars_str)
else:
raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!")
# 4. Optional_Parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'time_end')
else:
raise ValueError("[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.")
try:
# UTCTIME
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \
cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'):
tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
else:
self.vali_stime = None
self.vali_etime = None
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!")
# 5. Switches
self.lang_cn = False
if 'SWITCH' in cf.sections():
self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def __init__(self, cf=None):
# type: (Optional[ConfigParser]) -> None
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.subbasin_id = 0
self.simu_stime = None
self.simu_etime = None
self.out_stime = None
self.out_etime = None
self.config_dict = dict()
# Running time counted by time.time() of Python, in case of GetTimespan() function failed
self.runtime = 0.
if cf is None:
return
sec_name = 'SEIMS_Model'
if sec_name not in cf.sections():
raise ValueError('[%s] section MUST be existed in *.ini file.' %
sec_name)
if cf.has_option(sec_name, 'hostname'):
self.host = cf.get(sec_name, 'hostname')
if cf.has_option(sec_name, 'port'):
self.port = cf.getint(sec_name, 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [%s] is illegal!' % sec_name)
self.bin_dir = cf.get(sec_name, 'bin_dir')
self.model_dir = cf.get(sec_name, 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [%s]. '
'BIN_DIR and MODEL_DIR are required!' % sec_name)
self.db_name = os.path.split(
self.model_dir)[1] # Defaultly, spatial dbname equals dirname
if cf.has_option(sec_name, 'spatial_dbname'):
self.db_name = cf.get(sec_name, 'spatial_dbname')
if cf.has_option(sec_name, 'version'):
self.version = cf.get(sec_name, 'version')
if cf.has_option(sec_name,
'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get(sec_name, 'mpi_bin')
if cf.has_option(sec_name, 'hostopt'):
self.hosts_opt = cf.get(sec_name, 'hostopt')
if cf.has_option(sec_name, 'hostfile'):
self.hostfile = cf.get(sec_name, 'hostfile')
if cf.has_option(sec_name, 'processnum'):
self.nprocess = cf.getint(sec_name, 'processnum')
if cf.has_option(sec_name, 'threadsnum'):
self.nthread = cf.getint(sec_name, 'threadsnum')
if cf.has_option(sec_name, 'layeringmethod'):
self.lyrmtd = cf.getint(sec_name, 'layeringmethod')
if cf.has_option(sec_name, 'scenarioid'):
self.scenario_id = cf.getint(sec_name, 'scenarioid')
if cf.has_option(sec_name, 'calibrationid'):
self.calibration_id = cf.getint(sec_name, 'calibrationid')
if cf.has_option(sec_name, 'subbasinid'):
self.subbasin_id = cf.getint(sec_name, 'subbasinid')
self.simu_stime = parse_datetime_from_ini(cf, sec_name,
'sim_time_start')
self.simu_etime = parse_datetime_from_ini(cf, sec_name, 'sim_time_end')
if self.simu_stime and self.simu_etime and self.simu_stime >= self.simu_etime:
raise ValueError('Wrong simulation time settings in [%s]!' %
sec_name)
self.out_stime = parse_datetime_from_ini(cf, sec_name,
'output_time_start')
self.out_etime = parse_datetime_from_ini(cf, sec_name,
'output_time_end')
if self.out_stime and self.out_etime and self.out_stime >= self.out_etime:
raise ValueError('Wrong output time settings in [%s]!' % sec_name)
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.base_dir = None
self.clim_dir = None
self.spatial_dir = None
self.observe_dir = None
self.scenario_dir = None
self.model_dir = None
self.txt_db_dir = None
self.preproc_script_dir = None
self.seims_bin = None
self.mpi_bin = None
self.workspace = None
# 1.1. Directory determined flags
self.use_observed = True
self.use_scernario = True
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
# 3. Switch for building SEIMS. These switches should be removed! By lj.
# self.gen_cn = True
# self.gen_runoff_coef = True
# self.gen_crop = True
# self.gen_iuh = True
# 4. Climate inputs
self.hydro_climate_vars = None
self.prec_sites = None
self.prec_data = None
self.Meteo_sites = None
self.Meteo_data = None
self.thiessen_field = 'ID'
# 5. Spatial inputs
self.prec_sites_thiessen = None
self.meteo_sites_thiessen = None
self.dem = None
self.outlet_file = None
self.landuse = None
self.landcover_init_param = None
self.soil = None
self.soil_property = None
self.fields_partition = False
self.fields_partition_thresh = list()
self.additional_rs = dict()
# 6. Option parameters
self.d8acc_threshold = 0
self.np = 4
self.d8down_method = 's'
self.dorm_hr = -1.
self.temp_base = 0.
self.imper_perc_in_urban = 0.
self.default_landuse = -1
self.default_soil = -1
# 1. Directories
if 'PATH' in cf.sections():
self.base_dir = cf.get('PATH', 'base_data_dir')
self.clim_dir = cf.get('PATH', 'climate_data_dir')
self.spatial_dir = cf.get('PATH', 'spatial_data_dir')
self.observe_dir = cf.get('PATH', 'measurement_data_dir')
self.scenario_dir = cf.get('PATH', 'bmp_data_dir')
self.model_dir = cf.get('PATH', 'model_dir')
self.txt_db_dir = cf.get('PATH', 'txt_db_dir')
self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir')
self.seims_bin = cf.get('PATH', 'cpp_program_dir')
self.mpi_bin = cf.get('PATH', 'mpiexec_dir')
self.workspace = cf.get('PATH', 'working_dir')
else:
raise ValueError('[PATH] section MUST be existed in *.ini file.')
if not (FileClass.is_dir_exists(self.base_dir)
and FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.txt_db_dir)
and FileClass.is_dir_exists(self.preproc_script_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, '
'and CPP_PROGRAM_DIR are required!')
if not FileClass.is_dir_exists(self.mpi_bin):
self.mpi_bin = None
if not FileClass.is_dir_exists(self.workspace):
try: # first try to make dirs
UtilClass.mkdir(self.workspace)
# os.mkdir(self.workspace)
except OSError as exc:
self.workspace = self.model_dir + os.path.sep + 'preprocess_output'
print('WARNING: Make WORKING_DIR failed: %s. '
'Use the default: %s' % (exc.message, self.workspace))
if not os.path.exists(self.workspace):
UtilClass.mkdir(self.workspace)
self.dirs = DirNameUtils(self.workspace)
self.logs = LogNameUtils(self.dirs.log)
self.vecs = VectorNameUtils(self.dirs.geoshp)
self.taudems = TauDEMFilesUtils(self.dirs.taudem)
self.spatials = SpatialNamesUtils(self.dirs.geodata2db)
self.modelcfgs = ModelCfgUtils(self.model_dir)
self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database')
if not FileClass.is_dir_exists(self.clim_dir):
print('The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".')
self.clim_dir = self.base_dir + os.path.sep + 'climate'
if not FileClass.is_dir_exists(self.clim_dir):
raise IOError('Directories named "climate" MUST BE located in [base_dir]!')
if not FileClass.is_dir_exists(self.spatial_dir):
print('The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".')
self.spatial_dir = self.base_dir + os.path.sep + 'spatial'
raise IOError('Directories named "spatial" MUST BE located in [base_dir]!')
if not FileClass.is_dir_exists(self.observe_dir):
self.observe_dir = None
self.use_observed = False
if not FileClass.is_dir_exists(self.scenario_dir):
self.scenario_dir = None
self.use_scernario = False
# 2. MongoDB related
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError('[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Model related switch. The SWITCH section should be removed! By lj.
# by default, OpenMP version and daily (longterm) mode will be built
# if 'SWITCH' in cf.sections():
# self.gen_cn = cf.getboolean('SWITCH', 'gencn')
# self.gen_runoff_coef = cf.getboolean('SWITCH', 'genrunoffcoef')
# self.gen_crop = cf.getboolean('SWITCH', 'gencrop')
#
# if self.storm_mode:
# self.gen_iuh = False
# self.climate_db = ModelNameUtils.standardize_climate_dbname(self.climate_db)
# 4. Climate Input
if 'CLIMATE' in cf.sections():
self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get('CLIMATE',
'hydroclimatevarfile')
self.prec_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precsitefile')
self.prec_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'precdatafile')
self.Meteo_sites = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteositefile')
self.Meteo_data = self.clim_dir + os.path.sep + cf.get('CLIMATE', 'meteodatafile')
self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield')
else:
raise ValueError('Climate input file names MUST be provided in [CLIMATE]!')
# 5. Spatial Input
if 'SPATIAL' in cf.sections():
self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL',
'precsitesthiessen')
self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get('SPATIAL',
'meteositesthiessen')
self.dem = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'dem')
self.outlet_file = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'outlet_file')
if not os.path.exists(self.outlet_file):
self.outlet_file = None
self.landuse = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'landusefile')
self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get('SPATIAL',
'landcoverinitfile')
self.soil = self.spatial_dir + os.path.sep + cf.get('SPATIAL', 'soilseqnfile')
self.soil_property = self.txt_db_dir + os.path.sep + cf.get('SPATIAL', 'soilseqntext')
if cf.has_option('SPATIAL', 'additionalfile'):
additional_dict_str = cf.get('SPATIAL', 'additionalfile')
tmpdict = json.loads(additional_dict_str)
tmpdict = {str(k): (str(v) if isinstance(v, str) else v) for k, v in
list(tmpdict.items())}
for k, v in list(tmpdict.items()):
# Existence check has been moved to mask_origin_delineated_data()
# in sp_delineation.py
self.additional_rs[k] = v
# Field partition
if cf.has_option('SPATIAL', 'field_partition_thresh'):
ths = cf.get('SPATIAL', 'field_partition_thresh')
thsv = StringClass.extract_numeric_values_from_string(ths)
if thsv is not None:
self.fields_partition_thresh = [int(v) for v in thsv]
self.fields_partition = True
else:
raise ValueError('Spatial input file names MUST be provided in [SPATIAL]!')
# 6. Option parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold')
self.np = cf.getint('OPTIONAL_PARAMETERS', 'np')
self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod')
if StringClass.string_match(self.d8down_method, 'surface'):
self.d8down_method = 's'
elif StringClass.string_match(self.d8down_method, 'horizontal'):
self.d8down_method = 'h'
elif StringClass.string_match(self.d8down_method, 'pythagoras'):
self.d8down_method = 'p'
elif StringClass.string_match(self.d8down_method, 'vertical'):
self.d8down_method = 'v'
else:
self.d8down_method = self.d8down_method.lower()
if self.d8down_method not in ['s', 'h', 'p', 'v']:
self.d8down_method = 'h'
self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr')
self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base')
self.imper_perc_in_urban = cf.getfloat('OPTIONAL_PARAMETERS',
'imperviouspercinurbancell')
self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse')
self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf):
# Default arguments
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.config_dict = dict()
if 'SEIMS_Model' not in cf.sections():
raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.")
self.host = cf.get('SEIMS_Model', 'hostname')
self.port = cf.getint('SEIMS_Model', 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!')
self.bin_dir = cf.get('SEIMS_Model', 'bin_dir')
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [SEIMS_Model]. '
'BIN_DIR and MODEL_DIR are required!')
self.db_name = os.path.split(self.model_dir)[1]
if cf.has_option('SEIMS_Model', 'version'):
self.version = cf.get('SEIMS_Model', 'version')
if cf.has_option('SEIMS_Model', 'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin')
if cf.has_option('SEIMS_Model', 'hostopt'):
self.hosts_opt = cf.get('SEIMS_Model', 'hostopt')
if cf.has_option('SEIMS_Model', 'hostfile'):
self.hostfile = cf.get('SEIMS_Model', 'hostfile')
if cf.has_option('SEIMS_Model', 'processnum'):
self.nprocess = cf.getint('SEIMS_Model', 'processnum')
if cf.has_option('SEIMS_Model', 'threadsnum'):
self.nthread = cf.getint('SEIMS_Model', 'threadsnum')
if cf.has_option('SEIMS_Model', 'layeringmethod'):
self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod')
if cf.has_option('SEIMS_Model', 'scenarioid'):
self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid')
if cf.has_option('SEIMS_Model', 'calibrationid'):
self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid')
if not (cf.has_option('SEIMS_Model', 'sim_time_start') and
cf.has_option('SEIMS_Model', 'sim_time_end')):
raise ValueError("Start and end time MUST be specified in [SEIMS_Model].")
try:
# UTCTIME
tstart = cf.get('SEIMS_Model', 'sim_time_start')
tend = cf.get('SEIMS_Model', 'sim_time_end')
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time settings in [SEIMS_Model]!")
# Running time counted by time.time() of Python, in case of failed of GetTimespan()
self.runtime = 0.
def __init__(self, cf):
# Default arguments
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.config_dict = dict()
if 'SEIMS_Model' not in cf.sections():
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
self.host = cf.get('SEIMS_Model', 'hostname')
self.port = cf.getint('SEIMS_Model', 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!')
self.bin_dir = cf.get('SEIMS_Model', 'bin_dir')
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [SEIMS_Model]. '
'BIN_DIR and MODEL_DIR are required!')
self.db_name = os.path.split(self.model_dir)[1]
if cf.has_option('SEIMS_Model', 'version'):
self.version = cf.get('SEIMS_Model', 'version')
if cf.has_option('SEIMS_Model',
'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin')
if cf.has_option('SEIMS_Model', 'hostopt'):
self.hosts_opt = cf.get('SEIMS_Model', 'hostopt')
if cf.has_option('SEIMS_Model', 'hostfile'):
self.hostfile = cf.get('SEIMS_Model', 'hostfile')
if cf.has_option('SEIMS_Model', 'processnum'):
self.nprocess = cf.getint('SEIMS_Model', 'processnum')
if cf.has_option('SEIMS_Model', 'threadsnum'):
self.nthread = cf.getint('SEIMS_Model', 'threadsnum')
if cf.has_option('SEIMS_Model', 'layeringmethod'):
self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod')
if cf.has_option('SEIMS_Model', 'scenarioid'):
self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid')
if cf.has_option('SEIMS_Model', 'calibrationid'):
self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid')
if not (cf.has_option('SEIMS_Model', 'sim_time_start')
and cf.has_option('SEIMS_Model', 'sim_time_end')):
raise ValueError(
"Start and end time MUST be specified in [SEIMS_Model].")
try:
# UTCTIME
tstart = cf.get('SEIMS_Model', 'sim_time_start')
tend = cf.get('SEIMS_Model', 'sim_time_end')
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time settings in [SEIMS_Model]!")
# Running time counted by time.time() of Python, in case of failed of GetTimespan()
self.runtime = 0.
def __init__(self, cf):
"""Initialization."""
# 1. NSGA-II related parameters
self.nsga2_ngens = 1
self.nsga2_npop = 4
self.nsga2_rcross = 0.75
self.nsga2_pmut = 0.05
self.nsga2_rmut = 0.1
self.nsga2_rsel = 0.8
if 'NSGA2' in cf.sections():
self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum')
self.nsga2_npop = cf.getint('NSGA2', 'populationsize')
self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate')
self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc')
self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate')
self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate')
else:
raise ValueError('[NSGA2] section MUST be existed in *.ini file.')
if self.nsga2_npop % 4 != 0:
raise ValueError('PopulationSize must be a multiple of 4.')
# 2. MongoDB
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.spatial_db = ''
self.bmp_scenario_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. SEIMS_Model
self.model_dir = ''
self.seims_bin = ''
self.seims_nthread = 1
self.seims_lyrmethod = 0
if 'SEIMS_Model' in cf.sections():
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
self.seims_bin = cf.get('SEIMS_Model', 'bin_dir')
self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum')
self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod')
else:
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BIN_DIR and MODEL_DIR are required!')
# 4. Application specific setting section [BMPs]
self.bmps_info = dict()
self.bmps_rule = False
self.rule_method = 1
self.bmps_retain = dict()
self.export_sce_txt = False
self.export_sce_tif = False
if 'BMPs' in cf.sections():
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule')
if cf.has_option('BMPs', 'rule_method'):
self.rule_method = cf.getint('BMPs', 'rule_method')
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(
self.bmps_retain)
if cf.has_option('BMPs', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('BMPs',
'export_scenario_txt')
if cf.has_option('BMPs', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('BMPs',
'export_scenario_tif')
else:
raise ValueError("[BMPs] section MUST be existed for specific SA.")
self.bmps_info = json.loads(bmpsinfostr)
self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info)
# 5. Application specific setting section [Effectiveness]
self.worst_econ = 0
self.worst_env = 0
self.runtime_years = 0
if 'Effectiveness' in cf.sections():
self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy')
self.worst_env = cf.getfloat('Effectiveness', 'worst_environment')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
# 6. define gene_values
fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop)
fn += '_rule' if self.bmps_rule else '_random'
self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn
self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.nsga2_dir)
UtilClass.rmmkdir(self.scenario_dir)
self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt'
self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt'
self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log'
self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'
def __init__(self, cf):
"""Initialization."""
# 1. NSGA-II related parameters
self.nsga2_ngens = 1
self.nsga2_npop = 4
self.nsga2_rcross = 0.75
self.nsga2_pmut = 0.05
self.nsga2_rmut = 0.1
self.nsga2_rsel = 0.8
if 'NSGA2' in cf.sections():
self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum')
self.nsga2_npop = cf.getint('NSGA2', 'populationsize')
self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate')
self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc')
self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate')
self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate')
else:
raise ValueError('[NSGA2] section MUST be existed in *.ini file.')
if self.nsga2_npop % 4 != 0:
raise ValueError('PopulationSize must be a multiple of 4.')
# 2. MongoDB
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.spatial_db = ''
self.bmp_scenario_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
else:
raise ValueError('[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. SEIMS_Model
self.model_dir = ''
self.seims_bin = ''
self.seims_nthread = 1
self.seims_lyrmethod = 0
if 'SEIMS_Model' in cf.sections():
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
self.seims_bin = cf.get('SEIMS_Model', 'bin_dir')
self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum')
self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod')
else:
raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.")
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BIN_DIR and MODEL_DIR are required!')
# 4. Application specific setting section [BMPs]
self.bmps_info = dict()
self.bmps_rule = False
self.rule_method = 1
self.bmps_retain = dict()
self.export_sce_txt = False
self.export_sce_tif = False
if 'BMPs' in cf.sections():
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule')
if cf.has_option('BMPs', 'rule_method'):
self.rule_method = cf.getint('BMPs', 'rule_method')
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain)
if cf.has_option('BMPs', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('BMPs', 'export_scenario_txt')
if cf.has_option('BMPs', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('BMPs', 'export_scenario_tif')
else:
raise ValueError("[BMPs] section MUST be existed for specific SA.")
self.bmps_info = json.loads(bmpsinfostr)
self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info)
# 5. Application specific setting section [Effectiveness]
self.worst_econ = 0
self.worst_env = 0
self.runtime_years = 0
if 'Effectiveness' in cf.sections():
self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy')
self.worst_env = cf.getfloat('Effectiveness', 'worst_environment')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
# 6. define gene_values
fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop)
fn += '_rule' if self.bmps_rule else '_random'
self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn
self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.nsga2_dir)
UtilClass.rmmkdir(self.scenario_dir)
self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt'
self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt'
self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log'
self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'
