def __init__(self, cf, method='morris'):
"""Initialization."""
self.method = method
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of parameters sensitivity analysis
if 'PSA_Settings' not in cf.sections():
raise ValueError(
"[PSA_Settings] section MUST be existed in *.ini file.")
self.evaluate_params = list()
if cf.has_option('PSA_Settings', 'evaluateparam'):
eva_str = cf.get('PSA_Settings', 'evaluateparam')
self.evaluate_params = StringClass.split_string(eva_str, ',')
else:
self.evaluate_params = ['Q'] # Default
self.param_range_def = 'morris_param_rng.def' # Default
if cf.has_option('PSA_Settings', 'paramrngdef'):
self.param_range_def = cf.get('PSA_Settings', 'paramrngdef')
self.param_range_def = self.model.model_dir + os.path.sep + self.param_range_def
if not FileClass.is_file_exists(self.param_range_def):
raise IOError('Ranges of parameters MUST be provided!')
if not (cf.has_option('PSA_Settings', 'psa_time_start')
and cf.has_option('PSA_Settings', 'psa_time_end')):
raise ValueError(
"Start and end time of PSA MUST be specified in [PSA_Settings]."
)
try:
# UTCTIME
tstart = cf.get('PSA_Settings', 'psa_time_start')
tend = cf.get('PSA_Settings', 'psa_time_end')
self.psa_stime = StringClass.get_datetime(tstart)
self.psa_etime = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be"YYYY-MM-DD HH:MM:SS".')
if self.psa_stime >= self.psa_etime:
raise ValueError("Wrong time settings in [PSA_Settings]!")
# 3. Parameters settings for specific sensitivity analysis methods
self.morris = None
self.fast = None
if self.method == 'fast':
self.fast = FASTConfig(cf)
self.psa_outpath = '%s/PSA_FAST_N%dM%d' % (
self.model.model_dir, self.fast.N, self.fast.M)
elif self.method == 'morris':
self.morris = MorrisConfig(cf)
self.psa_outpath = '%s/PSA_Morris_N%dL%d' % (
self.model.model_dir, self.morris.N, self.morris.num_levels)
# 4. (Optional) Plot settings for matplotlib
self.plot_cfg = PlotConfig(cf)
# Do not remove psa_outpath if already existed
UtilClass.mkdir(self.psa_outpath)
self.outfiles = PSAOutputs(self.psa_outpath)
def __init__(self, cf, method='morris'):
"""Initialization."""
self.method = method
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of parameters sensitivity analysis
if 'PSA_Settings' not in cf.sections():
raise ValueError("[PSA_Settings] section MUST be existed in *.ini file.")
self.evaluate_params = list()
if cf.has_option('PSA_Settings', 'evaluateparam'):
eva_str = cf.get('PSA_Settings', 'evaluateparam')
self.evaluate_params = StringClass.split_string(eva_str, ',')
else:
self.evaluate_params = ['Q'] # Default
self.param_range_def = 'morris_param_rng.def' # Default
if cf.has_option('PSA_Settings', 'paramrngdef'):
self.param_range_def = cf.get('PSA_Settings', 'paramrngdef')
self.param_range_def = self.model.model_dir + os.path.sep + self.param_range_def
if not FileClass.is_file_exists(self.param_range_def):
raise IOError('Ranges of parameters MUST be provided!')
if not (cf.has_option('PSA_Settings', 'psa_time_start') and
cf.has_option('PSA_Settings', 'psa_time_end')):
raise ValueError("Start and end time of PSA MUST be specified in [PSA_Settings].")
try:
# UTCTIME
tstart = cf.get('PSA_Settings', 'psa_time_start')
tend = cf.get('PSA_Settings', 'psa_time_end')
self.psa_stime = StringClass.get_datetime(tstart)
self.psa_etime = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be"YYYY-MM-DD HH:MM:SS".')
if self.psa_stime >= self.psa_etime:
raise ValueError("Wrong time settings in [PSA_Settings]!")
# 3. Parameters settings for specific sensitivity analysis methods
self.morris = None
self.fast = None
if self.method == 'fast':
self.fast = FASTConfig(cf)
self.psa_outpath = '%s/PSA-FAST-N%dM%d' % (self.model.model_dir,
self.fast.N, self.fast.M)
elif self.method == 'morris':
self.morris = MorrisConfig(cf)
self.psa_outpath = '%s/PSA-Morris-N%dL%dJ%d' % (self.model.model_dir,
self.morris.N,
self.morris.num_levels,
self.morris.grid_jump)
# Do not remove psa_outpath if already existed
UtilClass.mkdir(self.psa_outpath)
self.outfiles = PSAOutputs(self.psa_outpath)
def SimulationPeriod(self):
if self._stime is not None and self._etime is not None:
return self._stime, self._etime
st = self.filein_tab.find_one(
{ModelCfgFields.tag: ModelCfgFields.stime})[ModelCfgFields.value]
et = self.filein_tab.find_one(
{ModelCfgFields.tag: ModelCfgFields.etime})[ModelCfgFields.value]
st = StringClass.get_datetime(st)
et = StringClass.get_datetime(et)
if self._stime is None or st > self._stime:
self._stime = st
if self._etime is None or et < self._etime:
self._etime = et
if st > self._etime > self._stime:
self._stime = st
self._etime = et
return self._stime, self._etime
def __init__(self, cf, method='nsga2'):
"""Initialization."""
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of auto-calibration
if 'CALI_Settings' not in cf.sections():
raise ValueError(
"[CALI_Settings] section MUST be existed in *.ini file.")
self.param_range_def = 'cali_param_rng.def'
if cf.has_option('CALI_Settings', 'paramrngdef'):
self.param_range_def = cf.get('CALI_Settings', 'paramrngdef')
self.param_range_def = self.model.model_dir + os.path.sep + self.param_range_def
if not FileClass.is_file_exists(self.param_range_def):
raise IOError('Ranges of parameters MUST be provided!')
if not (cf.has_option('CALI_Settings', 'cali_time_start')
and cf.has_option('CALI_Settings', 'cali_time_end')):
raise ValueError("Start and end time of Calibration "
"MUST be specified in [CALI_Settings].")
try: # UTCTIME
tstart = cf.get('CALI_Settings', 'cali_time_start')
tend = cf.get('CALI_Settings', 'cali_time_end')
self.cali_stime = StringClass.get_datetime(tstart)
self.cali_etime = StringClass.get_datetime(tend)
self.calc_validation = False
if cf.has_option('CALI_Settings', 'vali_time_start') and \
cf.has_option('CALI_Settings', 'vali_time_end'):
tstart = cf.get('CALI_Settings', 'vali_time_start')
tend = cf.get('CALI_Settings', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
self.calc_validation = True
except ValueError:
raise ValueError(
'The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".'
)
if self.cali_stime >= self.cali_etime or (
self.calc_validation and self.vali_stime >= self.vali_etime):
raise ValueError("Wrong time setted in [CALI_Settings]!")
# 3. Parameters settings for specific optimization algorithm
self.opt_mtd = method
self.opt = None
if self.opt_mtd == 'nsga2':
self.opt = ParseNSGA2Config(cf, self.model.model_dir)
def read_simulation_timerange(self):
"""Read simulation time range from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.main_db]
collection = db['FILE_IN']
try:
stime_str = collection.find_one({'TAG': 'STARTTIME'}, no_cursor_timeout=True)['VALUE']
etime_str = collection.find_one({'TAG': 'ENDTIME'}, no_cursor_timeout=True)['VALUE']
stime = StringClass.get_datetime(stime_str)
etime = StringClass.get_datetime(etime_str)
dlt = etime - stime + timedelta(seconds=1)
self.timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365.
except NetworkTimeout or Exception:
# In case of unexpected raise
self.timerange = 1. # set default
pass
client.close()
def __init__(self, cf, method='nsga2'):
"""Initialization."""
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of auto-calibration
if 'CALI_Settings' not in cf.sections():
raise ValueError("[CALI_Settings] section MUST be existed in *.ini file.")
self.param_range_def = 'cali_param_rng.def'
if cf.has_option('CALI_Settings', 'paramrngdef'):
self.param_range_def = cf.get('CALI_Settings', 'paramrngdef')
self.param_range_def = self.model.model_dir + os.path.sep + self.param_range_def
if not FileClass.is_file_exists(self.param_range_def):
raise IOError('Ranges of parameters MUST be provided!')
if not (cf.has_option('CALI_Settings', 'cali_time_start') and
cf.has_option('CALI_Settings', 'cali_time_end')):
raise ValueError("Start and end time of Calibration "
"MUST be specified in [CALI_Settings].")
try: # UTCTIME
tstart = cf.get('CALI_Settings', 'cali_time_start')
tend = cf.get('CALI_Settings', 'cali_time_end')
self.cali_stime = StringClass.get_datetime(tstart)
self.cali_etime = StringClass.get_datetime(tend)
self.calc_validation = False
if cf.has_option('CALI_Settings', 'vali_time_start') and \
cf.has_option('CALI_Settings', 'vali_time_end'):
tstart = cf.get('CALI_Settings', 'vali_time_start')
tend = cf.get('CALI_Settings', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
self.calc_validation = True
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".')
if self.cali_stime >= self.cali_etime or (self.calc_validation and
self.vali_stime >= self.vali_etime):
raise ValueError("Wrong time setted in [CALI_Settings]!")
# 3. Parameters settings for specific optimization algorithm
self.opt_mtd = method
self.opt = None
if self.opt_mtd == 'nsga2':
self.opt = ParseNSGA2Config(cf, self.model.model_dir)
def read_simulation_timerange(self):
"""Read simulation time range from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.main_db]
collection = db['FILE_IN']
try:
stime_str = collection.find_one({'TAG': 'STARTTIME'},
no_cursor_timeout=True)['VALUE']
etime_str = collection.find_one({'TAG': 'ENDTIME'},
no_cursor_timeout=True)['VALUE']
stime = StringClass.get_datetime(stime_str)
etime = StringClass.get_datetime(etime_str)
dlt = etime - stime + timedelta(seconds=1)
self.timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365.
except NetworkTimeout or Exception:
# In case of unexpected raise
self.timerange = 1. # set default
pass
client.close()
def parse_datetime_from_ini(cf, section_name, option_name):
# type: (ConfigParser, AnyStr, AnyStr) -> Optional[datetime]
"""Parse datetime from the `ConfigParser` object."""
if not check_config_option(cf, section_name, option_name):
return None
time_str = cf.get(section_name, option_name)
try: # UTCTIME
return StringClass.get_datetime(time_str)
except ValueError:
print('Warning: The time string %s is invalid, which MUST be '
'"YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".' % time_str)
return None
def read_simulation_from_txt(
ws, # type: AnyStr
plot_vars, # type: List[AnyStr]
subbsnID, # type: int
stime, # type: datetime
etime # type: datetime
):
# type: (...) -> (List[AnyStr], Dict[datetime, List[float]])
"""
Read simulation data from text file according to subbasin ID.
Returns:
1. Matched variable names, [var1, var2, ...]
2. Simulation data dict of all plotted variables, with UTCDATETIME.
{Datetime: [value_of_var1, value_of_var2, ...], ...}
"""
plot_vars_existed = list()
sim_data_dict = OrderedDict()
for i, v in enumerate(plot_vars):
txtfile = ws + os.path.sep + v + '.txt'
if not FileClass.is_file_exists(txtfile):
print('WARNING: Simulation variable file: %s is not existed!' %
txtfile)
continue
data_items = read_data_items_from_txt(txtfile)
found = False
data_available = False
for item in data_items:
item_vs = StringClass.split_string(item[0], ' ', elim_empty=True)
if len(item_vs) == 2:
if int(item_vs[1]) == subbsnID and not found:
found = True
elif int(item_vs[1]) != subbsnID and found:
break
if not found:
continue
if len(item_vs) != 3:
continue
date_str = '%s %s' % (item_vs[0], item_vs[1])
sim_datetime = StringClass.get_datetime(date_str,
"%Y-%m-%d %H:%M:%S")
if stime <= sim_datetime <= etime:
if sim_datetime not in sim_data_dict:
sim_data_dict[sim_datetime] = list()
sim_data_dict[sim_datetime].append(float(item_vs[2]))
data_available = True
if data_available:
plot_vars_existed.append(v)
print('Read simulation from %s to %s done.' %
(stime.strftime('%c'), etime.strftime('%c')))
return plot_vars_existed, sim_data_dict
def get_utcdatetime_from_field_values(flds, values, tsys, tzone=None):
"""Get datetime from field-value lists.
Returns:
utctime
"""
cur_y = 0
cur_m = 0
cur_d = 0
cur_hh = 0
cur_mm = 0
cur_ss = 0
dt = None
for i, fld in enumerate(flds):
if StringClass.string_match(fld, DataValueFields.dt):
dt = StringClass.get_datetime(values[i])
elif StringClass.string_match(fld, DataValueFields.y):
cur_y = int(values[i])
elif StringClass.string_match(fld, DataValueFields.m):
cur_m = int(values[i])
elif StringClass.string_match(fld, DataValueFields.d):
cur_d = int(values[i])
elif StringClass.string_match(fld, DataValueFields.hour):
cur_hh = int(values[i])
elif StringClass.string_match(fld, DataValueFields.minute):
cur_mm = int(values[i])
elif StringClass.string_match(fld, DataValueFields.second):
cur_ss = int(values[i])
# Get datetime and utc/local transformation
if dt is None: # 'DATETIME' is not existed
if cur_y < 1900 or cur_m <= 0 and cur_d <= 0:
raise ValueError("Can not find TIME information from "
"fields: %s" % ' '.join(fld for fld in flds))
else:
dt = datetime(cur_y, cur_m, cur_d, cur_hh, cur_mm, cur_ss)
if not StringClass.string_match(tsys, 'UTCTIME'):
if tzone is None:
tzone = time.timezone // -3600 # positive value for EAST
dt -= timedelta(minutes=tzone * 60)
return dt
def main():
"""TEST CODE"""
lat_station = 31.45
ssd_txt = r'C:\z_data\zhongTianShe\model_data_swat\climate\ssd_LY.txt'
sr_txt = r'C:\z_data\zhongTianShe\model_data_swat\climate\sr_LY.txt'
sr = list()
f = open(ssd_txt, 'r')
ssd_items = f.readlines()
f.close()
st_str = ssd_items[0].strip()
st_time = StringClass.get_datetime(st_str)
for i in range(1, len(ssd_items)):
ssd_tmp = StringClass.extract_numeric_values_from_string(ssd_items[i])
time_tmp = st_time + timedelta(days=i - 1)
sr_tmp = ([round(rs(DateClass.day_of_year(time_tmp), v, lat_station * PI / 180.), 1)]
for v in ssd_tmp)
sr.extend(sr_tmp)
f = open(sr_txt, 'w')
f.write(st_str + '\n')
for sr_tmp in sr:
f.write(','.join(str(v) for v in sr_tmp) + '\n')
f.close()
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.model_dir = None
self.scenario_id = -1
if 'PATH' in cf.sections():
self.model_dir = cf.get('PATH', 'model_dir')
self.scenario_id = cf.getint('PATH', 'scenarioid')
if self.scenario_id < 0:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT'
else:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str(
self.scenario_id)
else:
raise ValueError("[PATH] section MUST be existed in *.ini file.")
if not FileClass.is_dir_exists(self.model_dir):
raise ValueError("Please Check Directories defined in [PATH]")
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Parameters
self.plt_subbsnid = -1
self.plt_vars = list()
if 'PARAMETERS' in cf.sections():
self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid')
plt_vars_str = cf.get('PARAMETERS', 'plot_variables')
else:
raise ValueError(
"[PARAMETERS] section MUST be existed in *.ini file.")
if self.plt_subbsnid < 0:
raise ValueError(
"PLOT_SUBBASINID must be greater or equal than 0.")
if plt_vars_str != '':
self.plt_vars = StringClass.split_string(plt_vars_str)
else:
raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!")
# 4. Optional_Parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'time_end')
else:
raise ValueError(
"[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.")
try:
# UTCTIME
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \
cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'):
tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
else:
self.vali_stime = None
self.vali_etime = None
except ValueError:
raise ValueError(
'The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".'
)
if self.time_start >= self.time_end:
raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!")
# 5. Switches
self.lang_cn = False
if 'SWITCH' in cf.sections():
self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def __init__(
self,
bin_dir='', # type: AnyStr # The directory of SEIMS binary
model_dir='', # type: AnyStr # The directory of SEIMS model
nthread=4, # type: int # Thread number for OpenMP
lyrmtd=0, # type: int # Layering method, can be 0 (UP_DOWN) or 1 (DOWN_UP)
host='127.0.0.1', # type: AnyStr # MongoDB host address, default is `localhost`
port=27017, # type: int # MongoDB port, default is 27017
db_name='', # type: AnyStr # Main spatial dbname which can diff from dirname
scenario_id=-1, # type: int # Scenario ID defined in `<model>_Scenario` database
calibration_id=-1, # type: int # Calibration ID used for model auto-calibration
subbasin_id=0, # type: int # Subbasin ID, 0 for whole watershed, 9999 for field version
version='OMP', # type: AnyStr # SEIMS version, can be `MPI` or `OMP` (default)
nprocess=1, # type: int # Process number for MPI
mpi_bin='', # type: AnyStr # Full path of MPI executable file, e.g., './mpirun`
hosts_opt='-f', # type: AnyStr # Option for assigning hosts,
# e.g., `-f`, `-hostfile`, `-machine`, `-machinefile`
hostfile='', # type: AnyStr # File containing host names,
# or file mapping process numbers to machines
simu_stime=None, # type: Optional[datetime, AnyStr] # Start time of simulation
simu_etime=None, # type: Optional[datetime, AnyStr] # End time of simulation
out_stime=None, # type: Optional[datetime, AnyStr] # Start time of outputs
out_etime=None, # type: Optional[datetime, AnyStr] # End time of outputs
args_dict=None # type: Dict[AnyStr, Optional[AnyStr, datetime, int]]
):
# type: (...) -> None
# Derived from input arguments
if args_dict is None: # Preferred to use 'args_dict' if existed.
args_dict = dict()
bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir
model_dir = args_dict[
'model_dir'] if 'model_dir' in args_dict else model_dir
self.version = args_dict[
'version'] if 'version' in args_dict else version
suffix = '.exe' if sysstr == 'Windows' else ''
if self.version == 'MPI':
self.seims_exec = '%s/seims_mpi%s' % (bin_dir, suffix)
else:
self.seims_exec = '%s/seims_omp%s' % (bin_dir, suffix)
if not FileClass.is_file_exists(
self.seims_exec): # If not support OpenMP, use `seims`!
self.seims_exec = '%s/seims%s' % (bin_dir, suffix)
self.seims_exec = os.path.abspath(self.seims_exec)
self.model_dir = os.path.abspath(model_dir)
self.nthread = args_dict[
'nthread'] if 'nthread' in args_dict else nthread
self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd
self.host = args_dict['host'] if 'host' in args_dict else host
self.port = args_dict['port'] if 'port' in args_dict else port
self.db_name = args_dict['db_name'] if 'db_name' in args_dict \
else os.path.split(self.model_dir)[1]
self.scenario_id = args_dict[
'scenario_id'] if 'scenario_id' in args_dict else scenario_id
self.calibration_id = args_dict['calibration_id'] \
if 'calibration_id' in args_dict else calibration_id
self.subbasin_id = args_dict[
'subbasin_id'] if 'subbasin_id' in args_dict else subbasin_id
self.nprocess = args_dict[
'nprocess'] if 'nprocess' in args_dict else nprocess
self.mpi_bin = args_dict[
'mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin
self.hosts_opt = args_dict[
'hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt
self.hostfile = args_dict[
'hostfile'] if 'hostfile' in args_dict else hostfile
self.simu_stime = args_dict[
'simu_stime'] if 'simu_stime' in args_dict else simu_stime
self.simu_etime = args_dict[
'simu_etime'] if 'simu_etime' in args_dict else simu_etime
self.out_stime = args_dict[
'out_stime'] if 'out_stime' in args_dict else out_stime
self.out_etime = args_dict[
'out_etime'] if 'out_etime' in args_dict else out_etime
if is_string(
self.simu_stime) and not isinstance(self.simu_stime, datetime):
self.simu_stime = StringClass.get_datetime(self.simu_stime)
if is_string(
self.simu_etime) and not isinstance(self.simu_etime, datetime):
self.simu_etime = StringClass.get_datetime(self.simu_etime)
if is_string(
self.out_stime) and not isinstance(self.out_stime, datetime):
self.out_stime = StringClass.get_datetime(self.out_stime)
if is_string(
self.out_etime) and not isinstance(self.out_etime, datetime):
self.out_etime = StringClass.get_datetime(self.out_etime)
# Concatenate executable command
self.cmd = self.Command
self.run_success = False
self.output_dir = self.OutputDirectory
# Model data read from MongoDB
self.outlet_id = -1
self.subbasin_count = -1
self.scenario_dbname = ''
self.start_time = None
self.end_time = None
self.output_ids = list() # type: List[AnyStr]
self.output_items = dict() # type: Dict[AnyStr, Union[List[AnyStr]]]
# Data maybe used after model run
self.timespan = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, Dict[AnyStr, float]]]]
self.obs_vars = list(
) # type: List[AnyStr] # Observation types at the outlet
self.obs_value = dict(
) # type: Dict[datetime, List[float]] # Observation value
self.sim_vars = list(
) # type: List[AnyStr] # Simulation types, part of `obs_vars`
self.sim_value = dict(
) # type: Dict[datetime, List[float]] # Simulation value
# The format of sim_obs_dict:
# {VarName: {'UTCDATETIME': [t1, t2, ..., tn],
# 'Obs': [o1, o2, ..., on],
# 'Sim': [s1, s2, ..., sn]},
# ...
# }
self.sim_obs_dict = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, List[Union[datetime, float]]]]]
self.runtime = 0.
self.runlogs = list() # type: List[AnyStr]
self.mongoclient = None # type: Union[MongoClient, None] # Set to None after use
def calculate_95ppu(sim_obs_data, sim_data, outdir, gen_num,
vali_sim_obs_data=None, vali_sim_data=None,
plot_cfg=None # type: Optional[PlotConfig]
):
"""Calculate 95% prediction uncertainty and plot the hydrographs."""
if plot_cfg is None:
plot_cfg = PlotConfig()
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'
plt.rcParams['font.family'] = plot_cfg.font_name
plt.rcParams['timezone'] = 'UTC'
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.it'] = 'STIXGeneral:italic'
plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold'
if len(sim_data) < 2:
return
var_name = sim_obs_data[0]['var_name']
for idx, var in enumerate(var_name):
plot_validation = False
if vali_sim_obs_data and vali_sim_data and var in vali_sim_obs_data[0]['var_name']:
plot_validation = True
ylabel_str = var
if var in ['Q', 'QI', 'QG', 'QS']:
ylabel_str += ' (m$^3$/s)'
elif 'CONC' in var.upper(): # Concentrate
if 'SED' in var.upper():
ylabel_str += ' (g/L)'
else:
ylabel_str += ' (mg/L)'
elif 'SED' in var.upper(): # amount
ylabel_str += ' (kg)'
cali_obs_dates = sim_obs_data[0][var]['UTCDATETIME'][:]
if is_string(cali_obs_dates[0]):
cali_obs_dates = [StringClass.get_datetime(s) for s in cali_obs_dates]
obs_dates = cali_obs_dates[:]
order = 1 # By default, the calibration period is before the validation period.
if plot_validation:
vali_obs_dates = vali_sim_obs_data[0][var]['UTCDATETIME']
if is_string(vali_obs_dates[0]):
vali_obs_dates = [StringClass.get_datetime(s) for s in vali_obs_dates]
if vali_obs_dates[-1] <= cali_obs_dates[0]:
order = 0
obs_dates = vali_obs_dates + obs_dates
else:
obs_dates += vali_obs_dates
obs_data = sim_obs_data[0][var]['Obs'][:]
if plot_validation:
if order:
obs_data += vali_sim_obs_data[0][var]['Obs'][:]
else:
obs_data = vali_sim_obs_data[0][var]['Obs'][:] + obs_data
cali_sim_dates = list(sim_data[0].keys())
if is_string(cali_sim_dates[0]):
cali_sim_dates = [StringClass.get_datetime(s) for s in cali_sim_dates]
sim_dates = cali_sim_dates[:]
if plot_validation:
vali_sim_dates = list(vali_sim_data[0].keys())
if is_string(vali_sim_dates[0]):
vali_sim_dates = [StringClass.get_datetime(s) for s in vali_sim_dates]
if order:
sim_dates += vali_sim_dates
else:
sim_dates = vali_sim_dates + sim_dates
sim_data_list = list()
caliBestIdx = -1
caliBestNSE = -9999.
for idx2, ind in enumerate(sim_data):
tmp = numpy.array(list(ind.values()))
tmp = tmp[:, idx]
if sim_obs_data[idx2][var]['NSE'] > caliBestNSE:
caliBestNSE = sim_obs_data[idx2][var]['NSE']
caliBestIdx = idx2
tmpsim = tmp.tolist()
if plot_validation:
tmp_data = numpy.array(list(vali_sim_data[idx2].values()))[:, idx].tolist()
if order:
tmpsim += tmp_data
else:
tmpsim = tmp_data + tmpsim
sim_data_list.append(tmpsim)
sim_best = numpy.array(list(sim_data[caliBestIdx].values()))[:, idx]
sim_best = sim_best.tolist()
if plot_validation:
tmp_data = numpy.array(list(vali_sim_data[caliBestIdx].values()))[:, idx].tolist()
if order:
sim_best += tmp_data
else:
sim_best = tmp_data + sim_best
sim_data_list = numpy.array(sim_data_list)
ylows = numpy.percentile(sim_data_list, 2.5, 0, interpolation='nearest')
yups = numpy.percentile(sim_data_list, 97.5, 0, interpolation='nearest')
def calculate_95ppu_efficiency(obs_data_list, obs_dates_list, sim_dates_list):
# type: (...) -> (float, float)
count = 0
ylows_obs = list()
yups_obs = list()
for oi, ov in enumerate(obs_data_list):
try:
si = sim_dates_list.index(obs_dates_list[oi])
ylows_obs.append(ylows[si])
yups_obs.append(yups[si])
if ylows[si] <= ov <= yups[si]:
count += 1
except Exception:
continue
p = float(count) / len(obs_data_list)
ylows_obs = numpy.array(ylows_obs)
yups_obs = numpy.array(yups_obs)
r = numpy.mean(yups_obs - ylows_obs) / numpy.std(numpy.array(obs_data_list))
return p, r
# concatenate text
p_value, r_value = calculate_95ppu_efficiency(sim_obs_data[0][var]['Obs'],
cali_obs_dates,
list(sim_data[0].keys()))
txt = 'P-factor: %.2f\nR-factor: %.2f\n' % (p_value, r_value)
txt += u'某一最优模拟\n' if plot_cfg.plot_cn else 'One best simulation:\n'
txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (sim_obs_data[caliBestIdx][var]['NSE'],
sim_obs_data[caliBestIdx][var]['RSR'],
sim_obs_data[caliBestIdx][var]['PBIAS'],
sim_obs_data[caliBestIdx][var]['R-square'])
# concatenate text of validation if needed
vali_txt = ''
if plot_validation:
p_value, r_value = calculate_95ppu_efficiency(vali_sim_obs_data[0][var]['Obs'],
vali_obs_dates,
list(vali_sim_data[0].keys()))
vali_txt = 'P-factor: %.2f\nR-factor: %.2f\n\n' % (p_value, r_value)
vali_txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (vali_sim_obs_data[caliBestIdx][var]['NSE'],
vali_sim_obs_data[caliBestIdx][var]['RSR'],
vali_sim_obs_data[caliBestIdx][var]['PBIAS'],
vali_sim_obs_data[caliBestIdx][var]['R-square'])
# plot
fig, ax = plt.subplots(figsize=(12, 4))
ax.fill_between(sim_dates, ylows.tolist(), yups.tolist(),
color=(0.8, 0.8, 0.8), label='95PPU')
observed_label = u'实测值' if plot_cfg.plot_cn else 'Observed points'
ax.scatter(obs_dates, obs_data, marker='.', s=20,
color='g', label=observed_label)
besesim_label = u'最优模拟' if plot_cfg.plot_cn else 'Best simulation'
ax.plot(sim_dates, sim_best, linestyle='--', color='red', linewidth=1,
label=besesim_label)
ax.set_xlim(left=min(sim_dates), right=max(sim_dates))
ax.set_ylim(bottom=0.)
date_fmt = mdates.DateFormatter('%m-%d-%y')
ax.xaxis.set_major_formatter(date_fmt)
ax.tick_params(axis='x', bottom=True, top=False, length=5, width=2, which='major',
labelsize=plot_cfg.tick_fsize)
ax.tick_params(axis='y', left=True, right=False, length=5, width=2, which='major',
labelsize=plot_cfg.tick_fsize)
plt.xlabel(u'时间' if plot_cfg.plot_cn else 'Date time',
fontsize=plot_cfg.axislabel_fsize)
plt.ylabel(ylabel_str, fontsize=plot_cfg.axislabel_fsize)
# plot separate dash line
delta_dt = (sim_dates[-1] - sim_dates[0]) // 9
delta_dt2 = (sim_dates[-1] - sim_dates[0]) // 35
sep_time = sim_dates[-1]
time_pos = [sep_time - delta_dt]
time_pos2 = [sep_time - 2 * delta_dt]
ymax, ymin = ax.get_ylim()
yc = abs(ymax - ymin) * 0.9
if plot_validation:
sep_time = vali_sim_dates[0] if vali_sim_dates[0] >= cali_sim_dates[-1] \
else cali_sim_dates[0]
cali_vali_labels = [(u'验证期' if plot_cfg.plot_cn else 'Calibration'),
(u'率定期' if plot_cfg.plot_cn else 'Validation')]
if not order:
cali_vali_labels.reverse()
time_pos = [sep_time - delta_dt, sep_time + delta_dt2]
time_pos2 = [sep_time - 2 * delta_dt, sep_time + delta_dt2]
ax.axvline(sep_time, color='black', linestyle='dashed', linewidth=2)
plt.text(time_pos[0], yc, cali_vali_labels[0],
fontdict={'style': 'italic', 'weight': 'bold',
'size': plot_cfg.label_fsize},
color='black')
plt.text(time_pos[1], yc, cali_vali_labels[1],
fontdict={'style': 'italic', 'weight': 'bold',
'size': plot_cfg.label_fsize},
color='black')
# add legend
handles, labels = ax.get_legend_handles_labels()
figorders = [labels.index('95PPU'), labels.index(observed_label),
labels.index(besesim_label)]
ax.legend([handles[idx] for idx in figorders], [labels[idx] for idx in figorders],
fontsize=plot_cfg.legend_fsize, loc=2, framealpha=0.8)
# add text
cali_pos = time_pos[0] if order else time_pos[1]
plt.text(cali_pos, yc * 0.5, txt, color='red', fontsize=plot_cfg.label_fsize - 1)
if plot_validation:
vali_pos = time_pos[1] if order else time_pos[0]
plt.text(vali_pos, yc * 0.5, vali_txt, color='red', fontsize=plot_cfg.label_fsize - 1)
# fig.autofmt_xdate(rotation=0, ha='center')
plt.tight_layout()
save_png_eps(plt, outdir, 'Gen%d_95ppu_%s' % (gen_num, var), plot_cfg)
# close current plot in case of 'figure.max_open_warning'
plt.cla()
plt.clf()
plt.close()
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.model_dir = None
self.scenario_id = -1
if 'PATH' in cf.sections():
self.model_dir = cf.get('PATH', 'model_dir')
self.scenario_id = cf.getint('PATH', 'scenarioid')
if self.scenario_id < 0:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT'
else:
self.model_dir = self.model_dir + os.path.sep + 'OUTPUT' + str(self.scenario_id)
else:
raise ValueError("[PATH] section MUST be existed in *.ini file.")
if not FileClass.is_dir_exists(self.model_dir):
raise ValueError("Please Check Directories defined in [PATH]")
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError('[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Parameters
self.plt_subbsnid = -1
self.plt_vars = list()
if 'PARAMETERS' in cf.sections():
self.plt_subbsnid = cf.getint('PARAMETERS', 'plot_subbasinid')
plt_vars_str = cf.get('PARAMETERS', 'plot_variables')
else:
raise ValueError("[PARAMETERS] section MUST be existed in *.ini file.")
if self.plt_subbsnid < 0:
raise ValueError("PLOT_SUBBASINID must be greater or equal than 0.")
if plt_vars_str != '':
self.plt_vars = StringClass.split_string(plt_vars_str)
else:
raise ValueError("PLOT_VARIABLES illegal defined in [PARAMETERS]!")
# 4. Optional_Parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
tstart = cf.get('OPTIONAL_PARAMETERS', 'time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'time_end')
else:
raise ValueError("[OPTIONAL_PARAMETERS] section MUST be existed in *.ini file.")
try:
# UTCTIME
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
if cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_start') and \
cf.has_option('OPTIONAL_PARAMETERS', 'vali_time_end'):
tstart = cf.get('OPTIONAL_PARAMETERS', 'vali_time_start')
tend = cf.get('OPTIONAL_PARAMETERS', 'vali_time_end')
self.vali_stime = StringClass.get_datetime(tstart)
self.vali_etime = StringClass.get_datetime(tend)
else:
self.vali_stime = None
self.vali_etime = None
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD" or "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time setted in [OPTIONAL_PARAMETERS]!")
# 5. Switches
self.lang_cn = False
if 'SWITCH' in cf.sections():
self.lang_cn = cf.getboolean('SWITCH', 'lang_cn')
def calculate_95ppu(sim_obs_data,
sim_data,
outdir,
gen_num,
vali_sim_obs_data=None,
vali_sim_data=None):
"""Calculate 95% prediction uncertainty and plot the hydrographs."""
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'
plt.rcParams['font.family'] = 'Times New Roman'
plt.rcParams['timezone'] = 'UTC'
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.it'] = 'STIXGeneral:italic'
plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold'
if len(sim_data) < 2:
return
var_name = sim_obs_data[0]['var_name']
for idx, var in enumerate(var_name):
plot_validation = False
if vali_sim_obs_data and vali_sim_data and var in vali_sim_obs_data[0][
'var_name']:
plot_validation = True
ylabel_str = var
if var in ['Q', 'QI', 'QG', 'QS']:
ylabel_str += ' (m$^3$/s)'
elif 'CONC' in var.upper(): # Concentrate
if 'SED' in var.upper():
ylabel_str += ' (g/L)'
else:
ylabel_str += ' (mg/L)'
else: # amount
ylabel_str += ' (kg)'
cali_obs_dates = sim_obs_data[0][var]['UTCDATETIME'][:]
if isinstance(cali_obs_dates[0], str) or isinstance(
cali_obs_dates[0], text_type):
cali_obs_dates = [
StringClass.get_datetime(s) for s in cali_obs_dates
]
obs_dates = cali_obs_dates[:]
if plot_validation:
vali_obs_dates = vali_sim_obs_data[0][var]['UTCDATETIME']
if isinstance(vali_obs_dates[0], str) or isinstance(
vali_obs_dates[0], text_type):
vali_obs_dates = [
StringClass.get_datetime(s) for s in vali_obs_dates
]
obs_dates += vali_obs_dates
obs_data = sim_obs_data[0][var]['Obs'][:]
if plot_validation:
obs_data += vali_sim_obs_data[0][var]['Obs']
sim_dates = list(sim_data[0].keys())
if isinstance(sim_dates[0], str) or isinstance(sim_dates[0],
text_type):
sim_dates = [StringClass.get_datetime(s) for s in sim_dates]
if plot_validation:
vali_sim_dates = list(vali_sim_data[0].keys())
if isinstance(vali_sim_dates[0], str) or isinstance(
vali_sim_dates[0], text_type):
vali_sim_dates = [
StringClass.get_datetime(s) for s in vali_sim_dates
]
sim_dates += vali_sim_dates
sim_data_list = list()
caliBestIdx = -1
caliBestNSE = -9999.
for idx2, ind in enumerate(sim_data):
tmp = numpy.array(list(ind.values()))
tmp = tmp[:, idx]
if sim_obs_data[idx2][var]['NSE'] > caliBestNSE:
caliBestNSE = sim_obs_data[idx2][var]['NSE']
caliBestIdx = idx2
tmpsim = tmp.tolist()
if plot_validation:
tmpsim += numpy.array(list(
vali_sim_data[idx2].values()))[:, idx].tolist()
sim_data_list.append(tmpsim)
sim_best = numpy.array(list(sim_data[caliBestIdx].values()))[:, idx]
sim_best = sim_best.tolist()
if plot_validation:
sim_best += numpy.array(list(
vali_sim_data[caliBestIdx].values()))[:, idx].tolist()
sim_data_list = numpy.array(sim_data_list)
ylows = numpy.percentile(sim_data_list,
2.5,
0,
interpolation='nearest')
yups = numpy.percentile(sim_data_list,
97.5,
0,
interpolation='nearest')
def calculate_95ppu_efficiency(obs_data_list, obs_dates_list,
sim_dates_list):
count = 0
ylows_obs = list()
yups_obs = list()
for oi, ov in enumerate(obs_data_list):
try:
si = sim_dates_list.index(obs_dates_list[oi])
ylows_obs.append(ylows[si])
yups_obs.append(yups[si])
if ylows[si] <= ov <= yups[si]:
count += 1
except Exception:
continue
p = float(count) / len(obs_data_list)
ylows_obs = numpy.array(ylows_obs)
yups_obs = numpy.array(yups_obs)
r = numpy.mean(yups_obs - ylows_obs) / numpy.std(
numpy.array(obs_data_list))
return p, r
# concatenate text
p_value, r_value = calculate_95ppu_efficiency(
sim_obs_data[0][var]['Obs'], cali_obs_dates,
list(sim_data[0].keys()))
txt = 'P-factor: %.2f, R-factor: %.2f\n' % (p_value, r_value)
txt += 'One of the best simulations:\n' \
' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (sim_obs_data[caliBestIdx][var]['NSE'],
sim_obs_data[caliBestIdx][var]['RSR'],
sim_obs_data[caliBestIdx][var]['PBIAS'],
sim_obs_data[caliBestIdx][var]['R-square'])
# concatenate text of validation if needed
vali_txt = ''
if plot_validation:
p_value, r_value = calculate_95ppu_efficiency(
vali_sim_obs_data[0][var]['Obs'], vali_obs_dates,
list(vali_sim_data[0].keys()))
vali_txt = 'P-factor: %.2f, R-factor: %.2f\n\n' % (p_value,
r_value)
vali_txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (vali_sim_obs_data[caliBestIdx][var]['NSE'],
vali_sim_obs_data[caliBestIdx][var]['RSR'],
vali_sim_obs_data[caliBestIdx][var]['PBIAS'],
vali_sim_obs_data[caliBestIdx][var][
'R-square'])
# plot
fig, ax = plt.subplots(figsize=(12, 4))
ax.fill_between(sim_dates,
ylows.tolist(),
yups.tolist(),
color=(0.8, 0.8, 0.8),
label='95PPU')
ax.scatter(obs_dates,
obs_data,
marker='.',
s=20,
color='g',
label='Observed points')
ax.plot(sim_dates,
sim_best,
linestyle='--',
color='red',
label='Best simulation',
linewidth=1)
ax.set_xlim(left=min(sim_dates), right=max(sim_dates))
ax.set_ylim(bottom=0.)
date_fmt = mdates.DateFormatter('%m-%d-%y')
ax.xaxis.set_major_formatter(date_fmt)
ax.tick_params(axis='x',
bottom='on',
top='off',
length=5,
width=2,
which='major')
ax.tick_params(axis='y',
left='on',
right='off',
length=5,
width=2,
which='major')
plt.xlabel('Date', fontsize='small')
plt.ylabel(ylabel_str, fontsize='small')
# plot separate dash line
delta_dt = (max(sim_dates) - min(sim_dates)) / 9
delta_dt2 = (max(sim_dates) - min(sim_dates)) / 35
sep_time = max(sim_dates) - delta_dt
ymax, ymin = ax.get_ylim()
yc = abs(ymax - ymin) * 0.9
if plot_validation:
sep_time = vali_sim_dates[
0] if vali_sim_dates[0] > sim_dates[0] else sim_dates[0]
ax.axvline(sep_time,
color='black',
linestyle='dashed',
linewidth=2)
plt.text(sep_time - delta_dt,
yc,
'Calibration',
fontdict={
'style': 'italic',
'weight': 'bold'
},
color='black')
plt.text(sep_time + delta_dt2,
yc,
'Validation',
fontdict={
'style': 'italic',
'weight': 'bold'
},
color='black')
# add legend
handles, labels = ax.get_legend_handles_labels()
order = [
labels.index('95PPU'),
labels.index('Observed points'),
labels.index('Best simulation')
]
ax.legend([handles[idx] for idx in order],
[labels[idx] for idx in order],
fontsize='medium',
loc=2,
framealpha=0.8)
# add text
plt.text(sep_time - 2 * delta_dt, yc * 0.6, txt, color='red')
if plot_validation:
plt.text(sep_time + delta_dt2, yc * 0.6, vali_txt, color='red')
# fig.autofmt_xdate(rotation=0, ha='center')
plt.tight_layout()
save_png_eps(plt, outdir, 'Gen%d_95ppu_%s' % (gen_num, var))
# close current plot in case of 'figure.max_open_warning'
plt.cla()
plt.clf()
plt.close()
def interpolate_observed_data_to_regular_interval(in_file, time_interval, start_time, end_time,
eliminate_zero=False,
time_sys_output='UTCTIME', day_divided_hour=0):
"""
Interpolate not regular observed data to regular time interval data.
Todo: Not tested yet!
Args:
in_file: input data file, the basic format is as follows:
line 1: #<time_system> [<time_zone>], e.g., #LOCALTIME 8, #UTCTIME
line 2: DATETIME,field1,field2,...
line 3: YYYY-mm-dd HH:MM:SS,field1_value,field2_value,...
line 4: ...
...
Field name can be PCP, FLOW, SED
the unit is mm/h, m3/s, g/L (i.e., kg/m3), respectively.
time_interval: time interval, unit is minute, e.g., daily output is 1440
start_time: start time, the format must be 'YYYY-mm-dd HH:MM:SS', and the time system
is based on time_sys.
end_time: end time, see also start_time.
eliminate_zero: Boolean flag. If true, the time interval without original records will
not be output.
time_sys_output: time system of output time_system, the format must be
'<time_system> [<time_zone>]', e.g.,
'LOCALTIME'
'LOCALTIME 8'
'UTCTIME' (default)
day_divided_hour: If the time_interval is equal to N*1440, this parameter should be
carefully specified. The value must range from 0 to 23. e.g.,
day_divided_hour ==> day ranges (all expressed as 2013-02-03)
0 ==> 2013-02-03 00:00:00 to 2013-02-03 23:59:59 (default)
8 ==> 2013-02-03 08:00:00 to 2013-02-04 07:59:59
20 ==> 2013-02-03 20:00:00 to 2013-02-04 19:59:59
Returns:
The output data files are located in the same directory with the input file.
The nomenclature is: <field name>_<time system>_<time interval>_<nonzero>, e.g.,
pcp_utctime_1440_nonzero.csv, flow_localtime_60.csv.
Note that `.txt` format is also supported.
"""
FileClass.check_file_exists(in_file)
time_sys_input, time_zone_input = HydroClimateUtilClass.get_time_system_from_data_file(in_file)
data_items = read_data_items_from_txt(in_file)
flds = data_items[0][:]
data_items.remove(flds)
if not 0 <= day_divided_hour <= 23:
raise ValueError('Day divided hour must range from 0 to 23!')
try:
date_idx = flds.index('DATETIME')
flds.remove('DATETIME')
except ValueError:
raise ValueError('DATETIME must be one of the fields!')
# available field
available_flds = ['FLOW', 'SED', 'PCP']
def check_avaiable_field(cur_fld):
"""Check if the given field name is supported."""
support_flag = False
for fff in available_flds:
if fff.lower() in cur_fld.lower():
support_flag = True
break
return support_flag
ord_data = OrderedDict()
time_zone_output = time.timezone // 3600
if time_sys_output.lower().find('local') >= 0:
tmpstrs = StringClass.split_string(time_sys_output, [' '])
if len(tmpstrs) == 2 and MathClass.isnumerical(tmpstrs[1]):
time_zone_output = -1 * int(tmpstrs[1])
time_sys_output = 'LOCALTIME'
else:
time_sys_output = 'UTCTIME'
time_zone_output = 0
for item in data_items:
org_datetime = StringClass.get_datetime(item[date_idx])
if time_sys_input == 'LOCALTIME':
org_datetime += timedelta(hours=time_zone_input) # now, org_datetime is UTC time.
if time_sys_output == 'LOCALTIME':
org_datetime -= timedelta(hours=time_zone_output)
# now, org_datetime is consistent with the output time system
ord_data[org_datetime] = list()
for i, v in enumerate(item):
if i == date_idx:
continue
if MathClass.isnumerical(v):
ord_data[org_datetime].append(float(v))
else:
ord_data[org_datetime].append(v)
# print(ord_data)
itp_data = OrderedDict()
out_time_delta = timedelta(minutes=time_interval)
sdatetime = StringClass.get_datetime(start_time)
edatetime = StringClass.get_datetime(end_time)
item_dtime = sdatetime
if time_interval % 1440 == 0:
item_dtime = sdatetime.replace(hour=0, minute=0, second=0) + \
timedelta(minutes=day_divided_hour * 60)
while item_dtime <= edatetime:
# print(item_dtime)
# if item_dtime.month == 12 and item_dtime.day == 31:
# print("debug")
sdt = item_dtime # start datetime of records
edt = item_dtime + out_time_delta # end datetime of records
# get original data items
org_items = list()
pre_dt = list(ord_data.keys())[0]
pre_added = False
for i, v in list(ord_data.items()):
if sdt <= i < edt:
if not pre_added and pre_dt < sdt < i and sdt - pre_dt < out_time_delta:
# only add one item that less than sdt.
org_items.append([pre_dt] + ord_data.get(pre_dt))
pre_added = True
org_items.append([i] + v)
if i > edt:
break
pre_dt = i
if len(org_items) > 0:
org_items.append([edt]) # Just add end time for compute convenient
if org_items[0][0] < sdt:
org_items[0][0] = sdt # set the begin datetime of current time interval
# if eliminate time interval without original records
# initial interpolated list
itp_data[item_dtime] = [0.] * len(flds)
if len(org_items) == 0:
if eliminate_zero:
itp_data.popitem()
item_dtime += out_time_delta
continue
# core interpolation code
flow_idx = -1
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
if 'SED' in v_name.upper(): # FLOW must be existed
for v_idx2, v_name2 in enumerate(flds):
if 'FLOW' in v_name2.upper():
flow_idx = v_idx2
break
if flow_idx < 0:
raise RuntimeError('To interpolate SED, FLOW must be provided!')
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
itp_value = 0.
itp_auxiliary_value = 0.
for org_item_idx, org_item_dtv in enumerate(org_items):
if org_item_idx == 0:
continue
org_item_dt = org_item_dtv[0]
pre_item_dtv = org_items[org_item_idx - 1]
pre_item_dt = pre_item_dtv[0]
tmp_delta_dt = org_item_dt - pre_item_dt
tmp_delta_secs = tmp_delta_dt.days * 86400 + tmp_delta_dt.seconds
if 'SED' in v_name.upper():
itp_value += pre_item_dtv[v_idx + 1] * pre_item_dtv[flow_idx + 1] * \
tmp_delta_secs
itp_auxiliary_value += pre_item_dtv[flow_idx + 1] * tmp_delta_secs
else:
itp_value += pre_item_dtv[v_idx + 1] * tmp_delta_secs
if 'SED' in v_name.upper():
if MathClass.floatequal(itp_auxiliary_value, 0.):
itp_value = 0.
print('WARNING: Flow is 0 for %s, please check!' %
item_dtime.strftime('%Y-%m-%d %H:%M:%S'))
itp_value /= itp_auxiliary_value
elif 'FLOW' in v_name.upper():
itp_value /= (out_time_delta.days * 86400 + out_time_delta.seconds)
elif 'PCP' in v_name.upper(): # the input is mm/h, and output is mm
itp_value /= 3600.
itp_data[item_dtime][v_idx] = round(itp_value, 4)
item_dtime += out_time_delta
# for i, v in itp_data.items():
# print(i, v)
# output to files
work_path = os.path.dirname(in_file)
header_str = '#' + time_sys_output
if time_sys_output == 'LOCALTIME':
header_str = header_str + ' ' + str(time_zone_output)
for idx, fld in enumerate(flds):
if not check_avaiable_field(fld):
continue
file_name = fld + '_' + time_sys_output + '_' + str(time_interval)
if eliminate_zero:
file_name += '_nonzero'
file_name += '.csv'
out_file = work_path + os.path.sep + file_name
with open(out_file, 'w', encoding='utf-8') as f:
f.write(header_str + '\n')
f.write('DATETIME,' + fld + '\n')
for i, v in list(itp_data.items()):
cur_line = i.strftime('%Y-%m-%d %H:%M:%S') + ',' + str(v[idx]) + '\n'
f.write(cur_line)
def interpolate_observed_data_to_regular_interval(in_file, time_interval, start_time, end_time,
eliminate_zero=False,
time_sys_output='UTCTIME', day_divided_hour=0):
"""
Interpolate not regular observed data to regular time interval data.
Args:
in_file: input data file, the basic format is as follows:
line 1: #<time_system> [<time_zone>], e.g., #LOCALTIME 8, #UTCTIME
line 2: DATETIME,field1,field2,...
line 3: YYYY-mm-dd HH:MM:SS,field1_value,field2_value,...
line 4: ...
...
Field name can be PCP, FLOW, SED
the unit is mm/h, m3/s, g/L (i.e., kg/m3), respectively.
time_interval: time interval, unit is minute, e.g., daily output is 1440
start_time: start time, the format must be 'YYYY-mm-dd HH:MM:SS', and the time system
is based on time_sys.
end_time: end time, see also start_time.
eliminate_zero: Boolean flag. If true, the time interval without original records will
not be output.
time_sys_output: time system of output time_system, the format must be
'<time_system> [<time_zone>]', e.g.,
'LOCALTIME'
'LOCALTIME 8'
'UTCTIME' (default)
day_divided_hour: If the time_interval is equal to N*1440, this parameter should be
carefully specified. The value must range from 0 to 23. e.g.,
day_divided_hour ==> day ranges (all expressed as 2013-02-03)
0 ==> 2013-02-03 00:00:00 to 2013-02-03 23:59:59 (default)
8 ==> 2013-02-03 08:00:00 to 2013-02-04 07:59:59
20 ==> 2013-02-03 20:00:00 to 2013-02-04 19:59:59
Returns:
The output data files are located in the same directory with the input file.
The nomenclature is: <field name>_<time system>_<time interval>_<nonzero>, e.g.,
pcp_utctime_1440_nonzero.txt, flow_localtime_60.txt
"""
FileClass.check_file_exists(in_file)
time_sys_input, time_zone_input = HydroClimateUtilClass.get_time_system_from_data_file(in_file)
data_items = read_data_items_from_txt(in_file)
flds = data_items[0][:]
data_items.remove(flds)
if not 0 <= day_divided_hour <= 23:
raise ValueError('Day divided hour must range from 0 to 23!')
try:
date_idx = flds.index('DATETIME')
flds.remove('DATETIME')
except ValueError:
raise ValueError('DATETIME must be one of the fields!')
# available field
available_flds = ['FLOW', 'SED', 'PCP']
def check_avaiable_field(cur_fld):
"""Check if the given field name is supported."""
support_flag = False
for fff in available_flds:
if fff.lower() in cur_fld.lower():
support_flag = True
break
return support_flag
ord_data = OrderedDict()
time_zone_output = time.timezone / -3600
if time_sys_output.lower().find('local') >= 0:
tmpstrs = StringClass.split_string(time_sys_output, [' '])
if len(tmpstrs) == 2 and MathClass.isnumerical(tmpstrs[1]):
time_zone_output = int(tmpstrs[1])
time_sys_output = 'LOCALTIME'
else:
time_sys_output = 'UTCTIME'
time_zone_output = 0
for item in data_items:
org_datetime = StringClass.get_datetime(item[date_idx])
if time_sys_input == 'LOCALTIME':
org_datetime -= timedelta(hours=time_zone_input)
# now, org_datetime is UTC time.
if time_sys_output == 'LOCALTIME':
org_datetime += timedelta(hours=time_zone_output)
# now, org_datetime is consistent with the output time system
ord_data[org_datetime] = list()
for i, v in enumerate(item):
if i == date_idx:
continue
if MathClass.isnumerical(v):
ord_data[org_datetime].append(float(v))
else:
ord_data[org_datetime].append(v)
# print(ord_data)
itp_data = OrderedDict()
out_time_delta = timedelta(minutes=time_interval)
sdatetime = StringClass.get_datetime(start_time)
edatetime = StringClass.get_datetime(end_time)
item_dtime = sdatetime
if time_interval % 1440 == 0:
item_dtime = sdatetime.replace(hour=0, minute=0, second=0) + \
timedelta(minutes=day_divided_hour * 60)
while item_dtime <= edatetime:
# print(item_dtime)
# if item_dtime.month == 12 and item_dtime.day == 31:
# print("debug")
sdt = item_dtime # start datetime of records
edt = item_dtime + out_time_delta # end datetime of records
# get original data items
org_items = list()
pre_dt = list(ord_data.keys())[0]
pre_added = False
for i, v in list(ord_data.items()):
if sdt <= i < edt:
if not pre_added and pre_dt < sdt < i and sdt - pre_dt < out_time_delta:
# only add one item that less than sdt.
org_items.append([pre_dt] + ord_data.get(pre_dt))
pre_added = True
org_items.append([i] + v)
if i > edt:
break
pre_dt = i
if len(org_items) > 0:
org_items.append([edt]) # Just add end time for compute convenient
if org_items[0][0] < sdt:
org_items[0][0] = sdt # set the begin datetime of current time interval
# if eliminate time interval without original records
# initial interpolated list
itp_data[item_dtime] = [0.] * len(flds)
if len(org_items) == 0:
if eliminate_zero:
itp_data.popitem()
item_dtime += out_time_delta
continue
# core interpolation code
flow_idx = -1
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
if 'SED' in v_name.upper(): # FLOW must be existed
for v_idx2, v_name2 in enumerate(flds):
if 'FLOW' in v_name2.upper():
flow_idx = v_idx2
break
if flow_idx < 0:
raise RuntimeError('To interpolate SED, FLOW must be provided!')
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
itp_value = 0.
itp_auxiliary_value = 0.
for org_item_idx, org_item_dtv in enumerate(org_items):
if org_item_idx == 0:
continue
org_item_dt = org_item_dtv[0]
pre_item_dtv = org_items[org_item_idx - 1]
pre_item_dt = pre_item_dtv[0]
tmp_delta_dt = org_item_dt - pre_item_dt
tmp_delta_secs = tmp_delta_dt.days * 86400 + tmp_delta_dt.seconds
if 'SED' in v_name.upper():
itp_value += pre_item_dtv[v_idx + 1] * pre_item_dtv[flow_idx + 1] * \
tmp_delta_secs
itp_auxiliary_value += pre_item_dtv[flow_idx + 1] * tmp_delta_secs
else:
itp_value += pre_item_dtv[v_idx + 1] * tmp_delta_secs
if 'SED' in v_name.upper():
if MathClass.floatequal(itp_auxiliary_value, 0.):
itp_value = 0.
print('WARNING: Flow is 0 for %s, please check!' %
item_dtime.strftime('%Y-%m-%d %H:%M:%S'))
itp_value /= itp_auxiliary_value
elif 'FLOW' in v_name.upper():
itp_value /= (out_time_delta.days * 86400 + out_time_delta.seconds)
elif 'PCP' in v_name.upper(): # the input is mm/h, and output is mm
itp_value /= 3600.
itp_data[item_dtime][v_idx] = round(itp_value, 4)
item_dtime += out_time_delta
# for i, v in itp_data.items():
# print(i, v)
# output to files
work_path = os.path.dirname(in_file)
header_str = '#' + time_sys_output
if time_sys_output == 'LOCALTIME':
header_str = header_str + ' ' + str(time_zone_output)
for idx, fld in enumerate(flds):
if not check_avaiable_field(fld):
continue
file_name = fld + '_' + time_sys_output + '_' + str(time_interval)
if eliminate_zero:
file_name += '_nonzero'
file_name += '.txt'
out_file = work_path + os.path.sep + file_name
with open(out_file, 'w') as f:
f.write(header_str + '\n')
f.write('DATETIME,' + fld + '\n')
for i, v in list(itp_data.items()):
cur_line = i.strftime('%Y-%m-%d %H:%M:%S') + ',' + str(v[idx]) + '\n'
f.write(cur_line)
def __init__(self, cf):
# Default arguments
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.config_dict = dict()
if 'SEIMS_Model' not in cf.sections():
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
self.host = cf.get('SEIMS_Model', 'hostname')
self.port = cf.getint('SEIMS_Model', 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!')
self.bin_dir = cf.get('SEIMS_Model', 'bin_dir')
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [SEIMS_Model]. '
'BIN_DIR and MODEL_DIR are required!')
self.db_name = os.path.split(self.model_dir)[1]
if cf.has_option('SEIMS_Model', 'version'):
self.version = cf.get('SEIMS_Model', 'version')
if cf.has_option('SEIMS_Model',
'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin')
if cf.has_option('SEIMS_Model', 'hostopt'):
self.hosts_opt = cf.get('SEIMS_Model', 'hostopt')
if cf.has_option('SEIMS_Model', 'hostfile'):
self.hostfile = cf.get('SEIMS_Model', 'hostfile')
if cf.has_option('SEIMS_Model', 'processnum'):
self.nprocess = cf.getint('SEIMS_Model', 'processnum')
if cf.has_option('SEIMS_Model', 'threadsnum'):
self.nthread = cf.getint('SEIMS_Model', 'threadsnum')
if cf.has_option('SEIMS_Model', 'layeringmethod'):
self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod')
if cf.has_option('SEIMS_Model', 'scenarioid'):
self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid')
if cf.has_option('SEIMS_Model', 'calibrationid'):
self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid')
if not (cf.has_option('SEIMS_Model', 'sim_time_start')
and cf.has_option('SEIMS_Model', 'sim_time_end')):
raise ValueError(
"Start and end time MUST be specified in [SEIMS_Model].")
try:
# UTCTIME
tstart = cf.get('SEIMS_Model', 'sim_time_start')
tend = cf.get('SEIMS_Model', 'sim_time_end')
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time settings in [SEIMS_Model]!")
# Running time counted by time.time() of Python, in case of failed of GetTimespan()
self.runtime = 0.
def calculate_95ppu(sim_obs_data, sim_data, outdir, gen_num,
vali_sim_obs_data=None, vali_sim_data=None):
"""Calculate 95% prediction uncertainty and plot the hydrographs."""
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'
plt.rcParams['font.family'] = 'Times New Roman'
plt.rcParams['timezone'] = 'UTC'
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.it'] = 'STIXGeneral:italic'
plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold'
if len(sim_data) < 2:
return
var_name = sim_obs_data[0]['var_name']
for idx, var in enumerate(var_name):
plot_validation = False
if vali_sim_obs_data and vali_sim_data and var in vali_sim_obs_data[0]['var_name']:
plot_validation = True
ylabel_str = var
if var in ['Q', 'QI', 'QG', 'QS']:
ylabel_str += ' (m$^3$/s)'
elif 'CONC' in var.upper(): # Concentrate
if 'SED' in var.upper():
ylabel_str += ' (g/L)'
else:
ylabel_str += ' (mg/L)'
else: # amount
ylabel_str += ' (kg)'
cali_obs_dates = sim_obs_data[0][var]['UTCDATETIME'][:]
if isinstance(cali_obs_dates[0], str) or isinstance(cali_obs_dates[0], text_type):
cali_obs_dates = [StringClass.get_datetime(s) for s in cali_obs_dates]
obs_dates = cali_obs_dates[:]
if plot_validation:
vali_obs_dates = vali_sim_obs_data[0][var]['UTCDATETIME']
if isinstance(vali_obs_dates[0], str) or isinstance(vali_obs_dates[0], text_type):
vali_obs_dates = [StringClass.get_datetime(s) for s in vali_obs_dates]
obs_dates += vali_obs_dates
obs_data = sim_obs_data[0][var]['Obs'][:]
if plot_validation:
obs_data += vali_sim_obs_data[0][var]['Obs']
sim_dates = list(sim_data[0].keys())
if isinstance(sim_dates[0], str) or isinstance(sim_dates[0], text_type):
sim_dates = [StringClass.get_datetime(s) for s in sim_dates]
if plot_validation:
vali_sim_dates = list(vali_sim_data[0].keys())
if isinstance(vali_sim_dates[0], str) or isinstance(vali_sim_dates[0], text_type):
vali_sim_dates = [StringClass.get_datetime(s) for s in vali_sim_dates]
sim_dates += vali_sim_dates
sim_data_list = list()
caliBestIdx = -1
caliBestNSE = -9999.
for idx2, ind in enumerate(sim_data):
tmp = numpy.array(list(ind.values()))
tmp = tmp[:, idx]
if sim_obs_data[idx2][var]['NSE'] > caliBestNSE:
caliBestNSE = sim_obs_data[idx2][var]['NSE']
caliBestIdx = idx2
tmpsim = tmp.tolist()
if plot_validation:
tmpsim += numpy.array(list(vali_sim_data[idx2].values()))[:, idx].tolist()
sim_data_list.append(tmpsim)
sim_best = numpy.array(list(sim_data[caliBestIdx].values()))[:, idx]
sim_best = sim_best.tolist()
if plot_validation:
sim_best += numpy.array(list(vali_sim_data[caliBestIdx].values()))[:, idx].tolist()
sim_data_list = numpy.array(sim_data_list)
ylows = numpy.percentile(sim_data_list, 2.5, 0, interpolation='nearest')
yups = numpy.percentile(sim_data_list, 97.5, 0, interpolation='nearest')
def calculate_95ppu_efficiency(obs_data_list, obs_dates_list, sim_dates_list):
count = 0
ylows_obs = list()
yups_obs = list()
for oi, ov in enumerate(obs_data_list):
try:
si = sim_dates_list.index(obs_dates_list[oi])
ylows_obs.append(ylows[si])
yups_obs.append(yups[si])
if ylows[si] <= ov <= yups[si]:
count += 1
except Exception:
continue
p = float(count) / len(obs_data_list)
ylows_obs = numpy.array(ylows_obs)
yups_obs = numpy.array(yups_obs)
r = numpy.mean(yups_obs - ylows_obs) / numpy.std(numpy.array(obs_data_list))
return p, r
# concatenate text
p_value, r_value = calculate_95ppu_efficiency(sim_obs_data[0][var]['Obs'],
cali_obs_dates,
list(sim_data[0].keys()))
txt = 'P-factor: %.2f, R-factor: %.2f\n' % (p_value, r_value)
txt += 'One of the best simulations:\n' \
' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (sim_obs_data[caliBestIdx][var]['NSE'],
sim_obs_data[caliBestIdx][var]['RSR'],
sim_obs_data[caliBestIdx][var]['PBIAS'],
sim_obs_data[caliBestIdx][var]['R-square'])
# concatenate text of validation if needed
vali_txt = ''
if plot_validation:
p_value, r_value = calculate_95ppu_efficiency(vali_sim_obs_data[0][var]['Obs'],
vali_obs_dates,
list(vali_sim_data[0].keys()))
vali_txt = 'P-factor: %.2f, R-factor: %.2f\n\n' % (p_value, r_value)
vali_txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (vali_sim_obs_data[caliBestIdx][var]['NSE'],
vali_sim_obs_data[caliBestIdx][var]['RSR'],
vali_sim_obs_data[caliBestIdx][var]['PBIAS'],
vali_sim_obs_data[caliBestIdx][var][
'R-square'])
# plot
fig, ax = plt.subplots(figsize=(12, 4))
ax.fill_between(sim_dates, ylows.tolist(), yups.tolist(),
color=(0.8, 0.8, 0.8), label='95PPU')
ax.scatter(obs_dates, obs_data, marker='.', s=20,
color='g', label='Observed points')
ax.plot(sim_dates, sim_best, linestyle='--', color='red',
label='Best simulation', linewidth=1)
ax.set_xlim(left=min(sim_dates), right=max(sim_dates))
ax.set_ylim(bottom=0.)
date_fmt = mdates.DateFormatter('%m-%d-%y')
ax.xaxis.set_major_formatter(date_fmt)
ax.tick_params(axis='x', bottom='on', top='off', length=5, width=2, which='major')
ax.tick_params(axis='y', left='on', right='off', length=5, width=2, which='major')
plt.xlabel('Date', fontsize='small')
plt.ylabel(ylabel_str, fontsize='small')
# plot separate dash line
delta_dt = (max(sim_dates) - min(sim_dates)) / 9
delta_dt2 = (max(sim_dates) - min(sim_dates)) / 35
sep_time = max(sim_dates) - delta_dt
ymax, ymin = ax.get_ylim()
yc = abs(ymax - ymin) * 0.9
if plot_validation:
sep_time = vali_sim_dates[0] if vali_sim_dates[0] > sim_dates[0] else sim_dates[0]
ax.axvline(sep_time, color='black', linestyle='dashed', linewidth=2)
plt.text(sep_time - delta_dt, yc, 'Calibration',
fontdict={'style': 'italic', 'weight': 'bold'}, color='black')
plt.text(sep_time + delta_dt2, yc, 'Validation',
fontdict={'style': 'italic', 'weight': 'bold'}, color='black')
# add legend
handles, labels = ax.get_legend_handles_labels()
order = [labels.index('95PPU'), labels.index('Observed points'),
labels.index('Best simulation')]
ax.legend([handles[idx] for idx in order], [labels[idx] for idx in order],
fontsize='medium', loc=2, framealpha=0.8)
# add text
plt.text(sep_time - 2 * delta_dt, yc * 0.6, txt, color='red')
if plot_validation:
plt.text(sep_time + delta_dt2, yc * 0.6, vali_txt, color='red')
# fig.autofmt_xdate(rotation=0, ha='center')
plt.tight_layout()
save_png_eps(plt, outdir, 'Gen%d_95ppu_%s' % (gen_num, var))
# close current plot in case of 'figure.max_open_warning'
plt.cla()
plt.clf()
plt.close()
def __init__(self, cf):
# Default arguments
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.config_dict = dict()
if 'SEIMS_Model' not in cf.sections():
raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.")
self.host = cf.get('SEIMS_Model', 'hostname')
self.port = cf.getint('SEIMS_Model', 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!')
self.bin_dir = cf.get('SEIMS_Model', 'bin_dir')
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [SEIMS_Model]. '
'BIN_DIR and MODEL_DIR are required!')
self.db_name = os.path.split(self.model_dir)[1]
if cf.has_option('SEIMS_Model', 'version'):
self.version = cf.get('SEIMS_Model', 'version')
if cf.has_option('SEIMS_Model', 'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin')
if cf.has_option('SEIMS_Model', 'hostopt'):
self.hosts_opt = cf.get('SEIMS_Model', 'hostopt')
if cf.has_option('SEIMS_Model', 'hostfile'):
self.hostfile = cf.get('SEIMS_Model', 'hostfile')
if cf.has_option('SEIMS_Model', 'processnum'):
self.nprocess = cf.getint('SEIMS_Model', 'processnum')
if cf.has_option('SEIMS_Model', 'threadsnum'):
self.nthread = cf.getint('SEIMS_Model', 'threadsnum')
if cf.has_option('SEIMS_Model', 'layeringmethod'):
self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod')
if cf.has_option('SEIMS_Model', 'scenarioid'):
self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid')
if cf.has_option('SEIMS_Model', 'calibrationid'):
self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid')
if not (cf.has_option('SEIMS_Model', 'sim_time_start') and
cf.has_option('SEIMS_Model', 'sim_time_end')):
raise ValueError("Start and end time MUST be specified in [SEIMS_Model].")
try:
# UTCTIME
tstart = cf.get('SEIMS_Model', 'sim_time_start')
tend = cf.get('SEIMS_Model', 'sim_time_end')
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time settings in [SEIMS_Model]!")
# Running time counted by time.time() of Python, in case of failed of GetTimespan()
self.runtime = 0.
