def clean(self,
scenario_id=None,
calibration_id=None,
delete_scenario=False,
delete_spatial_gfs=False):
"""Clean model outputs in OUTPUT<ScenarioID>-<CalibrationID> directory and/or
GridFS files in OUTPUT collection.
Examples:
model.SetMongoClient()
model.clean()
model.UnsetMongoClient()
"""
rmtree(self.OutputDirectory, ignore_errors=True)
self.ConnectMongoDB()
read_model = ReadModelData(self.mongoclient, self.db_name)
if scenario_id is None:
scenario_id = self.scenario_id
if calibration_id is None:
calibration_id = self.calibration_id
read_model.CleanOutputGridFs(scenario_id, calibration_id)
if delete_scenario:
read_model.CleanScenariosConfiguration(scenario_id)
if delete_spatial_gfs:
read_model.CleanSpatialGridFs(scenario_id)
def OutputItems(self):
# type: (...) -> Dict[AnyStr, Optional[List[AnyStr]]]
"""Read output items from database."""
if self.output_items:
return self.output_items
read_model = ReadModelData(self.host, self.port, self.db_name)
self.output_ids, self.output_items = read_model.OutputItems()
return self.output_items
def ReadOutletObservations(self, vars_list):
# type: (List[AnyStr]) -> (List[AnyStr], Dict[datetime, List[float]])
"""
Examples:
model.SetMongoClient()
model.ReadOutletObservations()
model.UnsetMongoClient()
"""
self.ConnectMongoDB()
self.ReadMongoDBData()
read_model = ReadModelData(self.mongoclient, self.db_name)
self.obs_vars, self.obs_value = read_model.Observation(
self.outlet_id, vars_list, self.start_time, self.end_time)
return self.obs_vars, self.obs_value
def ResetOutputsPeriod(
self,
output_ids, # type: Union[AnyStr, List[AnyStr]]
stime, # type: Union[datetime, List[datetime]]
etime # type: Union[datetime, List[datetime]]
):
# type: (...) -> None
"""Reset the STARTTIME and ENDTIME of OUTPUTID(s)."""
if not isinstance(output_ids, list):
output_ids = [output_ids]
if not isinstance(stime, list):
stime = [stime]
if not isinstance(etime, list):
etime = [etime]
read_model = ReadModelData(self.host, self.port, self.db_name)
for idx, outputid in enumerate(output_ids):
cur_stime = stime[0]
if idx < len(stime):
cur_stime = stime[idx]
cur_etime = etime[0]
if idx < len(etime):
cur_etime = etime[idx]
cur_stime_str = cur_stime.strftime('%Y-%m-%d %H:%M:%S')
cur_etime_str = cur_etime.strftime('%Y-%m-%d %H:%M:%S')
db = read_model.maindb
db[DBTableNames.main_fileout].find_one_and_update(
{'OUTPUTID': outputid}, {
'$set': {
'STARTTIME': cur_stime_str,
'ENDTIME': cur_etime_str
}
})
def clean(self,
scenario_id=None,
calibration_id=None,
delete_scenario=False):
"""Clean model outputs in OUTPUT<ScenarioID>-<CalibrationID> directory and/or
GridFS files in OUTPUT collection.
"""
rmtree(self.OutputDirectory, ignore_errors=True)
read_model = ReadModelData(self.host, self.port, self.db_name)
if scenario_id is None:
scenario_id = self.scenario_id
if calibration_id is None:
calibration_id = self.calibration_id
read_model.CleanOutputGridFs(scenario_id, calibration_id)
if delete_scenario:
read_model.CleanScenariosConfiguration(scenario_id)
def ReadMongoDBData(self):
"""
Examples:
model.SetMongoClient()
model.ReadMongoDBData()
model.UnsetMongoClient()
"""
if self.outlet_id >= 0:
return
self.ConnectMongoDB()
read_model = ReadModelData(self.mongoclient, self.db_name)
self.outlet_id = read_model.OutletID
self.subbasin_count = read_model.SubbasinCount
self.scenario_dbname = read_model.ScenarioDBName
self.start_time, self.end_time = read_model.SimulationPeriod
self.output_ids, self.output_items = read_model.OutputItems()
def ResetSimulationPeriod(self):
"""Update simulation time range in MongoDB [FILE_IN]."""
read_model = ReadModelData(self.host, self.port, self.db_name)
if self.simu_stime and self.simu_etime:
stime_str = self.simu_stime.strftime('%Y-%m-%d %H:%M:%S')
etime_str = self.simu_etime.strftime('%Y-%m-%d %H:%M:%S')
db = read_model.maindb
db[DBTableNames.main_filein].find_one_and_update(
{'TAG': 'STARTTIME'}, {'$set': {
'VALUE': stime_str
}})
db[DBTableNames.main_filein].find_one_and_update(
{'TAG': 'ENDTIME'}, {'$set': {
'VALUE': etime_str
}})
self.start_time, self.end_time = read_model.SimulationPeriod
def ResetSimulationPeriod(self):
"""Update simulation time range in MongoDB [FILE_IN].
Examples:
model.SetMongoClient()
model.ResetSimulationPeriod()
model.UnsetMongoClient()
"""
self.ConnectMongoDB()
read_model = ReadModelData(self.mongoclient, self.db_name)
if self.simu_stime and self.simu_etime:
stime_str = self.simu_stime.strftime('%Y-%m-%d %H:%M:%S')
etime_str = self.simu_etime.strftime('%Y-%m-%d %H:%M:%S')
db = read_model.maindb
db[DBTableNames.main_filein].find_one_and_update(
{'TAG': 'STARTTIME'}, {'$set': {
'VALUE': stime_str
}})
db[DBTableNames.main_filein].find_one_and_update(
{'TAG': 'ENDTIME'}, {'$set': {
'VALUE': etime_str
}})
self.start_time, self.end_time = read_model.SimulationPeriod
def __init__(self, cfg):
# type: (PostConfig) -> None
"""Constructor"""
self.model = MainSEIMS(args_dict=cfg.model_cfg.ConfigDict)
self.ws = self.model.OutputDirectory
if not FileClass.is_dir_exists(self.ws):
raise ValueError('The output directory %s is not existed!' %
self.ws)
self.plot_vars = cfg.plot_vars
self.plot_cfg = cfg.plot_cfg # type: PlotConfig
# UTCTIME, calibration period
self.stime = cfg.cali_stime
self.etime = cfg.cali_etime
self.subbsnID = cfg.plt_subbsnid
# validation period
self.vali_stime = cfg.vali_stime
self.vali_etime = cfg.vali_etime
# Read model data from MongoDB, the time period of simulation is read from FILE_IN.
mongoclient = ConnectMongoDB(self.model.host,
self.model.port).get_conn()
self.readData = ReadModelData(mongoclient, self.model.db_name)
self.mode = self.readData.Mode
self.interval = self.readData.Interval
# check start and end time of calibration
st, et = self.readData.SimulationPeriod
self.plot_validation = True
if st > self.stime:
self.stime = st
if et < self.etime:
self.etime = et
if st > self.etime > self.stime:
self.stime = st
self.etime = et
# in this circumstance, no validation should be calculated.
self.vali_stime = None
self.vali_etime = None
self.plot_validation = False
# check validation time period
if self.vali_stime and self.vali_etime:
if self.vali_stime >= self.vali_etime or st > self.vali_etime > self.vali_stime \
or self.vali_stime >= et:
self.vali_stime = None
self.vali_etime = None
self.plot_validation = False
elif st > self.vali_stime:
self.vali_stime = st
elif et < self.vali_etime:
self.vali_etime = et
else:
self.plot_validation = False
# Set start time and end time of both calibration and validation periods
start = self.stime
end = self.etime
if self.plot_validation:
start = self.stime if self.stime < self.vali_stime else self.vali_stime
end = self.etime if self.etime > self.vali_etime else self.vali_etime
self.outletid = self.readData.OutletID
# read precipitation
self.pcp_date_value = self.readData.Precipitation(
self.subbsnID, start, end)
# read simulated data and update the available variables
self.plot_vars, self.sim_data_dict = read_simulation_from_txt(
self.ws, self.plot_vars, self.outletid, start, end)
self.sim_data_value = list(
) # type: List[List[Union[datetime, float]]]
for d, vs in self.sim_data_dict.items():
self.sim_data_value.append([d] + vs[:])
# reset start time and end time
if len(self.sim_data_value) == 0:
raise RuntimeError(
'No available simulate data, please check the start and end time!'
)
# read observation data from MongoDB
self.obs_vars, self.obs_data_dict = self.readData.Observation(
self.subbsnID, self.plot_vars, start, end)
# Calibration period
self.sim_obs_dict = match_simulation_observation(self.plot_vars,
self.sim_data_dict,
self.obs_vars,
self.obs_data_dict,
start_time=self.stime,
end_time=self.etime)
calculate_statistics(self.sim_obs_dict)
# Validation period if existed
self.vali_sim_obs_dict = dict()
if self.plot_validation:
self.vali_sim_obs_dict = match_simulation_observation(
self.plot_vars,
self.sim_data_dict,
self.obs_vars,
self.obs_data_dict,
start_time=self.vali_stime,
end_time=self.vali_etime)
calculate_statistics(self.vali_sim_obs_dict)
class TimeSeriesPlots(object):
"""Plot time series data, e.g., flow charge, sediment charge, etc.
"""
def __init__(self, cfg):
# type: (PostConfig) -> None
"""Constructor"""
self.model = MainSEIMS(args_dict=cfg.model_cfg.ConfigDict)
self.ws = self.model.OutputDirectory
if not FileClass.is_dir_exists(self.ws):
raise ValueError('The output directory %s is not existed!' %
self.ws)
self.plot_vars = cfg.plot_vars
self.plot_cfg = cfg.plot_cfg # type: PlotConfig
# UTCTIME, calibration period
self.stime = cfg.cali_stime
self.etime = cfg.cali_etime
self.subbsnID = cfg.plt_subbsnid
# validation period
self.vali_stime = cfg.vali_stime
self.vali_etime = cfg.vali_etime
# Read model data from MongoDB, the time period of simulation is read from FILE_IN.
mongoclient = ConnectMongoDB(self.model.host,
self.model.port).get_conn()
self.readData = ReadModelData(mongoclient, self.model.db_name)
self.mode = self.readData.Mode
self.interval = self.readData.Interval
# check start and end time of calibration
st, et = self.readData.SimulationPeriod
self.plot_validation = True
if st > self.stime:
self.stime = st
if et < self.etime:
self.etime = et
if st > self.etime > self.stime:
self.stime = st
self.etime = et
# in this circumstance, no validation should be calculated.
self.vali_stime = None
self.vali_etime = None
self.plot_validation = False
# check validation time period
if self.vali_stime and self.vali_etime:
if self.vali_stime >= self.vali_etime or st > self.vali_etime > self.vali_stime \
or self.vali_stime >= et:
self.vali_stime = None
self.vali_etime = None
self.plot_validation = False
elif st > self.vali_stime:
self.vali_stime = st
elif et < self.vali_etime:
self.vali_etime = et
else:
self.plot_validation = False
# Set start time and end time of both calibration and validation periods
start = self.stime
end = self.etime
if self.plot_validation:
start = self.stime if self.stime < self.vali_stime else self.vali_stime
end = self.etime if self.etime > self.vali_etime else self.vali_etime
self.outletid = self.readData.OutletID
# read precipitation
self.pcp_date_value = self.readData.Precipitation(
self.subbsnID, start, end)
# read simulated data and update the available variables
self.plot_vars, self.sim_data_dict = read_simulation_from_txt(
self.ws, self.plot_vars, self.outletid, start, end)
self.sim_data_value = list(
) # type: List[List[Union[datetime, float]]]
for d, vs in self.sim_data_dict.items():
self.sim_data_value.append([d] + vs[:])
# reset start time and end time
if len(self.sim_data_value) == 0:
raise RuntimeError(
'No available simulate data, please check the start and end time!'
)
# read observation data from MongoDB
self.obs_vars, self.obs_data_dict = self.readData.Observation(
self.subbsnID, self.plot_vars, start, end)
# Calibration period
self.sim_obs_dict = match_simulation_observation(self.plot_vars,
self.sim_data_dict,
self.obs_vars,
self.obs_data_dict,
start_time=self.stime,
end_time=self.etime)
calculate_statistics(self.sim_obs_dict)
# Validation period if existed
self.vali_sim_obs_dict = dict()
if self.plot_validation:
self.vali_sim_obs_dict = match_simulation_observation(
self.plot_vars,
self.sim_data_dict,
self.obs_vars,
self.obs_data_dict,
start_time=self.vali_stime,
end_time=self.vali_etime)
calculate_statistics(self.vali_sim_obs_dict)
def generate_plots(self):
"""Generate hydrographs of discharge, sediment, nutrient (amount or concentrate), etc."""
# set ticks direction, in or out
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'
plt.rcParams['font.family'] = self.plot_cfg.font_name
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.it'] = 'STIXGeneral:italic'
plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold'
obs_str = 'Observation'
sim_str = 'Simulation'
cali_str = 'Calibration'
vali_str = 'Validation'
pcp_str = 'Precipitation'
pcpaxis_str = 'Precipitation (mm)'
xaxis_str = 'Date time'
if self.plot_cfg.plot_cn:
plt.rcParams['axes.unicode_minus'] = False
obs_str = u'观测值'
sim_str = u'模拟值'
cali_str = u'率定期'
vali_str = u'验证期'
pcp_str = u'降水'
pcpaxis_str = u'降水 (mm)'
xaxis_str = u'时间'
sim_date = list(self.sim_data_dict.keys())
for i, param in enumerate(self.plot_vars):
# plt.figure(i)
fig, ax = plt.subplots(figsize=(12, 4))
ylabel_str = param
if param in ['Q', 'QI', 'QG', 'QS']:
ylabel_str += ' (m$^3$/s)'
elif 'CONC' in param.upper(): # Concentrate
if 'SED' in param.upper():
ylabel_str += ' (g/L)'
else:
ylabel_str += ' (mg/L)'
elif 'SED' in param.upper(): # amount
ylabel_str += ' (kg)'
obs_dates = None # type: List[datetime]
obs_values = None # type: List[float]
if self.sim_obs_dict and param in self.sim_obs_dict:
obs_dates = self.sim_obs_dict[param][DataValueFields.utc]
obs_values = self.sim_obs_dict[param]['Obs']
# append validation data
if self.vali_sim_obs_dict and param in self.vali_sim_obs_dict:
obs_dates += self.vali_sim_obs_dict[param][DataValueFields.utc]
obs_values += self.vali_sim_obs_dict[param]['Obs']
if obs_values is not None:
# TODO: if the observed data is continuous with datetime, plot line, otherwise, bar.
# bar graph
p1 = ax.bar(obs_dates,
obs_values,
label=obs_str,
color='none',
edgecolor='black',
linewidth=0.5,
align='center',
hatch='//')
# # line graph
# p1, = ax.plot(obs_dates, obs_values, label=obs_str, color='black', marker='+',
# markersize=2, linewidth=1)
sim_list = [v[i + 1] for v in self.sim_data_value]
p2, = ax.plot(sim_date,
sim_list,
label=sim_str,
color='red',
marker='+',
markersize=2,
linewidth=0.8)
plt.xlabel(xaxis_str,
fontdict={'size': self.plot_cfg.axislabel_fsize})
# format the ticks date axis
date_fmt = mdates.DateFormatter('%m-%d-%y')
# autodates = mdates.AutoDateLocator()
# days = mdates.DayLocator(bymonthday=range(1, 32), interval=4)
# months = mdates.MonthLocator()
# ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(date_fmt)
# ax.xaxis.set_minor_locator(days)
ax.tick_params('both',
length=5,
width=2,
which='major',
labelsize=self.plot_cfg.tick_fsize)
ax.tick_params('both',
length=3,
width=1,
which='minor',
labelsize=self.plot_cfg.tick_fsize)
ax.set_xlim(left=self.sim_data_value[0][0],
right=self.sim_data_value[-1][0])
fig.autofmt_xdate(rotation=0, ha='center')
plt.ylabel(ylabel_str,
fontdict={'size': self.plot_cfg.axislabel_fsize})
# plt.legend(bbox_to_anchor = (0.03, 0.85), loc = 2, shadow = True)
if obs_values is not None:
ymax = max(max(sim_list), max(obs_values)) * 1.6
ymin = min(min(sim_list), min(obs_values)) * 0.8
else:
ymax = max(sim_list) * 1.8
ymin = min(sim_list) * 0.8
ax.set_ylim(float(ymin), float(ymax))
ax2 = ax.twinx()
ax.tick_params(axis='x',
which='both',
bottom=True,
top=False,
labelsize=self.plot_cfg.tick_fsize)
ax2.tick_params(axis='y',
length=5,
width=2,
which='major',
labelsize=self.plot_cfg.tick_fsize)
ax2.set_ylabel(pcpaxis_str,
fontdict={'size': self.plot_cfg.axislabel_fsize})
pcp_date = [v[0] for v in self.pcp_date_value]
preci = [v[1] for v in self.pcp_date_value]
p3 = ax2.bar(pcp_date,
preci,
label=pcp_str,
color='blue',
linewidth=0,
align='center')
ax2.set_ylim(float(max(preci)) * 1.8, float(min(preci)) * 0.8)
# draw a dash line to separate calibration and validation period
delta_dt = (self.sim_data_value[-1][0] -
self.sim_data_value[0][0]) // 9
delta_dt2 = (self.sim_data_value[-1][0] -
self.sim_data_value[0][0]) // 35
# by default, separate time line is the end of calibration period
sep_time = self.etime
time_pos = [sep_time - delta_dt]
ymax, ymin = ax2.get_ylim()
yc = abs(ymax - ymin) / 4.
order = 1 # By default, calibration period is before validation period
if self.plot_validation:
sep_time = self.vali_stime if self.vali_stime >= self.etime else self.stime
cali_vali_labels = [cali_str, vali_str]
if self.vali_stime < self.stime:
order = 0
cali_vali_labels = [vali_str, cali_str]
time_pos = [sep_time - delta_dt, sep_time + delta_dt2]
ax.axvline(sep_time,
color='black',
linestyle='dashed',
linewidth=2)
plt.text(time_pos[0],
yc,
cali_vali_labels[0],
fontdict={
'style': 'italic',
'weight': 'bold',
'size': self.plot_cfg.label_fsize
},
color='black')
plt.text(time_pos[1],
yc,
cali_vali_labels[1],
fontdict={
'style': 'italic',
'weight': 'bold',
'size': self.plot_cfg.label_fsize
},
color='black')
# set legend and labels
if obs_values is None or len(obs_values) < 2:
leg = ax.legend([p3, p2], [pcp_str, sim_str],
ncol=2,
bbox_to_anchor=(0., 1.02, 1., 0.102),
borderaxespad=0.2,
loc='lower left',
fancybox=True,
fontsize=self.plot_cfg.legend_fsize)
else:
leg = ax.legend([p3, p1, p2], [pcp_str, obs_str, sim_str],
bbox_to_anchor=(0., 1.02, 1., 0.102),
borderaxespad=0.,
ncol=3,
loc='lower left',
fancybox=True,
fontsize=self.plot_cfg.legend_fsize)
try:
nse = self.sim_obs_dict[param]['NSE'] # type: float
r2 = self.sim_obs_dict[param]['R-square'] # type: float
pbias = self.sim_obs_dict[param]['PBIAS'] # type: float
rsr = self.sim_obs_dict[param]['RSR'] # type: float
cali_txt = '$\mathit{NSE}$: %.2f\n$\mathit{RSR}$: %.2f\n' \
'$\mathit{PBIAS}$: %.2f%%\n$\mathit{R^2}$: %.2f' % \
(nse, rsr, pbias, r2)
print_msg_header = 'Cali-%s-NSE,Cali-%s-RSR,' \
'Cali-%s-PBIAS,Cali-%s-R2,' % (param, param, param, param)
print_msg = '%.3f,%.3f,%.3f,%.3f,' % (nse, rsr, pbias, r2)
cali_pos = time_pos[0] if order else time_pos[1]
plt.text(cali_pos,
yc * 2.5,
cali_txt,
color='red',
fontsize=self.plot_cfg.label_fsize - 1)
if self.plot_validation and self.vali_sim_obs_dict:
nse = self.vali_sim_obs_dict[param]['NSE']
r2 = self.vali_sim_obs_dict[param]['R-square']
pbias = self.vali_sim_obs_dict[param]['PBIAS']
rsr = self.vali_sim_obs_dict[param]['RSR']
vali_txt = '$\mathit{NSE}$: %.2f\n$\mathit{RSR}$: %.2f\n' \
'$\mathit{PBIAS}$: %.2f%%\n$\mathit{R^2}$: %.2f' % \
(nse, rsr, pbias, r2)
print_msg_header += 'Vali-%s-NSE,Vali-%s-RSR,' \
'Vali-%s-PBIAS,' \
'Vali-%s-R2' % (param, param, param, param)
print_msg += '%.3f,%.3f,%.3f,%.3f' % (nse, rsr, pbias,
r2)
vali_pos = time_pos[1] if order else time_pos[0]
plt.text(vali_pos,
yc * 2.5,
vali_txt,
color='red',
fontsize=self.plot_cfg.label_fsize - 1)
print('%s\n%s\n' % (print_msg_header, print_msg))
except ValueError or Exception:
pass
plt.tight_layout()
leg.get_frame().set_alpha(0.5)
timerange = '%s-%s' % (self.sim_data_value[0][0].strftime(
'%Y-%m-%d'), self.sim_data_value[-1][0].strftime('%Y-%m-%d'))
save_png_eps(plt, self.ws, param + '-' + timerange, self.plot_cfg)
def ReadOutletObservations(self, vars_list):
# type: (List[AnyStr]) -> (List[AnyStr], Dict[datetime, List[float]])
read_model = ReadModelData(self.host, self.port, self.db_name)
self.obs_vars, self.obs_value = read_model.Observation(
self.outlet_id, vars_list, self.start_time, self.end_time)
return self.obs_vars, self.obs_value
def SimulatedPeriod(self):
# type: (...) -> (datetime, datetime)
read_model = ReadModelData(self.host, self.port, self.db_name)
return read_model.SimulationPeriod
def ScenarioDBName(self):
# type: (...) -> AnyStr
read_model = ReadModelData(self.host, self.port, self.db_name)
return read_model.ScenarioDBName
def SubbasinCount(self):
# type: (...) -> int
read_model = ReadModelData(self.host, self.port, self.db_name)
return read_model.SubbasinCount
def OutletID(self):
# type: (...) -> int
read_model = ReadModelData(self.host, self.port, self.db_name)
return read_model.OutletID
