def clean(self, scenario_id=None, calibration_id=None, delete_scenario=False, delete_spatial_gfs=False): """Clean model outputs in OUTPUT<ScenarioID>-<CalibrationID> directory and/or GridFS files in OUTPUT collection. Examples: model.SetMongoClient() model.clean() model.UnsetMongoClient() """ rmtree(self.OutputDirectory, ignore_errors=True) self.ConnectMongoDB() read_model = ReadModelData(self.mongoclient, self.db_name) if scenario_id is None: scenario_id = self.scenario_id if calibration_id is None: calibration_id = self.calibration_id read_model.CleanOutputGridFs(scenario_id, calibration_id) if delete_scenario: read_model.CleanScenariosConfiguration(scenario_id) if delete_spatial_gfs: read_model.CleanSpatialGridFs(scenario_id)
def OutputItems(self): # type: (...) -> Dict[AnyStr, Optional[List[AnyStr]]] """Read output items from database.""" if self.output_items: return self.output_items read_model = ReadModelData(self.host, self.port, self.db_name) self.output_ids, self.output_items = read_model.OutputItems() return self.output_items
def ReadOutletObservations(self, vars_list): # type: (List[AnyStr]) -> (List[AnyStr], Dict[datetime, List[float]]) """ Examples: model.SetMongoClient() model.ReadOutletObservations() model.UnsetMongoClient() """ self.ConnectMongoDB() self.ReadMongoDBData() read_model = ReadModelData(self.mongoclient, self.db_name) self.obs_vars, self.obs_value = read_model.Observation( self.outlet_id, vars_list, self.start_time, self.end_time) return self.obs_vars, self.obs_value
def ResetOutputsPeriod( self, output_ids, # type: Union[AnyStr, List[AnyStr]] stime, # type: Union[datetime, List[datetime]] etime # type: Union[datetime, List[datetime]] ): # type: (...) -> None """Reset the STARTTIME and ENDTIME of OUTPUTID(s).""" if not isinstance(output_ids, list): output_ids = [output_ids] if not isinstance(stime, list): stime = [stime] if not isinstance(etime, list): etime = [etime] read_model = ReadModelData(self.host, self.port, self.db_name) for idx, outputid in enumerate(output_ids): cur_stime = stime[0] if idx < len(stime): cur_stime = stime[idx] cur_etime = etime[0] if idx < len(etime): cur_etime = etime[idx] cur_stime_str = cur_stime.strftime('%Y-%m-%d %H:%M:%S') cur_etime_str = cur_etime.strftime('%Y-%m-%d %H:%M:%S') db = read_model.maindb db[DBTableNames.main_fileout].find_one_and_update( {'OUTPUTID': outputid}, { '$set': { 'STARTTIME': cur_stime_str, 'ENDTIME': cur_etime_str } })
def clean(self, scenario_id=None, calibration_id=None, delete_scenario=False): """Clean model outputs in OUTPUT<ScenarioID>-<CalibrationID> directory and/or GridFS files in OUTPUT collection. """ rmtree(self.OutputDirectory, ignore_errors=True) read_model = ReadModelData(self.host, self.port, self.db_name) if scenario_id is None: scenario_id = self.scenario_id if calibration_id is None: calibration_id = self.calibration_id read_model.CleanOutputGridFs(scenario_id, calibration_id) if delete_scenario: read_model.CleanScenariosConfiguration(scenario_id)
def ReadMongoDBData(self): """ Examples: model.SetMongoClient() model.ReadMongoDBData() model.UnsetMongoClient() """ if self.outlet_id >= 0: return self.ConnectMongoDB() read_model = ReadModelData(self.mongoclient, self.db_name) self.outlet_id = read_model.OutletID self.subbasin_count = read_model.SubbasinCount self.scenario_dbname = read_model.ScenarioDBName self.start_time, self.end_time = read_model.SimulationPeriod self.output_ids, self.output_items = read_model.OutputItems()
def ResetSimulationPeriod(self): """Update simulation time range in MongoDB [FILE_IN].""" read_model = ReadModelData(self.host, self.port, self.db_name) if self.simu_stime and self.simu_etime: stime_str = self.simu_stime.strftime('%Y-%m-%d %H:%M:%S') etime_str = self.simu_etime.strftime('%Y-%m-%d %H:%M:%S') db = read_model.maindb db[DBTableNames.main_filein].find_one_and_update( {'TAG': 'STARTTIME'}, {'$set': { 'VALUE': stime_str }}) db[DBTableNames.main_filein].find_one_and_update( {'TAG': 'ENDTIME'}, {'$set': { 'VALUE': etime_str }}) self.start_time, self.end_time = read_model.SimulationPeriod
def ResetSimulationPeriod(self): """Update simulation time range in MongoDB [FILE_IN]. Examples: model.SetMongoClient() model.ResetSimulationPeriod() model.UnsetMongoClient() """ self.ConnectMongoDB() read_model = ReadModelData(self.mongoclient, self.db_name) if self.simu_stime and self.simu_etime: stime_str = self.simu_stime.strftime('%Y-%m-%d %H:%M:%S') etime_str = self.simu_etime.strftime('%Y-%m-%d %H:%M:%S') db = read_model.maindb db[DBTableNames.main_filein].find_one_and_update( {'TAG': 'STARTTIME'}, {'$set': { 'VALUE': stime_str }}) db[DBTableNames.main_filein].find_one_and_update( {'TAG': 'ENDTIME'}, {'$set': { 'VALUE': etime_str }}) self.start_time, self.end_time = read_model.SimulationPeriod
def __init__(self, cfg): # type: (PostConfig) -> None """Constructor""" self.model = MainSEIMS(args_dict=cfg.model_cfg.ConfigDict) self.ws = self.model.OutputDirectory if not FileClass.is_dir_exists(self.ws): raise ValueError('The output directory %s is not existed!' % self.ws) self.plot_vars = cfg.plot_vars self.plot_cfg = cfg.plot_cfg # type: PlotConfig # UTCTIME, calibration period self.stime = cfg.cali_stime self.etime = cfg.cali_etime self.subbsnID = cfg.plt_subbsnid # validation period self.vali_stime = cfg.vali_stime self.vali_etime = cfg.vali_etime # Read model data from MongoDB, the time period of simulation is read from FILE_IN. mongoclient = ConnectMongoDB(self.model.host, self.model.port).get_conn() self.readData = ReadModelData(mongoclient, self.model.db_name) self.mode = self.readData.Mode self.interval = self.readData.Interval # check start and end time of calibration st, et = self.readData.SimulationPeriod self.plot_validation = True if st > self.stime: self.stime = st if et < self.etime: self.etime = et if st > self.etime > self.stime: self.stime = st self.etime = et # in this circumstance, no validation should be calculated. self.vali_stime = None self.vali_etime = None self.plot_validation = False # check validation time period if self.vali_stime and self.vali_etime: if self.vali_stime >= self.vali_etime or st > self.vali_etime > self.vali_stime \ or self.vali_stime >= et: self.vali_stime = None self.vali_etime = None self.plot_validation = False elif st > self.vali_stime: self.vali_stime = st elif et < self.vali_etime: self.vali_etime = et else: self.plot_validation = False # Set start time and end time of both calibration and validation periods start = self.stime end = self.etime if self.plot_validation: start = self.stime if self.stime < self.vali_stime else self.vali_stime end = self.etime if self.etime > self.vali_etime else self.vali_etime self.outletid = self.readData.OutletID # read precipitation self.pcp_date_value = self.readData.Precipitation( self.subbsnID, start, end) # read simulated data and update the available variables self.plot_vars, self.sim_data_dict = read_simulation_from_txt( self.ws, self.plot_vars, self.outletid, start, end) self.sim_data_value = list( ) # type: List[List[Union[datetime, float]]] for d, vs in self.sim_data_dict.items(): self.sim_data_value.append([d] + vs[:]) # reset start time and end time if len(self.sim_data_value) == 0: raise RuntimeError( 'No available simulate data, please check the start and end time!' ) # read observation data from MongoDB self.obs_vars, self.obs_data_dict = self.readData.Observation( self.subbsnID, self.plot_vars, start, end) # Calibration period self.sim_obs_dict = match_simulation_observation(self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.stime, end_time=self.etime) calculate_statistics(self.sim_obs_dict) # Validation period if existed self.vali_sim_obs_dict = dict() if self.plot_validation: self.vali_sim_obs_dict = match_simulation_observation( self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.vali_stime, end_time=self.vali_etime) calculate_statistics(self.vali_sim_obs_dict)
class TimeSeriesPlots(object): """Plot time series data, e.g., flow charge, sediment charge, etc. """ def __init__(self, cfg): # type: (PostConfig) -> None """Constructor""" self.model = MainSEIMS(args_dict=cfg.model_cfg.ConfigDict) self.ws = self.model.OutputDirectory if not FileClass.is_dir_exists(self.ws): raise ValueError('The output directory %s is not existed!' % self.ws) self.plot_vars = cfg.plot_vars self.plot_cfg = cfg.plot_cfg # type: PlotConfig # UTCTIME, calibration period self.stime = cfg.cali_stime self.etime = cfg.cali_etime self.subbsnID = cfg.plt_subbsnid # validation period self.vali_stime = cfg.vali_stime self.vali_etime = cfg.vali_etime # Read model data from MongoDB, the time period of simulation is read from FILE_IN. mongoclient = ConnectMongoDB(self.model.host, self.model.port).get_conn() self.readData = ReadModelData(mongoclient, self.model.db_name) self.mode = self.readData.Mode self.interval = self.readData.Interval # check start and end time of calibration st, et = self.readData.SimulationPeriod self.plot_validation = True if st > self.stime: self.stime = st if et < self.etime: self.etime = et if st > self.etime > self.stime: self.stime = st self.etime = et # in this circumstance, no validation should be calculated. self.vali_stime = None self.vali_etime = None self.plot_validation = False # check validation time period if self.vali_stime and self.vali_etime: if self.vali_stime >= self.vali_etime or st > self.vali_etime > self.vali_stime \ or self.vali_stime >= et: self.vali_stime = None self.vali_etime = None self.plot_validation = False elif st > self.vali_stime: self.vali_stime = st elif et < self.vali_etime: self.vali_etime = et else: self.plot_validation = False # Set start time and end time of both calibration and validation periods start = self.stime end = self.etime if self.plot_validation: start = self.stime if self.stime < self.vali_stime else self.vali_stime end = self.etime if self.etime > self.vali_etime else self.vali_etime self.outletid = self.readData.OutletID # read precipitation self.pcp_date_value = self.readData.Precipitation( self.subbsnID, start, end) # read simulated data and update the available variables self.plot_vars, self.sim_data_dict = read_simulation_from_txt( self.ws, self.plot_vars, self.outletid, start, end) self.sim_data_value = list( ) # type: List[List[Union[datetime, float]]] for d, vs in self.sim_data_dict.items(): self.sim_data_value.append([d] + vs[:]) # reset start time and end time if len(self.sim_data_value) == 0: raise RuntimeError( 'No available simulate data, please check the start and end time!' ) # read observation data from MongoDB self.obs_vars, self.obs_data_dict = self.readData.Observation( self.subbsnID, self.plot_vars, start, end) # Calibration period self.sim_obs_dict = match_simulation_observation(self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.stime, end_time=self.etime) calculate_statistics(self.sim_obs_dict) # Validation period if existed self.vali_sim_obs_dict = dict() if self.plot_validation: self.vali_sim_obs_dict = match_simulation_observation( self.plot_vars, self.sim_data_dict, self.obs_vars, self.obs_data_dict, start_time=self.vali_stime, end_time=self.vali_etime) calculate_statistics(self.vali_sim_obs_dict) def generate_plots(self): """Generate hydrographs of discharge, sediment, nutrient (amount or concentrate), etc.""" # set ticks direction, in or out plt.rcParams['xtick.direction'] = 'out' plt.rcParams['ytick.direction'] = 'out' plt.rcParams['font.family'] = self.plot_cfg.font_name plt.rcParams['mathtext.fontset'] = 'custom' plt.rcParams['mathtext.it'] = 'STIXGeneral:italic' plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold' obs_str = 'Observation' sim_str = 'Simulation' cali_str = 'Calibration' vali_str = 'Validation' pcp_str = 'Precipitation' pcpaxis_str = 'Precipitation (mm)' xaxis_str = 'Date time' if self.plot_cfg.plot_cn: plt.rcParams['axes.unicode_minus'] = False obs_str = u'观测值' sim_str = u'模拟值' cali_str = u'率定期' vali_str = u'验证期' pcp_str = u'降水' pcpaxis_str = u'降水 (mm)' xaxis_str = u'时间' sim_date = list(self.sim_data_dict.keys()) for i, param in enumerate(self.plot_vars): # plt.figure(i) fig, ax = plt.subplots(figsize=(12, 4)) ylabel_str = param if param in ['Q', 'QI', 'QG', 'QS']: ylabel_str += ' (m$^3$/s)' elif 'CONC' in param.upper(): # Concentrate if 'SED' in param.upper(): ylabel_str += ' (g/L)' else: ylabel_str += ' (mg/L)' elif 'SED' in param.upper(): # amount ylabel_str += ' (kg)' obs_dates = None # type: List[datetime] obs_values = None # type: List[float] if self.sim_obs_dict and param in self.sim_obs_dict: obs_dates = self.sim_obs_dict[param][DataValueFields.utc] obs_values = self.sim_obs_dict[param]['Obs'] # append validation data if self.vali_sim_obs_dict and param in self.vali_sim_obs_dict: obs_dates += self.vali_sim_obs_dict[param][DataValueFields.utc] obs_values += self.vali_sim_obs_dict[param]['Obs'] if obs_values is not None: # TODO: if the observed data is continuous with datetime, plot line, otherwise, bar. # bar graph p1 = ax.bar(obs_dates, obs_values, label=obs_str, color='none', edgecolor='black', linewidth=0.5, align='center', hatch='//') # # line graph # p1, = ax.plot(obs_dates, obs_values, label=obs_str, color='black', marker='+', # markersize=2, linewidth=1) sim_list = [v[i + 1] for v in self.sim_data_value] p2, = ax.plot(sim_date, sim_list, label=sim_str, color='red', marker='+', markersize=2, linewidth=0.8) plt.xlabel(xaxis_str, fontdict={'size': self.plot_cfg.axislabel_fsize}) # format the ticks date axis date_fmt = mdates.DateFormatter('%m-%d-%y') # autodates = mdates.AutoDateLocator() # days = mdates.DayLocator(bymonthday=range(1, 32), interval=4) # months = mdates.MonthLocator() # ax.xaxis.set_major_locator(months) ax.xaxis.set_major_formatter(date_fmt) # ax.xaxis.set_minor_locator(days) ax.tick_params('both', length=5, width=2, which='major', labelsize=self.plot_cfg.tick_fsize) ax.tick_params('both', length=3, width=1, which='minor', labelsize=self.plot_cfg.tick_fsize) ax.set_xlim(left=self.sim_data_value[0][0], right=self.sim_data_value[-1][0]) fig.autofmt_xdate(rotation=0, ha='center') plt.ylabel(ylabel_str, fontdict={'size': self.plot_cfg.axislabel_fsize}) # plt.legend(bbox_to_anchor = (0.03, 0.85), loc = 2, shadow = True) if obs_values is not None: ymax = max(max(sim_list), max(obs_values)) * 1.6 ymin = min(min(sim_list), min(obs_values)) * 0.8 else: ymax = max(sim_list) * 1.8 ymin = min(sim_list) * 0.8 ax.set_ylim(float(ymin), float(ymax)) ax2 = ax.twinx() ax.tick_params(axis='x', which='both', bottom=True, top=False, labelsize=self.plot_cfg.tick_fsize) ax2.tick_params(axis='y', length=5, width=2, which='major', labelsize=self.plot_cfg.tick_fsize) ax2.set_ylabel(pcpaxis_str, fontdict={'size': self.plot_cfg.axislabel_fsize}) pcp_date = [v[0] for v in self.pcp_date_value] preci = [v[1] for v in self.pcp_date_value] p3 = ax2.bar(pcp_date, preci, label=pcp_str, color='blue', linewidth=0, align='center') ax2.set_ylim(float(max(preci)) * 1.8, float(min(preci)) * 0.8) # draw a dash line to separate calibration and validation period delta_dt = (self.sim_data_value[-1][0] - self.sim_data_value[0][0]) // 9 delta_dt2 = (self.sim_data_value[-1][0] - self.sim_data_value[0][0]) // 35 # by default, separate time line is the end of calibration period sep_time = self.etime time_pos = [sep_time - delta_dt] ymax, ymin = ax2.get_ylim() yc = abs(ymax - ymin) / 4. order = 1 # By default, calibration period is before validation period if self.plot_validation: sep_time = self.vali_stime if self.vali_stime >= self.etime else self.stime cali_vali_labels = [cali_str, vali_str] if self.vali_stime < self.stime: order = 0 cali_vali_labels = [vali_str, cali_str] time_pos = [sep_time - delta_dt, sep_time + delta_dt2] ax.axvline(sep_time, color='black', linestyle='dashed', linewidth=2) plt.text(time_pos[0], yc, cali_vali_labels[0], fontdict={ 'style': 'italic', 'weight': 'bold', 'size': self.plot_cfg.label_fsize }, color='black') plt.text(time_pos[1], yc, cali_vali_labels[1], fontdict={ 'style': 'italic', 'weight': 'bold', 'size': self.plot_cfg.label_fsize }, color='black') # set legend and labels if obs_values is None or len(obs_values) < 2: leg = ax.legend([p3, p2], [pcp_str, sim_str], ncol=2, bbox_to_anchor=(0., 1.02, 1., 0.102), borderaxespad=0.2, loc='lower left', fancybox=True, fontsize=self.plot_cfg.legend_fsize) else: leg = ax.legend([p3, p1, p2], [pcp_str, obs_str, sim_str], bbox_to_anchor=(0., 1.02, 1., 0.102), borderaxespad=0., ncol=3, loc='lower left', fancybox=True, fontsize=self.plot_cfg.legend_fsize) try: nse = self.sim_obs_dict[param]['NSE'] # type: float r2 = self.sim_obs_dict[param]['R-square'] # type: float pbias = self.sim_obs_dict[param]['PBIAS'] # type: float rsr = self.sim_obs_dict[param]['RSR'] # type: float cali_txt = '$\mathit{NSE}$: %.2f\n$\mathit{RSR}$: %.2f\n' \ '$\mathit{PBIAS}$: %.2f%%\n$\mathit{R^2}$: %.2f' % \ (nse, rsr, pbias, r2) print_msg_header = 'Cali-%s-NSE,Cali-%s-RSR,' \ 'Cali-%s-PBIAS,Cali-%s-R2,' % (param, param, param, param) print_msg = '%.3f,%.3f,%.3f,%.3f,' % (nse, rsr, pbias, r2) cali_pos = time_pos[0] if order else time_pos[1] plt.text(cali_pos, yc * 2.5, cali_txt, color='red', fontsize=self.plot_cfg.label_fsize - 1) if self.plot_validation and self.vali_sim_obs_dict: nse = self.vali_sim_obs_dict[param]['NSE'] r2 = self.vali_sim_obs_dict[param]['R-square'] pbias = self.vali_sim_obs_dict[param]['PBIAS'] rsr = self.vali_sim_obs_dict[param]['RSR'] vali_txt = '$\mathit{NSE}$: %.2f\n$\mathit{RSR}$: %.2f\n' \ '$\mathit{PBIAS}$: %.2f%%\n$\mathit{R^2}$: %.2f' % \ (nse, rsr, pbias, r2) print_msg_header += 'Vali-%s-NSE,Vali-%s-RSR,' \ 'Vali-%s-PBIAS,' \ 'Vali-%s-R2' % (param, param, param, param) print_msg += '%.3f,%.3f,%.3f,%.3f' % (nse, rsr, pbias, r2) vali_pos = time_pos[1] if order else time_pos[0] plt.text(vali_pos, yc * 2.5, vali_txt, color='red', fontsize=self.plot_cfg.label_fsize - 1) print('%s\n%s\n' % (print_msg_header, print_msg)) except ValueError or Exception: pass plt.tight_layout() leg.get_frame().set_alpha(0.5) timerange = '%s-%s' % (self.sim_data_value[0][0].strftime( '%Y-%m-%d'), self.sim_data_value[-1][0].strftime('%Y-%m-%d')) save_png_eps(plt, self.ws, param + '-' + timerange, self.plot_cfg)
def ReadOutletObservations(self, vars_list): # type: (List[AnyStr]) -> (List[AnyStr], Dict[datetime, List[float]]) read_model = ReadModelData(self.host, self.port, self.db_name) self.obs_vars, self.obs_value = read_model.Observation( self.outlet_id, vars_list, self.start_time, self.end_time) return self.obs_vars, self.obs_value
def SimulatedPeriod(self): # type: (...) -> (datetime, datetime) read_model = ReadModelData(self.host, self.port, self.db_name) return read_model.SimulationPeriod
def ScenarioDBName(self): # type: (...) -> AnyStr read_model = ReadModelData(self.host, self.port, self.db_name) return read_model.ScenarioDBName
def SubbasinCount(self): # type: (...) -> int read_model = ReadModelData(self.host, self.port, self.db_name) return read_model.SubbasinCount
def OutletID(self): # type: (...) -> int read_model = ReadModelData(self.host, self.port, self.db_name) return read_model.OutletID