def workflow(cfg, maindb, climdb): """ This function mainly to import measurement data to MongoDB data type may include Q (discharge, m3/s), SED (mg/L), tn (mg/L), tp (mg/L), etc. the required parameters that defined in configuration file (*.ini) """ if not cfg.use_observed: return False c_list = climdb.collection_names() if not StringClass.string_in_list(DBTableNames.observes, c_list): climdb.create_collection(DBTableNames.observes) else: climdb.drop_collection(DBTableNames.observes) if not StringClass.string_in_list(DBTableNames.sites, c_list): climdb.create_collection(DBTableNames.sites) if not StringClass.string_in_list(DBTableNames.var_desc, c_list): climdb.create_collection(DBTableNames.var_desc) file_list = FileClass.get_full_filename_by_suffixes(cfg.observe_dir, ['.txt']) meas_file_list = [] site_loc = [] for fl in file_list: if StringClass.is_substring('observed_', fl): meas_file_list.append(fl) else: site_loc.append(fl) ImportObservedData.data_from_txt(maindb, climdb, meas_file_list, site_loc, cfg.spatials.subbsn) return True
def regular_data_from_txt(climdb, data_file): """Regular precipitation data from text file.""" # delete existed precipitation data climdb[DBTableNames.data_values].remove( {DataValueFields.type: DataType.p}) tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file( data_file) if tsysin == 'UTCTIME': tzonein = time.timezone / -3600 clim_data_items = read_data_items_from_txt(data_file) clim_flds = clim_data_items[0] station_id = list() bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() count = 0 for fld in clim_flds: if not StringClass.string_in_list(fld, [ DataValueFields.dt, DataValueFields.y, DataValueFields.m, DataValueFields.d, DataValueFields.hour, DataValueFields.minute, DataValueFields.second ]): station_id.append(fld) for i, clim_data_item in enumerate(clim_data_items): if i == 0: continue dic = dict() precipitation = list() for j, clim_data_v in enumerate(clim_data_item): if StringClass.string_in_list(clim_flds[j], station_id): precipitation.append(float(clim_data_v)) utc_time = HydroClimateUtilClass.get_utcdatetime_from_field_values( clim_flds, clim_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_time + timedelta( minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_time for j, cur_id in enumerate(station_id): cur_dic = dict() cur_dic[DataValueFields.value] = precipitation[j] cur_dic[DataValueFields.id] = int(cur_id) cur_dic[DataValueFields.type] = DataType.p cur_dic[DataValueFields.time_zone] = dic[ DataValueFields.time_zone] cur_dic[DataValueFields.local_time] = dic[ DataValueFields.local_time] cur_dic[DataValueFields.utc] = dic[DataValueFields.utc] bulk.insert(cur_dic) count += 1 if count % 500 == 0: # execute each 500 records MongoUtil.run_bulk(bulk) bulk = climdb[ DBTableNames.data_values].initialize_ordered_bulk_op() if count % 500 != 0: MongoUtil.run_bulk(bulk) # Create index climdb[DBTableNames.data_values].create_index([ (DataValueFields.id, ASCENDING), (DataValueFields.type, ASCENDING), (DataValueFields.utc, ASCENDING) ])
def workflow(cfg, maindb, climdb): """ This function mainly to import measurement data to MongoDB data type may include Q (discharge, m3/s), SED (mg/L), TN (mg/L), TP (mg/L), etc. the required parameters that defined in configuration file (*.ini) """ if not cfg.use_observed: return False c_list = climdb.collection_names() if not StringClass.string_in_list(DBTableNames.observes, c_list): climdb.create_collection(DBTableNames.observes) else: climdb.drop_collection(DBTableNames.observes) if not StringClass.string_in_list(DBTableNames.sites, c_list): climdb.create_collection(DBTableNames.sites) if not StringClass.string_in_list(DBTableNames.var_desc, c_list): climdb.create_collection(DBTableNames.var_desc) file_list = FileClass.get_full_filename_by_suffixes( cfg.observe_dir, ['.txt', '.csv']) meas_file_list = list() site_loc = list() for fl in file_list: if StringClass.is_substring('observed_', fl): meas_file_list.append(fl) else: site_loc.append(fl) ImportObservedData.data_from_txt(maindb, climdb, meas_file_list, site_loc, cfg.spatials.subbsn) return True
def regular_data_from_txt(climdb, data_file): """Regular precipitation data from text file.""" # delete existed precipitation data climdb[DBTableNames.data_values].remove({DataValueFields.type: DataType.p}) tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file(data_file) if tsysin == 'UTCTIME': tzonein = time.timezone / -3600 clim_data_items = read_data_items_from_txt(data_file) clim_flds = clim_data_items[0] station_id = list() bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() count = 0 for fld in clim_flds: if not StringClass.string_in_list(fld, [DataValueFields.dt, DataValueFields.y, DataValueFields.m, DataValueFields.d, DataValueFields.hour, DataValueFields.minute, DataValueFields.second]): station_id.append(fld) for i, clim_data_item in enumerate(clim_data_items): if i == 0: continue dic = dict() precipitation = list() for j, clim_data_v in enumerate(clim_data_item): if StringClass.string_in_list(clim_flds[j], station_id): precipitation.append(float(clim_data_v)) utc_time = HydroClimateUtilClass.get_utcdatetime_from_field_values(clim_flds, clim_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_time + timedelta(minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_time for j, cur_id in enumerate(station_id): cur_dic = dict() cur_dic[DataValueFields.value] = precipitation[j] cur_dic[DataValueFields.id] = int(cur_id) cur_dic[DataValueFields.type] = DataType.p cur_dic[DataValueFields.time_zone] = dic[DataValueFields.time_zone] cur_dic[DataValueFields.local_time] = dic[DataValueFields.local_time] cur_dic[DataValueFields.utc] = dic[DataValueFields.utc] bulk.insert(cur_dic) count += 1 if count % 500 == 0: # execute each 500 records MongoUtil.run_bulk(bulk) bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() if count % 500 != 0: MongoUtil.run_bulk(bulk) # Create index climdb[DBTableNames.data_values].create_index([(DataValueFields.id, ASCENDING), (DataValueFields.type, ASCENDING), (DataValueFields.utc, ASCENDING)])
def workflow(cfg, main_db, clim_db): """Workflow""" # 1. Find meteorology and precipitation sites in study area thiessen_file_list = [ cfg.meteo_sites_thiessen, cfg.prec_sites_thiessen ] type_list = [DataType.m, DataType.p] # The entire basin, used for OpenMP version ImportHydroClimateSites.find_sites(main_db, cfg.climate_db, cfg.vecs.bsn, FieldNames.basin, thiessen_file_list, cfg.thiessen_field, type_list) # The subbasins, used for MPI&OpenMP version ImportHydroClimateSites.find_sites(main_db, cfg.climate_db, cfg.vecs.subbsn, FieldNames.subbasin_id, thiessen_file_list, cfg.thiessen_field, type_list) # 2. Import geographic information of each sites to Hydro-Climate database c_list = clim_db.collection_names() tables = [DBTableNames.sites, DBTableNames.var_desc] for tb in tables: if not StringClass.string_in_list(tb, c_list): clim_db.create_collection(tb) ImportHydroClimateSites.variable_table(clim_db, cfg.hydro_climate_vars) site_m_loc = ImportHydroClimateSites.sites_table( clim_db, cfg.Meteo_sites, DataType.m) site_p_loc = ImportHydroClimateSites.sites_table( clim_db, cfg.prec_sites, DataType.p) # print(site_m_loc, site_p_loc) return site_m_loc, site_p_loc
def workflow(cfg, main_db, clim_db): """Workflow""" # 1. Find meteorology and precipitation sites in study area thiessen_file_list = [cfg.meteo_sites_thiessen, cfg.prec_sites_thiessen] type_list = [DataType.m, DataType.p] # The entire basin, used for OpenMP version ImportHydroClimateSites.find_sites(main_db, cfg.climate_db, cfg.vecs.bsn, FieldNames.basin, thiessen_file_list, cfg.thiessen_field, type_list) # The subbasins, used for MPI&OpenMP version ImportHydroClimateSites.find_sites(main_db, cfg.climate_db, cfg.vecs.subbsn, FieldNames.subbasin_id, thiessen_file_list, cfg.thiessen_field, type_list) # 2. Import geographic information of each sites to Hydro-Climate database c_list = clim_db.collection_names() tables = [DBTableNames.sites, DBTableNames.var_desc] for tb in tables: if not StringClass.string_in_list(tb, c_list): clim_db.create_collection(tb) ImportHydroClimateSites.variable_table(clim_db, cfg.hydro_climate_vars) site_m_loc = ImportHydroClimateSites.sites_table(clim_db, cfg.Meteo_sites, DataType.m) site_p_loc = ImportHydroClimateSites.sites_table(clim_db, cfg.prec_sites, DataType.p) # print(site_m_loc, site_p_loc) return site_m_loc, site_p_loc
def initial_params_from_txt(cfg, maindb): """ import initial calibration parameters from txt data file. Args: cfg: SEIMS config object maindb: MongoDB database object """ # delete if existed, initialize if not existed c_list = maindb.collection_names() if not StringClass.string_in_list(DBTableNames.main_parameter, c_list): maindb.create_collection(DBTableNames.main_parameter) else: maindb.drop_collection(DBTableNames.main_parameter) # initialize bulk operator bulk = maindb[DBTableNames.main_parameter].initialize_ordered_bulk_op() # read initial parameters from txt file data_items = read_data_items_from_txt(cfg.paramcfgs.init_params_file) field_names = data_items[0][0:] # print(field_names) for i, cur_data_item in enumerate(data_items): if i == 0: continue # print(cur_data_item) # initial one default blank parameter dict. data_import = {ModelParamFields.name: '', ModelParamFields.desc: '', ModelParamFields.unit: '', ModelParamFields.module: '', ModelParamFields.value: DEFAULT_NODATA, ModelParamFields.impact: DEFAULT_NODATA, ModelParamFields.change: 'NC', ModelParamFields.max: DEFAULT_NODATA, ModelParamFields.min: DEFAULT_NODATA, ModelParamFields.type: ''} for k, v in list(data_import.items()): idx = field_names.index(k) if cur_data_item[idx] == '': if StringClass.string_match(k, ModelParamFields.change_ac): data_import[k] = 0 elif StringClass.string_match(k, ModelParamFields.change_rc): data_import[k] = 1 elif StringClass.string_match(k, ModelParamFields.change_nc): data_import[k] = 0 elif StringClass.string_match(k, ModelParamFields.change_vc): data_import[k] = DEFAULT_NODATA # Be careful to check NODATA when use! else: if MathClass.isnumerical(cur_data_item[idx]): data_import[k] = float(cur_data_item[idx]) else: data_import[k] = cur_data_item[idx] bulk.insert(data_import) # execute import operators MongoUtil.run_bulk(bulk, 'No operation during initial_params_from_txt.') # initialize index by parameter's type and name by ascending order. maindb[DBTableNames.main_parameter].create_index([(ModelParamFields.type, ASCENDING), (ModelParamFields.name, ASCENDING)])
def read_optionaldta_section(self, _optdta): """Optional parameters settings of digital terrain analysis for topographic attributes""" if _optdta not in self.cf.sections(): return self.flow_model = self.cf.getint(_optdta, 'flowmodel') self.rpi_method = self.cf.getint(_optdta, 'rpimethod') self.dist_exp = self.cf.getint(_optdta, 'distanceexponentforidw') self.max_move_dist = self.cf.getfloat(_optdta, 'maxmovedist') self.numthresh = self.cf.getint(_optdta, 'numthresh') self.d8_stream_thresh = self.cf.getint(_optdta, 'd8streamthreshold') self.d8_down_method = self.cf.get(_optdta, 'd8downmethod') self.d8_stream_tag = self.cf.getint(_optdta, 'd8streamtag') self.d8_up_method = self.cf.get(_optdta, 'd8upmethod') self.dinf_stream_thresh = self.cf.getint(_optdta, 'dinfstreamthreshold') self.dinf_down_stat = self.cf.get(_optdta, 'dinfdownstat') self.dinf_down_method = self.cf.get(_optdta, 'dinfdownmethod') self.dinf_dist_down_wg = self.cf.get(_optdta, 'dinfdistdownwg') self.propthresh = self.cf.getfloat(_optdta, 'propthresh') self.dinf_up_stat = self.cf.get(_optdta, 'dinfupstat') self.dinf_up_method = self.cf.get(_optdta, 'dinfupmethod') if self.flow_model != 0: self.flow_model = 1 if self.rpi_method != 0: self.rpi_method = 1 if self.dist_exp < 0: self.dist_exp = 8 if self.max_move_dist < 0: self.max_move_dist = 50 if self.numthresh < 0: self.numthresh = 20 if self.d8_stream_thresh < 0: self.d8_stream_thresh = 0 distance_method = ['Horizontal', 'Vertical', 'Pythagoras', 'Surface'] stat_method = ['Average', 'Maximum', 'Minimum'] if not StringClass.string_in_list(self.d8_down_method, distance_method): self.d8_down_method = 'Surface' if self.d8_stream_tag < 0: self.d8_stream_tag = 1 if not StringClass.string_in_list(self.d8_up_method, distance_method): self.d8_up_method = 'Surface' if self.dinf_stream_thresh < 0: self.dinf_stream_thresh = 0 if StringClass.string_in_list(self.dinf_down_stat, stat_method): self.dinf_down_stat = 'Average' if StringClass.string_in_list(self.dinf_down_method, distance_method): self.dinf_down_method = 'Surface' self.dinf_dist_down_wg = AutoFuzSlpPosConfig.check_file_available( self.dinf_dist_down_wg) if self.propthresh < 0: self.propthresh = 0.0 if not StringClass.string_in_list(self.dinf_up_stat, stat_method): self.dinf_up_stat = 'Average' if not StringClass.string_in_list(self.dinf_up_method, distance_method): self.dinf_up_method = 'Surface' self.pretaudem = PreProcessAttrNames(self.ws.pre_dir, self.flow_model)
def read_ext_conf(ext_file): """Read extract typical location configuration file.""" with open(ext_file, 'r', encoding='utf-8') as f: lines = f.readlines() ext_conf_data = list() # Read the number of records rec_num = int(lines[1].split('\n')[0].split('\t')[1]) ext_conf_data.append(rec_num) for i in range(0, rec_num): temp_conf = lines[i + 2].split('\n')[0].split('\t') if StringClass.string_in_list(temp_conf[1], ['profc', 'horizc', 'planc']): min_v = 1000. * float(temp_conf[3]) max_v = 1000. * float(temp_conf[4]) else: min_v = float(temp_conf[3]) max_v = float(temp_conf[4]) min_s = str(round(min_v, 2)) max_s = str(round(max_v, 2)) ext_conf_data.append([temp_conf[1], min_s, max_s]) return ext_conf_data
def read_inf_conf(ext_file): """Read fuzzy inference configuration file.""" f = open(ext_file) lines = f.readlines() f.close() inf_conf_data = [] # Read the number of records rec_num = int(lines[2].split('\n')[0].split('\t')[1]) inf_conf_data.append(rec_num) for i in range(0, rec_num): temp_conf = lines[i + 3].split('\n')[0].split('\t') if StringClass.string_in_list(temp_conf[1], ['profc', 'horizc', 'planc']): w1_v = 1000. * float(temp_conf[4]) w2_v = 1000. * float(temp_conf[7]) else: w1_v = float(temp_conf[4]) w2_v = float(temp_conf[7]) w1S = str(round(w1_v, 2)) w2S = str(round(w2_v, 2)) inf_conf_data.append([temp_conf[1], temp_conf[3], w1S, w2S]) return inf_conf_data
def data_from_txt(maindb, hydro_clim_db, obs_txts_list, sites_info_txts_list, subbsn_file): """ Read observed data from txt file Args: maindb: Main spatial database hydro_clim_db: hydro-climate dababase obs_txts_list: txt file paths of observed data sites_info_txts_list: txt file paths of site information subbsn_file: subbasin raster file Returns: True or False """ # 1. Read monitor station information, and store variables information and station IDs variable_lists = [] site_ids = [] for site_file in sites_info_txts_list: site_data_items = read_data_items_from_txt(site_file) site_flds = site_data_items[0] for i in range(1, len(site_data_items)): dic = dict() for j, v in enumerate(site_data_items[i]): if StringClass.string_match(site_flds[j], StationFields.id): dic[StationFields.id] = int(v) site_ids.append(dic[StationFields.id]) elif StringClass.string_match(site_flds[j], StationFields.name): dic[StationFields.name] = v.strip() elif StringClass.string_match(site_flds[j], StationFields.type): types = StringClass.split_string(v.strip(), '-') elif StringClass.string_match(site_flds[j], StationFields.lat): dic[StationFields.lat] = float(v) elif StringClass.string_match(site_flds[j], StationFields.lon): dic[StationFields.lon] = float(v) elif StringClass.string_match(site_flds[j], StationFields.x): dic[StationFields.x] = float(v) elif StringClass.string_match(site_flds[j], StationFields.y): dic[StationFields.y] = float(v) elif StringClass.string_match(site_flds[j], StationFields.unit): dic[StationFields.unit] = v.strip() elif StringClass.string_match(site_flds[j], StationFields.elev): dic[StationFields.elev] = float(v) elif StringClass.string_match(site_flds[j], StationFields.outlet): dic[StationFields.outlet] = float(v) for j, cur_type in enumerate(types): site_dic = dict() site_dic[StationFields.id] = dic[StationFields.id] site_dic[StationFields.name] = dic[StationFields.name] site_dic[StationFields.type] = cur_type site_dic[StationFields.lat] = dic[StationFields.lat] site_dic[StationFields.lon] = dic[StationFields.lon] site_dic[StationFields.x] = dic[StationFields.x] site_dic[StationFields.y] = dic[StationFields.y] site_dic[StationFields.elev] = dic[StationFields.elev] site_dic[StationFields.outlet] = dic[StationFields.outlet] # Add SubbasinID field matched, cur_sids = ImportObservedData.match_subbasin(subbsn_file, site_dic, maindb) if not matched: break cur_subbsn_id_str = '' if len(cur_sids) == 1: # if only one subbasin ID, store integer cur_subbsn_id_str = cur_sids[0] else: cur_subbsn_id_str = ','.join(str(cid) for cid in cur_sids if cur_sids is None) site_dic[StationFields.subbsn] = cur_subbsn_id_str curfilter = {StationFields.id: site_dic[StationFields.id], StationFields.type: site_dic[StationFields.type]} # print(curfilter) hydro_clim_db[DBTableNames.sites].find_one_and_replace(curfilter, site_dic, upsert=True) var_dic = dict() var_dic[StationFields.type] = types[j] var_dic[StationFields.unit] = dic[StationFields.unit] if var_dic not in variable_lists: variable_lists.append(var_dic) site_ids = list(set(site_ids)) # 2. Read measurement data and import to MongoDB bulk = hydro_clim_db[DBTableNames.observes].initialize_ordered_bulk_op() count = 0 for measDataFile in obs_txts_list: # print(measDataFile) obs_data_items = read_data_items_from_txt(measDataFile) tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file(measDataFile) if tsysin == 'UTCTIME': tzonein = time.timezone / -3600 # If the data items is EMPTY or only have one header row, then goto # next data file. if obs_data_items == [] or len(obs_data_items) == 1: continue obs_flds = obs_data_items[0] required_flds = [StationFields.id, DataValueFields.type, DataValueFields.value] for fld in required_flds: if not StringClass.string_in_list(fld, obs_flds): # data can not meet the request! raise ValueError('The %s can not meet the required format!' % measDataFile) for i, cur_obs_data_item in enumerate(obs_data_items): dic = dict() if i == 0: continue for j, cur_data_value in enumerate(cur_obs_data_item): if StringClass.string_match(obs_flds[j], StationFields.id): dic[StationFields.id] = int(cur_data_value) # if current site ID is not included, goto next data item if dic[StationFields.id] not in site_ids: continue elif StringClass.string_match(obs_flds[j], DataValueFields.type): dic[DataValueFields.type] = cur_data_value elif StringClass.string_match(obs_flds[j], DataValueFields.value): dic[DataValueFields.value] = float(cur_data_value) utc_t = HydroClimateUtilClass.get_utcdatetime_from_field_values(obs_flds, cur_obs_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_t + timedelta(minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_t # curfilter = {StationFields.id: dic[StationFields.id], # DataValueFields.type: dic[DataValueFields.type], # DataValueFields.utc: dic[DataValueFields.utc]} # bulk.find(curfilter).replace_one(dic) bulk.insert(dic) count += 1 if count % 500 == 0: MongoUtil.run_bulk(bulk) bulk = hydro_clim_db[DBTableNames.observes].initialize_ordered_bulk_op() # db[DBTableNames.observes].find_one_and_replace(curfilter, dic, upsert=True) if count % 500 != 0: MongoUtil.run_bulk(bulk) # 3. Add measurement data with unit converted # loop variables list added_dics = [] for curVar in variable_lists: # print(curVar) # if the unit is mg/L, then change the Type name with the suffix 'Conc', # and convert the corresponding data to kg if the discharge data is # available. cur_type = curVar[StationFields.type] cur_unit = curVar[StationFields.unit] # Find data by Type for item in hydro_clim_db[DBTableNames.observes].find({StationFields.type: cur_type}): # print(item) dic = dict() dic[StationFields.id] = item[StationFields.id] dic[DataValueFields.value] = item[DataValueFields.value] dic[StationFields.type] = item[StationFields.type] dic[DataValueFields.local_time] = item[DataValueFields.local_time] dic[DataValueFields.time_zone] = item[DataValueFields.time_zone] dic[DataValueFields.utc] = item[DataValueFields.utc] if cur_unit == 'mg/L' or cur_unit == 'g/L': # update the Type name dic[StationFields.type] = cur_type + 'Conc' curfilter = {StationFields.id: dic[StationFields.id], DataValueFields.type: cur_type, DataValueFields.utc: dic[DataValueFields.utc]} hydro_clim_db[DBTableNames.observes].find_one_and_replace(curfilter, dic, upsert=True) dic[StationFields.type] = cur_type # find discharge on current day cur_filter = {StationFields.type: 'Q', DataValueFields.utc: dic[DataValueFields.utc], StationFields.id: dic[StationFields.id]} q_dic = hydro_clim_db[DBTableNames.observes].find_one(filter=cur_filter) q = -9999. if q_dic is not None: q = q_dic[DataValueFields.value] else: continue if cur_unit == 'mg/L': # convert mg/L to kg dic[DataValueFields.value] = round( dic[DataValueFields.value] * q * 86400. / 1000., 2) elif cur_unit == 'g/L': # convert g/L to kg dic[DataValueFields.value] = round( dic[DataValueFields.value] * q * 86400., 2) elif cur_unit == 'kg': dic[StationFields.type] = cur_type + 'Conc' # convert kg to mg/L dic[DataValueFields.value] = round( dic[DataValueFields.value] / q * 1000. / 86400., 2) # add new data item added_dics.append(dic) # import to MongoDB for dic in added_dics: curfilter = {StationFields.id: dic[StationFields.id], DataValueFields.type: dic[DataValueFields.type], DataValueFields.utc: dic[DataValueFields.utc]} hydro_clim_db[DBTableNames.observes].find_one_and_replace(curfilter, dic, upsert=True)
def data_from_txt(maindb, hydro_clim_db, obs_txts_list, sites_info_txts_list, subbsn_file): """ Read observed data from txt file Args: maindb: Main spatial database hydro_clim_db: hydro-climate dababase obs_txts_list: txt file paths of observed data sites_info_txts_list: txt file paths of site information subbsn_file: subbasin raster file Returns: True or False """ # 1. Read monitor station information, and store variables information and station IDs variable_lists = [] site_ids = [] for site_file in sites_info_txts_list: site_data_items = read_data_items_from_txt(site_file) site_flds = site_data_items[0] for i in range(1, len(site_data_items)): dic = dict() types = list() units = list() for j, v in enumerate(site_data_items[i]): if StringClass.string_match(site_flds[j], StationFields.id): dic[StationFields.id] = int(v) site_ids.append(dic[StationFields.id]) elif StringClass.string_match(site_flds[j], StationFields.name): dic[StationFields.name] = v.strip() elif StringClass.string_match(site_flds[j], StationFields.type): types = StringClass.split_string(v.strip(), '-') elif StringClass.string_match(site_flds[j], StationFields.lat): dic[StationFields.lat] = float(v) elif StringClass.string_match(site_flds[j], StationFields.lon): dic[StationFields.lon] = float(v) elif StringClass.string_match(site_flds[j], StationFields.x): dic[StationFields.x] = float(v) elif StringClass.string_match(site_flds[j], StationFields.y): dic[StationFields.y] = float(v) elif StringClass.string_match(site_flds[j], StationFields.unit): units = StringClass.split_string(v.strip(), '-') elif StringClass.string_match(site_flds[j], StationFields.elev): dic[StationFields.elev] = float(v) elif StringClass.string_match(site_flds[j], StationFields.outlet): dic[StationFields.outlet] = float(v) for j, cur_type in enumerate(types): site_dic = dict() site_dic[StationFields.id] = dic[StationFields.id] site_dic[StationFields.name] = dic[StationFields.name] site_dic[StationFields.type] = cur_type site_dic[StationFields.lat] = dic[StationFields.lat] site_dic[StationFields.lon] = dic[StationFields.lon] site_dic[StationFields.x] = dic[StationFields.x] site_dic[StationFields.y] = dic[StationFields.y] site_dic[StationFields.unit] = units[j] site_dic[StationFields.elev] = dic[StationFields.elev] site_dic[StationFields.outlet] = dic[StationFields.outlet] # Add SubbasinID field matched, cur_sids = ImportObservedData.match_subbasin( subbsn_file, site_dic, maindb) if not matched: break if len(cur_sids ) == 1: # if only one subbasin ID, store integer cur_subbsn_id_str = cur_sids[0] else: cur_subbsn_id_str = ','.join( str(cid) for cid in cur_sids if cur_sids is None) site_dic[StationFields.subbsn] = cur_subbsn_id_str curfilter = { StationFields.id: site_dic[StationFields.id], StationFields.type: site_dic[StationFields.type] } # print(curfilter) hydro_clim_db[DBTableNames.sites].find_one_and_replace( curfilter, site_dic, upsert=True) var_dic = dict() var_dic[StationFields.type] = types[j] var_dic[StationFields.unit] = units[j] if var_dic not in variable_lists: variable_lists.append(var_dic) site_ids = list(set(site_ids)) # 2. Read measurement data and import to MongoDB bulk = hydro_clim_db[ DBTableNames.observes].initialize_ordered_bulk_op() count = 0 for measDataFile in obs_txts_list: # print(measDataFile) obs_data_items = read_data_items_from_txt(measDataFile) tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file( measDataFile) # If the data items is EMPTY or only have one header row, then goto # next data file. if obs_data_items == [] or len(obs_data_items) == 1: continue obs_flds = obs_data_items[0] required_flds = [ StationFields.id, DataValueFields.type, DataValueFields.value ] for fld in required_flds: if not StringClass.string_in_list( fld, obs_flds): # data can not meet the request! raise ValueError( 'The %s can not meet the required format!' % measDataFile) for i, cur_obs_data_item in enumerate(obs_data_items): dic = dict() if i == 0: continue for j, cur_data_value in enumerate(cur_obs_data_item): if StringClass.string_match(obs_flds[j], StationFields.id): dic[StationFields.id] = int(cur_data_value) # if current site ID is not included, goto next data item if dic[StationFields.id] not in site_ids: continue elif StringClass.string_match(obs_flds[j], DataValueFields.type): dic[DataValueFields.type] = cur_data_value elif StringClass.string_match(obs_flds[j], DataValueFields.value): dic[DataValueFields.value] = float(cur_data_value) utc_t = HydroClimateUtilClass.get_utcdatetime_from_field_values( obs_flds, cur_obs_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_t - timedelta( minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_t # curfilter = {StationFields.id: dic[StationFields.id], # DataValueFields.type: dic[DataValueFields.type], # DataValueFields.utc: dic[DataValueFields.utc]} # bulk.find(curfilter).replace_one(dic) bulk.insert(dic) count += 1 if count % 500 == 0: MongoUtil.run_bulk(bulk) bulk = hydro_clim_db[ DBTableNames.observes].initialize_ordered_bulk_op() # db[DBTableNames.observes].find_one_and_replace(curfilter, dic, upsert=True) if count % 500 != 0: MongoUtil.run_bulk(bulk) # 3. Add measurement data with unit converted # loop variables list added_dics = list() for curVar in variable_lists: # print(curVar) # if the unit is mg/L, then change the Type name with the suffix 'Conc', # and convert the corresponding data to kg if the discharge data is # available. cur_type = curVar[StationFields.type] cur_unit = curVar[StationFields.unit] # Find data by Type for item in hydro_clim_db[DBTableNames.observes].find( {StationFields.type: cur_type}): # print(item) dic = dict() dic[StationFields.id] = item[StationFields.id] dic[DataValueFields.value] = item[DataValueFields.value] dic[StationFields.type] = item[StationFields.type] dic[DataValueFields.local_time] = item[ DataValueFields.local_time] dic[DataValueFields.time_zone] = item[ DataValueFields.time_zone] dic[DataValueFields.utc] = item[DataValueFields.utc] if cur_unit == 'mg/L' or cur_unit == 'g/L': # update the Type name dic[StationFields.type] = '%sConc' % cur_type curfilter = { StationFields.id: dic[StationFields.id], DataValueFields.type: cur_type, DataValueFields.utc: dic[DataValueFields.utc] } hydro_clim_db[DBTableNames.observes].find_one_and_replace( curfilter, dic, upsert=True) dic[StationFields.type] = cur_type # find discharge on current day cur_filter = { StationFields.type: 'Q', DataValueFields.utc: dic[DataValueFields.utc], StationFields.id: dic[StationFields.id] } q_dic = hydro_clim_db[DBTableNames.observes].find_one( filter=cur_filter) if q_dic is not None: q = q_dic[DataValueFields.value] else: continue if cur_unit == 'mg/L': # convert mg/L to kg dic[DataValueFields.value] = round( dic[DataValueFields.value] * q * 86400. / 1000., 2) elif cur_unit == 'g/L': # convert g/L to kg dic[DataValueFields.value] = round( dic[DataValueFields.value] * q * 86400., 2) elif cur_unit == 'kg': dic[StationFields.type] = '%sConc' % cur_type # convert kg to mg/L dic[DataValueFields.value] = round( dic[DataValueFields.value] / q * 1000. / 86400., 2) # add new data item added_dics.append(dic) # import to MongoDB for dic in added_dics: curfilter = { StationFields.id: dic[StationFields.id], DataValueFields.type: dic[DataValueFields.type], DataValueFields.utc: dic[DataValueFields.utc] } hydro_clim_db[DBTableNames.observes].find_one_and_replace( curfilter, dic, upsert=True)
def model_io_configuration(cfg, maindb): """ Import Input and Output Configuration of SEIMS, i.e., file.in and file.out Args: cfg: SEIMS config object maindb: MongoDB database object """ file_in_path = cfg.modelcfgs.filein file_out_path = cfg.paramcfgs.init_outputs_file # initialize if collection not existed c_list = maindb.collection_names() conf_tabs = [DBTableNames.main_filein, DBTableNames.main_fileout] for item in conf_tabs: if not StringClass.string_in_list(item, c_list): maindb.create_collection(item) else: maindb.drop_collection(item) file_in_items = read_data_items_from_txt(file_in_path) for item in file_in_items: file_in_dict = dict() values = StringClass.split_string(item[0].strip(), ['|']) if len(values) != 2: raise ValueError( 'One item should only have one Tag and one value string,' ' split by "|"') file_in_dict[ModelCfgFields.tag] = values[0] file_in_dict[ModelCfgFields.value] = values[1] maindb[DBTableNames.main_filein].insert(file_in_dict) # begin to import initial outputs settings file_out_items = read_data_items_from_txt(file_out_path) bulk = maindb[DBTableNames.main_fileout].initialize_unordered_bulk_op() out_field_array = file_out_items[0] # print(out_data_array) def read_output_item(output_fields, item): file_out_dict = dict() for i, v in enumerate(output_fields): if StringClass.string_match(ModelCfgFields.mod_cls, v): file_out_dict[ModelCfgFields.mod_cls] = item[i] elif StringClass.string_match(ModelCfgFields.output_id, v): file_out_dict[ModelCfgFields.output_id] = item[i] elif StringClass.string_match(ModelCfgFields.desc, v): file_out_dict[ModelCfgFields.desc] = item[i] elif StringClass.string_match(ModelCfgFields.unit, v): file_out_dict[ModelCfgFields.unit] = item[i] elif StringClass.string_match(ModelCfgFields.type, v): file_out_dict[ModelCfgFields.type] = item[i] elif StringClass.string_match(ModelCfgFields.stime, v): file_out_dict[ModelCfgFields.stime] = item[i] elif StringClass.string_match(ModelCfgFields.etime, v): file_out_dict[ModelCfgFields.etime] = item[i] elif StringClass.string_match(ModelCfgFields.interval, v): file_out_dict[ModelCfgFields.interval] = item[i] elif StringClass.string_match(ModelCfgFields.interval_unit, v): file_out_dict[ModelCfgFields.interval_unit] = item[i] elif StringClass.string_match(ModelCfgFields.filename, v): file_out_dict[ModelCfgFields.filename] = item[i] elif StringClass.string_match(ModelCfgFields.use, v): file_out_dict[ModelCfgFields.use] = item[i] elif StringClass.string_match(ModelCfgFields.subbsn, v): file_out_dict[ModelCfgFields.subbsn] = item[i] if not list(file_out_dict.keys()): raise ValueError( 'There are not any valid output item stored in file.out!') return file_out_dict for idx, iitem in enumerate(file_out_items): if idx == 0: continue iitem_dict = read_output_item(out_field_array, iitem) bulk.insert(iitem_dict) MongoUtil.run_bulk( bulk, 'No operations to execute when import initial outputs settings.') # begin to import the desired outputs # initialize bulk operator bulk = maindb[DBTableNames.main_fileout].initialize_ordered_bulk_op() # read initial parameters from txt file data_items = read_data_items_from_txt(cfg.modelcfgs.fileout) # print(field_names) user_out_field_array = data_items[0] if ModelCfgFields.output_id not in user_out_field_array: if len(data_items[0]) != 7: # For the compatibility of old code! raise RuntimeError( 'If header information is not provided,' 'items in file.out must have 7 columns, i.e., OUTPUTID,' 'TYPE,STARTTIME,ENDTIME,INTERVAL,INTERVAL_UNIT,SUBBASIN.' 'Otherwise, the OUTPUTID MUST existed in the header!') user_out_field_array = [ 'OUTPUTID', 'TYPE', 'STARTTIME', 'ENDTIME', 'INTERVAL', 'INTERVAL_UNIT', 'SUBBASIN' ] data_items.insert(0, user_out_field_array) for idx, iitem in enumerate(data_items): if idx == 0: continue data_import = read_output_item(user_out_field_array, iitem) data_import[ModelCfgFields.use] = 1 cur_filter = dict() cur_filter[ModelCfgFields.output_id] = data_import[ ModelCfgFields.output_id] bulk.find(cur_filter).update({'$set': data_import}) # execute import operators MongoUtil.run_bulk( bulk, 'No operations to excute when import the desired outputs.')
def lookup_tables_as_collection_and_gridfs(cfg, maindb): """Import lookup tables (from txt file) as Collection and GridFS Args: cfg: SEIMS config object maindb: workflow model database """ for tablename, txt_file in list( cfg.paramcfgs.lookup_tabs_dict.items()): # import each lookup table as a collection and GridFS file. c_list = maindb.collection_names() if not StringClass.string_in_list(tablename.upper(), c_list): maindb.create_collection(tablename.upper()) else: maindb.drop_collection(tablename.upper()) # initial bulk operator bulk = maindb[tablename.upper()].initialize_ordered_bulk_op() # delete if the tablename gridfs file existed spatial = GridFS(maindb, DBTableNames.gridfs_spatial) if spatial.exists(filename=tablename.upper()): x = spatial.get_version(filename=tablename.upper()) spatial.delete(x._id) # read data items data_items = read_data_items_from_txt(txt_file) field_names = data_items[0][0:] item_values = list() # import as gridfs file for i, cur_data_item in enumerate(data_items): if i == 0: continue data_import = dict() # import as Collection item_value = list() # import as gridfs file for idx, fld in enumerate(field_names): if MathClass.isnumerical(cur_data_item[idx]): tmp_value = float(cur_data_item[idx]) data_import[fld] = tmp_value item_value.append(tmp_value) else: data_import[fld] = cur_data_item[idx] bulk.insert(data_import) if len(item_value) > 0: item_values.append(item_value) MongoUtil.run_bulk(bulk, 'No operations during import %s.' % tablename) # begin import gridfs file n_row = len(item_values) # print(item_values) if n_row >= 1: n_col = len(item_values[0]) for i in range(n_row): if n_col != len(item_values[i]): raise ValueError( 'Please check %s to make sure each item has ' 'the same numeric dimension. The size of first ' 'row is: %d, and the current data item is: %d' % (tablename, n_col, len(item_values[i]))) else: item_values[i].insert(0, n_col) metadic = { ModelParamDataUtils.item_count: n_row, ModelParamDataUtils.field_count: n_col } cur_lookup_gridfs = spatial.new_file( filename=tablename.upper(), metadata=metadic) header = [n_row] fmt = '%df' % 1 s = pack(fmt, *header) cur_lookup_gridfs.write(s) fmt = '%df' % (n_col + 1) for i in range(n_row): s = pack(fmt, *item_values[i]) cur_lookup_gridfs.write(s) cur_lookup_gridfs.close()
def scenario_from_texts(cfg, main_db, scenario_db): """Import BMPs Scenario data to MongoDB Args: cfg: SEIMS configuration object main_db: climate database scenario_db: scenario database Returns: False if failed, otherwise True. """ if not cfg.use_scernario: return False print('Import BMP Scenario Data... ') bmp_files = FileClass.get_filename_by_suffixes(cfg.scenario_dir, ['.txt']) bmp_tabs = list() bmp_tabs_path = list() for f in bmp_files: bmp_tabs.append(f.split('.')[0]) bmp_tabs_path.append(cfg.scenario_dir + os.path.sep + f) # initialize if collection not existed c_list = scenario_db.collection_names() for item in bmp_tabs: if not StringClass.string_in_list(item.upper(), c_list): scenario_db.create_collection(item.upper()) else: scenario_db.drop_collection(item.upper()) # Read subbasin.tif and dist2Stream.tif subbasin_r = RasterUtilClass.read_raster(cfg.spatials.subbsn) dist2stream_r = RasterUtilClass.read_raster( cfg.spatials.dist2stream_d8) # End reading for j, bmp_txt in enumerate(bmp_tabs_path): bmp_tab_name = bmp_tabs[j] data_array = read_data_items_from_txt(bmp_txt) field_array = data_array[0] data_array = data_array[1:] for item in data_array: dic = dict() for i, field_name in enumerate(field_array): if MathClass.isnumerical(item[i]): v = float(item[i]) if v % 1. == 0.: v = int(v) dic[field_name.upper()] = v else: dic[field_name.upper()] = str(item[i]).upper() if StringClass.string_in_list(ImportScenario2Mongo._LocalX, list(dic.keys())) and \ StringClass.string_in_list(ImportScenario2Mongo._LocalY, list(dic.keys())): subbsn_id = subbasin_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) distance = dist2stream_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) if subbsn_id is not None and distance is not None: dic[ImportScenario2Mongo._SUBBASINID] = int(subbsn_id) dic[ImportScenario2Mongo._DISTDOWN] = float(distance) scenario_db[bmp_tab_name.upper()].find_one_and_replace( dic, dic, upsert=True) else: scenario_db[bmp_tab_name.upper()].find_one_and_replace( dic, dic, upsert=True) # print('BMP tables are imported.') # Write BMP database name into Model workflow database c_list = main_db.collection_names() if not StringClass.string_in_list(DBTableNames.main_scenario, c_list): main_db.create_collection(DBTableNames.main_scenario) bmp_info_dic = dict() bmp_info_dic[ImportScenario2Mongo._FLD_DB] = cfg.bmp_scenario_db main_db[DBTableNames.main_scenario].find_one_and_replace(bmp_info_dic, bmp_info_dic, upsert=True) return True
def daily_data_from_txt(climdb, data_txt_file, sites_info_dict): """Import climate data table""" tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file(data_txt_file) if tsysin == 'UTCTIME': tzonein = time.timezone / -3600 clim_data_items = read_data_items_from_txt(data_txt_file) clim_flds = clim_data_items[0] # PHUCalDic is used for Calculating potential heat units (PHU) # for each climate station and each year. # format is {StationID:{Year1:[values],Year2:[Values]...}, ...} # PHUCalDic = {} # format: {StationID1: climateStats1, ...} hydro_climate_stats = dict() required_flds = [DataType.max_tmp, DataType.min_tmp, DataType.rm, DataType.ws] output_flds = [DataType.mean_tmp, DataType.max_tmp, DataType.min_tmp, DataType.rm, DataType.pet, DataType.ws, DataType.sr] # remove existed records for fld in output_flds: climdb[DBTableNames.data_values].remove({'TYPE': fld}) for fld in required_flds: if not StringClass.string_in_list(fld, clim_flds): raise ValueError('Meteorological Daily data MUST contain %s!' % fld) # Create bulk object bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() count = 0 for i, cur_clim_data_item in enumerate(clim_data_items): if i == 0: continue dic = dict() cur_ssd = DEFAULT_NODATA for j, clim_data_v in enumerate(cur_clim_data_item): if StringClass.string_match(clim_flds[j], DataValueFields.id): dic[DataValueFields.id] = int(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.mean_tmp): dic[DataType.mean_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.min_tmp): dic[DataType.min_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.max_tmp): dic[DataType.max_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.pet): dic[DataType.pet] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.sr): dic[DataType.sr] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.ws): dic[DataType.ws] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.rm): dic[DataType.rm] = float(clim_data_v) * 0.01 elif StringClass.string_match(clim_flds[j], DataType.ssd): cur_ssd = float(clim_data_v) # Get datetime and utc/local transformation utc_time = HydroClimateUtilClass.get_utcdatetime_from_field_values(clim_flds, cur_clim_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_time + timedelta(minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_time dic[DataValueFields.y] = utc_time.year # Do if some of these data are not provided if DataType.mean_tmp not in list(dic.keys()): dic[DataType.mean_tmp] = (dic[DataType.max_tmp] + dic[DataType.min_tmp]) / 2. if DataType.sr not in list(dic.keys()): if cur_ssd == DEFAULT_NODATA: raise ValueError(DataType.sr + ' or ' + DataType.ssd + ' must be provided!') else: if dic[DataValueFields.id] in list(sites_info_dict.keys()): cur_lon, cur_lat = sites_info_dict[dic[DataValueFields.id]].lon_lat() sr = round(HydroClimateUtilClass.rs(DateClass.day_of_year(utc_time), float(cur_ssd), cur_lat * PI / 180.), 1) dic[DataType.sr] = sr for fld in output_flds: cur_dic = dict() if fld in list(dic.keys()): cur_dic[DataValueFields.value] = dic[fld] cur_dic[DataValueFields.id] = dic[ DataValueFields.id] cur_dic[DataValueFields.utc] = dic[DataValueFields.utc] cur_dic[DataValueFields.time_zone] = dic[DataValueFields.time_zone] cur_dic[DataValueFields.local_time] = dic[DataValueFields.local_time] cur_dic[DataValueFields.type] = fld # Old code, insert or update one item a time, which is quite inefficiency # Update by using bulk operation interface. lj # # find old records and remove (deprecated because of low efficiency, lj.) # curfilter = {DataValueFields.type: fld, # DataValueFields.utc: dic[DataValueFields.utc]} # bulk.find(curfilter).upsert().replace_one(cur_dic) bulk.insert(cur_dic) count += 1 if count % 500 == 0: # execute each 500 records MongoUtil.run_bulk(bulk) bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() if dic[DataValueFields.id] not in list(hydro_climate_stats.keys()): hydro_climate_stats[dic[DataValueFields.id]] = ClimateStats() hydro_climate_stats[dic[DataValueFields.id]].add_item(dic) # execute the remained records if count % 500 != 0: MongoUtil.run_bulk(bulk) for item, cur_climate_stats in list(hydro_climate_stats.items()): cur_climate_stats.annual_stats() # Create index climdb[DBTableNames.data_values].create_index([(DataValueFields.id, ASCENDING), (DataValueFields.type, ASCENDING), (DataValueFields.utc, ASCENDING)]) # prepare dic for MongoDB for s_id, stats_v in list(hydro_climate_stats.items()): for YYYY in list(stats_v.Count.keys()): cur_dic = dict() cur_dic[DataValueFields.value] = stats_v.PHUTOT[YYYY] cur_dic[DataValueFields.id] = s_id cur_dic[DataValueFields.y] = YYYY cur_dic[VariableDesc.unit] = 'heat units' cur_dic[VariableDesc.type] = DataType.phu_tot curfilter = {DataValueFields.id: s_id, VariableDesc.type: DataType.phu_tot, DataValueFields.y: YYYY} climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True) # import annual mean temperature cur_dic[VariableDesc.type] = DataType.mean_tmp cur_dic[VariableDesc.unit] = 'deg C' cur_dic[DataValueFields.value] = stats_v.MeanTmp[YYYY] curfilter = {DataValueFields.id: s_id, VariableDesc.type: DataType.mean_tmp, DataValueFields.y: YYYY} climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True) cur_dic[DataValueFields.value] = stats_v.PHU0 cur_dic[DataValueFields.id] = s_id cur_dic[DataValueFields.y] = DEFAULT_NODATA cur_dic[VariableDesc.unit] = 'heat units' cur_dic[VariableDesc.type] = DataType.phu0 curfilter = {DataValueFields.id: s_id, VariableDesc.type: DataType.phu0, DataValueFields.y: DEFAULT_NODATA} climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True) # import annual mean temperature cur_dic[VariableDesc.type] = DataType.mean_tmp0 cur_dic[VariableDesc.unit] = 'deg C' cur_dic[DataValueFields.value] = stats_v.MeanTmp0 curfilter = {DataValueFields.id: s_id, VariableDesc.type: DataType.mean_tmp0, DataValueFields.y: DEFAULT_NODATA} climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True)
def scenario_from_texts(cfg, main_db, scenario_db): """Import BMPs Scenario data to MongoDB Args: cfg: SEIMS configuration object main_db: climate database scenario_db: scenario database Returns: False if failed, otherwise True. """ if not cfg.use_scernario: return False print('Import BMP Scenario Data... ') bmp_files = FileClass.get_filename_by_suffixes(cfg.scenario_dir, ['.txt']) bmp_tabs = list() bmp_tabs_path = list() for f in bmp_files: bmp_tabs.append(f.split('.')[0]) bmp_tabs_path.append(cfg.scenario_dir + os.path.sep + f) # initialize if collection not existed c_list = scenario_db.collection_names() for item in bmp_tabs: if not StringClass.string_in_list(item.upper(), c_list): scenario_db.create_collection(item.upper()) else: scenario_db.drop_collection(item.upper()) # Read subbasin.tif and dist2Stream.tif subbasin_r = RasterUtilClass.read_raster(cfg.spatials.subbsn) dist2stream_r = RasterUtilClass.read_raster(cfg.spatials.dist2stream_d8) # End reading for j, bmp_txt in enumerate(bmp_tabs_path): bmp_tab_name = bmp_tabs[j] data_array = read_data_items_from_txt(bmp_txt) field_array = data_array[0] data_array = data_array[1:] for item in data_array: dic = dict() for i, field_name in enumerate(field_array): if MathClass.isnumerical(item[i]): v = float(item[i]) if v % 1. == 0.: v = int(v) dic[field_name.upper()] = v else: dic[field_name.upper()] = str(item[i]).upper() if StringClass.string_in_list(ImportScenario2Mongo._LocalX, list(dic.keys())) and \ StringClass.string_in_list(ImportScenario2Mongo._LocalY, list(dic.keys())): subbsn_id = subbasin_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) distance = dist2stream_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) if subbsn_id is not None and distance is not None: dic[ImportScenario2Mongo._SUBBASINID] = int(subbsn_id) dic[ImportScenario2Mongo._DISTDOWN] = float(distance) scenario_db[bmp_tab_name.upper()].find_one_and_replace(dic, dic, upsert=True) else: scenario_db[bmp_tab_name.upper()].find_one_and_replace(dic, dic, upsert=True) # print('BMP tables are imported.') # Write BMP database name into Model workflow database c_list = main_db.collection_names() if not StringClass.string_in_list(DBTableNames.main_scenario, c_list): main_db.create_collection(DBTableNames.main_scenario) bmp_info_dic = dict() bmp_info_dic[ImportScenario2Mongo._FLD_DB] = cfg.bmp_scenario_db main_db[DBTableNames.main_scenario].find_one_and_replace(bmp_info_dic, bmp_info_dic, upsert=True) return True
def daily_data_from_txt(climdb, data_txt_file, sites_info_dict): """Import climate data table""" tsysin, tzonein = HydroClimateUtilClass.get_time_system_from_data_file( data_txt_file) if tsysin == 'UTCTIME': tzonein = time.timezone / -3600 clim_data_items = read_data_items_from_txt(data_txt_file) clim_flds = clim_data_items[0] # PHUCalDic is used for Calculating potential heat units (PHU) # for each climate station and each year. # format is {StationID:{Year1:[values],Year2:[Values]...}, ...} # PHUCalDic = {} # format: {StationID1: climateStats1, ...} hydro_climate_stats = dict() required_flds = [ DataType.max_tmp, DataType.min_tmp, DataType.rm, DataType.ws ] output_flds = [ DataType.mean_tmp, DataType.max_tmp, DataType.min_tmp, DataType.rm, DataType.pet, DataType.ws, DataType.sr ] # remove existed records for fld in output_flds: climdb[DBTableNames.data_values].remove({'TYPE': fld}) for fld in required_flds: if not StringClass.string_in_list(fld, clim_flds): raise ValueError('Meteorological Daily data MUST contain %s!' % fld) # Create bulk object bulk = climdb[DBTableNames.data_values].initialize_ordered_bulk_op() count = 0 for i, cur_clim_data_item in enumerate(clim_data_items): if i == 0: continue dic = dict() cur_ssd = DEFAULT_NODATA for j, clim_data_v in enumerate(cur_clim_data_item): if StringClass.string_match(clim_flds[j], DataValueFields.id): dic[DataValueFields.id] = int(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.mean_tmp): dic[DataType.mean_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.min_tmp): dic[DataType.min_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.max_tmp): dic[DataType.max_tmp] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.pet): dic[DataType.pet] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.sr): dic[DataType.sr] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.ws): dic[DataType.ws] = float(clim_data_v) elif StringClass.string_match(clim_flds[j], DataType.rm): dic[DataType.rm] = float(clim_data_v) * 0.01 elif StringClass.string_match(clim_flds[j], DataType.ssd): cur_ssd = float(clim_data_v) # Get datetime and utc/local transformation utc_time = HydroClimateUtilClass.get_utcdatetime_from_field_values( clim_flds, cur_clim_data_item, tsysin, tzonein) dic[DataValueFields.local_time] = utc_time + timedelta( minutes=tzonein * 60) dic[DataValueFields.time_zone] = tzonein dic[DataValueFields.utc] = utc_time dic[DataValueFields.y] = utc_time.year # Do if some of these data are not provided if DataType.mean_tmp not in list(dic.keys()): dic[DataType.mean_tmp] = (dic[DataType.max_tmp] + dic[DataType.min_tmp]) / 2. if DataType.sr not in list(dic.keys()): if cur_ssd == DEFAULT_NODATA: raise ValueError(DataType.sr + ' or ' + DataType.ssd + ' must be provided!') else: if dic[DataValueFields.id] in list(sites_info_dict.keys()): cur_lon, cur_lat = sites_info_dict[dic[ DataValueFields.id]].lon_lat() sr = round( HydroClimateUtilClass.rs( DateClass.day_of_year(utc_time), float(cur_ssd), cur_lat * PI / 180.), 1) dic[DataType.sr] = sr for fld in output_flds: cur_dic = dict() if fld in list(dic.keys()): cur_dic[DataValueFields.value] = dic[fld] cur_dic[DataValueFields.id] = dic[DataValueFields.id] cur_dic[DataValueFields.utc] = dic[DataValueFields.utc] cur_dic[DataValueFields.time_zone] = dic[ DataValueFields.time_zone] cur_dic[DataValueFields.local_time] = dic[ DataValueFields.local_time] cur_dic[DataValueFields.type] = fld # Old code, insert or update one item a time, which is quite inefficiency # Update by using bulk operation interface. lj # # find old records and remove (deprecated because of low efficiency, lj.) # curfilter = {DataValueFields.type: fld, # DataValueFields.utc: dic[DataValueFields.utc]} # bulk.find(curfilter).upsert().replace_one(cur_dic) bulk.insert(cur_dic) count += 1 if count % 500 == 0: # execute each 500 records MongoUtil.run_bulk(bulk) bulk = climdb[ DBTableNames. data_values].initialize_ordered_bulk_op() if dic[DataValueFields.id] not in list(hydro_climate_stats.keys()): hydro_climate_stats[dic[DataValueFields.id]] = ClimateStats() hydro_climate_stats[dic[DataValueFields.id]].add_item(dic) # execute the remained records if count % 500 != 0: MongoUtil.run_bulk(bulk) for item, cur_climate_stats in list(hydro_climate_stats.items()): cur_climate_stats.annual_stats() # Create index climdb[DBTableNames.data_values].create_index([ (DataValueFields.id, ASCENDING), (DataValueFields.type, ASCENDING), (DataValueFields.utc, ASCENDING) ]) # prepare dic for MongoDB for s_id, stats_v in list(hydro_climate_stats.items()): for YYYY in list(stats_v.Count.keys()): cur_dic = dict() cur_dic[DataValueFields.value] = stats_v.PHUTOT[YYYY] cur_dic[DataValueFields.id] = s_id cur_dic[DataValueFields.y] = YYYY cur_dic[VariableDesc.unit] = 'heat units' cur_dic[VariableDesc.type] = DataType.phu_tot curfilter = { DataValueFields.id: s_id, VariableDesc.type: DataType.phu_tot, DataValueFields.y: YYYY } climdb[DBTableNames.annual_stats].find_one_and_replace( curfilter, cur_dic, upsert=True) # import annual mean temperature cur_dic[VariableDesc.type] = DataType.mean_tmp cur_dic[VariableDesc.unit] = 'deg C' cur_dic[DataValueFields.value] = stats_v.MeanTmp[YYYY] curfilter = { DataValueFields.id: s_id, VariableDesc.type: DataType.mean_tmp, DataValueFields.y: YYYY } climdb[DBTableNames.annual_stats].find_one_and_replace( curfilter, cur_dic, upsert=True) cur_dic[DataValueFields.value] = stats_v.PHU0 cur_dic[DataValueFields.id] = s_id cur_dic[DataValueFields.y] = DEFAULT_NODATA cur_dic[VariableDesc.unit] = 'heat units' cur_dic[VariableDesc.type] = DataType.phu0 curfilter = { DataValueFields.id: s_id, VariableDesc.type: DataType.phu0, DataValueFields.y: DEFAULT_NODATA } climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True) # import annual mean temperature cur_dic[VariableDesc.type] = DataType.mean_tmp0 cur_dic[VariableDesc.unit] = 'deg C' cur_dic[DataValueFields.value] = stats_v.MeanTmp0 curfilter = { DataValueFields.id: s_id, VariableDesc.type: DataType.mean_tmp0, DataValueFields.y: DEFAULT_NODATA } climdb[DBTableNames.annual_stats].find_one_and_replace(curfilter, cur_dic, upsert=True)
def lookup_tables_as_collection_and_gridfs(cfg, maindb): """Import lookup tables (from txt file) as Collection and GridFS Args: cfg: SEIMS config object maindb: workflow model database """ for tablename, txt_file in list(cfg.paramcfgs.lookup_tabs_dict.items()): # import each lookup table as a collection and GridFS file. c_list = maindb.collection_names() if not StringClass.string_in_list(tablename.upper(), c_list): maindb.create_collection(tablename.upper()) else: maindb.drop_collection(tablename.upper()) # initial bulk operator bulk = maindb[tablename.upper()].initialize_ordered_bulk_op() # delete if the tablename gridfs file existed spatial = GridFS(maindb, DBTableNames.gridfs_spatial) if spatial.exists(filename=tablename.upper()): x = spatial.get_version(filename=tablename.upper()) spatial.delete(x._id) # read data items data_items = read_data_items_from_txt(txt_file) field_names = data_items[0][0:] item_values = list() # import as gridfs file for i, cur_data_item in enumerate(data_items): if i == 0: continue data_import = dict() # import as Collection item_value = list() # import as gridfs file for idx, fld in enumerate(field_names): if MathClass.isnumerical(cur_data_item[idx]): tmp_value = float(cur_data_item[idx]) data_import[fld] = tmp_value item_value.append(tmp_value) else: data_import[fld] = cur_data_item[idx] bulk.insert(data_import) if len(item_value) > 0: item_values.append(item_value) MongoUtil.run_bulk(bulk, 'No operations during import %s.' % tablename) # begin import gridfs file n_row = len(item_values) # print(item_values) if n_row >= 1: n_col = len(item_values[0]) for i in range(n_row): if n_col != len(item_values[i]): raise ValueError('Please check %s to make sure each item has ' 'the same numeric dimension. The size of first ' 'row is: %d, and the current data item is: %d' % (tablename, n_col, len(item_values[i]))) else: item_values[i].insert(0, n_col) metadic = {ModelParamDataUtils.item_count: n_row, ModelParamDataUtils.field_count: n_col} cur_lookup_gridfs = spatial.new_file(filename=tablename.upper(), metadata=metadic) header = [n_row] fmt = '%df' % 1 s = pack(fmt, *header) cur_lookup_gridfs.write(s) fmt = '%df' % (n_col + 1) for i in range(n_row): s = pack(fmt, *item_values[i]) cur_lookup_gridfs.write(s) cur_lookup_gridfs.close()
def model_io_configuration(cfg, maindb): """ Import Input and Output Configuration of SEIMS, i.e., file.in and file.out Args: cfg: SEIMS config object maindb: MongoDB database object """ file_in_path = cfg.modelcfgs.filein file_out_path = cfg.paramcfgs.init_outputs_file # initialize if collection not existed c_list = maindb.collection_names() conf_tabs = [DBTableNames.main_filein, DBTableNames.main_fileout] for item in conf_tabs: if not StringClass.string_in_list(item, c_list): maindb.create_collection(item) else: maindb.drop_collection(item) file_in_items = read_data_items_from_txt(file_in_path) file_out_items = read_data_items_from_txt(file_out_path) for item in file_in_items: file_in_dict = dict() values = StringClass.split_string(item[0].strip(), ['|']) if len(values) != 2: raise ValueError('One item should only have one Tag and one value string,' ' split by "|"') file_in_dict[ModelCfgFields.tag] = values[0] file_in_dict[ModelCfgFields.value] = values[1] maindb[DBTableNames.main_filein].insert(file_in_dict) # begin to import initial outputs settings bulk = maindb[DBTableNames.main_fileout].initialize_unordered_bulk_op() out_field_array = file_out_items[0] out_data_array = file_out_items[1:] # print(out_data_array) for item in out_data_array: file_out_dict = dict() for i, v in enumerate(out_field_array): if StringClass.string_match(ModelCfgFields.mod_cls, v): file_out_dict[ModelCfgFields.mod_cls] = item[i] elif StringClass.string_match(ModelCfgFields.output_id, v): file_out_dict[ModelCfgFields.output_id] = item[i] elif StringClass.string_match(ModelCfgFields.desc, v): file_out_dict[ModelCfgFields.desc] = item[i] elif StringClass.string_match(ModelCfgFields.unit, v): file_out_dict[ModelCfgFields.unit] = item[i] elif StringClass.string_match(ModelCfgFields.type, v): file_out_dict[ModelCfgFields.type] = item[i] elif StringClass.string_match(ModelCfgFields.stime, v): file_out_dict[ModelCfgFields.stime] = item[i] elif StringClass.string_match(ModelCfgFields.etime, v): file_out_dict[ModelCfgFields.etime] = item[i] elif StringClass.string_match(ModelCfgFields.interval, v): file_out_dict[ModelCfgFields.interval] = item[i] elif StringClass.string_match(ModelCfgFields.interval_unit, v): file_out_dict[ModelCfgFields.interval_unit] = item[i] elif StringClass.string_match(ModelCfgFields.filename, v): file_out_dict[ModelCfgFields.filename] = item[i] elif StringClass.string_match(ModelCfgFields.use, v): file_out_dict[ModelCfgFields.use] = item[i] elif StringClass.string_match(ModelCfgFields.subbsn, v): file_out_dict[ModelCfgFields.subbsn] = item[i] if not list(file_out_dict.keys()): raise ValueError('There are not any valid output item stored in file.out!') bulk.insert(file_out_dict) MongoUtil.run_bulk(bulk, 'No operations to excute when import initial outputs settings.') # begin to import the desired outputs # initialize bulk operator bulk = maindb[DBTableNames.main_fileout].initialize_ordered_bulk_op() # read initial parameters from txt file data_items = read_data_items_from_txt(cfg.modelcfgs.fileout) # print(field_names) for i, cur_data_item in enumerate(data_items): data_import = dict() cur_filter = dict() # print(cur_data_item) if len(cur_data_item) == 7: data_import[ModelCfgFields.output_id] = cur_data_item[0] data_import[ModelCfgFields.type] = cur_data_item[1] data_import[ModelCfgFields.stime] = cur_data_item[2] data_import[ModelCfgFields.etime] = cur_data_item[3] data_import[ModelCfgFields.interval] = cur_data_item[4] data_import[ModelCfgFields.interval_unit] = cur_data_item[5] data_import[ModelCfgFields.subbsn] = cur_data_item[6] data_import[ModelCfgFields.use] = 1 cur_filter[ModelCfgFields.output_id] = cur_data_item[0] else: raise RuntimeError('Items in file.out must have 7 columns, i.e., OUTPUTID,' 'TYPE,STARTTIME,ENDTIME,INTERVAL,INTERVAL_UNIT,SUBBASIN.') bulk.find(cur_filter).update({'$set': data_import}) # execute import operators MongoUtil.run_bulk(bulk, 'No operations to excute when import the desired outputs.')
def initial_params_from_txt(cfg, maindb): """ import initial calibration parameters from txt data file. Args: cfg: SEIMS config object maindb: MongoDB database object """ # delete if existed, initialize if not existed c_list = maindb.collection_names() if not StringClass.string_in_list(DBTableNames.main_parameter, c_list): maindb.create_collection(DBTableNames.main_parameter) else: maindb.drop_collection(DBTableNames.main_parameter) # initialize bulk operator bulk = maindb[DBTableNames.main_parameter].initialize_ordered_bulk_op() # read initial parameters from txt file data_items = read_data_items_from_txt(cfg.paramcfgs.init_params_file) field_names = data_items[0][0:] # print(field_names) for i, cur_data_item in enumerate(data_items): if i == 0: continue # print(cur_data_item) # initial one default blank parameter dict. data_import = { ModelParamFields.name: '', ModelParamFields.desc: '', ModelParamFields.unit: '', ModelParamFields.module: '', ModelParamFields.value: DEFAULT_NODATA, ModelParamFields.impact: DEFAULT_NODATA, ModelParamFields.change: 'NC', ModelParamFields.max: DEFAULT_NODATA, ModelParamFields.min: DEFAULT_NODATA, ModelParamFields.type: '' } for k, v in list(data_import.items()): idx = field_names.index(k) if cur_data_item[idx] == '': if StringClass.string_match(k, ModelParamFields.change_ac): data_import[k] = 0 elif StringClass.string_match(k, ModelParamFields.change_rc): data_import[k] = 1 elif StringClass.string_match(k, ModelParamFields.change_nc): data_import[k] = 0 elif StringClass.string_match(k, ModelParamFields.change_vc): data_import[ k] = DEFAULT_NODATA # Be careful to check NODATA when use! else: if MathClass.isnumerical(cur_data_item[idx]): data_import[k] = float(cur_data_item[idx]) else: data_import[k] = cur_data_item[idx] bulk.insert(data_import) # execute import operators MongoUtil.run_bulk(bulk, 'No operation during initial_params_from_txt.') # initialize index by parameter's type and name by ascending order. maindb[DBTableNames.main_parameter].create_index([ (ModelParamFields.type, ASCENDING), (ModelParamFields.name, ASCENDING) ])