def main(): from preprocess.config import parse_ini_configuration seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() db_model = conn[seims_cfg.spatial_db] spatial_gfs = GridFS(db_model, DBTableNames.gridfs_spatial) csv_path = r'C:\z_data\zhongTianShe\model_data_seims\field_scale_params' csv_files = FileClass.get_full_filename_by_suffixes(csv_path, ['.csv']) field_count = 7419 prefix = 9999 # Create mask file mask_name = '%d_MASK' % prefix mask_array = [[1] * field_count] import_array_to_mongodb(spatial_gfs, mask_array, mask_name) # Create spatial parameters for csv_file in csv_files: print('Import %s...' % csv_file) param_arrays = read_field_arrays_from_csv(csv_file) for key, value in list(param_arrays.items()): import_array_to_mongodb(spatial_gfs, value, '%d_%s' % (prefix, key))
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() main_db = conn[seims_cfg.spatial_db] TerrainUtilClass.parameters_extraction(seims_cfg, main_db)
def delete_scenarios_by_ids(hostname, port, dbname, sids): """Delete scenario data by ID in MongoDB.""" client = ConnectMongoDB(hostname, port) conn = client.get_conn() db = conn[dbname] collection = db['BMP_SCENARIOS'] for _id in sids: collection.remove({'ID': _id}) print('Delete scenario: %d in MongoDB completed!' % _id) client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() ImportWeightData.workflow(seims_cfg, conn, 0) client.close()
def write_param_values_to_mongodb(hostname, port, spatial_db, param_defs, param_values): # update Parameters collection in MongoDB client = ConnectMongoDB(hostname, port) conn = client.get_conn() db = conn[spatial_db] collection = db['PARAMETERS'] collection.update_many({}, {'$unset': {'CALI_VALUES': ''}}) for idx, pname in enumerate(param_defs['names']): v2str = ','.join(str(v) for v in param_values[:, idx]) collection.find_one_and_update({'NAME': pname}, {'$set': {'CALI_VALUES': v2str}}) client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() main_db = conn[seims_cfg.spatial_db] ImportParam2Mongo.workflow(seims_cfg, main_db) client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() maindb = conn[seims_cfg.spatial_db] ImportReaches2Mongo.generate_reach_table(seims_cfg, maindb) client.close()
def reset_simulation_timerange(self): """Update simulation time range in MongoDB [FILE_IN].""" client = ConnectMongoDB(self.model.host, self.model.port) conn = client.get_conn() db = conn[self.model.db_name] stime_str = self.model.time_start.strftime('%Y-%m-%d %H:%M:%S') etime_str = self.model.time_end.strftime('%Y-%m-%d %H:%M:%S') db[DBTableNames.main_filein].find_one_and_update({'TAG': 'STARTTIME'}, {'$set': {'VALUE': stime_str}}) db[DBTableNames.main_filein].find_one_and_update({'TAG': 'ENDTIME'}, {'$set': {'VALUE': etime_str}}) client.close()
def reset_simulation_timerange(self): """Update simulation time range in MongoDB [FILE_IN].""" client = ConnectMongoDB(self.model.host, self.model.port) conn = client.get_conn() db = conn[self.model.db_name] stime_str = self.model.simu_stime.strftime('%Y-%m-%d %H:%M:%S') etime_str = self.model.simu_etime.strftime('%Y-%m-%d %H:%M:%S') db[DBTableNames.main_filein].find_one_and_update({'TAG': 'STARTTIME'}, {'$set': {'VALUE': stime_str}}) db[DBTableNames.main_filein].find_one_and_update({'TAG': 'ENDTIME'}, {'$set': {'VALUE': etime_str}}) client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() maindb = conn[seims_cfg.spatial_db] scenariodb = conn[seims_cfg.bmp_scenario_db] ImportScenario2Mongo.scenario_from_texts(seims_cfg, maindb, scenariodb) client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() hydroclim_db = conn[seims_cfg.climate_db] site_m = HydroClimateUtilClass.query_climate_sites(hydroclim_db, 'M') site_p = HydroClimateUtilClass.query_climate_sites(hydroclim_db, 'P') client.close()
def main(): """TEST CODE""" from preprocess.config import parse_ini_configuration from preprocess.db_mongodb import ConnectMongoDB seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() db = conn[seims_cfg.climate_db] import time st = time.time() ImportMeteoData.workflow(seims_cfg, db) et = time.time() print(et - st) client.close()
def __init__(self, host, port, dbname): """Initialization.""" client = ConnectMongoDB(host, port) conn = client.get_conn() self.maindb = conn[dbname] self.filein_tab = self.maindb[DBTableNames.main_filein] self._climdb_name = self.HydroClimateDBName self.climatedb = conn[self._climdb_name] self._mode = '' self._interval = -1 # UTCTIME self._stime = None self._etime = None self._outletid = -1
def ParamDefs(self): """Read cali_param_rng.def file name,lower_bound,upper_bound e.g., Param1,0,1 Param2,0.5,1.2 Param3,-1.0,1.0 Returns: a dictionary containing: - names - the names of the parameters - bounds - a list of lists of lower and upper bounds - num_vars - a scalar indicating the number of variables (the length of names) """ # read param_defs.json if already existed if self.param_defs: return self.param_defs # read param_range_def file and output to json file client = ConnectMongoDB(self.cfg.model.host, self.cfg.model.port) conn = client.get_conn() db = conn[self.cfg.model.db_name] collection = db['PARAMETERS'] names = list() bounds = list() num_vars = 0 if not FileClass.is_file_exists(self.cfg.param_range_def): raise ValueError('Parameters definition file: %s is not' ' existed!' % self.cfg.param_range_def) items = read_data_items_from_txt(self.cfg.param_range_def) for item in items: if len(item) < 3: continue # find parameter name, print warning message if not existed cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True) if not cursor.count(): print('WARNING: parameter %s is not existed!' % item[0]) continue num_vars += 1 names.append(item[0]) bounds.append([float(item[1]), float(item[2])]) self.param_defs = { 'names': names, 'bounds': bounds, 'num_vars': num_vars } return self.param_defs
def write_param_values_to_mongodb(hostname, port, spatial_db, param_defs, param_values): # update Parameters collection in MongoDB client = ConnectMongoDB(hostname, port) conn = client.get_conn() db = conn[spatial_db] collection = db['PARAMETERS'] collection.update_many({}, {'$unset': {'CALI_VALUES': ''}}) for idx, pname in enumerate(param_defs['names']): v2str = ','.join(str(v) for v in param_values[:, idx]) collection.find_one_and_update({'NAME': pname}, {'$set': { 'CALI_VALUES': v2str }}) client.close()
def main(): from preprocess.config import parse_ini_configuration seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() db_model = conn[seims_cfg.spatial_db] sitelist_coll = db_model[DBTableNames.main_sitelist] # Add an item in SITELIST collection, in which the SUBBASINID is 9999 bsn_item = sitelist_coll.find_one({FieldNames.subbasin_id: 0}) field_bsn_item = deepcopy(bsn_item) del field_bsn_item['_id'] field_bsn_item[FieldNames.subbasin_id] = 9999 sitelist_coll.insert_one(field_bsn_item)
def ParamDefs(self): """Read cali_param_rng.def file name,lower_bound,upper_bound e.g., Param1,0,1 Param2,0.5,1.2 Param3,-1.0,1.0 Returns: a dictionary containing: - names - the names of the parameters - bounds - a list of lists of lower and upper bounds - num_vars - a scalar indicating the number of variables (the length of names) """ # read param_defs.json if already existed if self.param_defs: return self.param_defs # read param_range_def file and output to json file client = ConnectMongoDB(self.cfg.model.host, self.cfg.model.port) conn = client.get_conn() db = conn[self.cfg.model.db_name] collection = db['PARAMETERS'] names = list() bounds = list() num_vars = 0 if not FileClass.is_file_exists(self.cfg.param_range_def): raise ValueError('Parameters definition file: %s is not' ' existed!' % self.cfg.param_range_def) items = read_data_items_from_txt(self.cfg.param_range_def) for item in items: if len(item) < 3: continue # find parameter name, print warning message if not existed cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True) if not cursor.count(): print('WARNING: parameter %s is not existed!' % item[0]) continue num_vars += 1 names.append(item[0]) bounds.append([float(item[1]), float(item[2])]) self.param_defs = {'names': names, 'bounds': bounds, 'num_vars': num_vars} return self.param_defs
def read_simulation_timerange(self): """Read simulation time range from MongoDB.""" client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() db = conn[self.main_db] collection = db['FILE_IN'] try: stime_str = collection.find_one({'TAG': 'STARTTIME'}, no_cursor_timeout=True)['VALUE'] etime_str = collection.find_one({'TAG': 'ENDTIME'}, no_cursor_timeout=True)['VALUE'] stime = StringClass.get_datetime(stime_str) etime = StringClass.get_datetime(etime_str) dlt = etime - stime + timedelta(seconds=1) self.timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365. except NetworkTimeout or Exception: # In case of unexpected raise self.timerange = 1. # set default pass client.close()
def write_param_values_to_mongodb(self): """Update Parameters collection in MongoDB. Notes: The field value of 'CALI_VALUES' of all parameters will be deleted first. """ if not self.param_defs: self.read_param_ranges() if self.param_values is None or len(self.param_values) == 0: self.generate_samples() client = ConnectMongoDB(self.model.host, self.model.port) conn = client.get_conn() db = conn[self.model.db_name] collection = db['PARAMETERS'] collection.update_many({}, {'$unset': {'CALI_VALUES': ''}}) for idx, pname in enumerate(self.param_defs['names']): v2str = ','.join(str(v) for v in self.param_values[:, idx]) collection.find_one_and_update({'NAME': pname}, {'$set': {'CALI_VALUES': v2str}}) client.close()
def read_bmp_parameters(self): """Read BMP configuration from MongoDB. Each BMP is stored in Collection as one item identified by 'SUBSCENARIO' field, so the `self.bmps_params` is dict with BMP_ID ('SUBSCENARIO') as key. """ client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port) conn = client.get_conn() scenariodb = conn[self.scenario_db] bmpcoll = scenariodb[self.cfg.bmps_coll] findbmps = bmpcoll.find({}, no_cursor_timeout=True) for fb in findbmps: fb = UtilClass.decode_strs_in_dict(fb) if 'SUBSCENARIO' not in fb: continue curid = fb['SUBSCENARIO'] if curid not in self.cfg.bmps_subids: continue if curid not in self.bmps_params: self.bmps_params[curid] = dict() for k, v in fb.items(): if k == 'SUBSCENARIO': continue elif k == 'LANDUSE': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = None else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] elif k == 'SLPPOS': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = list(self.cfg.slppos_tags.keys()) else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] else: self.bmps_params[curid][k] = v client.close()
def export_to_mongodb(self): """Export current scenario to MongoDB. Delete the same ScenarioID if existed. """ client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() db = conn[self.scenario_db] collection = db['BMP_SCENARIOS'] try: # find ScenarioID, remove if existed. if collection.find({'ID': self.ID}, no_cursor_timeout=True).count(): collection.remove({'ID': self.ID}) except NetworkTimeout or Exception: # In case of unexpected raise pass for objid, bmp_item in self.bmp_items.items(): bmp_item['_id'] = ObjectId() collection.insert_one(bmp_item) client.close()
def read_bmp_parameters(self): """Read BMP configuration from MongoDB.""" client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() scenariodb = conn[self.bmp_scenario_db] bmpcoll = scenariodb[self.bmps_coll] findbmps = bmpcoll.find({}, no_cursor_timeout=True) for fb in findbmps: fb = UtilClass.decode_strs_in_dict(fb) if 'SUBSCENARIO' not in fb: continue curid = fb['SUBSCENARIO'] if curid not in self.bmps_subids: continue if curid not in self.bmps_params: self.bmps_params[curid] = dict() for k, v in fb.items(): if k == 'SUBSCENARIO': continue elif k == 'LANDUSE': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = None else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] elif k == 'SLPPOS': if isinstance(v, int): v = [v] elif v == 'ALL' or v == '': v = list(self.slppos_tags.keys()) else: v = StringClass.extract_numeric_values_from_string(v) v = [int(abs(nv)) for nv in v] self.bmps_params[curid][k] = v[:] else: self.bmps_params[curid][k] = v client.close()
def __init__(self, host, port, dbname): # type: (AnyStr, int, AnyStr) -> None """Initialization.""" client = ConnectMongoDB(host, port) conn = client.get_conn() self.maindb = conn[dbname] self.filein_tab = self.maindb[DBTableNames.main_filein] self.fileout_tab = self.maindb[DBTableNames.main_fileout] self._climdb_name = self.HydroClimateDBName self.climatedb = conn[self._climdb_name] self._scenariodb_name = self.ScenarioDBName self.scenariodb = conn[self._scenariodb_name] self._mode = '' self._interval = -1 # UTCTIME self._stime = None self._etime = None self._outletid = -1 # OUTPUT items self._output_items = dict() # type: Dict[AnyStr, Union[List[AnyStr]]]
def export_to_mongodb(self): """Export current scenario to MongoDB. Delete the same ScenarioID if existed. """ client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() db = conn[self.scenario_db] collection = db['BMP_SCENARIOS'] try: # find ScenarioID, remove if existed. if collection.find({ 'ID': self.ID }, no_cursor_timeout=True).count(): collection.remove({'ID': self.ID}) except NetworkTimeout or Exception: # In case of unexpected raise pass for objid, bmp_item in self.bmp_items.items(): bmp_item['_id'] = ObjectId() collection.insert_one(bmp_item) client.close()
def ReadRasterFromMongoDB(ip, port, db_name, gfsname, gfilename): client = ConnectMongoDB(ip, port) conn = client.get_conn() maindb = conn[db_name] spatial_gfs = GridFS(maindb, gfsname) if not spatial_gfs.exists(filename=gfilename): raise ValueError('WARNING: %s is not existed in %s:%s!' % (gfilename, db_name, gfsname)) try: gfsdata = maindb[DBTableNames.gridfs_spatial].files.find( {'filename': gfilename}, no_cursor_timeout=True)[0] except NetworkTimeout or Exception: # In case of unexpected raise client.close() return None ysize = int(gfsdata['metadata'][RasterMetadata.nrows]) xsize = int(gfsdata['metadata'][RasterMetadata.ncols]) xll = gfsdata['metadata'][RasterMetadata.xll] yll = gfsdata['metadata'][RasterMetadata.yll] cellsize = gfsdata['metadata'][RasterMetadata.cellsize] nodata = gfsdata['metadata'][RasterMetadata.nodata] srs = gfsdata['metadata'][RasterMetadata.srs] if is_string(srs): srs = str(srs) srs = osr.GetUserInputAsWKT(srs) geotransform = [0] * 6 geotransform[0] = xll - 0.5 * cellsize geotransform[1] = cellsize geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax geotransform[5] = -cellsize array_data = spatial_gfs.get(gfsdata['_id']) total_len = xsize * ysize fmt = '%df' % (total_len, ) array_data = unpack(fmt, array_data.read()) array_data = numpy.reshape(array_data, (ysize, xsize)) return Raster(ysize, xsize, array_data, nodata, geotransform, srs)
def workflow(cfg): """Building MongoDB workflow""" f = cfg.logs.build_mongo # build a connection to mongodb database client = ConnectMongoDB(cfg.hostname, cfg.port) conn = client.get_conn() maindb = conn[cfg.spatial_db] climatedb = conn[cfg.climate_db] scenariodb = None if cfg.use_scernario: scenariodb = conn[cfg.bmp_scenario_db] # import model parameters information to MongoDB status_output('Import model parameters', 10, f) ImportParam2Mongo.workflow(cfg, maindb) n_subbasins = MongoQuery.get_init_parameter_value(maindb, SubbsnStatsName.subbsn_num) print('Number of subbasins: %d' % n_subbasins) # Extract spatial parameters for reaches, landuse, soil, etc. status_output('Extract spatial parameters for reaches, landuse, soil, etc...', 20, f) extract_spatial_parameters(cfg, maindb) # import stream parameters status_output('Generating reach table with initialized parameters...', 40, f) ImportReaches2Mongo.generate_reach_table(cfg, maindb) # import raster data to MongoDB status_output('Importing raster to MongoDB....', 50, f) ImportMongodbClass.spatial_rasters(cfg, 0) ImportMongodbClass.spatial_rasters(cfg, n_subbasins) # Import IUH status_output('Generating and importing IUH (Instantaneous Unit Hydrograph)....', 60, f) ImportMongodbClass.iuh(cfg, 0) ImportMongodbClass.iuh(cfg, n_subbasins) # Import grid layering data status_output('Generating and importing grid layering....', 70, f) ImportMongodbClass.grid_layering(cfg, 0) ImportMongodbClass.grid_layering(cfg, n_subbasins) # Import hydro-climate data status_output('Import climate data....', 80, f) ImportMongodbClass.climate_data(cfg, maindb, climatedb) # Import weight and related data, this should after ImportMongodbClass.climate_data() status_output('Generating weight data for interpolation of meteorology data ' 'and weight dependent parameters....', 85, f) ImportWeightData.workflow(cfg, conn, 0) ImportWeightData.workflow(cfg, conn, n_subbasins) # Measurement Data, such as discharge, sediment yield. status_output('Import observed data, such as discharge, sediment yield....', 90, f) ImportObservedData.workflow(cfg, maindb, climatedb) # Import BMP scenario database to MongoDB status_output('Importing bmp scenario....', 95, f) ImportScenario2Mongo.scenario_from_texts(cfg, maindb, scenariodb) status_output('Build DB: %s finished!' % cfg.spatial_db, 100, f) # close connection to MongoDB client.close()
def read_param_ranges(self): """Read param_rng.def file name,lower_bound,upper_bound,group,dist (group and dist are optional) e.g., Param1,0,1[,Group1][,dist1] Param2,0,1[,Group2][,dist2] Param3,0,1[,Group3][,dist3] Returns: a dictionary containing: - names - the names of the parameters - bounds - a list of lists of lower and upper bounds - num_vars - a scalar indicating the number of variables (the length of names) - groups - a list of group names (strings) for each variable - dists - a list of distributions for the problem, None if not specified or all uniform """ # read param_defs.json if already existed if not self.param_defs: if FileClass.is_file_exists(self.cfg.outfiles.param_defs_json): with open(self.cfg.outfiles.param_defs_json, 'r', encoding='utf-8') as f: self.param_defs = UtilClass.decode_strs_in_dict(json.load(f)) return # read param_range_def file and output to json file client = ConnectMongoDB(self.model.host, self.model.port) conn = client.get_conn() db = conn[self.model.db_name] collection = db['PARAMETERS'] names = list() bounds = list() groups = list() dists = list() num_vars = 0 items = read_data_items_from_txt(self.cfg.param_range_def) for item in items: if len(item) < 3: continue # find parameter name, print warning message if not existed cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True) if not cursor.count(): print('WARNING: parameter %s is not existed!' % item[0]) continue num_vars += 1 names.append(item[0]) bounds.append([float(item[1]), float(item[2])]) # If the fourth column does not contain a group name, use # the parameter name if len(item) >= 4: groups.append(item[3]) else: groups.append(item[0]) if len(item) >= 5: dists.append(item[4]) else: dists.append('unif') if groups == names: groups = None elif len(set(groups)) == 1: raise ValueError('Only one group defined, results will not bemeaningful') # setting dists to none if all are uniform # because non-uniform scaling is not needed if all([d == 'unif' for d in dists]): dists = None self.param_defs = {'names': names, 'bounds': bounds, 'num_vars': num_vars, 'groups': groups, 'dists': dists} # Save as json, which can be loaded by json.load() json_data = json.dumps(self.param_defs, indent=4, cls=SpecialJsonEncoder) with open(self.cfg.outfiles.param_defs_json, 'w', encoding='utf-8') as f: f.write('%s' % json_data)
def workflow(cfg): """Building MongoDB workflow""" f = cfg.logs.build_mongo # build a connection to mongodb database client = ConnectMongoDB(cfg.hostname, cfg.port) conn = client.get_conn() maindb = conn[cfg.spatial_db] climatedb = conn[cfg.climate_db] scenariodb = None if cfg.use_scernario: scenariodb = conn[cfg.bmp_scenario_db] # import model parameters information to MongoDB status_output('Import model parameters', 10, f) ImportParam2Mongo.workflow(cfg, maindb) n_subbasins = MongoQuery.get_init_parameter_value( maindb, SubbsnStatsName.subbsn_num) print('Number of subbasins: %d' % n_subbasins) # Extract spatial parameters for reaches, landuse, soil, etc. status_output( 'Extract spatial parameters for reaches, landuse, soil, etc...', 20, f) extract_spatial_parameters(cfg, maindb) # import stream parameters status_output('Generating reach table with initialized parameters...', 40, f) ImportReaches2Mongo.generate_reach_table(cfg, maindb) # import raster data to MongoDB status_output('Importing raster to MongoDB....', 50, f) ImportMongodbClass.spatial_rasters(cfg, 0) ImportMongodbClass.spatial_rasters(cfg, n_subbasins) # Import IUH status_output( 'Generating and importing IUH (Instantaneous Unit Hydrograph)....', 60, f) ImportMongodbClass.iuh(cfg, 0) ImportMongodbClass.iuh(cfg, n_subbasins) # Import grid layering data status_output('Generating and importing grid layering....', 70, f) ImportMongodbClass.grid_layering(cfg, 0) ImportMongodbClass.grid_layering(cfg, n_subbasins) # Import hydro-climate data status_output('Import climate data....', 80, f) ImportMongodbClass.climate_data(cfg, maindb, climatedb) # Import weight and related data, this should after ImportMongodbClass.climate_data() status_output( 'Generating weight data for interpolation of meteorology data ' 'and weight dependent parameters....', 85, f) ImportWeightData.workflow(cfg, conn, 0) ImportWeightData.workflow(cfg, conn, n_subbasins) # Measurement Data, such as discharge, sediment yield. status_output( 'Import observed data, such as discharge, sediment yield....', 90, f) ImportObservedData.workflow(cfg, maindb, climatedb) # Import BMP scenario database to MongoDB status_output('Importing bmp scenario....', 95, f) ImportScenario2Mongo.scenario_from_texts(cfg, maindb, scenariodb) status_output('Build DB: %s finished!' % cfg.spatial_db, 100, f) # close connection to MongoDB client.close()
def read_param_ranges(self): """Read param_rng.def file name,lower_bound,upper_bound,group,dist (group and dist are optional) e.g., Param1,0,1[,Group1][,dist1] Param2,0,1[,Group2][,dist2] Param3,0,1[,Group3][,dist3] Returns: a dictionary containing: - names - the names of the parameters - bounds - a list of lists of lower and upper bounds - num_vars - a scalar indicating the number of variables (the length of names) - groups - a list of group names (strings) for each variable - dists - a list of distributions for the problem, None if not specified or all uniform """ # read param_defs.json if already existed if not self.param_defs: if FileClass.is_file_exists(self.cfg.outfiles.param_defs_json): with open(self.cfg.outfiles.param_defs_json, 'r') as f: self.param_defs = UtilClass.decode_strs_in_dict(json.load(f)) return # read param_range_def file and output to json file client = ConnectMongoDB(self.model.host, self.model.port) conn = client.get_conn() db = conn[self.model.db_name] collection = db['PARAMETERS'] names = list() bounds = list() groups = list() dists = list() num_vars = 0 items = read_data_items_from_txt(self.cfg.param_range_def) for item in items: if len(item) < 3: continue # find parameter name, print warning message if not existed cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True) if not cursor.count(): print('WARNING: parameter %s is not existed!' % item[0]) continue num_vars += 1 names.append(item[0]) bounds.append([float(item[1]), float(item[2])]) # If the fourth column does not contain a group name, use # the parameter name if len(item) >= 4: groups.append(item[3]) else: groups.append(item[0]) if len(item) >= 5: dists.append(item[4]) else: dists.append('unif') if groups == names: groups = None elif len(set(groups)) == 1: raise ValueError('Only one group defined, results will not bemeaningful') # setting dists to none if all are uniform # because non-uniform scaling is not needed if all([d == 'unif' for d in dists]): dists = None self.param_defs = {'names': names, 'bounds': bounds, 'num_vars': num_vars, 'groups': groups, 'dists': dists} # Save as json, which can be loaded by json.load() json_data = json.dumps(self.param_defs, indent=4, cls=SpecialJsonEncoder) with open(self.cfg.outfiles.param_defs_json, 'w') as f: f.write(json_data)
def DelinateSlopePositionByThreshold( modelcfg, # type: ParseSEIMSConfig thresholds, # type: Dict[int, List] fuzzyslppos_fnames, # type: List[Tuple[int, AnyStr, AnyStr]] outfname, # type: AnyStr subbsn_id=0 # type: int ): # type: (...) -> Dict """ Args: model_cfg: Configuration of SEIMS-based model thresholds: {HillslopeID: {rdgID, bksID, vlyID, T_bks2rdg, T_bks2vly}, ...} fuzzyslppos_fnames: [(1, 'summit', 'rdgInf'), ...] outfname: output GridFS name subbsn_id: By default use the whole watershed data Returns: hillslp_data(dict): {} """ # 1. Read raster data from MongoDB hillslpr = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port, modelcfg.db_name, DBTableNames.gridfs_spatial, '%d_HILLSLOPE_MERGED' % subbsn_id) landuser = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port, modelcfg.db_name, DBTableNames.gridfs_spatial, '%d_LANDUSE' % subbsn_id) fuzslppos_rs = list() for tag, tagname, gfsname in fuzzyslppos_fnames: fuzslppos_rs.append( ReadRasterFromMongoDB(modelcfg.host, modelcfg.port, modelcfg.db_name, DBTableNames.gridfs_spatial, '%d_%s' % (subbsn_id, gfsname.upper()))) # Output for test # out_dir = r'D:\data_m\youwuzhen\seims_models_phd\data_prepare\spatial\spatial_units\tmp' # out_hillslp = out_dir + os.sep + 'hillslope.tif' # RasterUtilClass.write_gtiff_file(out_hillslp, hillslpr.nRows, hillslpr.nCols, # hillslpr.data, hillslpr.geotrans, hillslpr.srs, # hillslpr.noDataValue) # out_landuse = out_dir + os.sep + 'landuse.tif' # RasterUtilClass.write_gtiff_file(out_landuse, landuser.nRows, landuser.nCols, # landuser.data, landuser.geotrans, landuser.srs, # landuser.noDataValue) # for i, (tag, tagname, gfsname) in enumerate(fuzzyslppos_fnames): # curname = out_dir + os.sep + '%s.tif' % gfsname # RasterUtilClass.write_gtiff_file(curname, fuzslppos_rs[i].nRows, fuzslppos_rs[i].nCols, # fuzslppos_rs[i].data, fuzslppos_rs[i].geotrans, # fuzslppos_rs[i].srs, # fuzslppos_rs[i].noDataValue) # 2. Initialize output outgfsname = '%d_%s' % (subbsn_id, outfname.upper()) outdict = dict( ) # type: Dict[AnyStr, Dict[int, Dict[AnyStr, Union[float, Dict[int, float]]]]] slppos_cls = numpy.ones( (hillslpr.nRows, hillslpr.nCols)) * hillslpr.noDataValue valid_cells = 0 # Get the fuzzy slope position values from up to bottom def GetFuzzySlopePositionValues(i_row, i_col): seqvalues = [-9999] * len(fuzslppos_rs) for iseq, fuzdata in enumerate(fuzslppos_rs): curv = fuzdata.data[i_row][i_col] if MathClass.floatequal(curv, fuzdata.noDataValue): return None if curv < 0: return None seqvalues[iseq] = curv return seqvalues # ACTUAL ALGORITHM for row in range(hillslpr.nRows): for col in range(hillslpr.nCols): # Exclude invalid situation hillslp_id = hillslpr.data[row][col] if MathClass.floatequal(hillslp_id, hillslpr.noDataValue): continue if hillslp_id not in thresholds: continue landuse_id = landuser.data[row][col] if MathClass.floatequal(landuse_id, landuser.noDataValue): continue fuzzyvalues = GetFuzzySlopePositionValues(row, col) if fuzzyvalues is None: continue # THIS PART SHOULD BE REVIEWED CAREFULLY LATER! --START # Step 1. Get the index of slope position with maximum similarity max_fuz = max(fuzzyvalues) max_idx = fuzzyvalues.index(max_fuz) tmpfuzzyvalues = fuzzyvalues[:] tmpfuzzyvalues.remove(max_fuz) sec_fuz = max(tmpfuzzyvalues) sec_idx = fuzzyvalues.index(sec_fuz) sel_idx = max_idx # Select the maximum by default cur_threshs = thresholds[hillslp_id][1 - len(fuzzyvalues):] if max_idx == len(fuzzyvalues) - 1: # the bottom position if sec_idx == len( fuzzyvalues ) - 2 and 0 < max_fuz - sec_fuz < cur_threshs[-1]: sel_idx = sec_idx # change valley to backslope elif max_idx == 0: # the upper position if sec_idx == 1 and 0 < max_fuz - sec_fuz < cur_threshs[0]: sel_idx = sec_idx # change ridge to backslope else: # the middle positions # Two thresholds could be applied, # i.e., cur_threshs[max_idx-1] and cur_threshs[max_idx] if sec_idx == max_idx - 1 and 0. > sec_fuz - max_fuz > cur_threshs[ max_idx - 1]: sel_idx = sec_idx elif sec_idx == max_idx + 1 and 0. > sec_fuz - max_fuz > cur_threshs[ max_idx]: sel_idx = sec_idx # Exception: if sec_fuz < 0.1 and sel_idx == sec_idx: sel_idx = max_idx # if sel_idx != max_idx: # boundary has been adapted # print('fuzzy values: %s, thresholds: %s, ' # 'sel_idx: %d' % (fuzzyvalues.__str__(), cur_threshs.__str__(), sel_idx)) slppos_id = thresholds[hillslp_id][sel_idx] # THIS PART SHOULD BE REVIEWED CAREFULLY LATER! --END slppos_cls[row][col] = slppos_id sel_tagname = fuzzyslppos_fnames[sel_idx][1] if sel_tagname not in outdict: outdict[sel_tagname] = dict() if slppos_id not in outdict[sel_tagname]: outdict[sel_tagname][slppos_id] = { 'area': 0, 'landuse': dict() } outdict[sel_tagname][slppos_id]['area'] += 1 if landuse_id not in outdict[sel_tagname][slppos_id]['landuse']: outdict[sel_tagname][slppos_id]['landuse'][landuse_id] = 0. outdict[sel_tagname][slppos_id]['landuse'][landuse_id] += 1. valid_cells += 1 # Change cell counts to area area_km2 = hillslpr.dx * hillslpr.dx * 1.e-6 for tagname, slpposdict in viewitems(outdict): for sid, datadict in viewitems(slpposdict): outdict[tagname][sid]['area'] *= area_km2 for luid in outdict[tagname][sid]['landuse']: outdict[tagname][sid]['landuse'][luid] *= area_km2 # 3. Write the classified slope positions data back to mongodb metadata = dict() metadata[RasterMetadata.subbasin] = subbsn_id metadata['ID'] = outgfsname metadata['TYPE'] = outfname.upper() metadata[RasterMetadata.cellsize] = hillslpr.dx metadata[RasterMetadata.nodata] = hillslpr.noDataValue metadata[RasterMetadata.ncols] = hillslpr.nCols metadata[RasterMetadata.nrows] = hillslpr.nRows metadata[RasterMetadata.xll] = hillslpr.xMin + 0.5 * hillslpr.dx metadata[RasterMetadata.yll] = hillslpr.yMin + 0.5 * hillslpr.dx metadata['LAYERS'] = 1. metadata[RasterMetadata.cellnum] = valid_cells metadata[RasterMetadata.srs] = hillslpr.srs client = ConnectMongoDB(modelcfg.host, modelcfg.port) conn = client.get_conn() maindb = conn[modelcfg.db_name] spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial) # delete if the tablename gridfs file existed if spatial_gfs.exists(filename=outgfsname): x = spatial_gfs.get_version(filename=outgfsname) spatial_gfs.delete(x._id) # create and write new GridFS file new_gridfs = spatial_gfs.new_file(filename=outgfsname, metadata=metadata) new_gridfs_array = slppos_cls.reshape( (1, hillslpr.nCols * hillslpr.nRows)).tolist()[0] fmt = '%df' % hillslpr.nCols * hillslpr.nRows s = pack(fmt, *new_gridfs_array) new_gridfs.write(s) new_gridfs.close() # Read and output for test # slpposcls_r = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port, # modelcfg.db_name, DBTableNames.gridfs_spatial, outgfsname) # out_slpposcls = out_dir + os.sep + '%s.tif' % outgfsname # RasterUtilClass.write_gtiff_file(out_slpposcls, slpposcls_r.nRows, slpposcls_r.nCols, # slpposcls_r.data, slpposcls_r.geotrans, slpposcls_r.srs, # slpposcls_r.noDataValue) client.close() return outdict
def export_scenario_to_gtiff(self, outpath=None): """Export scenario to GTiff. TODO: Read Raster from MongoDB should be extracted to pygeoc. """ if not self.export_sce_tif: return dist = self.bmps_info['DISTRIBUTION'] dist_list = StringClass.split_string(dist, '|') if len(dist_list) >= 2 and dist_list[0] == 'RASTER': dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin # read dist_name from MongoDB client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() maindb = conn[self.main_db] spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial) # read file from mongodb if not spatial_gfs.exists(filename=dist_name): print('WARNING: %s is not existed, export scenario failed!' % dist_name) return try: slpposf = maindb[DBTableNames.gridfs_spatial].files.find( {'filename': dist_name}, no_cursor_timeout=True)[0] except NetworkTimeout or Exception: # In case of unexpected raise client.close() return ysize = int(slpposf['metadata'][RasterMetadata.nrows]) xsize = int(slpposf['metadata'][RasterMetadata.ncols]) xll = slpposf['metadata'][RasterMetadata.xll] yll = slpposf['metadata'][RasterMetadata.yll] cellsize = slpposf['metadata'][RasterMetadata.cellsize] nodata_value = slpposf['metadata'][RasterMetadata.nodata] srs = slpposf['metadata'][RasterMetadata.srs] if isinstance(srs, text_type): srs = str(srs) srs = osr.GetUserInputAsWKT(srs) geotransform = [0] * 6 geotransform[0] = xll - 0.5 * cellsize geotransform[1] = cellsize geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax geotransform[5] = -cellsize slppos_data = spatial_gfs.get(slpposf['_id']) total_len = xsize * ysize fmt = '%df' % (total_len, ) slppos_data = unpack(fmt, slppos_data.read()) slppos_data = numpy.reshape(slppos_data, (ysize, xsize)) v_dict = dict() for idx, gene_v in enumerate(self.gene_values): v_dict[self.gene_to_unit[idx]] = gene_v for k, v in v_dict.items(): slppos_data[slppos_data == k] = v if outpath is None: outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID RasterUtilClass.write_gtiff_file(outpath, ysize, xsize, slppos_data, geotransform, srs, nodata_value) client.close()
def export_scenario_to_gtiff(self, outpath=None): """Export scenario to GTiff. TODO: Read Raster from MongoDB should be extracted to pygeoc. """ if not self.export_sce_tif: return dist = self.bmps_info['DISTRIBUTION'] dist_list = StringClass.split_string(dist, '|') if len(dist_list) >= 2 and dist_list[0] == 'RASTER': dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin # read dist_name from MongoDB client = ConnectMongoDB(self.hostname, self.port) conn = client.get_conn() maindb = conn[self.main_db] spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial) # read file from mongodb if not spatial_gfs.exists(filename=dist_name): print('WARNING: %s is not existed, export scenario failed!' % dist_name) return try: slpposf = maindb[DBTableNames.gridfs_spatial].files.find({'filename': dist_name}, no_cursor_timeout=True)[0] except NetworkTimeout or Exception: # In case of unexpected raise client.close() return ysize = int(slpposf['metadata'][RasterMetadata.nrows]) xsize = int(slpposf['metadata'][RasterMetadata.ncols]) xll = slpposf['metadata'][RasterMetadata.xll] yll = slpposf['metadata'][RasterMetadata.yll] cellsize = slpposf['metadata'][RasterMetadata.cellsize] nodata_value = slpposf['metadata'][RasterMetadata.nodata] srs = slpposf['metadata'][RasterMetadata.srs] if isinstance(srs, text_type): srs = str(srs) srs = osr.GetUserInputAsWKT(srs) geotransform = [0] * 6 geotransform[0] = xll - 0.5 * cellsize geotransform[1] = cellsize geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax geotransform[5] = -cellsize slppos_data = spatial_gfs.get(slpposf['_id']) total_len = xsize * ysize fmt = '%df' % (total_len,) slppos_data = unpack(fmt, slppos_data.read()) slppos_data = numpy.reshape(slppos_data, (ysize, xsize)) v_dict = dict() for idx, gene_v in enumerate(self.gene_values): v_dict[self.gene_to_unit[idx]] = gene_v for k, v in v_dict.items(): slppos_data[slppos_data == k] = v if outpath is None: outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID RasterUtilClass.write_gtiff_file(outpath, ysize, xsize, slppos_data, geotransform, srs, nodata_value) client.close()