def delete_scenarios_by_ids(hostname, port, dbname, sids):
"""Delete scenario data by ID in MongoDB."""
client = ConnectMongoDB(hostname, port)
conn = client.get_conn()
db = conn[dbname]
collection = db['BMP_SCENARIOS']
for _id in sids:
collection.remove({'ID': _id})
print('Delete scenario: %d in MongoDB completed!' % _id)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
ImportWeightData.workflow(seims_cfg, conn, 0)
client.close()
def write_param_values_to_mongodb(hostname, port, spatial_db, param_defs, param_values):
# update Parameters collection in MongoDB
client = ConnectMongoDB(hostname, port)
conn = client.get_conn()
db = conn[spatial_db]
collection = db['PARAMETERS']
collection.update_many({}, {'$unset': {'CALI_VALUES': ''}})
for idx, pname in enumerate(param_defs['names']):
v2str = ','.join(str(v) for v in param_values[:, idx])
collection.find_one_and_update({'NAME': pname}, {'$set': {'CALI_VALUES': v2str}})
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
ImportWeightData.workflow(seims_cfg, conn, 0)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
maindb = conn[seims_cfg.spatial_db]
ImportReaches2Mongo.generate_reach_table(seims_cfg, maindb)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
main_db = conn[seims_cfg.spatial_db]
LanduseUtilClass.parameters_extraction(seims_cfg, main_db)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
main_db = conn[seims_cfg.spatial_db]
LanduseUtilClass.parameters_extraction(seims_cfg, main_db)
client.close()
def reset_simulation_timerange(self):
"""Update simulation time range in MongoDB [FILE_IN]."""
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
stime_str = self.model.time_start.strftime('%Y-%m-%d %H:%M:%S')
etime_str = self.model.time_end.strftime('%Y-%m-%d %H:%M:%S')
db[DBTableNames.main_filein].find_one_and_update({'TAG': 'STARTTIME'},
{'$set': {'VALUE': stime_str}})
db[DBTableNames.main_filein].find_one_and_update({'TAG': 'ENDTIME'},
{'$set': {'VALUE': etime_str}})
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
maindb = conn[seims_cfg.spatial_db]
ImportReaches2Mongo.generate_reach_table(seims_cfg, maindb)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
main_db = conn[seims_cfg.spatial_db]
ImportParam2Mongo.workflow(seims_cfg, main_db)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
main_db = conn[seims_cfg.spatial_db]
ImportParam2Mongo.workflow(seims_cfg, main_db)
client.close()
def reset_simulation_timerange(self):
"""Update simulation time range in MongoDB [FILE_IN]."""
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
stime_str = self.model.simu_stime.strftime('%Y-%m-%d %H:%M:%S')
etime_str = self.model.simu_etime.strftime('%Y-%m-%d %H:%M:%S')
db[DBTableNames.main_filein].find_one_and_update({'TAG': 'STARTTIME'},
{'$set': {'VALUE': stime_str}})
db[DBTableNames.main_filein].find_one_and_update({'TAG': 'ENDTIME'},
{'$set': {'VALUE': etime_str}})
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
hydroclim_db = conn[seims_cfg.climate_db]
site_m = HydroClimateUtilClass.query_climate_sites(hydroclim_db, 'M')
site_p = HydroClimateUtilClass.query_climate_sites(hydroclim_db, 'P')
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
maindb = conn[seims_cfg.spatial_db]
scenariodb = conn[seims_cfg.bmp_scenario_db]
ImportScenario2Mongo.scenario_from_texts(seims_cfg, maindb, scenariodb)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
maindb = conn[seims_cfg.spatial_db]
scenariodb = conn[seims_cfg.bmp_scenario_db]
ImportScenario2Mongo.scenario_from_texts(seims_cfg, maindb, scenariodb)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
db = conn[seims_cfg.climate_db]
import time
st = time.time()
ImportMeteoData.workflow(seims_cfg, db)
et = time.time()
print(et - st)
client.close()
def main():
"""TEST CODE"""
from preprocess.config import parse_ini_configuration
from preprocess.db_mongodb import ConnectMongoDB
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
db = conn[seims_cfg.climate_db]
import time
st = time.time()
ImportMeteoData.workflow(seims_cfg, db)
et = time.time()
print(et - st)
client.close()
def write_param_values_to_mongodb(hostname, port, spatial_db, param_defs,
param_values):
# update Parameters collection in MongoDB
client = ConnectMongoDB(hostname, port)
conn = client.get_conn()
db = conn[spatial_db]
collection = db['PARAMETERS']
collection.update_many({}, {'$unset': {'CALI_VALUES': ''}})
for idx, pname in enumerate(param_defs['names']):
v2str = ','.join(str(v) for v in param_values[:, idx])
collection.find_one_and_update({'NAME': pname},
{'$set': {
'CALI_VALUES': v2str
}})
client.close()
def read_simulation_timerange(self):
"""Read simulation time range from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.main_db]
collection = db['FILE_IN']
try:
stime_str = collection.find_one({'TAG': 'STARTTIME'}, no_cursor_timeout=True)['VALUE']
etime_str = collection.find_one({'TAG': 'ENDTIME'}, no_cursor_timeout=True)['VALUE']
stime = StringClass.get_datetime(stime_str)
etime = StringClass.get_datetime(etime_str)
dlt = etime - stime + timedelta(seconds=1)
self.timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365.
except NetworkTimeout or Exception:
# In case of unexpected raise
self.timerange = 1. # set default
pass
client.close()
def write_param_values_to_mongodb(self):
"""Update Parameters collection in MongoDB.
Notes:
The field value of 'CALI_VALUES' of all parameters will be deleted first.
"""
if not self.param_defs:
self.read_param_ranges()
if self.param_values is None or len(self.param_values) == 0:
self.generate_samples()
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
collection = db['PARAMETERS']
collection.update_many({}, {'$unset': {'CALI_VALUES': ''}})
for idx, pname in enumerate(self.param_defs['names']):
v2str = ','.join(str(v) for v in self.param_values[:, idx])
collection.find_one_and_update({'NAME': pname}, {'$set': {'CALI_VALUES': v2str}})
client.close()
def write_param_values_to_mongodb(self):
"""Update Parameters collection in MongoDB.
Notes:
The field value of 'CALI_VALUES' of all parameters will be deleted first.
"""
if not self.param_defs:
self.read_param_ranges()
if self.param_values is None or len(self.param_values) == 0:
self.generate_samples()
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
collection = db['PARAMETERS']
collection.update_many({}, {'$unset': {'CALI_VALUES': ''}})
for idx, pname in enumerate(self.param_defs['names']):
v2str = ','.join(str(v) for v in self.param_values[:, idx])
collection.find_one_and_update({'NAME': pname}, {'$set': {'CALI_VALUES': v2str}})
client.close()
def read_bmp_parameters(self):
"""Read BMP configuration from MongoDB.
Each BMP is stored in Collection as one item identified by 'SUBSCENARIO' field,
so the `self.bmps_params` is dict with BMP_ID ('SUBSCENARIO') as key.
"""
client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port)
conn = client.get_conn()
scenariodb = conn[self.scenario_db]
bmpcoll = scenariodb[self.cfg.bmps_coll]
findbmps = bmpcoll.find({}, no_cursor_timeout=True)
for fb in findbmps:
fb = UtilClass.decode_strs_in_dict(fb)
if 'SUBSCENARIO' not in fb:
continue
curid = fb['SUBSCENARIO']
if curid not in self.cfg.bmps_subids:
continue
if curid not in self.bmps_params:
self.bmps_params[curid] = dict()
for k, v in fb.items():
if k == 'SUBSCENARIO':
continue
elif k == 'LANDUSE':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = None
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
elif k == 'SLPPOS':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = list(self.cfg.slppos_tags.keys())
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
else:
self.bmps_params[curid][k] = v
client.close()
def export_to_mongodb(self):
"""Export current scenario to MongoDB.
Delete the same ScenarioID if existed.
"""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.scenario_db]
collection = db['BMP_SCENARIOS']
try:
# find ScenarioID, remove if existed.
if collection.find({'ID': self.ID}, no_cursor_timeout=True).count():
collection.remove({'ID': self.ID})
except NetworkTimeout or Exception:
# In case of unexpected raise
pass
for objid, bmp_item in self.bmp_items.items():
bmp_item['_id'] = ObjectId()
collection.insert_one(bmp_item)
client.close()
def read_bmp_parameters(self):
"""Read BMP configuration from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
scenariodb = conn[self.bmp_scenario_db]
bmpcoll = scenariodb[self.bmps_coll]
findbmps = bmpcoll.find({}, no_cursor_timeout=True)
for fb in findbmps:
fb = UtilClass.decode_strs_in_dict(fb)
if 'SUBSCENARIO' not in fb:
continue
curid = fb['SUBSCENARIO']
if curid not in self.bmps_subids:
continue
if curid not in self.bmps_params:
self.bmps_params[curid] = dict()
for k, v in fb.items():
if k == 'SUBSCENARIO':
continue
elif k == 'LANDUSE':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = None
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
elif k == 'SLPPOS':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = list(self.slppos_tags.keys())
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
else:
self.bmps_params[curid][k] = v
client.close()
def read_simulation_timerange(self):
"""Read simulation time range from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.main_db]
collection = db['FILE_IN']
try:
stime_str = collection.find_one({'TAG': 'STARTTIME'},
no_cursor_timeout=True)['VALUE']
etime_str = collection.find_one({'TAG': 'ENDTIME'},
no_cursor_timeout=True)['VALUE']
stime = StringClass.get_datetime(stime_str)
etime = StringClass.get_datetime(etime_str)
dlt = etime - stime + timedelta(seconds=1)
self.timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365.
except NetworkTimeout or Exception:
# In case of unexpected raise
self.timerange = 1. # set default
pass
client.close()
def export_to_mongodb(self):
"""Export current scenario to MongoDB.
Delete the same ScenarioID if existed.
"""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
db = conn[self.scenario_db]
collection = db['BMP_SCENARIOS']
try:
# find ScenarioID, remove if existed.
if collection.find({
'ID': self.ID
}, no_cursor_timeout=True).count():
collection.remove({'ID': self.ID})
except NetworkTimeout or Exception:
# In case of unexpected raise
pass
for objid, bmp_item in self.bmp_items.items():
bmp_item['_id'] = ObjectId()
collection.insert_one(bmp_item)
client.close()
def ReadRasterFromMongoDB(ip, port, db_name, gfsname, gfilename):
client = ConnectMongoDB(ip, port)
conn = client.get_conn()
maindb = conn[db_name]
spatial_gfs = GridFS(maindb, gfsname)
if not spatial_gfs.exists(filename=gfilename):
raise ValueError('WARNING: %s is not existed in %s:%s!' %
(gfilename, db_name, gfsname))
try:
gfsdata = maindb[DBTableNames.gridfs_spatial].files.find(
{'filename': gfilename}, no_cursor_timeout=True)[0]
except NetworkTimeout or Exception:
# In case of unexpected raise
client.close()
return None
ysize = int(gfsdata['metadata'][RasterMetadata.nrows])
xsize = int(gfsdata['metadata'][RasterMetadata.ncols])
xll = gfsdata['metadata'][RasterMetadata.xll]
yll = gfsdata['metadata'][RasterMetadata.yll]
cellsize = gfsdata['metadata'][RasterMetadata.cellsize]
nodata = gfsdata['metadata'][RasterMetadata.nodata]
srs = gfsdata['metadata'][RasterMetadata.srs]
if is_string(srs):
srs = str(srs)
srs = osr.GetUserInputAsWKT(srs)
geotransform = [0] * 6
geotransform[0] = xll - 0.5 * cellsize
geotransform[1] = cellsize
geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax
geotransform[5] = -cellsize
array_data = spatial_gfs.get(gfsdata['_id'])
total_len = xsize * ysize
fmt = '%df' % (total_len, )
array_data = unpack(fmt, array_data.read())
array_data = numpy.reshape(array_data, (ysize, xsize))
return Raster(ysize, xsize, array_data, nodata, geotransform, srs)
def workflow(cfg):
"""Building MongoDB workflow"""
f = cfg.logs.build_mongo
# build a connection to mongodb database
client = ConnectMongoDB(cfg.hostname, cfg.port)
conn = client.get_conn()
maindb = conn[cfg.spatial_db]
climatedb = conn[cfg.climate_db]
scenariodb = None
if cfg.use_scernario:
scenariodb = conn[cfg.bmp_scenario_db]
# import model parameters information to MongoDB
status_output('Import model parameters', 10, f)
ImportParam2Mongo.workflow(cfg, maindb)
n_subbasins = MongoQuery.get_init_parameter_value(
maindb, SubbsnStatsName.subbsn_num)
print('Number of subbasins: %d' % n_subbasins)
# Extract spatial parameters for reaches, landuse, soil, etc.
status_output(
'Extract spatial parameters for reaches, landuse, soil, etc...',
20, f)
extract_spatial_parameters(cfg, maindb)
# import stream parameters
status_output('Generating reach table with initialized parameters...',
40, f)
ImportReaches2Mongo.generate_reach_table(cfg, maindb)
# import raster data to MongoDB
status_output('Importing raster to MongoDB....', 50, f)
ImportMongodbClass.spatial_rasters(cfg, 0)
ImportMongodbClass.spatial_rasters(cfg, n_subbasins)
# Import IUH
status_output(
'Generating and importing IUH (Instantaneous Unit Hydrograph)....',
60, f)
ImportMongodbClass.iuh(cfg, 0)
ImportMongodbClass.iuh(cfg, n_subbasins)
# Import grid layering data
status_output('Generating and importing grid layering....', 70, f)
ImportMongodbClass.grid_layering(cfg, 0)
ImportMongodbClass.grid_layering(cfg, n_subbasins)
# Import hydro-climate data
status_output('Import climate data....', 80, f)
ImportMongodbClass.climate_data(cfg, maindb, climatedb)
# Import weight and related data, this should after ImportMongodbClass.climate_data()
status_output(
'Generating weight data for interpolation of meteorology data '
'and weight dependent parameters....', 85, f)
ImportWeightData.workflow(cfg, conn, 0)
ImportWeightData.workflow(cfg, conn, n_subbasins)
# Measurement Data, such as discharge, sediment yield.
status_output(
'Import observed data, such as discharge, sediment yield....', 90,
f)
ImportObservedData.workflow(cfg, maindb, climatedb)
# Import BMP scenario database to MongoDB
status_output('Importing bmp scenario....', 95, f)
ImportScenario2Mongo.scenario_from_texts(cfg, maindb, scenariodb)
status_output('Build DB: %s finished!' % cfg.spatial_db, 100, f)
# close connection to MongoDB
client.close()
def DelinateSlopePositionByThreshold(
modelcfg, # type: ParseSEIMSConfig
thresholds, # type: Dict[int, List]
fuzzyslppos_fnames, # type: List[Tuple[int, AnyStr, AnyStr]]
outfname, # type: AnyStr
subbsn_id=0 # type: int
):
# type: (...) -> Dict
"""
Args:
model_cfg: Configuration of SEIMS-based model
thresholds: {HillslopeID: {rdgID, bksID, vlyID, T_bks2rdg, T_bks2vly}, ...}
fuzzyslppos_fnames: [(1, 'summit', 'rdgInf'), ...]
outfname: output GridFS name
subbsn_id: By default use the whole watershed data
Returns:
hillslp_data(dict): {}
"""
# 1. Read raster data from MongoDB
hillslpr = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port,
modelcfg.db_name,
DBTableNames.gridfs_spatial,
'%d_HILLSLOPE_MERGED' % subbsn_id)
landuser = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port,
modelcfg.db_name,
DBTableNames.gridfs_spatial,
'%d_LANDUSE' % subbsn_id)
fuzslppos_rs = list()
for tag, tagname, gfsname in fuzzyslppos_fnames:
fuzslppos_rs.append(
ReadRasterFromMongoDB(modelcfg.host, modelcfg.port,
modelcfg.db_name,
DBTableNames.gridfs_spatial,
'%d_%s' % (subbsn_id, gfsname.upper())))
# Output for test
# out_dir = r'D:\data_m\youwuzhen\seims_models_phd\data_prepare\spatial\spatial_units\tmp'
# out_hillslp = out_dir + os.sep + 'hillslope.tif'
# RasterUtilClass.write_gtiff_file(out_hillslp, hillslpr.nRows, hillslpr.nCols,
# hillslpr.data, hillslpr.geotrans, hillslpr.srs,
# hillslpr.noDataValue)
# out_landuse = out_dir + os.sep + 'landuse.tif'
# RasterUtilClass.write_gtiff_file(out_landuse, landuser.nRows, landuser.nCols,
# landuser.data, landuser.geotrans, landuser.srs,
# landuser.noDataValue)
# for i, (tag, tagname, gfsname) in enumerate(fuzzyslppos_fnames):
# curname = out_dir + os.sep + '%s.tif' % gfsname
# RasterUtilClass.write_gtiff_file(curname, fuzslppos_rs[i].nRows, fuzslppos_rs[i].nCols,
# fuzslppos_rs[i].data, fuzslppos_rs[i].geotrans,
# fuzslppos_rs[i].srs,
# fuzslppos_rs[i].noDataValue)
# 2. Initialize output
outgfsname = '%d_%s' % (subbsn_id, outfname.upper())
outdict = dict(
) # type: Dict[AnyStr, Dict[int, Dict[AnyStr, Union[float, Dict[int, float]]]]]
slppos_cls = numpy.ones(
(hillslpr.nRows, hillslpr.nCols)) * hillslpr.noDataValue
valid_cells = 0
# Get the fuzzy slope position values from up to bottom
def GetFuzzySlopePositionValues(i_row, i_col):
seqvalues = [-9999] * len(fuzslppos_rs)
for iseq, fuzdata in enumerate(fuzslppos_rs):
curv = fuzdata.data[i_row][i_col]
if MathClass.floatequal(curv, fuzdata.noDataValue):
return None
if curv < 0:
return None
seqvalues[iseq] = curv
return seqvalues
# ACTUAL ALGORITHM
for row in range(hillslpr.nRows):
for col in range(hillslpr.nCols):
# Exclude invalid situation
hillslp_id = hillslpr.data[row][col]
if MathClass.floatequal(hillslp_id, hillslpr.noDataValue):
continue
if hillslp_id not in thresholds:
continue
landuse_id = landuser.data[row][col]
if MathClass.floatequal(landuse_id, landuser.noDataValue):
continue
fuzzyvalues = GetFuzzySlopePositionValues(row, col)
if fuzzyvalues is None:
continue
# THIS PART SHOULD BE REVIEWED CAREFULLY LATER! --START
# Step 1. Get the index of slope position with maximum similarity
max_fuz = max(fuzzyvalues)
max_idx = fuzzyvalues.index(max_fuz)
tmpfuzzyvalues = fuzzyvalues[:]
tmpfuzzyvalues.remove(max_fuz)
sec_fuz = max(tmpfuzzyvalues)
sec_idx = fuzzyvalues.index(sec_fuz)
sel_idx = max_idx # Select the maximum by default
cur_threshs = thresholds[hillslp_id][1 - len(fuzzyvalues):]
if max_idx == len(fuzzyvalues) - 1: # the bottom position
if sec_idx == len(
fuzzyvalues
) - 2 and 0 < max_fuz - sec_fuz < cur_threshs[-1]:
sel_idx = sec_idx # change valley to backslope
elif max_idx == 0: # the upper position
if sec_idx == 1 and 0 < max_fuz - sec_fuz < cur_threshs[0]:
sel_idx = sec_idx # change ridge to backslope
else: # the middle positions
# Two thresholds could be applied,
# i.e., cur_threshs[max_idx-1] and cur_threshs[max_idx]
if sec_idx == max_idx - 1 and 0. > sec_fuz - max_fuz > cur_threshs[
max_idx - 1]:
sel_idx = sec_idx
elif sec_idx == max_idx + 1 and 0. > sec_fuz - max_fuz > cur_threshs[
max_idx]:
sel_idx = sec_idx
# Exception:
if sec_fuz < 0.1 and sel_idx == sec_idx:
sel_idx = max_idx
# if sel_idx != max_idx: # boundary has been adapted
# print('fuzzy values: %s, thresholds: %s, '
# 'sel_idx: %d' % (fuzzyvalues.__str__(), cur_threshs.__str__(), sel_idx))
slppos_id = thresholds[hillslp_id][sel_idx]
# THIS PART SHOULD BE REVIEWED CAREFULLY LATER! --END
slppos_cls[row][col] = slppos_id
sel_tagname = fuzzyslppos_fnames[sel_idx][1]
if sel_tagname not in outdict:
outdict[sel_tagname] = dict()
if slppos_id not in outdict[sel_tagname]:
outdict[sel_tagname][slppos_id] = {
'area': 0,
'landuse': dict()
}
outdict[sel_tagname][slppos_id]['area'] += 1
if landuse_id not in outdict[sel_tagname][slppos_id]['landuse']:
outdict[sel_tagname][slppos_id]['landuse'][landuse_id] = 0.
outdict[sel_tagname][slppos_id]['landuse'][landuse_id] += 1.
valid_cells += 1
# Change cell counts to area
area_km2 = hillslpr.dx * hillslpr.dx * 1.e-6
for tagname, slpposdict in viewitems(outdict):
for sid, datadict in viewitems(slpposdict):
outdict[tagname][sid]['area'] *= area_km2
for luid in outdict[tagname][sid]['landuse']:
outdict[tagname][sid]['landuse'][luid] *= area_km2
# 3. Write the classified slope positions data back to mongodb
metadata = dict()
metadata[RasterMetadata.subbasin] = subbsn_id
metadata['ID'] = outgfsname
metadata['TYPE'] = outfname.upper()
metadata[RasterMetadata.cellsize] = hillslpr.dx
metadata[RasterMetadata.nodata] = hillslpr.noDataValue
metadata[RasterMetadata.ncols] = hillslpr.nCols
metadata[RasterMetadata.nrows] = hillslpr.nRows
metadata[RasterMetadata.xll] = hillslpr.xMin + 0.5 * hillslpr.dx
metadata[RasterMetadata.yll] = hillslpr.yMin + 0.5 * hillslpr.dx
metadata['LAYERS'] = 1.
metadata[RasterMetadata.cellnum] = valid_cells
metadata[RasterMetadata.srs] = hillslpr.srs
client = ConnectMongoDB(modelcfg.host, modelcfg.port)
conn = client.get_conn()
maindb = conn[modelcfg.db_name]
spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial)
# delete if the tablename gridfs file existed
if spatial_gfs.exists(filename=outgfsname):
x = spatial_gfs.get_version(filename=outgfsname)
spatial_gfs.delete(x._id)
# create and write new GridFS file
new_gridfs = spatial_gfs.new_file(filename=outgfsname, metadata=metadata)
new_gridfs_array = slppos_cls.reshape(
(1, hillslpr.nCols * hillslpr.nRows)).tolist()[0]
fmt = '%df' % hillslpr.nCols * hillslpr.nRows
s = pack(fmt, *new_gridfs_array)
new_gridfs.write(s)
new_gridfs.close()
# Read and output for test
# slpposcls_r = ReadRasterFromMongoDB(modelcfg.host, modelcfg.port,
# modelcfg.db_name, DBTableNames.gridfs_spatial, outgfsname)
# out_slpposcls = out_dir + os.sep + '%s.tif' % outgfsname
# RasterUtilClass.write_gtiff_file(out_slpposcls, slpposcls_r.nRows, slpposcls_r.nCols,
# slpposcls_r.data, slpposcls_r.geotrans, slpposcls_r.srs,
# slpposcls_r.noDataValue)
client.close()
return outdict
def export_scenario_to_gtiff(self, outpath=None):
"""Export scenario to GTiff.
TODO: Read Raster from MongoDB should be extracted to pygeoc.
"""
if not self.export_sce_tif:
return
dist = self.bmps_info['DISTRIBUTION']
dist_list = StringClass.split_string(dist, '|')
if len(dist_list) >= 2 and dist_list[0] == 'RASTER':
dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin
# read dist_name from MongoDB
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
maindb = conn[self.main_db]
spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial)
# read file from mongodb
if not spatial_gfs.exists(filename=dist_name):
print('WARNING: %s is not existed, export scenario failed!' %
dist_name)
return
try:
slpposf = maindb[DBTableNames.gridfs_spatial].files.find(
{'filename': dist_name}, no_cursor_timeout=True)[0]
except NetworkTimeout or Exception:
# In case of unexpected raise
client.close()
return
ysize = int(slpposf['metadata'][RasterMetadata.nrows])
xsize = int(slpposf['metadata'][RasterMetadata.ncols])
xll = slpposf['metadata'][RasterMetadata.xll]
yll = slpposf['metadata'][RasterMetadata.yll]
cellsize = slpposf['metadata'][RasterMetadata.cellsize]
nodata_value = slpposf['metadata'][RasterMetadata.nodata]
srs = slpposf['metadata'][RasterMetadata.srs]
if isinstance(srs, text_type):
srs = str(srs)
srs = osr.GetUserInputAsWKT(srs)
geotransform = [0] * 6
geotransform[0] = xll - 0.5 * cellsize
geotransform[1] = cellsize
geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax
geotransform[5] = -cellsize
slppos_data = spatial_gfs.get(slpposf['_id'])
total_len = xsize * ysize
fmt = '%df' % (total_len, )
slppos_data = unpack(fmt, slppos_data.read())
slppos_data = numpy.reshape(slppos_data, (ysize, xsize))
v_dict = dict()
for idx, gene_v in enumerate(self.gene_values):
v_dict[self.gene_to_unit[idx]] = gene_v
for k, v in v_dict.items():
slppos_data[slppos_data == k] = v
if outpath is None:
outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID
RasterUtilClass.write_gtiff_file(outpath, ysize, xsize,
slppos_data, geotransform, srs,
nodata_value)
client.close()
def workflow(cfg):
"""Building MongoDB workflow"""
f = cfg.logs.build_mongo
# build a connection to mongodb database
client = ConnectMongoDB(cfg.hostname, cfg.port)
conn = client.get_conn()
maindb = conn[cfg.spatial_db]
climatedb = conn[cfg.climate_db]
scenariodb = None
if cfg.use_scernario:
scenariodb = conn[cfg.bmp_scenario_db]
# import model parameters information to MongoDB
status_output('Import model parameters', 10, f)
ImportParam2Mongo.workflow(cfg, maindb)
n_subbasins = MongoQuery.get_init_parameter_value(maindb, SubbsnStatsName.subbsn_num)
print('Number of subbasins: %d' % n_subbasins)
# Extract spatial parameters for reaches, landuse, soil, etc.
status_output('Extract spatial parameters for reaches, landuse, soil, etc...', 20, f)
extract_spatial_parameters(cfg, maindb)
# import stream parameters
status_output('Generating reach table with initialized parameters...', 40, f)
ImportReaches2Mongo.generate_reach_table(cfg, maindb)
# import raster data to MongoDB
status_output('Importing raster to MongoDB....', 50, f)
ImportMongodbClass.spatial_rasters(cfg, 0)
ImportMongodbClass.spatial_rasters(cfg, n_subbasins)
# Import IUH
status_output('Generating and importing IUH (Instantaneous Unit Hydrograph)....', 60, f)
ImportMongodbClass.iuh(cfg, 0)
ImportMongodbClass.iuh(cfg, n_subbasins)
# Import grid layering data
status_output('Generating and importing grid layering....', 70, f)
ImportMongodbClass.grid_layering(cfg, 0)
ImportMongodbClass.grid_layering(cfg, n_subbasins)
# Import hydro-climate data
status_output('Import climate data....', 80, f)
ImportMongodbClass.climate_data(cfg, maindb, climatedb)
# Import weight and related data, this should after ImportMongodbClass.climate_data()
status_output('Generating weight data for interpolation of meteorology data '
'and weight dependent parameters....', 85, f)
ImportWeightData.workflow(cfg, conn, 0)
ImportWeightData.workflow(cfg, conn, n_subbasins)
# Measurement Data, such as discharge, sediment yield.
status_output('Import observed data, such as discharge, sediment yield....', 90, f)
ImportObservedData.workflow(cfg, maindb, climatedb)
# Import BMP scenario database to MongoDB
status_output('Importing bmp scenario....', 95, f)
ImportScenario2Mongo.scenario_from_texts(cfg, maindb, scenariodb)
status_output('Build DB: %s finished!' % cfg.spatial_db, 100, f)
# close connection to MongoDB
client.close()
def export_scenario_to_gtiff(self, outpath=None):
"""Export scenario to GTiff.
TODO: Read Raster from MongoDB should be extracted to pygeoc.
"""
if not self.export_sce_tif:
return
dist = self.bmps_info['DISTRIBUTION']
dist_list = StringClass.split_string(dist, '|')
if len(dist_list) >= 2 and dist_list[0] == 'RASTER':
dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin
# read dist_name from MongoDB
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
maindb = conn[self.main_db]
spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial)
# read file from mongodb
if not spatial_gfs.exists(filename=dist_name):
print('WARNING: %s is not existed, export scenario failed!' % dist_name)
return
try:
slpposf = maindb[DBTableNames.gridfs_spatial].files.find({'filename': dist_name},
no_cursor_timeout=True)[0]
except NetworkTimeout or Exception:
# In case of unexpected raise
client.close()
return
ysize = int(slpposf['metadata'][RasterMetadata.nrows])
xsize = int(slpposf['metadata'][RasterMetadata.ncols])
xll = slpposf['metadata'][RasterMetadata.xll]
yll = slpposf['metadata'][RasterMetadata.yll]
cellsize = slpposf['metadata'][RasterMetadata.cellsize]
nodata_value = slpposf['metadata'][RasterMetadata.nodata]
srs = slpposf['metadata'][RasterMetadata.srs]
if isinstance(srs, text_type):
srs = str(srs)
srs = osr.GetUserInputAsWKT(srs)
geotransform = [0] * 6
geotransform[0] = xll - 0.5 * cellsize
geotransform[1] = cellsize
geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax
geotransform[5] = -cellsize
slppos_data = spatial_gfs.get(slpposf['_id'])
total_len = xsize * ysize
fmt = '%df' % (total_len,)
slppos_data = unpack(fmt, slppos_data.read())
slppos_data = numpy.reshape(slppos_data, (ysize, xsize))
v_dict = dict()
for idx, gene_v in enumerate(self.gene_values):
v_dict[self.gene_to_unit[idx]] = gene_v
for k, v in v_dict.items():
slppos_data[slppos_data == k] = v
if outpath is None:
outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID
RasterUtilClass.write_gtiff_file(outpath, ysize, xsize, slppos_data, geotransform,
srs, nodata_value)
client.close()