def Mode(self):
# type: (...) -> AnyStr
"""Get simulation mode."""
if self._mode != '':
return self._mode.upper()
mode_dict = self.filein_tab.find_one({ModelCfgFields.tag: FieldNames.mode})
self._mode = mode_dict[ModelCfgFields.value]
if is_string(self._mode):
self._mode = str(self._mode)
return self._mode.upper()
def mask_raster(in_raster, mask, out_raster):
"""
Mask raster data.
Args:
in_raster: list or one raster
mask: Mask raster data
out_raster: list or one raster
"""
if is_string(in_raster) and is_string(out_raster):
in_raster = [str(in_raster)]
out_raster = [str(out_raster)]
if len(in_raster) != len(out_raster):
raise RuntimeError(
'input raster and output raster must have the same size.')
maskr = RasterUtilClass.read_raster(mask)
rows = maskr.nRows
cols = maskr.nCols
maskdata = maskr.data
temp = maskdata == maskr.noDataValue
for inr, outr in zip(in_raster, out_raster):
origin = RasterUtilClass.read_raster(inr)
if origin.nRows == rows and origin.nCols == cols:
masked = numpy.where(temp, origin.noDataValue, origin.data)
else:
masked = numpy.ones((rows, cols)) * origin.noDataValue
# TODO, the following loop should be optimized by numpy or numba
for i in range(rows):
for j in range(cols):
if maskdata[i][j] == maskr.noDataValue:
continue
# get the center point coordinate of current cell
tempx, tempy = maskr.get_central_coors(i, j)
tempv = origin.get_value_by_xy(tempx, tempy)
if tempv is None:
continue
masked[i][j] = tempv
RasterUtilClass.write_gtiff_file(outr, maskr.nRows, maskr.nCols,
masked, maskr.geotrans, maskr.srs,
origin.noDataValue,
origin.dataType)
def mask_raster(in_raster, mask, out_raster):
"""
Mask raster data.
Args:
in_raster: list or one raster
mask: Mask raster data
out_raster: list or one raster
"""
if is_string(in_raster) and is_string(out_raster):
in_raster = [str(in_raster)]
out_raster = [str(out_raster)]
if len(in_raster) != len(out_raster):
raise RuntimeError('input raster and output raster must have the same size.')
maskr = RasterUtilClass.read_raster(mask)
rows = maskr.nRows
cols = maskr.nCols
maskdata = maskr.data
temp = maskdata == maskr.noDataValue
for inr, outr in zip(in_raster, out_raster):
origin = RasterUtilClass.read_raster(inr)
if origin.nRows == rows and origin.nCols == cols:
masked = numpy.where(temp, origin.noDataValue, origin.data)
else:
masked = numpy.ones((rows, cols)) * origin.noDataValue
# TODO, the following loop should be optimized by numpy or numba
for i in range(rows):
for j in range(cols):
if maskdata[i][j] == maskr.noDataValue:
continue
# get the center point coordinate of current cell
tempx, tempy = maskr.get_central_coors(i, j)
tempv = origin.get_value_by_xy(tempx, tempy)
if tempv is None:
continue
masked[i][j] = tempv
RasterUtilClass.write_gtiff_file(outr, maskr.nRows, maskr.nCols, masked,
maskr.geotrans, maskr.srs,
origin.noDataValue, origin.dataType)
def raster_dilation(rasterfile):
"""Dilate the raster image.
Find the max pixel's value in 8-neighborhood. Then change the compute
pixel's value into the max pixel's value.
Args:
rasterfile: input original raster image, type can be filename(string,
like "test1.tif"), rasterfile(class Raster) or numpy.ndarray.
Returns:
dilation_raster: raster image after dilation, type is numpy.ndarray.
"""
if is_string(rasterfile):
origin_raster = RasterUtilClass.read_raster(str(rasterfile))
elif isinstance(rasterfile, Raster):
origin_raster = rasterfile.data
elif isinstance(rasterfile, numpy.ndarray):
origin_raster = rasterfile
else:
return 'Your rasterfile has a wrong type. Type must be string or ' \
'numpy.array or class Raster in pygeoc.'
min_value_raster = origin_raster.min()
dilation_raster = numpy.zeros(
(origin_raster.shape[0], origin_raster.shape[1]))
# In order to compute the raster edges, we need to expand the original
# raster's rows and cols. We need to add the edges whose pixels' value is
# the max pixel's value in raster.
add_row = numpy.full((1, origin_raster.shape[1]), min_value_raster)
temp_origin_raster = numpy.vstack((numpy.vstack(
(add_row, origin_raster)), add_row))
add_col = numpy.full((origin_raster.shape[0] + 2, 1), min_value_raster)
expand_origin_raster = numpy.hstack((numpy.hstack(
(add_col, temp_origin_raster)), add_col))
# Dilate the raster.
for i in range(origin_raster.shape[0]):
for j in range(origin_raster.shape[1]):
max_pixel_value = min_value_raster
# Find the max pixel value in the 8-neighborhood.
for k in range(3):
for l in range(3):
if expand_origin_raster[i + k,
j + l] >= max_pixel_value:
max_pixel_value = expand_origin_raster[i + k,
j + l]
# After this loop, we get the max pixel's value of the
# 8-neighborhood. Then we change the compute pixel's value into
# the max pixel's value.
dilation_raster[i, j] = max_pixel_value
# Return the result.
return dilation_raster
def raster_dilation(rasterfile):
"""Dilate the raster image.
Find the max pixel's value in 8-neighborhood. Then change the compute
pixel's value into the max pixel's value.
Args:
rasterfile: input original raster image, type can be filename(string,
like "test1.tif"), rasterfile(class Raster) or numpy.ndarray.
Returns:
dilation_raster: raster image after dilation, type is numpy.ndarray.
"""
if is_string(rasterfile):
origin_raster = RasterUtilClass.read_raster(str(rasterfile))
elif isinstance(rasterfile, Raster):
origin_raster = rasterfile.data
elif isinstance(rasterfile, numpy.ndarray):
origin_raster = rasterfile
else:
return 'Your rasterfile has a wrong type. Type must be string or ' \
'numpy.array or class Raster in pygeoc.'
min_value_raster = origin_raster.min()
dilation_raster = numpy.zeros((origin_raster.shape[0], origin_raster.shape[1]))
# In order to compute the raster edges, we need to expand the original
# raster's rows and cols. We need to add the edges whose pixels' value is
# the max pixel's value in raster.
add_row = numpy.full((1, origin_raster.shape[1]), min_value_raster)
temp_origin_raster = numpy.vstack((numpy.vstack((add_row, origin_raster)), add_row))
add_col = numpy.full((origin_raster.shape[0] + 2, 1), min_value_raster)
expand_origin_raster = numpy.hstack((numpy.hstack((add_col, temp_origin_raster)), add_col))
# Dilate the raster.
for i in range(origin_raster.shape[0]):
for j in range(origin_raster.shape[1]):
max_pixel_value = min_value_raster
# Find the max pixel value in the 8-neighborhood.
for k in range(3):
for l in range(3):
if expand_origin_raster[i + k, j + l] >= max_pixel_value:
max_pixel_value = expand_origin_raster[i + k, j + l]
# After this loop, we get the max pixel's value of the
# 8-neighborhood. Then we change the compute pixel's value into
# the max pixel's value.
dilation_raster[i, j] = max_pixel_value
# Return the result.
return dilation_raster
def export_landuse_lookup_files_from_mongodb(cfg, maindb):
"""export landuse lookup tables to txt file from MongoDB."""
lookup_dir = cfg.dirs.lookup
property_namelist = ModelParamDataUtils.landuse_fields
property_map = dict()
property_namelist.append('USLE_P')
query_result = maindb['LANDUSELOOKUP'].find()
if query_result is None:
raise RuntimeError(
'LanduseLoop Collection is not existed or empty!')
count = 0
for row in query_result:
# print(row)
value_map = dict()
for i, p_name in enumerate(property_namelist):
if StringClass.string_match(p_name, 'USLE_P'):
# Currently, USLE_P is set as 1 for all landuse.
value_map[p_name] = 1
else:
# I do not know why manning * 10 here. Just uncommented now. lj
# if StringClass.string_match(p_name, "Manning"):
# value_map[p_name] = row.get(p_name) * 10
# else:
v = row.get(p_name)
if is_string(v):
v = StringClass.extract_numeric_values_from_string(
v)[0]
value_map[p_name] = v
count += 1
property_map[count] = value_map
n = len(property_map)
UtilClass.rmmkdir(lookup_dir)
for propertyName in property_namelist:
with open('%s/%s.txt' % (
lookup_dir,
propertyName,
),
'w',
encoding='utf-8') as f:
f.write('%d\n' % n)
for prop_id in property_map:
s = '%d %f\n' % (int(property_map[prop_id]['LANDUSE_ID']),
property_map[prop_id][propertyName])
f.write('%s' % s)
def get_suitable_bmps(self, types='LANDUSE'):
# type: (Union[AnyStr, List[AnyStr]]) -> None
"""Construct the suitable BMPs for each slope position."""
if is_string(types):
types = [types]
for bid, bdict in self.bmps_params.items():
for type in types:
if type not in bdict:
continue
if type not in self.suit_bmps:
self.suit_bmps.setdefault(type, dict())
suitsp = bdict[type]
for sp in suitsp:
if sp not in self.suit_bmps[type]:
self.suit_bmps[type][sp] = [bid]
elif bid not in self.suit_bmps[type][sp]:
self.suit_bmps[type][sp].append(bid)
if 'EFFECTIVENESS' in bdict:
self.bmps_grade[bid] = bdict['EFFECTIVENESS']
def ReadRasterFromMongoDB(ip, port, db_name, gfsname, gfilename):
client = ConnectMongoDB(ip, port)
conn = client.get_conn()
maindb = conn[db_name]
spatial_gfs = GridFS(maindb, gfsname)
if not spatial_gfs.exists(filename=gfilename):
raise ValueError('WARNING: %s is not existed in %s:%s!' %
(gfilename, db_name, gfsname))
try:
gfsdata = maindb[DBTableNames.gridfs_spatial].files.find(
{'filename': gfilename}, no_cursor_timeout=True)[0]
except NetworkTimeout or Exception:
# In case of unexpected raise
client.close()
return None
ysize = int(gfsdata['metadata'][RasterMetadata.nrows])
xsize = int(gfsdata['metadata'][RasterMetadata.ncols])
xll = gfsdata['metadata'][RasterMetadata.xll]
yll = gfsdata['metadata'][RasterMetadata.yll]
cellsize = gfsdata['metadata'][RasterMetadata.cellsize]
nodata = gfsdata['metadata'][RasterMetadata.nodata]
srs = gfsdata['metadata'][RasterMetadata.srs]
if is_string(srs):
srs = str(srs)
srs = osr.GetUserInputAsWKT(srs)
geotransform = [0] * 6
geotransform[0] = xll - 0.5 * cellsize
geotransform[1] = cellsize
geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax
geotransform[5] = -cellsize
array_data = spatial_gfs.get(gfsdata['_id'])
total_len = xsize * ysize
fmt = '%df' % (total_len, )
array_data = unpack(fmt, array_data.read())
array_data = numpy.reshape(array_data, (ysize, xsize))
return Raster(ysize, xsize, array_data, nodata, geotransform, srs)
def ogrwkt2shapely(input_shape, id_field):
"""Return shape objects list and ids list"""
# CAUTION, IMPORTANT
# Because shapely is dependent on sqlite, and the version is not consistent
# with GDAL executable (e.g., located in C:\GDAL_x64\bin), thus the shapely
# must be locally imported here.
from shapely.wkt import loads as shapely_loads
shapely_objects = list()
shape_area = list()
id_list = list()
# print(input_shape)
shp = ogr_Open(input_shape)
if shp is None:
raise RuntimeError(
'The input ESRI Shapefile: %s is not existed or has '
'no read permission!' % input_shape)
lyr = shp.GetLayer()
for n in range(0, lyr.GetFeatureCount()):
feat = lyr.GetFeature(n)
# This function may print Failed `CDLL(/opt/local/lib/libgeos_c.dylib)` in macOS
# Don't worry about that!
wkt_feat = shapely_loads(feat.geometry().ExportToWkt())
if is_string(id_field):
id_field = str(id_field)
id_index = feat.GetFieldIndex(id_field)
fldid = feat.GetField(id_index)
if fldid not in id_list:
id_list.append(fldid)
shapely_objects.append(wkt_feat)
shape_area.append(wkt_feat.area)
else: # if multipolygon, take the polygon part with largest area.
exist_id_idx = id_list.index(fldid)
if shape_area[exist_id_idx] < wkt_feat.area:
shape_area[exist_id_idx] = wkt_feat.area
shapely_objects[exist_id_idx] = wkt_feat
return shapely_objects, id_list
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.base_dir = None
self.clim_dir = None
self.spatial_dir = None
self.observe_dir = None
self.scenario_dir = None
self.model_dir = None
self.txt_db_dir = None
self.preproc_script_dir = None
self.seims_bin = None
self.mpi_bin = None
self.workspace = None
# 1.1. Directory determined flags
self.use_observed = True
self.use_scernario = True
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
# 3. Climate inputs
self.hydro_climate_vars = None
self.prec_sites = None
self.prec_data = None
self.Meteo_sites = None
self.Meteo_data = None
self.thiessen_field = 'ID'
# 4. Spatial inputs
self.prec_sites_thiessen = None
self.meteo_sites_thiessen = None
self.dem = None
self.outlet_file = None
self.landuse = None
self.landcover_init_param = None
self.soil = None
self.soil_property = None
self.fields_partition = False
self.fields_partition_thresh = list()
self.additional_rs = dict()
# 5. Option parameters
self.d8acc_threshold = 0
self.np = 4
self.d8down_method = 's'
self.dorm_hr = -1.
self.temp_base = 0.
self.imper_perc_in_urban = 0.
self.default_landuse = -1
self.default_soil = -1
# 1. Directories
if 'PATH' in cf.sections():
self.base_dir = cf.get('PATH', 'base_data_dir')
self.clim_dir = cf.get('PATH', 'climate_data_dir')
self.spatial_dir = cf.get('PATH', 'spatial_data_dir')
self.observe_dir = cf.get('PATH', 'measurement_data_dir')
self.scenario_dir = cf.get('PATH', 'bmp_data_dir')
self.model_dir = cf.get('PATH', 'model_dir')
self.txt_db_dir = cf.get('PATH', 'txt_db_dir')
self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir')
self.seims_bin = cf.get('PATH', 'cpp_program_dir')
self.mpi_bin = cf.get('PATH', 'mpiexec_dir')
self.workspace = cf.get('PATH', 'working_dir')
else:
raise ValueError('[PATH] section MUST be existed in *.ini file.')
if not (FileClass.is_dir_exists(self.base_dir)
and FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.txt_db_dir)
and FileClass.is_dir_exists(self.preproc_script_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError(
'Please Check Directories defined in [PATH]. '
'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, '
'and CPP_PROGRAM_DIR are required!')
if not FileClass.is_dir_exists(self.mpi_bin):
self.mpi_bin = None
if not FileClass.is_dir_exists(self.workspace):
try: # first try to make dirs
UtilClass.mkdir(self.workspace)
# os.mkdir(self.workspace)
except OSError as exc:
self.workspace = self.model_dir + os.path.sep + 'preprocess_output'
print('WARNING: Make WORKING_DIR failed! Use the default: %s' %
self.workspace)
if not os.path.exists(self.workspace):
UtilClass.mkdir(self.workspace)
self.dirs = DirNameUtils(self.workspace)
self.logs = LogNameUtils(self.dirs.log)
self.vecs = VectorNameUtils(self.dirs.geoshp)
self.taudems = TauDEMFilesUtils(self.dirs.taudem)
self.spatials = SpatialNamesUtils(self.dirs.geodata2db)
self.modelcfgs = ModelCfgUtils(self.model_dir)
self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir +
os.path.sep + 'database')
if not FileClass.is_dir_exists(self.clim_dir):
print(
'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".'
)
self.clim_dir = self.base_dir + os.path.sep + 'climate'
if not FileClass.is_dir_exists(self.clim_dir):
raise IOError(
'Directories named "climate" MUST BE located in [base_dir]!'
)
if not FileClass.is_dir_exists(self.spatial_dir):
print(
'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".'
)
self.spatial_dir = self.base_dir + os.path.sep + 'spatial'
raise IOError(
'Directories named "spatial" MUST BE located in [base_dir]!')
if not FileClass.is_dir_exists(self.observe_dir):
self.observe_dir = None
self.use_observed = False
if not FileClass.is_dir_exists(self.scenario_dir):
self.scenario_dir = None
self.use_scernario = False
# 2. MongoDB related
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Climate Input
if 'CLIMATE' in cf.sections():
self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'hydroclimatevarfile')
self.prec_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precsitefile')
self.prec_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precdatafile')
self.Meteo_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteositefile')
self.Meteo_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteodatafile')
self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield')
else:
raise ValueError(
'Climate input file names MUST be provided in [CLIMATE]!')
# 4. Spatial Input
if 'SPATIAL' in cf.sections():
self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'precsitesthiessen')
self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'meteositesthiessen')
self.dem = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'dem')
self.outlet_file = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'outlet_file')
if not os.path.exists(self.outlet_file):
self.outlet_file = None
self.landuse = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'landusefile')
self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'landcoverinitfile')
self.soil = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqnfile')
self.soil_property = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqntext')
if cf.has_option('SPATIAL', 'additionalfile'):
additional_dict_str = cf.get('SPATIAL', 'additionalfile')
tmpdict = json.loads(additional_dict_str)
tmpdict = {
str(k): (str(v) if is_string(v) else v)
for k, v in list(tmpdict.items())
}
for k, v in list(tmpdict.items()):
# Existence check has been moved to mask_origin_delineated_data()
# in sp_delineation.py
self.additional_rs[k] = v
# Field partition
if cf.has_option('SPATIAL', 'field_partition_thresh'):
ths = cf.get('SPATIAL', 'field_partition_thresh')
thsv = StringClass.extract_numeric_values_from_string(ths)
if thsv is not None:
self.fields_partition_thresh = [int(v) for v in thsv]
self.fields_partition = True
else:
raise ValueError(
'Spatial input file names MUST be provided in [SPATIAL]!')
# 5. Optional parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS',
'd8accthreshold')
self.np = cf.getint('OPTIONAL_PARAMETERS', 'np')
self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod')
if StringClass.string_match(self.d8down_method, 'surface'):
self.d8down_method = 's'
elif StringClass.string_match(self.d8down_method, 'horizontal'):
self.d8down_method = 'h'
elif StringClass.string_match(self.d8down_method, 'pythagoras'):
self.d8down_method = 'p'
elif StringClass.string_match(self.d8down_method, 'vertical'):
self.d8down_method = 'v'
else:
self.d8down_method = self.d8down_method.lower()
if self.d8down_method not in ['s', 'h', 'p', 'v']:
self.d8down_method = 's'
self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr')
self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base')
self.imper_perc_in_urban = cf.getfloat(
'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell')
self.default_landuse = cf.getint('OPTIONAL_PARAMETERS',
'defaultlanduse')
self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(
self,
bin_dir='', # type: AnyStr # The directory of SEIMS binary
model_dir='', # type: AnyStr # The directory of SEIMS model
nthread=4, # type: int # Thread number for OpenMP
lyrmtd=0, # type: int # Layering method, can be 0 (UP_DOWN) or 1 (DOWN_UP)
host='127.0.0.1', # type: AnyStr # MongoDB host address, default is `localhost`
port=27017, # type: int # MongoDB port, default is 27017
db_name='', # type: AnyStr # Main spatial dbname which can diff from dirname
scenario_id=-1, # type: int # Scenario ID defined in `<model>_Scenario` database
calibration_id=-1, # type: int # Calibration ID used for model auto-calibration
subbasin_id=0, # type: int # Subbasin ID, 0 for whole watershed, 9999 for field version
version='OMP', # type: AnyStr # SEIMS version, can be `MPI` or `OMP` (default)
nprocess=1, # type: int # Process number for MPI
mpi_bin='', # type: AnyStr # Full path of MPI executable file, e.g., './mpirun`
hosts_opt='-f', # type: AnyStr # Option for assigning hosts,
# e.g., `-f`, `-hostfile`, `-machine`, `-machinefile`
hostfile='', # type: AnyStr # File containing host names,
# or file mapping process numbers to machines
simu_stime=None, # type: Optional[datetime, AnyStr] # Start time of simulation
simu_etime=None, # type: Optional[datetime, AnyStr] # End time of simulation
out_stime=None, # type: Optional[datetime, AnyStr] # Start time of outputs
out_etime=None, # type: Optional[datetime, AnyStr] # End time of outputs
args_dict=None # type: Dict[AnyStr, Optional[AnyStr, datetime, int]]
):
# type: (...) -> None
# Derived from input arguments
if args_dict is None: # Preferred to use 'args_dict' if existed.
args_dict = dict()
bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir
model_dir = args_dict[
'model_dir'] if 'model_dir' in args_dict else model_dir
self.version = args_dict[
'version'] if 'version' in args_dict else version
suffix = '.exe' if sysstr == 'Windows' else ''
if self.version == 'MPI':
self.seims_exec = '%s/seims_mpi%s' % (bin_dir, suffix)
else:
self.seims_exec = '%s/seims_omp%s' % (bin_dir, suffix)
if not FileClass.is_file_exists(
self.seims_exec): # If not support OpenMP, use `seims`!
self.seims_exec = '%s/seims%s' % (bin_dir, suffix)
self.seims_exec = os.path.abspath(self.seims_exec)
self.model_dir = os.path.abspath(model_dir)
self.nthread = args_dict[
'nthread'] if 'nthread' in args_dict else nthread
self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd
self.host = args_dict['host'] if 'host' in args_dict else host
self.port = args_dict['port'] if 'port' in args_dict else port
self.db_name = args_dict['db_name'] if 'db_name' in args_dict \
else os.path.split(self.model_dir)[1]
self.scenario_id = args_dict[
'scenario_id'] if 'scenario_id' in args_dict else scenario_id
self.calibration_id = args_dict['calibration_id'] \
if 'calibration_id' in args_dict else calibration_id
self.subbasin_id = args_dict[
'subbasin_id'] if 'subbasin_id' in args_dict else subbasin_id
self.nprocess = args_dict[
'nprocess'] if 'nprocess' in args_dict else nprocess
self.mpi_bin = args_dict[
'mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin
self.hosts_opt = args_dict[
'hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt
self.hostfile = args_dict[
'hostfile'] if 'hostfile' in args_dict else hostfile
self.simu_stime = args_dict[
'simu_stime'] if 'simu_stime' in args_dict else simu_stime
self.simu_etime = args_dict[
'simu_etime'] if 'simu_etime' in args_dict else simu_etime
self.out_stime = args_dict[
'out_stime'] if 'out_stime' in args_dict else out_stime
self.out_etime = args_dict[
'out_etime'] if 'out_etime' in args_dict else out_etime
if is_string(
self.simu_stime) and not isinstance(self.simu_stime, datetime):
self.simu_stime = StringClass.get_datetime(self.simu_stime)
if is_string(
self.simu_etime) and not isinstance(self.simu_etime, datetime):
self.simu_etime = StringClass.get_datetime(self.simu_etime)
if is_string(
self.out_stime) and not isinstance(self.out_stime, datetime):
self.out_stime = StringClass.get_datetime(self.out_stime)
if is_string(
self.out_etime) and not isinstance(self.out_etime, datetime):
self.out_etime = StringClass.get_datetime(self.out_etime)
# Concatenate executable command
self.cmd = self.Command
self.run_success = False
self.output_dir = self.OutputDirectory
# Model data read from MongoDB
self.outlet_id = -1
self.subbasin_count = -1
self.scenario_dbname = ''
self.start_time = None
self.end_time = None
self.output_ids = list() # type: List[AnyStr]
self.output_items = dict() # type: Dict[AnyStr, Union[List[AnyStr]]]
# Data maybe used after model run
self.timespan = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, Dict[AnyStr, float]]]]
self.obs_vars = list(
) # type: List[AnyStr] # Observation types at the outlet
self.obs_value = dict(
) # type: Dict[datetime, List[float]] # Observation value
self.sim_vars = list(
) # type: List[AnyStr] # Simulation types, part of `obs_vars`
self.sim_value = dict(
) # type: Dict[datetime, List[float]] # Simulation value
# The format of sim_obs_dict:
# {VarName: {'UTCDATETIME': [t1, t2, ..., tn],
# 'Obs': [o1, o2, ..., on],
# 'Sim': [s1, s2, ..., sn]},
# ...
# }
self.sim_obs_dict = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, List[Union[datetime, float]]]]]
self.runtime = 0.
self.runlogs = list() # type: List[AnyStr]
self.mongoclient = None # type: Union[MongoClient, None] # Set to None after use
def calculate_95ppu(sim_obs_data, sim_data, outdir, gen_num,
vali_sim_obs_data=None, vali_sim_data=None,
plot_cfg=None # type: Optional[PlotConfig]
):
"""Calculate 95% prediction uncertainty and plot the hydrographs."""
if plot_cfg is None:
plot_cfg = PlotConfig()
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'
plt.rcParams['font.family'] = plot_cfg.font_name
plt.rcParams['timezone'] = 'UTC'
plt.rcParams['mathtext.fontset'] = 'custom'
plt.rcParams['mathtext.it'] = 'STIXGeneral:italic'
plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold'
if len(sim_data) < 2:
return
var_name = sim_obs_data[0]['var_name']
for idx, var in enumerate(var_name):
plot_validation = False
if vali_sim_obs_data and vali_sim_data and var in vali_sim_obs_data[0]['var_name']:
plot_validation = True
ylabel_str = var
if var in ['Q', 'QI', 'QG', 'QS']:
ylabel_str += ' (m$^3$/s)'
elif 'CONC' in var.upper(): # Concentrate
if 'SED' in var.upper():
ylabel_str += ' (g/L)'
else:
ylabel_str += ' (mg/L)'
elif 'SED' in var.upper(): # amount
ylabel_str += ' (kg)'
cali_obs_dates = sim_obs_data[0][var]['UTCDATETIME'][:]
if is_string(cali_obs_dates[0]):
cali_obs_dates = [StringClass.get_datetime(s) for s in cali_obs_dates]
obs_dates = cali_obs_dates[:]
order = 1 # By default, the calibration period is before the validation period.
if plot_validation:
vali_obs_dates = vali_sim_obs_data[0][var]['UTCDATETIME']
if is_string(vali_obs_dates[0]):
vali_obs_dates = [StringClass.get_datetime(s) for s in vali_obs_dates]
if vali_obs_dates[-1] <= cali_obs_dates[0]:
order = 0
obs_dates = vali_obs_dates + obs_dates
else:
obs_dates += vali_obs_dates
obs_data = sim_obs_data[0][var]['Obs'][:]
if plot_validation:
if order:
obs_data += vali_sim_obs_data[0][var]['Obs'][:]
else:
obs_data = vali_sim_obs_data[0][var]['Obs'][:] + obs_data
cali_sim_dates = list(sim_data[0].keys())
if is_string(cali_sim_dates[0]):
cali_sim_dates = [StringClass.get_datetime(s) for s in cali_sim_dates]
sim_dates = cali_sim_dates[:]
if plot_validation:
vali_sim_dates = list(vali_sim_data[0].keys())
if is_string(vali_sim_dates[0]):
vali_sim_dates = [StringClass.get_datetime(s) for s in vali_sim_dates]
if order:
sim_dates += vali_sim_dates
else:
sim_dates = vali_sim_dates + sim_dates
sim_data_list = list()
caliBestIdx = -1
caliBestNSE = -9999.
for idx2, ind in enumerate(sim_data):
tmp = numpy.array(list(ind.values()))
tmp = tmp[:, idx]
if sim_obs_data[idx2][var]['NSE'] > caliBestNSE:
caliBestNSE = sim_obs_data[idx2][var]['NSE']
caliBestIdx = idx2
tmpsim = tmp.tolist()
if plot_validation:
tmp_data = numpy.array(list(vali_sim_data[idx2].values()))[:, idx].tolist()
if order:
tmpsim += tmp_data
else:
tmpsim = tmp_data + tmpsim
sim_data_list.append(tmpsim)
sim_best = numpy.array(list(sim_data[caliBestIdx].values()))[:, idx]
sim_best = sim_best.tolist()
if plot_validation:
tmp_data = numpy.array(list(vali_sim_data[caliBestIdx].values()))[:, idx].tolist()
if order:
sim_best += tmp_data
else:
sim_best = tmp_data + sim_best
sim_data_list = numpy.array(sim_data_list)
ylows = numpy.percentile(sim_data_list, 2.5, 0, interpolation='nearest')
yups = numpy.percentile(sim_data_list, 97.5, 0, interpolation='nearest')
def calculate_95ppu_efficiency(obs_data_list, obs_dates_list, sim_dates_list):
# type: (...) -> (float, float)
count = 0
ylows_obs = list()
yups_obs = list()
for oi, ov in enumerate(obs_data_list):
try:
si = sim_dates_list.index(obs_dates_list[oi])
ylows_obs.append(ylows[si])
yups_obs.append(yups[si])
if ylows[si] <= ov <= yups[si]:
count += 1
except Exception:
continue
p = float(count) / len(obs_data_list)
ylows_obs = numpy.array(ylows_obs)
yups_obs = numpy.array(yups_obs)
r = numpy.mean(yups_obs - ylows_obs) / numpy.std(numpy.array(obs_data_list))
return p, r
# concatenate text
p_value, r_value = calculate_95ppu_efficiency(sim_obs_data[0][var]['Obs'],
cali_obs_dates,
list(sim_data[0].keys()))
txt = 'P-factor: %.2f\nR-factor: %.2f\n' % (p_value, r_value)
txt += u'某一最优模拟\n' if plot_cfg.plot_cn else 'One best simulation:\n'
txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (sim_obs_data[caliBestIdx][var]['NSE'],
sim_obs_data[caliBestIdx][var]['RSR'],
sim_obs_data[caliBestIdx][var]['PBIAS'],
sim_obs_data[caliBestIdx][var]['R-square'])
# concatenate text of validation if needed
vali_txt = ''
if plot_validation:
p_value, r_value = calculate_95ppu_efficiency(vali_sim_obs_data[0][var]['Obs'],
vali_obs_dates,
list(vali_sim_data[0].keys()))
vali_txt = 'P-factor: %.2f\nR-factor: %.2f\n\n' % (p_value, r_value)
vali_txt += ' $\mathit{NSE}$: %.2f\n' \
' $\mathit{RSR}$: %.2f\n' \
' $\mathit{PBIAS}$: %.2f%%\n' \
' $\mathit{R^2}$: %.2f' % (vali_sim_obs_data[caliBestIdx][var]['NSE'],
vali_sim_obs_data[caliBestIdx][var]['RSR'],
vali_sim_obs_data[caliBestIdx][var]['PBIAS'],
vali_sim_obs_data[caliBestIdx][var]['R-square'])
# plot
fig, ax = plt.subplots(figsize=(12, 4))
ax.fill_between(sim_dates, ylows.tolist(), yups.tolist(),
color=(0.8, 0.8, 0.8), label='95PPU')
observed_label = u'实测值' if plot_cfg.plot_cn else 'Observed points'
ax.scatter(obs_dates, obs_data, marker='.', s=20,
color='g', label=observed_label)
besesim_label = u'最优模拟' if plot_cfg.plot_cn else 'Best simulation'
ax.plot(sim_dates, sim_best, linestyle='--', color='red', linewidth=1,
label=besesim_label)
ax.set_xlim(left=min(sim_dates), right=max(sim_dates))
ax.set_ylim(bottom=0.)
date_fmt = mdates.DateFormatter('%m-%d-%y')
ax.xaxis.set_major_formatter(date_fmt)
ax.tick_params(axis='x', bottom=True, top=False, length=5, width=2, which='major',
labelsize=plot_cfg.tick_fsize)
ax.tick_params(axis='y', left=True, right=False, length=5, width=2, which='major',
labelsize=plot_cfg.tick_fsize)
plt.xlabel(u'时间' if plot_cfg.plot_cn else 'Date time',
fontsize=plot_cfg.axislabel_fsize)
plt.ylabel(ylabel_str, fontsize=plot_cfg.axislabel_fsize)
# plot separate dash line
delta_dt = (sim_dates[-1] - sim_dates[0]) // 9
delta_dt2 = (sim_dates[-1] - sim_dates[0]) // 35
sep_time = sim_dates[-1]
time_pos = [sep_time - delta_dt]
time_pos2 = [sep_time - 2 * delta_dt]
ymax, ymin = ax.get_ylim()
yc = abs(ymax - ymin) * 0.9
if plot_validation:
sep_time = vali_sim_dates[0] if vali_sim_dates[0] >= cali_sim_dates[-1] \
else cali_sim_dates[0]
cali_vali_labels = [(u'验证期' if plot_cfg.plot_cn else 'Calibration'),
(u'率定期' if plot_cfg.plot_cn else 'Validation')]
if not order:
cali_vali_labels.reverse()
time_pos = [sep_time - delta_dt, sep_time + delta_dt2]
time_pos2 = [sep_time - 2 * delta_dt, sep_time + delta_dt2]
ax.axvline(sep_time, color='black', linestyle='dashed', linewidth=2)
plt.text(time_pos[0], yc, cali_vali_labels[0],
fontdict={'style': 'italic', 'weight': 'bold',
'size': plot_cfg.label_fsize},
color='black')
plt.text(time_pos[1], yc, cali_vali_labels[1],
fontdict={'style': 'italic', 'weight': 'bold',
'size': plot_cfg.label_fsize},
color='black')
# add legend
handles, labels = ax.get_legend_handles_labels()
figorders = [labels.index('95PPU'), labels.index(observed_label),
labels.index(besesim_label)]
ax.legend([handles[idx] for idx in figorders], [labels[idx] for idx in figorders],
fontsize=plot_cfg.legend_fsize, loc=2, framealpha=0.8)
# add text
cali_pos = time_pos[0] if order else time_pos[1]
plt.text(cali_pos, yc * 0.5, txt, color='red', fontsize=plot_cfg.label_fsize - 1)
if plot_validation:
vali_pos = time_pos[1] if order else time_pos[0]
plt.text(vali_pos, yc * 0.5, vali_txt, color='red', fontsize=plot_cfg.label_fsize - 1)
# fig.autofmt_xdate(rotation=0, ha='center')
plt.tight_layout()
save_png_eps(plt, outdir, 'Gen%d_95ppu_%s' % (gen_num, var), plot_cfg)
# close current plot in case of 'figure.max_open_warning'
plt.cla()
plt.clf()
plt.close()
def export_scenario_to_gtiff(self, outpath=None):
# type: (Optional[str]) -> None
"""Export scenario to GTiff.
TODO: Read Raster from MongoDB should be extracted to pygeoc.
"""
if not self.export_sce_tif:
return
dist = self.bmps_info[self.cfg.bmpid]['DISTRIBUTION']
dist_list = StringClass.split_string(dist, '|')
if len(dist_list) >= 2 and dist_list[0] == 'RASTER':
dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin
# read dist_name from MongoDB
# client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port)
# conn = client.get_conn()
conn = MongoDBObj.client
maindb = conn[self.modelcfg.db_name]
spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial)
# read file from mongodb
if not spatial_gfs.exists(filename=dist_name):
print('WARNING: %s is not existed, export scenario failed!' %
dist_name)
return
try:
slpposf = maindb[DBTableNames.gridfs_spatial].files.find(
{'filename': dist_name}, no_cursor_timeout=True)[0]
except NetworkTimeout or Exception:
# In case of unexpected raise
# client.close()
return
ysize = int(slpposf['metadata'][RasterMetadata.nrows])
xsize = int(slpposf['metadata'][RasterMetadata.ncols])
xll = slpposf['metadata'][RasterMetadata.xll]
yll = slpposf['metadata'][RasterMetadata.yll]
cellsize = slpposf['metadata'][RasterMetadata.cellsize]
nodata_value = slpposf['metadata'][RasterMetadata.nodata]
srs = slpposf['metadata'][RasterMetadata.srs]
if is_string(srs):
srs = str(srs)
srs = osr.GetUserInputAsWKT(srs)
geotransform = [0] * 6
geotransform[0] = xll - 0.5 * cellsize
geotransform[1] = cellsize
geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax
geotransform[5] = -cellsize
slppos_data = spatial_gfs.get(slpposf['_id'])
total_len = xsize * ysize
fmt = '%df' % (total_len, )
slppos_data = unpack(fmt, slppos_data.read())
slppos_data = numpy.reshape(slppos_data, (ysize, xsize))
v_dict = dict()
for unitidx, geneidx in viewitems(self.cfg.unit_to_gene):
v_dict[unitidx] = self.gene_values[geneidx]
# Deprecated and replaced by using self.cfg.unit_to_gene. 03/14/2019. ljzhu.
# for idx, gene_v in enumerate(self.gene_values):
# v_dict[self.cfg.gene_to_unit[idx]] = gene_v
for k, v in v_dict.items():
slppos_data[slppos_data == k] = v
if outpath is None:
outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID
RasterUtilClass.write_gtiff_file(outpath, ysize, xsize,
slppos_data, geotransform, srs,
nodata_value)
