def Mode(self): # type: (...) -> AnyStr """Get simulation mode.""" if self._mode != '': return self._mode.upper() mode_dict = self.filein_tab.find_one({ModelCfgFields.tag: FieldNames.mode}) self._mode = mode_dict[ModelCfgFields.value] if is_string(self._mode): self._mode = str(self._mode) return self._mode.upper()
def mask_raster(in_raster, mask, out_raster): """ Mask raster data. Args: in_raster: list or one raster mask: Mask raster data out_raster: list or one raster """ if is_string(in_raster) and is_string(out_raster): in_raster = [str(in_raster)] out_raster = [str(out_raster)] if len(in_raster) != len(out_raster): raise RuntimeError( 'input raster and output raster must have the same size.') maskr = RasterUtilClass.read_raster(mask) rows = maskr.nRows cols = maskr.nCols maskdata = maskr.data temp = maskdata == maskr.noDataValue for inr, outr in zip(in_raster, out_raster): origin = RasterUtilClass.read_raster(inr) if origin.nRows == rows and origin.nCols == cols: masked = numpy.where(temp, origin.noDataValue, origin.data) else: masked = numpy.ones((rows, cols)) * origin.noDataValue # TODO, the following loop should be optimized by numpy or numba for i in range(rows): for j in range(cols): if maskdata[i][j] == maskr.noDataValue: continue # get the center point coordinate of current cell tempx, tempy = maskr.get_central_coors(i, j) tempv = origin.get_value_by_xy(tempx, tempy) if tempv is None: continue masked[i][j] = tempv RasterUtilClass.write_gtiff_file(outr, maskr.nRows, maskr.nCols, masked, maskr.geotrans, maskr.srs, origin.noDataValue, origin.dataType)
def mask_raster(in_raster, mask, out_raster): """ Mask raster data. Args: in_raster: list or one raster mask: Mask raster data out_raster: list or one raster """ if is_string(in_raster) and is_string(out_raster): in_raster = [str(in_raster)] out_raster = [str(out_raster)] if len(in_raster) != len(out_raster): raise RuntimeError('input raster and output raster must have the same size.') maskr = RasterUtilClass.read_raster(mask) rows = maskr.nRows cols = maskr.nCols maskdata = maskr.data temp = maskdata == maskr.noDataValue for inr, outr in zip(in_raster, out_raster): origin = RasterUtilClass.read_raster(inr) if origin.nRows == rows and origin.nCols == cols: masked = numpy.where(temp, origin.noDataValue, origin.data) else: masked = numpy.ones((rows, cols)) * origin.noDataValue # TODO, the following loop should be optimized by numpy or numba for i in range(rows): for j in range(cols): if maskdata[i][j] == maskr.noDataValue: continue # get the center point coordinate of current cell tempx, tempy = maskr.get_central_coors(i, j) tempv = origin.get_value_by_xy(tempx, tempy) if tempv is None: continue masked[i][j] = tempv RasterUtilClass.write_gtiff_file(outr, maskr.nRows, maskr.nCols, masked, maskr.geotrans, maskr.srs, origin.noDataValue, origin.dataType)
def raster_dilation(rasterfile): """Dilate the raster image. Find the max pixel's value in 8-neighborhood. Then change the compute pixel's value into the max pixel's value. Args: rasterfile: input original raster image, type can be filename(string, like "test1.tif"), rasterfile(class Raster) or numpy.ndarray. Returns: dilation_raster: raster image after dilation, type is numpy.ndarray. """ if is_string(rasterfile): origin_raster = RasterUtilClass.read_raster(str(rasterfile)) elif isinstance(rasterfile, Raster): origin_raster = rasterfile.data elif isinstance(rasterfile, numpy.ndarray): origin_raster = rasterfile else: return 'Your rasterfile has a wrong type. Type must be string or ' \ 'numpy.array or class Raster in pygeoc.' min_value_raster = origin_raster.min() dilation_raster = numpy.zeros( (origin_raster.shape[0], origin_raster.shape[1])) # In order to compute the raster edges, we need to expand the original # raster's rows and cols. We need to add the edges whose pixels' value is # the max pixel's value in raster. add_row = numpy.full((1, origin_raster.shape[1]), min_value_raster) temp_origin_raster = numpy.vstack((numpy.vstack( (add_row, origin_raster)), add_row)) add_col = numpy.full((origin_raster.shape[0] + 2, 1), min_value_raster) expand_origin_raster = numpy.hstack((numpy.hstack( (add_col, temp_origin_raster)), add_col)) # Dilate the raster. for i in range(origin_raster.shape[0]): for j in range(origin_raster.shape[1]): max_pixel_value = min_value_raster # Find the max pixel value in the 8-neighborhood. for k in range(3): for l in range(3): if expand_origin_raster[i + k, j + l] >= max_pixel_value: max_pixel_value = expand_origin_raster[i + k, j + l] # After this loop, we get the max pixel's value of the # 8-neighborhood. Then we change the compute pixel's value into # the max pixel's value. dilation_raster[i, j] = max_pixel_value # Return the result. return dilation_raster
def raster_dilation(rasterfile): """Dilate the raster image. Find the max pixel's value in 8-neighborhood. Then change the compute pixel's value into the max pixel's value. Args: rasterfile: input original raster image, type can be filename(string, like "test1.tif"), rasterfile(class Raster) or numpy.ndarray. Returns: dilation_raster: raster image after dilation, type is numpy.ndarray. """ if is_string(rasterfile): origin_raster = RasterUtilClass.read_raster(str(rasterfile)) elif isinstance(rasterfile, Raster): origin_raster = rasterfile.data elif isinstance(rasterfile, numpy.ndarray): origin_raster = rasterfile else: return 'Your rasterfile has a wrong type. Type must be string or ' \ 'numpy.array or class Raster in pygeoc.' min_value_raster = origin_raster.min() dilation_raster = numpy.zeros((origin_raster.shape[0], origin_raster.shape[1])) # In order to compute the raster edges, we need to expand the original # raster's rows and cols. We need to add the edges whose pixels' value is # the max pixel's value in raster. add_row = numpy.full((1, origin_raster.shape[1]), min_value_raster) temp_origin_raster = numpy.vstack((numpy.vstack((add_row, origin_raster)), add_row)) add_col = numpy.full((origin_raster.shape[0] + 2, 1), min_value_raster) expand_origin_raster = numpy.hstack((numpy.hstack((add_col, temp_origin_raster)), add_col)) # Dilate the raster. for i in range(origin_raster.shape[0]): for j in range(origin_raster.shape[1]): max_pixel_value = min_value_raster # Find the max pixel value in the 8-neighborhood. for k in range(3): for l in range(3): if expand_origin_raster[i + k, j + l] >= max_pixel_value: max_pixel_value = expand_origin_raster[i + k, j + l] # After this loop, we get the max pixel's value of the # 8-neighborhood. Then we change the compute pixel's value into # the max pixel's value. dilation_raster[i, j] = max_pixel_value # Return the result. return dilation_raster
def export_landuse_lookup_files_from_mongodb(cfg, maindb): """export landuse lookup tables to txt file from MongoDB.""" lookup_dir = cfg.dirs.lookup property_namelist = ModelParamDataUtils.landuse_fields property_map = dict() property_namelist.append('USLE_P') query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError( 'LanduseLoop Collection is not existed or empty!') count = 0 for row in query_result: # print(row) value_map = dict() for i, p_name in enumerate(property_namelist): if StringClass.string_match(p_name, 'USLE_P'): # Currently, USLE_P is set as 1 for all landuse. value_map[p_name] = 1 else: # I do not know why manning * 10 here. Just uncommented now. lj # if StringClass.string_match(p_name, "Manning"): # value_map[p_name] = row.get(p_name) * 10 # else: v = row.get(p_name) if is_string(v): v = StringClass.extract_numeric_values_from_string( v)[0] value_map[p_name] = v count += 1 property_map[count] = value_map n = len(property_map) UtilClass.rmmkdir(lookup_dir) for propertyName in property_namelist: with open('%s/%s.txt' % ( lookup_dir, propertyName, ), 'w', encoding='utf-8') as f: f.write('%d\n' % n) for prop_id in property_map: s = '%d %f\n' % (int(property_map[prop_id]['LANDUSE_ID']), property_map[prop_id][propertyName]) f.write('%s' % s)
def get_suitable_bmps(self, types='LANDUSE'): # type: (Union[AnyStr, List[AnyStr]]) -> None """Construct the suitable BMPs for each slope position.""" if is_string(types): types = [types] for bid, bdict in self.bmps_params.items(): for type in types: if type not in bdict: continue if type not in self.suit_bmps: self.suit_bmps.setdefault(type, dict()) suitsp = bdict[type] for sp in suitsp: if sp not in self.suit_bmps[type]: self.suit_bmps[type][sp] = [bid] elif bid not in self.suit_bmps[type][sp]: self.suit_bmps[type][sp].append(bid) if 'EFFECTIVENESS' in bdict: self.bmps_grade[bid] = bdict['EFFECTIVENESS']
def ReadRasterFromMongoDB(ip, port, db_name, gfsname, gfilename): client = ConnectMongoDB(ip, port) conn = client.get_conn() maindb = conn[db_name] spatial_gfs = GridFS(maindb, gfsname) if not spatial_gfs.exists(filename=gfilename): raise ValueError('WARNING: %s is not existed in %s:%s!' % (gfilename, db_name, gfsname)) try: gfsdata = maindb[DBTableNames.gridfs_spatial].files.find( {'filename': gfilename}, no_cursor_timeout=True)[0] except NetworkTimeout or Exception: # In case of unexpected raise client.close() return None ysize = int(gfsdata['metadata'][RasterMetadata.nrows]) xsize = int(gfsdata['metadata'][RasterMetadata.ncols]) xll = gfsdata['metadata'][RasterMetadata.xll] yll = gfsdata['metadata'][RasterMetadata.yll] cellsize = gfsdata['metadata'][RasterMetadata.cellsize] nodata = gfsdata['metadata'][RasterMetadata.nodata] srs = gfsdata['metadata'][RasterMetadata.srs] if is_string(srs): srs = str(srs) srs = osr.GetUserInputAsWKT(srs) geotransform = [0] * 6 geotransform[0] = xll - 0.5 * cellsize geotransform[1] = cellsize geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax geotransform[5] = -cellsize array_data = spatial_gfs.get(gfsdata['_id']) total_len = xsize * ysize fmt = '%df' % (total_len, ) array_data = unpack(fmt, array_data.read()) array_data = numpy.reshape(array_data, (ysize, xsize)) return Raster(ysize, xsize, array_data, nodata, geotransform, srs)
def ogrwkt2shapely(input_shape, id_field): """Return shape objects list and ids list""" # CAUTION, IMPORTANT # Because shapely is dependent on sqlite, and the version is not consistent # with GDAL executable (e.g., located in C:\GDAL_x64\bin), thus the shapely # must be locally imported here. from shapely.wkt import loads as shapely_loads shapely_objects = list() shape_area = list() id_list = list() # print(input_shape) shp = ogr_Open(input_shape) if shp is None: raise RuntimeError( 'The input ESRI Shapefile: %s is not existed or has ' 'no read permission!' % input_shape) lyr = shp.GetLayer() for n in range(0, lyr.GetFeatureCount()): feat = lyr.GetFeature(n) # This function may print Failed `CDLL(/opt/local/lib/libgeos_c.dylib)` in macOS # Don't worry about that! wkt_feat = shapely_loads(feat.geometry().ExportToWkt()) if is_string(id_field): id_field = str(id_field) id_index = feat.GetFieldIndex(id_field) fldid = feat.GetField(id_index) if fldid not in id_list: id_list.append(fldid) shapely_objects.append(wkt_feat) shape_area.append(wkt_feat.area) else: # if multipolygon, take the polygon part with largest area. exist_id_idx = id_list.index(fldid) if shape_area[exist_id_idx] < wkt_feat.area: shape_area[exist_id_idx] = wkt_feat.area shapely_objects[exist_id_idx] = wkt_feat return shapely_objects, id_list
def __init__(self, cf): """Initialization.""" # 1. Directories self.base_dir = None self.clim_dir = None self.spatial_dir = None self.observe_dir = None self.scenario_dir = None self.model_dir = None self.txt_db_dir = None self.preproc_script_dir = None self.seims_bin = None self.mpi_bin = None self.workspace = None # 1.1. Directory determined flags self.use_observed = True self.use_scernario = True # 2. MongoDB configuration and database, collation, GridFS names self.hostname = '127.0.0.1' # localhost by default self.port = 27017 self.climate_db = '' self.bmp_scenario_db = '' self.spatial_db = '' # 3. Climate inputs self.hydro_climate_vars = None self.prec_sites = None self.prec_data = None self.Meteo_sites = None self.Meteo_data = None self.thiessen_field = 'ID' # 4. Spatial inputs self.prec_sites_thiessen = None self.meteo_sites_thiessen = None self.dem = None self.outlet_file = None self.landuse = None self.landcover_init_param = None self.soil = None self.soil_property = None self.fields_partition = False self.fields_partition_thresh = list() self.additional_rs = dict() # 5. Option parameters self.d8acc_threshold = 0 self.np = 4 self.d8down_method = 's' self.dorm_hr = -1. self.temp_base = 0. self.imper_perc_in_urban = 0. self.default_landuse = -1 self.default_soil = -1 # 1. Directories if 'PATH' in cf.sections(): self.base_dir = cf.get('PATH', 'base_data_dir') self.clim_dir = cf.get('PATH', 'climate_data_dir') self.spatial_dir = cf.get('PATH', 'spatial_data_dir') self.observe_dir = cf.get('PATH', 'measurement_data_dir') self.scenario_dir = cf.get('PATH', 'bmp_data_dir') self.model_dir = cf.get('PATH', 'model_dir') self.txt_db_dir = cf.get('PATH', 'txt_db_dir') self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir') self.seims_bin = cf.get('PATH', 'cpp_program_dir') self.mpi_bin = cf.get('PATH', 'mpiexec_dir') self.workspace = cf.get('PATH', 'working_dir') else: raise ValueError('[PATH] section MUST be existed in *.ini file.') if not (FileClass.is_dir_exists(self.base_dir) and FileClass.is_dir_exists(self.model_dir) and FileClass.is_dir_exists(self.txt_db_dir) and FileClass.is_dir_exists(self.preproc_script_dir) and FileClass.is_dir_exists(self.seims_bin)): raise IOError( 'Please Check Directories defined in [PATH]. ' 'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, ' 'and CPP_PROGRAM_DIR are required!') if not FileClass.is_dir_exists(self.mpi_bin): self.mpi_bin = None if not FileClass.is_dir_exists(self.workspace): try: # first try to make dirs UtilClass.mkdir(self.workspace) # os.mkdir(self.workspace) except OSError as exc: self.workspace = self.model_dir + os.path.sep + 'preprocess_output' print('WARNING: Make WORKING_DIR failed! Use the default: %s' % self.workspace) if not os.path.exists(self.workspace): UtilClass.mkdir(self.workspace) self.dirs = DirNameUtils(self.workspace) self.logs = LogNameUtils(self.dirs.log) self.vecs = VectorNameUtils(self.dirs.geoshp) self.taudems = TauDEMFilesUtils(self.dirs.taudem) self.spatials = SpatialNamesUtils(self.dirs.geodata2db) self.modelcfgs = ModelCfgUtils(self.model_dir) self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir + os.path.sep + 'database') if not FileClass.is_dir_exists(self.clim_dir): print( 'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".' ) self.clim_dir = self.base_dir + os.path.sep + 'climate' if not FileClass.is_dir_exists(self.clim_dir): raise IOError( 'Directories named "climate" MUST BE located in [base_dir]!' ) if not FileClass.is_dir_exists(self.spatial_dir): print( 'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".' ) self.spatial_dir = self.base_dir + os.path.sep + 'spatial' raise IOError( 'Directories named "spatial" MUST BE located in [base_dir]!') if not FileClass.is_dir_exists(self.observe_dir): self.observe_dir = None self.use_observed = False if not FileClass.is_dir_exists(self.scenario_dir): self.scenario_dir = None self.use_scernario = False # 2. MongoDB related if 'MONGODB' in cf.sections(): self.hostname = cf.get('MONGODB', 'hostname') self.port = cf.getint('MONGODB', 'port') self.climate_db = cf.get('MONGODB', 'climatedbname') self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname') self.spatial_db = cf.get('MONGODB', 'spatialdbname') else: raise ValueError( '[MONGODB] section MUST be existed in *.ini file.') if not StringClass.is_valid_ip_addr(self.hostname): raise ValueError('HOSTNAME illegal defined in [MONGODB]!') # 3. Climate Input if 'CLIMATE' in cf.sections(): self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'hydroclimatevarfile') self.prec_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precsitefile') self.prec_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'precdatafile') self.Meteo_sites = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteositefile') self.Meteo_data = self.clim_dir + os.path.sep + cf.get( 'CLIMATE', 'meteodatafile') self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield') else: raise ValueError( 'Climate input file names MUST be provided in [CLIMATE]!') # 4. Spatial Input if 'SPATIAL' in cf.sections(): self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'precsitesthiessen') self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'meteositesthiessen') self.dem = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'dem') self.outlet_file = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'outlet_file') if not os.path.exists(self.outlet_file): self.outlet_file = None self.landuse = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'landusefile') self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'landcoverinitfile') self.soil = self.spatial_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqnfile') self.soil_property = self.txt_db_dir + os.path.sep + cf.get( 'SPATIAL', 'soilseqntext') if cf.has_option('SPATIAL', 'additionalfile'): additional_dict_str = cf.get('SPATIAL', 'additionalfile') tmpdict = json.loads(additional_dict_str) tmpdict = { str(k): (str(v) if is_string(v) else v) for k, v in list(tmpdict.items()) } for k, v in list(tmpdict.items()): # Existence check has been moved to mask_origin_delineated_data() # in sp_delineation.py self.additional_rs[k] = v # Field partition if cf.has_option('SPATIAL', 'field_partition_thresh'): ths = cf.get('SPATIAL', 'field_partition_thresh') thsv = StringClass.extract_numeric_values_from_string(ths) if thsv is not None: self.fields_partition_thresh = [int(v) for v in thsv] self.fields_partition = True else: raise ValueError( 'Spatial input file names MUST be provided in [SPATIAL]!') # 5. Optional parameters if 'OPTIONAL_PARAMETERS' in cf.sections(): self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS', 'd8accthreshold') self.np = cf.getint('OPTIONAL_PARAMETERS', 'np') self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod') if StringClass.string_match(self.d8down_method, 'surface'): self.d8down_method = 's' elif StringClass.string_match(self.d8down_method, 'horizontal'): self.d8down_method = 'h' elif StringClass.string_match(self.d8down_method, 'pythagoras'): self.d8down_method = 'p' elif StringClass.string_match(self.d8down_method, 'vertical'): self.d8down_method = 'v' else: self.d8down_method = self.d8down_method.lower() if self.d8down_method not in ['s', 'h', 'p', 'v']: self.d8down_method = 's' self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr') self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base') self.imper_perc_in_urban = cf.getfloat( 'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell') self.default_landuse = cf.getint('OPTIONAL_PARAMETERS', 'defaultlanduse') self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__( self, bin_dir='', # type: AnyStr # The directory of SEIMS binary model_dir='', # type: AnyStr # The directory of SEIMS model nthread=4, # type: int # Thread number for OpenMP lyrmtd=0, # type: int # Layering method, can be 0 (UP_DOWN) or 1 (DOWN_UP) host='127.0.0.1', # type: AnyStr # MongoDB host address, default is `localhost` port=27017, # type: int # MongoDB port, default is 27017 db_name='', # type: AnyStr # Main spatial dbname which can diff from dirname scenario_id=-1, # type: int # Scenario ID defined in `<model>_Scenario` database calibration_id=-1, # type: int # Calibration ID used for model auto-calibration subbasin_id=0, # type: int # Subbasin ID, 0 for whole watershed, 9999 for field version version='OMP', # type: AnyStr # SEIMS version, can be `MPI` or `OMP` (default) nprocess=1, # type: int # Process number for MPI mpi_bin='', # type: AnyStr # Full path of MPI executable file, e.g., './mpirun` hosts_opt='-f', # type: AnyStr # Option for assigning hosts, # e.g., `-f`, `-hostfile`, `-machine`, `-machinefile` hostfile='', # type: AnyStr # File containing host names, # or file mapping process numbers to machines simu_stime=None, # type: Optional[datetime, AnyStr] # Start time of simulation simu_etime=None, # type: Optional[datetime, AnyStr] # End time of simulation out_stime=None, # type: Optional[datetime, AnyStr] # Start time of outputs out_etime=None, # type: Optional[datetime, AnyStr] # End time of outputs args_dict=None # type: Dict[AnyStr, Optional[AnyStr, datetime, int]] ): # type: (...) -> None # Derived from input arguments if args_dict is None: # Preferred to use 'args_dict' if existed. args_dict = dict() bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir model_dir = args_dict[ 'model_dir'] if 'model_dir' in args_dict else model_dir self.version = args_dict[ 'version'] if 'version' in args_dict else version suffix = '.exe' if sysstr == 'Windows' else '' if self.version == 'MPI': self.seims_exec = '%s/seims_mpi%s' % (bin_dir, suffix) else: self.seims_exec = '%s/seims_omp%s' % (bin_dir, suffix) if not FileClass.is_file_exists( self.seims_exec): # If not support OpenMP, use `seims`! self.seims_exec = '%s/seims%s' % (bin_dir, suffix) self.seims_exec = os.path.abspath(self.seims_exec) self.model_dir = os.path.abspath(model_dir) self.nthread = args_dict[ 'nthread'] if 'nthread' in args_dict else nthread self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd self.host = args_dict['host'] if 'host' in args_dict else host self.port = args_dict['port'] if 'port' in args_dict else port self.db_name = args_dict['db_name'] if 'db_name' in args_dict \ else os.path.split(self.model_dir)[1] self.scenario_id = args_dict[ 'scenario_id'] if 'scenario_id' in args_dict else scenario_id self.calibration_id = args_dict['calibration_id'] \ if 'calibration_id' in args_dict else calibration_id self.subbasin_id = args_dict[ 'subbasin_id'] if 'subbasin_id' in args_dict else subbasin_id self.nprocess = args_dict[ 'nprocess'] if 'nprocess' in args_dict else nprocess self.mpi_bin = args_dict[ 'mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin self.hosts_opt = args_dict[ 'hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt self.hostfile = args_dict[ 'hostfile'] if 'hostfile' in args_dict else hostfile self.simu_stime = args_dict[ 'simu_stime'] if 'simu_stime' in args_dict else simu_stime self.simu_etime = args_dict[ 'simu_etime'] if 'simu_etime' in args_dict else simu_etime self.out_stime = args_dict[ 'out_stime'] if 'out_stime' in args_dict else out_stime self.out_etime = args_dict[ 'out_etime'] if 'out_etime' in args_dict else out_etime if is_string( self.simu_stime) and not isinstance(self.simu_stime, datetime): self.simu_stime = StringClass.get_datetime(self.simu_stime) if is_string( self.simu_etime) and not isinstance(self.simu_etime, datetime): self.simu_etime = StringClass.get_datetime(self.simu_etime) if is_string( self.out_stime) and not isinstance(self.out_stime, datetime): self.out_stime = StringClass.get_datetime(self.out_stime) if is_string( self.out_etime) and not isinstance(self.out_etime, datetime): self.out_etime = StringClass.get_datetime(self.out_etime) # Concatenate executable command self.cmd = self.Command self.run_success = False self.output_dir = self.OutputDirectory # Model data read from MongoDB self.outlet_id = -1 self.subbasin_count = -1 self.scenario_dbname = '' self.start_time = None self.end_time = None self.output_ids = list() # type: List[AnyStr] self.output_items = dict() # type: Dict[AnyStr, Union[List[AnyStr]]] # Data maybe used after model run self.timespan = dict( ) # type: Dict[AnyStr, Dict[AnyStr, Union[float, Dict[AnyStr, float]]]] self.obs_vars = list( ) # type: List[AnyStr] # Observation types at the outlet self.obs_value = dict( ) # type: Dict[datetime, List[float]] # Observation value self.sim_vars = list( ) # type: List[AnyStr] # Simulation types, part of `obs_vars` self.sim_value = dict( ) # type: Dict[datetime, List[float]] # Simulation value # The format of sim_obs_dict: # {VarName: {'UTCDATETIME': [t1, t2, ..., tn], # 'Obs': [o1, o2, ..., on], # 'Sim': [s1, s2, ..., sn]}, # ... # } self.sim_obs_dict = dict( ) # type: Dict[AnyStr, Dict[AnyStr, Union[float, List[Union[datetime, float]]]]] self.runtime = 0. self.runlogs = list() # type: List[AnyStr] self.mongoclient = None # type: Union[MongoClient, None] # Set to None after use
def calculate_95ppu(sim_obs_data, sim_data, outdir, gen_num, vali_sim_obs_data=None, vali_sim_data=None, plot_cfg=None # type: Optional[PlotConfig] ): """Calculate 95% prediction uncertainty and plot the hydrographs.""" if plot_cfg is None: plot_cfg = PlotConfig() plt.rcParams['xtick.direction'] = 'out' plt.rcParams['ytick.direction'] = 'out' plt.rcParams['font.family'] = plot_cfg.font_name plt.rcParams['timezone'] = 'UTC' plt.rcParams['mathtext.fontset'] = 'custom' plt.rcParams['mathtext.it'] = 'STIXGeneral:italic' plt.rcParams['mathtext.bf'] = 'STIXGeneral:italic:bold' if len(sim_data) < 2: return var_name = sim_obs_data[0]['var_name'] for idx, var in enumerate(var_name): plot_validation = False if vali_sim_obs_data and vali_sim_data and var in vali_sim_obs_data[0]['var_name']: plot_validation = True ylabel_str = var if var in ['Q', 'QI', 'QG', 'QS']: ylabel_str += ' (m$^3$/s)' elif 'CONC' in var.upper(): # Concentrate if 'SED' in var.upper(): ylabel_str += ' (g/L)' else: ylabel_str += ' (mg/L)' elif 'SED' in var.upper(): # amount ylabel_str += ' (kg)' cali_obs_dates = sim_obs_data[0][var]['UTCDATETIME'][:] if is_string(cali_obs_dates[0]): cali_obs_dates = [StringClass.get_datetime(s) for s in cali_obs_dates] obs_dates = cali_obs_dates[:] order = 1 # By default, the calibration period is before the validation period. if plot_validation: vali_obs_dates = vali_sim_obs_data[0][var]['UTCDATETIME'] if is_string(vali_obs_dates[0]): vali_obs_dates = [StringClass.get_datetime(s) for s in vali_obs_dates] if vali_obs_dates[-1] <= cali_obs_dates[0]: order = 0 obs_dates = vali_obs_dates + obs_dates else: obs_dates += vali_obs_dates obs_data = sim_obs_data[0][var]['Obs'][:] if plot_validation: if order: obs_data += vali_sim_obs_data[0][var]['Obs'][:] else: obs_data = vali_sim_obs_data[0][var]['Obs'][:] + obs_data cali_sim_dates = list(sim_data[0].keys()) if is_string(cali_sim_dates[0]): cali_sim_dates = [StringClass.get_datetime(s) for s in cali_sim_dates] sim_dates = cali_sim_dates[:] if plot_validation: vali_sim_dates = list(vali_sim_data[0].keys()) if is_string(vali_sim_dates[0]): vali_sim_dates = [StringClass.get_datetime(s) for s in vali_sim_dates] if order: sim_dates += vali_sim_dates else: sim_dates = vali_sim_dates + sim_dates sim_data_list = list() caliBestIdx = -1 caliBestNSE = -9999. for idx2, ind in enumerate(sim_data): tmp = numpy.array(list(ind.values())) tmp = tmp[:, idx] if sim_obs_data[idx2][var]['NSE'] > caliBestNSE: caliBestNSE = sim_obs_data[idx2][var]['NSE'] caliBestIdx = idx2 tmpsim = tmp.tolist() if plot_validation: tmp_data = numpy.array(list(vali_sim_data[idx2].values()))[:, idx].tolist() if order: tmpsim += tmp_data else: tmpsim = tmp_data + tmpsim sim_data_list.append(tmpsim) sim_best = numpy.array(list(sim_data[caliBestIdx].values()))[:, idx] sim_best = sim_best.tolist() if plot_validation: tmp_data = numpy.array(list(vali_sim_data[caliBestIdx].values()))[:, idx].tolist() if order: sim_best += tmp_data else: sim_best = tmp_data + sim_best sim_data_list = numpy.array(sim_data_list) ylows = numpy.percentile(sim_data_list, 2.5, 0, interpolation='nearest') yups = numpy.percentile(sim_data_list, 97.5, 0, interpolation='nearest') def calculate_95ppu_efficiency(obs_data_list, obs_dates_list, sim_dates_list): # type: (...) -> (float, float) count = 0 ylows_obs = list() yups_obs = list() for oi, ov in enumerate(obs_data_list): try: si = sim_dates_list.index(obs_dates_list[oi]) ylows_obs.append(ylows[si]) yups_obs.append(yups[si]) if ylows[si] <= ov <= yups[si]: count += 1 except Exception: continue p = float(count) / len(obs_data_list) ylows_obs = numpy.array(ylows_obs) yups_obs = numpy.array(yups_obs) r = numpy.mean(yups_obs - ylows_obs) / numpy.std(numpy.array(obs_data_list)) return p, r # concatenate text p_value, r_value = calculate_95ppu_efficiency(sim_obs_data[0][var]['Obs'], cali_obs_dates, list(sim_data[0].keys())) txt = 'P-factor: %.2f\nR-factor: %.2f\n' % (p_value, r_value) txt += u'某一最优模拟\n' if plot_cfg.plot_cn else 'One best simulation:\n' txt += ' $\mathit{NSE}$: %.2f\n' \ ' $\mathit{RSR}$: %.2f\n' \ ' $\mathit{PBIAS}$: %.2f%%\n' \ ' $\mathit{R^2}$: %.2f' % (sim_obs_data[caliBestIdx][var]['NSE'], sim_obs_data[caliBestIdx][var]['RSR'], sim_obs_data[caliBestIdx][var]['PBIAS'], sim_obs_data[caliBestIdx][var]['R-square']) # concatenate text of validation if needed vali_txt = '' if plot_validation: p_value, r_value = calculate_95ppu_efficiency(vali_sim_obs_data[0][var]['Obs'], vali_obs_dates, list(vali_sim_data[0].keys())) vali_txt = 'P-factor: %.2f\nR-factor: %.2f\n\n' % (p_value, r_value) vali_txt += ' $\mathit{NSE}$: %.2f\n' \ ' $\mathit{RSR}$: %.2f\n' \ ' $\mathit{PBIAS}$: %.2f%%\n' \ ' $\mathit{R^2}$: %.2f' % (vali_sim_obs_data[caliBestIdx][var]['NSE'], vali_sim_obs_data[caliBestIdx][var]['RSR'], vali_sim_obs_data[caliBestIdx][var]['PBIAS'], vali_sim_obs_data[caliBestIdx][var]['R-square']) # plot fig, ax = plt.subplots(figsize=(12, 4)) ax.fill_between(sim_dates, ylows.tolist(), yups.tolist(), color=(0.8, 0.8, 0.8), label='95PPU') observed_label = u'实测值' if plot_cfg.plot_cn else 'Observed points' ax.scatter(obs_dates, obs_data, marker='.', s=20, color='g', label=observed_label) besesim_label = u'最优模拟' if plot_cfg.plot_cn else 'Best simulation' ax.plot(sim_dates, sim_best, linestyle='--', color='red', linewidth=1, label=besesim_label) ax.set_xlim(left=min(sim_dates), right=max(sim_dates)) ax.set_ylim(bottom=0.) date_fmt = mdates.DateFormatter('%m-%d-%y') ax.xaxis.set_major_formatter(date_fmt) ax.tick_params(axis='x', bottom=True, top=False, length=5, width=2, which='major', labelsize=plot_cfg.tick_fsize) ax.tick_params(axis='y', left=True, right=False, length=5, width=2, which='major', labelsize=plot_cfg.tick_fsize) plt.xlabel(u'时间' if plot_cfg.plot_cn else 'Date time', fontsize=plot_cfg.axislabel_fsize) plt.ylabel(ylabel_str, fontsize=plot_cfg.axislabel_fsize) # plot separate dash line delta_dt = (sim_dates[-1] - sim_dates[0]) // 9 delta_dt2 = (sim_dates[-1] - sim_dates[0]) // 35 sep_time = sim_dates[-1] time_pos = [sep_time - delta_dt] time_pos2 = [sep_time - 2 * delta_dt] ymax, ymin = ax.get_ylim() yc = abs(ymax - ymin) * 0.9 if plot_validation: sep_time = vali_sim_dates[0] if vali_sim_dates[0] >= cali_sim_dates[-1] \ else cali_sim_dates[0] cali_vali_labels = [(u'验证期' if plot_cfg.plot_cn else 'Calibration'), (u'率定期' if plot_cfg.plot_cn else 'Validation')] if not order: cali_vali_labels.reverse() time_pos = [sep_time - delta_dt, sep_time + delta_dt2] time_pos2 = [sep_time - 2 * delta_dt, sep_time + delta_dt2] ax.axvline(sep_time, color='black', linestyle='dashed', linewidth=2) plt.text(time_pos[0], yc, cali_vali_labels[0], fontdict={'style': 'italic', 'weight': 'bold', 'size': plot_cfg.label_fsize}, color='black') plt.text(time_pos[1], yc, cali_vali_labels[1], fontdict={'style': 'italic', 'weight': 'bold', 'size': plot_cfg.label_fsize}, color='black') # add legend handles, labels = ax.get_legend_handles_labels() figorders = [labels.index('95PPU'), labels.index(observed_label), labels.index(besesim_label)] ax.legend([handles[idx] for idx in figorders], [labels[idx] for idx in figorders], fontsize=plot_cfg.legend_fsize, loc=2, framealpha=0.8) # add text cali_pos = time_pos[0] if order else time_pos[1] plt.text(cali_pos, yc * 0.5, txt, color='red', fontsize=plot_cfg.label_fsize - 1) if plot_validation: vali_pos = time_pos[1] if order else time_pos[0] plt.text(vali_pos, yc * 0.5, vali_txt, color='red', fontsize=plot_cfg.label_fsize - 1) # fig.autofmt_xdate(rotation=0, ha='center') plt.tight_layout() save_png_eps(plt, outdir, 'Gen%d_95ppu_%s' % (gen_num, var), plot_cfg) # close current plot in case of 'figure.max_open_warning' plt.cla() plt.clf() plt.close()
def export_scenario_to_gtiff(self, outpath=None): # type: (Optional[str]) -> None """Export scenario to GTiff. TODO: Read Raster from MongoDB should be extracted to pygeoc. """ if not self.export_sce_tif: return dist = self.bmps_info[self.cfg.bmpid]['DISTRIBUTION'] dist_list = StringClass.split_string(dist, '|') if len(dist_list) >= 2 and dist_list[0] == 'RASTER': dist_name = '0_' + dist_list[1] # prefix 0_ means the whole basin # read dist_name from MongoDB # client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port) # conn = client.get_conn() conn = MongoDBObj.client maindb = conn[self.modelcfg.db_name] spatial_gfs = GridFS(maindb, DBTableNames.gridfs_spatial) # read file from mongodb if not spatial_gfs.exists(filename=dist_name): print('WARNING: %s is not existed, export scenario failed!' % dist_name) return try: slpposf = maindb[DBTableNames.gridfs_spatial].files.find( {'filename': dist_name}, no_cursor_timeout=True)[0] except NetworkTimeout or Exception: # In case of unexpected raise # client.close() return ysize = int(slpposf['metadata'][RasterMetadata.nrows]) xsize = int(slpposf['metadata'][RasterMetadata.ncols]) xll = slpposf['metadata'][RasterMetadata.xll] yll = slpposf['metadata'][RasterMetadata.yll] cellsize = slpposf['metadata'][RasterMetadata.cellsize] nodata_value = slpposf['metadata'][RasterMetadata.nodata] srs = slpposf['metadata'][RasterMetadata.srs] if is_string(srs): srs = str(srs) srs = osr.GetUserInputAsWKT(srs) geotransform = [0] * 6 geotransform[0] = xll - 0.5 * cellsize geotransform[1] = cellsize geotransform[3] = yll + (ysize - 0.5) * cellsize # yMax geotransform[5] = -cellsize slppos_data = spatial_gfs.get(slpposf['_id']) total_len = xsize * ysize fmt = '%df' % (total_len, ) slppos_data = unpack(fmt, slppos_data.read()) slppos_data = numpy.reshape(slppos_data, (ysize, xsize)) v_dict = dict() for unitidx, geneidx in viewitems(self.cfg.unit_to_gene): v_dict[unitidx] = self.gene_values[geneidx] # Deprecated and replaced by using self.cfg.unit_to_gene. 03/14/2019. ljzhu. # for idx, gene_v in enumerate(self.gene_values): # v_dict[self.cfg.gene_to_unit[idx]] = gene_v for k, v in v_dict.items(): slppos_data[slppos_data == k] = v if outpath is None: outpath = self.scenario_dir + os.path.sep + 'Scenario_%d.tif' % self.ID RasterUtilClass.write_gtiff_file(outpath, ysize, xsize, slppos_data, geotransform, srs, nodata_value)