def initialize_landcover_parameters(landcover_file, landcover_initial_fields_file, dst_dir): """generate initial landcover_init_param parameters""" lc_data_items = read_data_items_from_txt(landcover_initial_fields_file) # print lc_data_items field_names = lc_data_items[0] lu_id = -1 for i, v in enumerate(field_names): if StringClass.string_match(v, 'LANDUSE_ID'): lu_id = i break data_items = lc_data_items[1:] replace_dicts = dict() for item in data_items: for i, v in enumerate(item): if i != lu_id: if field_names[i].upper() not in replace_dicts.keys(): replace_dicts[field_names[i].upper()] = { float(item[lu_id]): float(v) } else: replace_dicts[field_names[i].upper()][float( item[lu_id])] = float(v) # print replace_dicts # Generate GTIFF for item, v in replace_dicts.items(): filename = dst_dir + SEP + item + '.tif' print(filename) RasterUtilClass.raster_reclassify(landcover_file, v, filename) return replace_dicts['LANDCOVER'].values()
def reclassify_landcover_parameters(landuse_file, landcover_file, landcover_initial_fields_file, landcover_lookup_file, attr_names, dst_dir): """relassify landcover_init_param parameters""" land_cover_codes = LanduseUtilClass.initialize_landcover_parameters( landuse_file, landcover_initial_fields_file, dst_dir) attr_map = LanduseUtilClass.read_crop_lookup_table( landcover_lookup_file) n = len(attr_names) replace_dicts = [] dst_crop_tifs = [] for i in range(n): cur_attr = attr_names[i] cur_dict = dict() dic = attr_map[cur_attr] for code in land_cover_codes: if MathClass.floatequal(code, DEFAULT_NODATA): continue if code not in cur_dict.keys(): cur_dict[code] = dic.get(code) replace_dicts.append(cur_dict) dst_crop_tifs.append(dst_dir + SEP + cur_attr + '.tif') # print replace_dicts # print(len(replace_dicts)) # print dst_crop_tifs # print(len(dst_crop_tifs)) # Generate GTIFF for i, v in enumerate(dst_crop_tifs): # print dst_crop_tifs[i] RasterUtilClass.raster_reclassify(landcover_file, replace_dicts[i], v)
def output_hillslope(method_id): """Output hillslope according different stream cell value method.""" for (tmp_row, tmp_col) in stream_coors: tmp_hillslp_ids = cal_hillslope_codes( max_id, stream_data[tmp_row][tmp_col]) if 0 < method_id < 3: hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[ method_id] # is head stream cell? if (tmp_row, tmp_col) in headstream_coors: hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[0] elif method_id == 3: hillslope_mtx[tmp_row][tmp_col] = DEFAULT_NODATA # Output to raster file hillslope_out_new = hillslope_out dirpath = os.path.dirname(hillslope_out_new) + os.sep corename = FileClass.get_core_name_without_suffix( hillslope_out_new) if method_id == 1: hillslope_out_new = dirpath + corename + '_right.tif' elif method_id == 2: hillslope_out_new = dirpath + corename + '_left.tif' elif method_id == 3: hillslope_out_new = dirpath + corename + '_nodata.tif' RasterUtilClass.write_gtiff_file(hillslope_out_new, nrows, ncols, hillslope_mtx, geotrans, srs, DEFAULT_NODATA, datatype)
def calculate_channel_width(acc_file, chwidth_file): """Calculate channel width.""" acc_r = RasterUtilClass.read_raster(acc_file) xsize = acc_r.nCols ysize = acc_r.nRows dx = acc_r.dx cell_area = dx * dx # storm frequency a b # 2 1 0.56 # 10 1.2 0.56 # 100 1.4 0.56 a = 1.2 b = 0.56 # TODO: Figure out what's means, and move it to text.py or config.py. LJ tmp_ones = numpy.ones((ysize, xsize)) width = tmp_ones * DEFAULT_NODATA valid_values = numpy.where(acc_r.validZone, acc_r.data, tmp_ones) width = numpy.where( acc_r.validZone, numpy.power((a * (valid_values + 1) * cell_area / 1000000.), b), width) RasterUtilClass.write_gtiff_file(chwidth_file, ysize, xsize, width, acc_r.geotrans, acc_r.srs, DEFAULT_NODATA, GDT_Float32) return width
def flow_time_to_stream(streamlink, velocity, flow_dir_file, t0_s_file, flow_dir_code="TauDEM"): """Calculate flow time to the workflow channel from each grid cell.""" strlk_data = RasterUtilClass.read_raster(streamlink).data vel_r = RasterUtilClass.read_raster(velocity) vel_data = vel_r.data xsize = vel_r.nCols ysize = vel_r.nRows # noDataValue = vel_r.noDataValue weight = numpy.where(strlk_data <= 0, numpy.ones((ysize, xsize)), numpy.zeros((ysize, xsize))) traveltime = numpy.where(vel_r.validZone, numpy.zeros((ysize, xsize)), vel_data) flowlen = TerrainUtilClass.calculate_flow_length( flow_dir_file, weight, flow_dir_code) traveltime = numpy.where(vel_r.validZone, flowlen / (vel_data * 5. / 3.) / 3600., traveltime) RasterUtilClass.write_gtiff_file(t0_s_file, ysize, xsize, traveltime, vel_r.geotrans, vel_r.srs, DEFAULT_NODATA, GDT_Float32)
def flow_velocity(slope_file, radius_file, manning_file, velocity_file): """velocity.""" slp_r = RasterUtilClass.read_raster(slope_file) slp_data = slp_r.data xsize = slp_r.nCols ysize = slp_r.nRows nodata_value = slp_r.noDataValue rad_data = RasterUtilClass.read_raster(radius_file).data man_data = RasterUtilClass.read_raster(manning_file).data vel_max = 3.0 vel_min = 0.0001 def velocity_cal(rad, man, slp): """Calculate velocity""" if abs(slp - nodata_value) < UTIL_ZERO: return DEFAULT_NODATA # print rad,man,slp tmp = numpy.power(man, -1) * numpy.power( rad, 2. / 3.) * numpy.power(slp, 0.5) # print tmp if tmp < vel_min: return vel_min if tmp > vel_max: return vel_max return tmp velocity_cal_numpy = numpy.frompyfunc(velocity_cal, 3, 1) velocity = velocity_cal_numpy(rad_data, man_data, slp_data) RasterUtilClass.write_gtiff_file(velocity_file, ysize, xsize, velocity, slp_r.geotrans, slp_r.srs, DEFAULT_NODATA, GDT_Float32)
def hydrological_radius(acc_file, radius_file, storm_probability="T2"): """Calculate hydrological radius.""" acc_r = RasterUtilClass.read_raster(acc_file) xsize = acc_r.nCols ysize = acc_r.nRows nodata_value = acc_r.noDataValue cellsize = acc_r.dx data = acc_r.data coe_table = { "T2": [0.05, 0.48], "T10": [0.12, 0.52], "T100": [0.18, 0.55] } ap = coe_table[storm_probability][0] bp = coe_table[storm_probability][1] def radius_cal(acc): """Calculate hydrological radius""" if abs(acc - nodata_value) < UTIL_ZERO: return DEFAULT_NODATA return numpy.power( ap * ((acc + 1) * cellsize * cellsize / 1000000.), bp) radius_cal_numpy = numpy.frompyfunc(radius_cal, 1, 1) radius = radius_cal_numpy(data) RasterUtilClass.write_gtiff_file(radius_file, ysize, xsize, radius, acc_r.geotrans, acc_r.srs, DEFAULT_NODATA, GDT_Float32)
def calculate_latitude_dependent_parameters(lat_file, min_dayl_file, dormhr_file, dorm_hr): """ Calculate latitude dependent parameters, include: 1. minimum daylength (daylmn), 2. day length threshold for dormancy (dormhr) """ # calculate minimum daylength, from readwgn.f of SWAT # daylength=2*acos(-tan(sd)*tan(lat))/omega # where solar declination, sd, = -23.5 degrees for minimum daylength in # northern hemisphere and -tan(sd) = .4348 # absolute value is taken of tan(lat) to convert southern hemisphere # values to northern hemisphere # the angular velocity of the earth's rotation, omega, = 15 deg/hr or # 0.2618 rad/hr and 2/0.2618 = 7.6394 cell_lat_r = RasterUtilClass.read_raster(lat_file) lat_data = cell_lat_r.data # daylmn_data = cell_lat_r.data zero = numpy.zeros((cell_lat_r.nRows, cell_lat_r.nCols)) # nodata = numpy.ones((cell_lat_r.nRows, cell_lat_r.nCols)) * cell_lat_r.noDataValue # convert degrees to radians (2pi/360=1/57.296) daylmn_data = 0.4348 * numpy.abs(numpy.tan(lat_data / 57.296)) condition = daylmn_data < 1. daylmn_data = numpy.where(condition, numpy.arccos(daylmn_data), zero) # condition2 = lat_data != cell_lat_r.noDataValue daylmn_data *= 7.6394 daylmn_data = numpy.where(cell_lat_r.validZone, daylmn_data, lat_data) RasterUtilClass.write_gtiff_file(min_dayl_file, cell_lat_r.nRows, cell_lat_r.nCols, daylmn_data, cell_lat_r.geotrans, cell_lat_r.srs, cell_lat_r.noDataValue, GDT_Float32) def cal_dorm_hr(lat): """calculate day length threshold for dormancy""" if lat == cell_lat_r.noDataValue: return cell_lat_r.noDataValue else: if 20. <= lat <= 40: return (numpy.abs(lat - 20.)) / 20. elif lat > 40.: return 1. elif lat < 20.: return -1. cal_dorm_hr_numpy = numpy.frompyfunc(cal_dorm_hr, 1, 1) # dormhr_data = numpy.copy(lat_data) if dorm_hr < -UTIL_ZERO: dormhr_data = cal_dorm_hr_numpy(lat_data) else: dormhr_data = numpy.where( cell_lat_r.validZone, numpy.ones((cell_lat_r.nRows, cell_lat_r.nCols)) * dorm_hr, lat_data) RasterUtilClass.write_gtiff_file(dormhr_file, cell_lat_r.nRows, cell_lat_r.nCols, dormhr_data, cell_lat_r.geotrans, cell_lat_r.srs, cell_lat_r.noDataValue, GDT_Float32)
def mask_origin_delineated_data(cfg): """Mask the original delineated data by Subbasin raster.""" subbasin_tau_file = cfg.taudems.subbsn geodata2dbdir = cfg.dirs.geodata2db UtilClass.mkdir(geodata2dbdir) mask_file = cfg.spatials.mask RasterUtilClass.get_mask_from_raster(subbasin_tau_file, mask_file) # Total 12 raster files original_files = [ cfg.taudems.subbsn, cfg.taudems.d8flow, cfg.taudems.stream_raster, cfg.taudems.slp, cfg.taudems.filldem, cfg.taudems.d8acc, cfg.taudems.stream_order, cfg.taudems.dinf, cfg.taudems.dinf_d8dir, cfg.taudems.dinf_slp, cfg.taudems.dinf_weight, cfg.taudems.dist2stream_d8 ] # output masked files output_files = [ cfg.taudems.subbsn_m, cfg.taudems.d8flow_m, cfg.taudems.stream_m, cfg.spatials.slope, cfg.spatials.filldem, cfg.spatials.d8acc, cfg.spatials.stream_order, cfg.spatials.dinf, cfg.spatials.dinf_d8dir, cfg.spatials.dinf_slp, cfg.spatials.dinf_weight, cfg.spatials.dist2stream_d8 ] default_values = [] for i in range(len(original_files)): default_values.append(DEFAULT_NODATA) # other input rasters need to be masked # soil and landuse FileClass.check_file_exists(cfg.soil) FileClass.check_file_exists(cfg.landuse) original_files.append(cfg.soil) output_files.append(cfg.spatials.soil_type) default_values.append(cfg.default_soil) original_files.append(cfg.landuse) output_files.append(cfg.spatials.landuse) default_values.append(cfg.default_landuse) # management fields if cfg.mgt_field is not None and FileClass.is_file_exists( cfg.mgt_field): original_files.append(cfg.mgt_field) output_files.append(cfg.spatials.mgt_field) default_values.append(DEFAULT_NODATA) config_file = cfg.logs.mask_cfg # run mask operation print("Mask original delineated data by Subbasin raster...") SpatialDelineation.mask_raster_cpp(cfg.seims_bin, mask_file, original_files, output_files, default_values, config_file)
def post_process_of_delineated_data(cfg): """Do some necessary transfer for subbasin, stream, and flow direction raster.""" # inputs stream_net_file = cfg.taudems.streamnet_shp subbasin_file = cfg.taudems.subbsn_m flow_dir_file_tau = cfg.taudems.d8flow_m stream_raster_file = cfg.taudems.stream_m # outputs # -- shapefile shp_dir = cfg.dirs.geoshp UtilClass.mkdir(shp_dir) # ---- outlet, copy from DirNameUtils.TauDEM FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet) # ---- reaches output_reach_file = cfg.vecs.reach # ---- subbasins subbasin_vector_file = cfg.vecs.subbsn # -- raster file output_subbasin_file = cfg.spatials.subbsn output_flow_dir_file = cfg.spatials.d8flow output_stream_link_file = cfg.spatials.stream_link output_hillslope_file = cfg.spatials.hillslope id_map = StreamnetUtil.serialize_streamnet(stream_net_file, output_reach_file) RasterUtilClass.raster_reclassify(subbasin_file, id_map, output_subbasin_file, GDT_Int32) StreamnetUtil.assign_stream_id_raster(stream_raster_file, output_subbasin_file, output_stream_link_file) # Convert D8 encoding rule to ArcGIS if cfg.is_TauDEM: shutil_copy(flow_dir_file_tau, output_flow_dir_file) else: D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file) # convert raster to shapefile (for subbasin and basin) print "Generating subbasin vector..." VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file, "subbasin", FieldNames.subbasin_id) mask_file = cfg.spatials.mask basin_vector = cfg.vecs.bsn print "Generating basin vector..." VectorUtilClass.raster2shp(mask_file, basin_vector, "basin", FieldNames.basin) # delineate hillslope DelineateHillslope.downstream_method_whitebox(output_stream_link_file, flow_dir_file_tau, output_hillslope_file)
def match_subbasin(subbsn_file, site_dict): """ Match the ID of subbasin 1. Read the coordinates of each subbasin's outlet, and the outlet ID of the whole basin (not finished yet) 2. If the isOutlet field equals to 2.1 - 0, then return the subbasin_id of the site's location 2.2 - 1, then return the outlet ID of the whole basiin 2.3 - 2, then return the outlet ID of nearest subbasin 2.4 - 3, then return the outlet IDs of the conjunct subbasins """ subbasin_raster = RasterUtilClass.read_raster(subbsn_file) localx = site_dict.get(StationFields.x) localy = site_dict.get(StationFields.y) site_type = site_dict.get(StationFields.outlet) subbasin_id = subbasin_raster.get_value_by_xy(localx, localy) if subbasin_id is None and site_type != 1: # the site is not inside the basin and not the outlet either. return False, None if site_type == 0: return True, [subbasin_id] elif site_type == 1: return True, [subbasin_id ] # TODO, get outlet ID of the whole basin elif site_type == 2: return True, [subbasin_id] # TODO
def add_channel_width_to_shp(reach_shp_file, stream_link_file, width_data, default_depth=5.): """Add channel/reach width and default depth to ESRI shapefile""" stream_link = RasterUtilClass.read_raster(stream_link_file) n_rows = stream_link.nRows n_cols = stream_link.nCols nodata_value = stream_link.noDataValue data_stream = stream_link.data ch_width_dic = dict() ch_num_dic = dict() for i in range(n_rows): for j in range(n_cols): if abs(data_stream[i][j] - nodata_value) > UTIL_ZERO: tmpid = int(data_stream[i][j]) ch_num_dic.setdefault(tmpid, 0) ch_width_dic.setdefault(tmpid, 0) ch_num_dic[tmpid] += 1 ch_width_dic[tmpid] += width_data[i][j] for k in ch_num_dic: ch_width_dic[k] /= ch_num_dic[k] # add channel width_data field to reach shp file ds_reach = ogr_Open(reach_shp_file, update=True) layer_reach = ds_reach.GetLayer(0) layer_def = layer_reach.GetLayerDefn() i_link = layer_def.GetFieldIndex(ImportReaches2Mongo._LINKNO) i_width = layer_def.GetFieldIndex(ImportReaches2Mongo._WIDTH) i_depth = layer_def.GetFieldIndex(ImportReaches2Mongo._DEPTH) if i_width < 0: new_field = ogr_FieldDefn(ImportReaches2Mongo._WIDTH, OFTReal) layer_reach.CreateField(new_field) if i_depth < 0: new_field = ogr_FieldDefn(ImportReaches2Mongo._DEPTH, OFTReal) layer_reach.CreateField(new_field) # grid_code:feature map # ftmap = {} layer_reach.ResetReading() ft = layer_reach.GetNextFeature() while ft is not None: tmpid = ft.GetFieldAsInteger(i_link) w = 1 if tmpid in ch_width_dic.keys(): w = ch_width_dic[tmpid] ft.SetField(ImportReaches2Mongo._WIDTH, w) ft.SetField(ImportReaches2Mongo._DEPTH, default_depth) layer_reach.SetFeature(ft) ft = layer_reach.GetNextFeature() layer_reach.SyncToDisk() ds_reach.Destroy() del ds_reach
def generate_cn2(maindb, landuse_file, hydrogroup_file, cn2_filename): """Generate CN2 raster.""" query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError( "LanduseLoop Collection is not existed or empty!") # cn2 list for each landuse type and hydrological soil group cn2_map = dict() for row in query_result: lu_id = row.get('LANDUSE_ID') cn2_list = [ row.get('CN2A'), row.get('CN2B'), row.get('CN2C'), row.get('CN2D') ] cn2_map[lu_id] = cn2_list # print (cn2Map) lu_r = RasterUtilClass.read_raster(landuse_file) data_landuse = lu_r.data xsize = lu_r.nCols ysize = lu_r.nRows nodata_value = lu_r.noDataValue hg_r = RasterUtilClass.read_raster(hydrogroup_file) data_hg = hg_r.data def cal_cn2(lucc_id, hg): """Calculate CN2 value from landuse ID and Hydro Group number.""" lucc_id = int(lucc_id) if lucc_id < 0 or MathClass.floatequal(lucc_id, nodata_value): return DEFAULT_NODATA else: hg = int(hg) - 1 return cn2_map[lucc_id][hg] cal_cn2_numpy = np_frompyfunc(cal_cn2, 2, 1) data_prop = cal_cn2_numpy(data_landuse, data_hg) RasterUtilClass.write_gtiff_file(cn2_filename, ysize, xsize, data_prop, lu_r.geotrans, lu_r.srs, nodata_value, GDT_Float32)
def std_of_flow_time_to_stream(streamlink, flow_dir_file, slope, radius, velocity, delta_s_file, flow_dir_code="TauDEM"): """Generate standard deviation of t0_s (flow time to the workflow channel from each cell).""" strlk_r = RasterUtilClass.read_raster(streamlink) strlk_data = strlk_r.data rad_data = RasterUtilClass.read_raster(radius).data slo_data = RasterUtilClass.read_raster(slope).data vel_r = RasterUtilClass.read_raster(velocity) vel_data = vel_r.data xsize = vel_r.nCols ysize = vel_r.nRows nodata_value = vel_r.noDataValue def initial_variables(vel, strlk, slp, rad): """initial variables""" if abs(vel - nodata_value) < UTIL_ZERO: return DEFAULT_NODATA if strlk <= 0: tmp_weight = 1 else: tmp_weight = 0 # 0 is river if slp < 0.0005: slp = 0.0005 # dampGrid = vel * rad / (slp / 100. * 2.) # No need to divide 100 # in my view. By LJ damp_grid = vel * rad / (slp * 2.) celerity = vel * 5. / 3. tmp_weight *= damp_grid * 2. / numpy.power(celerity, 3.) return tmp_weight initial_variables_numpy = numpy.frompyfunc(initial_variables, 4, 1) weight = initial_variables_numpy(vel_data, strlk_data, slo_data, rad_data) delta_s_sqr = TerrainUtilClass.calculate_flow_length( flow_dir_file, weight, flow_dir_code) def cal_delta_s(vel, sqr): """Calculate delta s""" if abs(vel - nodata_value) < UTIL_ZERO: return nodata_value else: return sqrt(sqr) / 3600. cal_delta_s_numpy = numpy.frompyfunc(cal_delta_s, 2, 1) delta_s = cal_delta_s_numpy(vel_data, delta_s_sqr) RasterUtilClass.write_gtiff_file(delta_s_file, ysize, xsize, delta_s, strlk_r.geotrans, strlk_r.srs, DEFAULT_NODATA, GDT_Float32)
def generate_lat_raster(cfg): """Generate latitude raster""" dem_file = cfg.spatials.filldem ds = RasterUtilClass.read_raster(dem_file) src_srs = ds.srs if not src_srs.ExportToProj4(): raise ValueError("The source raster %s has not coordinate, " "which is required!" % dem_file) dst_srs = osr_SpatialReference() dst_srs.ImportFromEPSG(4326) # WGS84 # dst_wkt = dst_srs.ExportToWkt() transform = osr_CoordinateTransformation(src_srs, dst_srs) point_ll = ogr_CreateGeometryFromWkt("POINT (%f %f)" % (ds.xMin, ds.yMin)) point_ur = ogr_CreateGeometryFromWkt("POINT (%f %f)" % (ds.xMax, ds.yMax)) point_ll.Transform(transform) point_ur.Transform(transform) lower_lat = point_ll.GetY() up_lat = point_ur.GetY() rows = ds.nRows cols = ds.nCols delta_lat = (up_lat - lower_lat) / float(rows) def cal_cell_lat(row, col): """calculate latitude of cell by row number""" return up_lat - (row + 0.5) * delta_lat data_lat = fromfunction(cal_cell_lat, (rows, cols)) data_lat = where(ds.validZone, data_lat, ds.data) RasterUtilClass.write_gtiff_file(cfg.spatials.cell_lat, rows, cols, data_lat, ds.geotrans, ds.srs, ds.noDataValue, GDT_Float32)
def calculate_flow_length(flow_dir_file, weight, flow_dir_code="TauDEM"): """Generate flow length with weight.""" flow_dir_raster = RasterUtilClass.read_raster(flow_dir_file) fdir_data = flow_dir_raster.data xsize = flow_dir_raster.nCols ysize = flow_dir_raster.nRows nodata_value = flow_dir_raster.noDataValue # geotransform = flow_dir_raster.srs cellsize = flow_dir_raster.dx length = numpy.zeros((ysize, xsize)) for i in range(0, ysize): for j in range(0, xsize): if abs(fdir_data[i][j] - nodata_value) < UTIL_ZERO: length[i][j] = nodata_value continue TerrainUtilClass.flow_length_cell(i, j, ysize, xsize, fdir_data, cellsize, weight, length, flow_dir_code) return length
def output_wgs84_geojson(cfg): """Convert ESRI shapefile to GeoJson based on WGS84 coordinate.""" src_srs = RasterUtilClass.read_raster(cfg.dem).srs proj_srs = src_srs.ExportToProj4() if not proj_srs: raise ValueError("The source raster %s has not " "coordinate, which is required!" % cfg.dem) # print proj_srs wgs84_srs = "EPSG:4326" geo_json_dict = { "reach": [cfg.vecs.reach, cfg.vecs.json_reach], "subbasin": [cfg.vecs.subbsn, cfg.vecs.json_subbsn], "basin": [cfg.vecs.bsn, cfg.vecs.json_bsn], "outlet": [cfg.vecs.outlet, cfg.vecs.json_outlet] } for jsonName, shp_json_list in geo_json_dict.items(): # delete if geojson file already existed if FileClass.is_file_exists(shp_json_list[1]): remove(shp_json_list[1]) VectorUtilClass.convert2geojson(shp_json_list[1], proj_srs, wgs84_srs, shp_json_list[0])
def get_subbasin_cell_count(subbsn_file): """Get cell number of each subbasin. Args: subbsn_file: subbasin raster file. Returns: subbasin cell count dict and cell width """ num_dic = dict() wtsd_raster = RasterUtilClass.read_raster(subbsn_file) data = wtsd_raster.data xsize = wtsd_raster.nCols ysize = wtsd_raster.nRows dx = wtsd_raster.dx nodata_value = wtsd_raster.noDataValue for i in range(ysize): for j in range(xsize): k = int(data[i][j]) if abs(k - nodata_value) > UTIL_ZERO: num_dic[k] = num_dic.setdefault(k, 0) + 1 return num_dic, dx
def downstream_method_whitebox(stream_raster, flow_dir_raster, hillslope_out, d8alg="taudem", stream_value_method=-1): """Algorithm modified from Whitebox GAT v3.4.0. source code: https://github.com/jblindsay/whitebox-geospatial-analysis-tools/ blob/master/HydroTools/src/plugins/Hillslopes.java Args: stream_raster: Stream cell value greater than 0 is identified by stream The input stream are recommended sequenced as 1, 2, 3... flow_dir_raster: D8 flow direction in TauDEM code hillslope_out: With the sequenced stream IDs, the output hillslope will be numbered: - Header hillslope: MaxStreamID + (current_id - 1) * 3 + 1 - Right hillslope: MaxStreamID + (current_id - 1) * 3 + 2 - Left hillslope: MaxStreamID + (current_id - 1) * 3 + 3 d8alg: Currently, "TauDEM", "ArcGIS", and "Whitebox" are supported. stream_value_method: stream value assigned method, depend on this parameter, the output hillslope will be appended as follows: -1 - all the four files will be output. 0 - keep stream link code, which has the default file name 1 - Set to the value of right hillslope and head hillslope, <name>_r.tif 2 - Set to the value of left hillslope and head hillslope, <name>_l.tif 3 - Set stream cell to NoData, <name>_n.tif """ print("Delineating hillslopes (header, left, and right hillslope)...") streamr = RasterUtilClass.read_raster(stream_raster) stream_data = streamr.data stream_nodata = streamr.noDataValue geotrans = streamr.geotrans srs = streamr.srs nrows = streamr.nRows ncols = streamr.nCols datatype = streamr.dataType flowd8r = RasterUtilClass.read_raster(flow_dir_raster) flowd8_data = flowd8r.data flowd8_nodata = flowd8r.noDataValue if flowd8r.nRows != nrows or flowd8r.nCols != ncols: raise ValueError("The input extent of D8 flow direction is not " "consistent with stream data!") # definition of utility functions def cal_hillslope_codes(maxid, curid): """Set hillslope encode IDs.""" return [ maxid + (curid - 1) * 3 + 1, # head maxid + (curid - 1) * 3 + 2, # right maxid + (curid - 1) * 3 + 3 ] # left def inflow_stream_number(vrow, vcol, flowmodel="taudem"): """ Count the inflow stream cell number and coordinates of all inflow cells Args: vrow: row number vcol: col number flowmodel: D8 flow direction algorithm. Returns: neighb_stream_cell_num: inflow cells number that is stream cell_coors: inflow cell coordinates, the size() is equal or greater than neighb_stream_cell_num """ neighb_stream_cell_num = 0 cell_coors = [] for c in range(8): newrow = vrow + FlowModelConst.ccw_drow[c] newcol = vcol + FlowModelConst.ccw_dcol[c] if newrow < 0 or newrow >= nrows or newcol < 0 or newcol >= ncols: continue if flowd8_data[newrow][ newcol] == FlowModelConst.d8_inflows.get(flowmodel)[c]: cell_coors.append((newrow, newcol)) if stream_data[newrow][newcol] > 0 \ and stream_data[newrow][newcol] != stream_nodata: neighb_stream_cell_num += 1 return neighb_stream_cell_num, cell_coors def assign_sequenced_stream_ids(c_id, vrow, vcol, flowmodel="taudem"): """set sequenced stream IDs""" in_strm_num, in_coors = inflow_stream_number(vrow, vcol, flowmodel) if in_strm_num == 0: # it's a headwater location so start a downstream flowpath c_id += 1 tmp_row = vrow tmp_col = vcol sequenced_stream_d[tmp_row][tmp_col] = c_id searched_flag = True while searched_flag: # find the downslope neighbour tmpflowd8 = flowd8_data[tmp_row][tmp_col] if tmpflowd8 < 0 or tmpflowd8 == flowd8_nodata: if stream_data[tmp_row][tmp_col] > 0 \ and stream_data[tmp_row][tmp_col] != stream_nodata: # it is a valid stream cell and probably just has no downslope # neighbour (e.g. at the edge of the grid) sequenced_stream_d[tmp_row][tmp_col] = c_id break tmp_row, tmp_col = D8Util.downstream_index( tmpflowd8, tmp_row, tmp_col, flowmodel) if tmp_row < 0 or tmp_row >= nrows or tmp_col < 0 or tmp_col >= ncols: break if stream_data[tmp_row][tmp_col] <= 0: searched_flag = False # it is not a stream cell else: if sequenced_stream_d[tmp_row][tmp_col] > 0: # run into a larger stream, end the downstream search break # is it a confluence (conjunction node) in_strm_num, in_coors = inflow_stream_number( tmp_row, tmp_col, flowmodel) if in_strm_num >= 2: c_id += 1 sequenced_stream_d[tmp_row][tmp_col] = c_id return c_id def assign_hillslope_code_of_neighbors(vrow, vcol, flowmodel="taudem"): """set hillslope code for neighbors of current stream cell.""" stream_coors.append((vrow, vcol)) in_strm_num, in_coors = inflow_stream_number(vrow, vcol, flowmodel) strm_id = stream_data[vrow][vcol] # print ("Assign hillslope code for stream cell, r: %d, c: %d, ID: %d" % (vrow, vcol, # int(strm_id))) # set hillslope IDs hillslp_ids = cal_hillslope_codes(max_id, strm_id) cur_d8_value = flowd8_data[vrow][vcol] if in_strm_num == 0: # it is a one-order stream head headstream_coors.append((vrow, vcol)) for (in_nostrm_row, in_nostrm_col) in in_coors: hillslope_mtx[in_nostrm_row][in_nostrm_col] = hillslp_ids[ 0] else: # search the 3*3 neighbors by clockwise and counterclockwise separately if cur_d8_value <= 0 or cur_d8_value == flowd8_nodata: return dirv = int(cur_d8_value) # direction code d_idx = FlowModelConst.d8_dirs.get(flowmodel).index( dirv) # direction index # look to the right side, i.e. clockwise d_idx_r = d_idx while True and len(in_coors) > 0: d_idx_r -= 1 if d_idx_r > 7: d_idx_r = 0 if d_idx_r < 0: d_idx_r = 7 tmp_row = vrow + FlowModelConst.ccw_drow[d_idx_r] tmp_col = vcol + FlowModelConst.ccw_dcol[d_idx_r] if (tmp_row, tmp_col ) not in in_coors: # not inflow to this cell continue tmpstream = stream_data[tmp_row][tmp_col] in_coors.remove((tmp_row, tmp_col)) if tmpstream <= 0 or tmpstream == stream_nodata: hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[ 1] # right hillslope else: # encounter another in flow stream break # look to the left side, i.e. counterclockwise d_idx_l = d_idx while True and len(in_coors) > 0: d_idx_l += 1 if d_idx_l > 7: d_idx_l = 0 if d_idx_l < 0: d_idx_l = 7 tmp_row = vrow + FlowModelConst.ccw_drow[d_idx_l] tmp_col = vcol + FlowModelConst.ccw_dcol[d_idx_l] if (tmp_row, tmp_col ) not in in_coors: # not inflow to this cell continue tmpstream = stream_data[tmp_row][tmp_col] in_coors.remove((tmp_row, tmp_col)) if tmpstream <= 0 or tmpstream == stream_nodata: hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[ 2] # left hillslope else: # encounter another in flow stream break # if any inflow cells existed? if len(in_coors) > 0: for (tmp_row, tmp_col) in in_coors: tmpstream = stream_data[tmp_row][tmp_col] if tmpstream <= 0 or tmpstream == stream_nodata: hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[0] # add the current cell as head stream headstream_coors.append((vrow, vcol)) def output_hillslope(method_id): """Output hillslope according different stream cell value method.""" for (tmp_row, tmp_col) in stream_coors: tmp_hillslp_ids = cal_hillslope_codes( max_id, stream_data[tmp_row][tmp_col]) if 0 < method_id < 3: hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[ method_id] # is head stream cell? if (tmp_row, tmp_col) in headstream_coors: hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[0] elif method_id == 3: hillslope_mtx[tmp_row][tmp_col] = DEFAULT_NODATA # Output to raster file hillslope_out_new = hillslope_out dirpath = os.path.dirname(hillslope_out_new) + os.sep corename = FileClass.get_core_name_without_suffix( hillslope_out_new) if method_id == 1: hillslope_out_new = dirpath + corename + '_right.tif' elif method_id == 2: hillslope_out_new = dirpath + corename + '_left.tif' elif method_id == 3: hillslope_out_new = dirpath + corename + '_nodata.tif' RasterUtilClass.write_gtiff_file(hillslope_out_new, nrows, ncols, hillslope_mtx, geotrans, srs, DEFAULT_NODATA, datatype) # 1. assign a unique id to each link in the stream network if needed assign_stream_id = False tmp = numpy.where((stream_data > 0) & (stream_data != stream_nodata), stream_data, numpy.nan) max_id = int(numpy.nanmax(tmp)) # i.e., stream link number min_id = int(numpy.nanmin(tmp)) for i in range(min_id, max_id + 1): if i not in tmp: assign_stream_id = True break if max_id == min_id: assign_stream_id = True current_id = 0 if assign_stream_id: # calculate and output sequenced stream raster sequenced_stream_d = numpy.ones((nrows, ncols)) * DEFAULT_NODATA for row in range(nrows): for col in range(ncols): # if the cell is not a stream, or has been assigned an ID if stream_data[row][col] <= 0 or stream_data[row][col] == stream_nodata \ or sequenced_stream_d[row][col] > 0: continue current_id = assign_sequenced_stream_ids( current_id, row, col, d8alg) stream_data = numpy.copy(sequenced_stream_d) stream_nodata = DEFAULT_NODATA stream_core = FileClass.get_core_name_without_suffix(stream_raster) stream_seq_file = os.path.dirname( stream_raster) + os.sep + stream_core + '_seq.tif' RasterUtilClass.write_gtiff_file(stream_seq_file, nrows, ncols, sequenced_stream_d, geotrans, srs, DEFAULT_NODATA, datatype) max_id = current_id # 2. assign hillslope code according to the 3*3 neighbors of stream cells hillslope_mtx = numpy.copy(stream_data) hillslope_mtx[stream_data == stream_nodata] = DEFAULT_NODATA headstream_coors = [] # head stream cells stream_coors = [] # all stream cells, include head stream cells. for row in range(nrows): for col in range(ncols): # if not a stream cell, or hillslope code has been assigned if stream_data[row][col] <= 0 or stream_data[row][col] == stream_nodata \ or hillslope_mtx[row][col] < 0: continue assign_hillslope_code_of_neighbors(row, col, d8alg) # 3. From each cell, search downstream for not assigned hillslope for row in range(nrows): for col in range(ncols): if hillslope_mtx[row][col] > 0 or flowd8_data[row][ col] == flowd8_nodata: continue flag = False tmprow = row tmpcol = col tmpcoors = [(row, col)] hillslp_id = DEFAULT_NODATA while not flag: # find it's downslope neighbour curflowdir = flowd8_data[tmprow][tmpcol] if curflowdir <= 0 or curflowdir == flowd8_nodata: break curflowdir = int(curflowdir) tmprow, tmpcol = D8Util.downstream_index( curflowdir, tmprow, tmpcol, d8alg) if tmprow < 0 or tmprow >= nrows or tmpcol < 0 or tmpcol >= ncols: break # if the new cell already has a hillslope value, use that if hillslope_mtx[tmprow][tmpcol] > 0: hillslp_id = hillslope_mtx[tmprow][tmpcol] flag = True if not flag: tmpcoors.append((tmprow, tmpcol)) # set the source cells for (crow, ccol) in tmpcoors: hillslope_mtx[crow][ccol] = hillslp_id # 4. reassign stream cell's value according to stream_value_method, and output if stream_value_method < 0: # output output_hillslope(0) output_hillslope(1) output_hillslope(2) output_hillslope(3) else: output_hillslope(stream_value_method)
def generate_runoff_coefficent(maindb, landuse_file, slope_file, soil_texture_file, runoff_coeff_file, imper_perc=0.3): """Generate potential runoff coefficient.""" # read landuselookup table from MongoDB prc_fields = ["PRC_ST%d" % (i, ) for i in range(1, 13)] sc_fields = ["SC_ST%d" % (i, ) for i in range(1, 13)] query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError( "LanduseLoop Collection is not existed or empty!") runoff_c0 = dict() runoff_s0 = dict() for row in query_result: tmpid = row.get('LANDUSE_ID') runoff_c0[tmpid] = [float(row.get(item)) for item in prc_fields] runoff_s0[tmpid] = [float(row.get(item)) for item in sc_fields] landu_raster = RasterUtilClass.read_raster(landuse_file) landu_data = landu_raster.data nodata_value1 = landu_raster.noDataValue xsize = landu_raster.nCols ysize = landu_raster.nRows nodata_value2 = landu_raster.noDataValue slo_data = RasterUtilClass.read_raster(slope_file).data soil_texture_array = RasterUtilClass.read_raster( soil_texture_file).data id_omited = [] def coef_cal(lu_id, soil_texture, slope): """Calculate runoff coefficient by landuse, soil texture and slope.""" if abs(lu_id - nodata_value1) < UTIL_ZERO or int(lu_id) < 0: return nodata_value2 if int(lu_id) not in runoff_c0.keys(): if int(lu_id) not in id_omited: print('The landuse ID: %d does not exist.' % int(lu_id)) id_omited.append(int(lu_id)) stid = int(soil_texture) - 1 c0 = runoff_c0[int(lu_id)][stid] s0 = runoff_s0[int(lu_id)][stid] / 100. slp = slope if slp + s0 < 0.0001: return c0 coef1 = (1 - c0) * slp / (slp + s0) coef2 = c0 + coef1 # TODO, Check if it is (lu_id >= 98), by lj if int(lu_id) == 106 or int(lu_id) == 107 or int(lu_id) == 105: return coef2 * (1 - imper_perc) + imper_perc else: return coef2 coef_cal_numpy = np_frompyfunc(coef_cal, 3, 1) coef = coef_cal_numpy(landu_data, soil_texture_array, slo_data) RasterUtilClass.write_gtiff_file(runoff_coeff_file, ysize, xsize, coef, landu_raster.geotrans, landu_raster.srs, nodata_value2, GDT_Float32)
def subbasin_statistics(cfg, maindb): """ Import subbasin numbers, outlet ID, etc. to MongoDB. """ streamlink_r = cfg.spatials.stream_link flowdir_r = cfg.spatials.d8flow direction_items = dict() if cfg.is_TauDEM: direction_items = FlowModelConst.get_cell_shift("TauDEM") else: direction_items = FlowModelConst.get_cell_shift("ArcGIS") streamlink_d = RasterUtilClass.read_raster(streamlink_r) nodata = streamlink_d.noDataValue nrows = streamlink_d.nRows ncols = streamlink_d.nCols streamlink_data = streamlink_d.data max_subbasin_id = int(streamlink_d.get_max()) min_subbasin_id = int(streamlink_d.get_min()) subbasin_num = len(unique(streamlink_data)) - 1 # print max_subbasin_id, min_subbasin_id, subbasin_num flowdir_d = RasterUtilClass.read_raster(flowdir_r) flowdir_data = flowdir_d.data i_row = -1 i_col = -1 for row in range(nrows): for col in range(ncols): if streamlink_data[row][col] != nodata: i_row = row i_col = col # print row, col break else: continue break if i_row == -1 or i_col == -1: raise ValueError("Stream link data invalid, please check and retry.") def flow_down_stream_idx(dir_value, i, j): """Return row and col of downstream direction.""" drow, dcol = direction_items[int(dir_value)] return i + drow, j + dcol def find_outlet_index(r, c): """Find outlet's coordinate""" flag = True while flag: fdir = flowdir_data[r][c] newr, newc = flow_down_stream_idx(fdir, r, c) if newr < 0 or newc < 0 or newr >= nrows or newc >= ncols \ or streamlink_data[newr][newc] == nodata: flag = False else: # print newr, newc, streamlink_data[newr][newc] r = newr c = newc return r, c o_row, o_col = find_outlet_index(i_row, i_col) outlet_bsn_id = int(streamlink_data[o_row][o_col]) import_stats_dict = {SubbsnStatsName.outlet: outlet_bsn_id, SubbsnStatsName.o_row: o_row, SubbsnStatsName.o_col: o_col, SubbsnStatsName.subbsn_max: max_subbasin_id, SubbsnStatsName.subbsn_min: min_subbasin_id, SubbsnStatsName.subbsn_num: subbasin_num} for stat, stat_v in import_stats_dict.items(): dic = {ModelParamFields.name: stat, ModelParamFields.desc: stat, ModelParamFields.unit: "NONE", ModelParamFields.module: "ALL", ModelParamFields.value: stat_v, ModelParamFields.impact: DEFAULT_NODATA, ModelParamFields.change: ModelParamFields.change_nc, ModelParamFields.max: DEFAULT_NODATA, ModelParamFields.min: DEFAULT_NODATA, ModelParamFields.type: "SUBBASIN"} curfilter = {ModelParamFields.name: dic[ModelParamFields.name]} # print (dic, curfilter) maindb[DBTableNames.main_parameter].find_one_and_replace(curfilter, dic, upsert=True) maindb[DBTableNames.main_parameter].create_index(ModelParamFields.name)
def scenario_from_texts(cfg, main_db, scenario_db): """Import BMPs Scenario data to MongoDB Args: cfg: SEIMS configuration object main_db: climate database scenario_db: scenario database Returns: False if failed, otherwise True. """ if not cfg.use_scernario: return False print ("Import BMP Scenario Data... ") bmp_files = FileClass.get_filename_by_suffixes(cfg.scenario_dir, ['.txt']) bmp_tabs = [] bmp_tabs_path = [] for f in bmp_files: bmp_tabs.append(f.split('.')[0]) bmp_tabs_path.append(cfg.scenario_dir + SEP + f) # create if collection not existed c_list = scenario_db.collection_names() for item in bmp_tabs: if not StringClass.string_in_list(item.upper(), c_list): scenario_db.create_collection(item.upper()) else: scenario_db.drop_collection(item.upper()) # Read subbasin.tif and dist2Stream.tif subbasin_r = RasterUtilClass.read_raster(cfg.spatials.subbsn) dist2stream_r = RasterUtilClass.read_raster(cfg.spatials.dist2stream_d8) # End reading for j, bmp_txt in enumerate(bmp_tabs_path): bmp_tab_name = bmp_tabs[j] data_array = read_data_items_from_txt(bmp_txt) field_array = data_array[0] data_array = data_array[1:] for item in data_array: dic = {} for i, field_name in enumerate(field_array): if MathClass.isnumerical(item[i]): dic[field_name.upper()] = float(item[i]) else: dic[field_name.upper()] = str(item[i]).upper() if StringClass.string_in_list(ImportScenario2Mongo._LocalX, dic.keys()) and \ StringClass.string_in_list(ImportScenario2Mongo._LocalY, dic.keys()): subbsn_id = subbasin_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) distance = dist2stream_r.get_value_by_xy( dic[ImportScenario2Mongo._LocalX.upper()], dic[ImportScenario2Mongo._LocalY.upper()]) if subbsn_id is not None and distance is not None: dic[ImportScenario2Mongo._SUBBASINID] = float(subbsn_id) dic[ImportScenario2Mongo._DISTDOWN] = float(distance) scenario_db[bmp_tab_name.upper()].find_one_and_replace(dic, dic, upsert=True) else: scenario_db[bmp_tab_name.upper()].find_one_and_replace(dic, dic, upsert=True) # print 'BMP tables are imported.' # Write BMP database name into Model workflow database c_list = main_db.collection_names() if not StringClass.string_in_list(DBTableNames.main_scenario, c_list): main_db.create_collection(DBTableNames.main_scenario) bmp_info_dic = dict() bmp_info_dic[ImportScenario2Mongo._FLD_DB] = cfg.bmp_scenario_db main_db[DBTableNames.main_scenario].find_one_and_replace(bmp_info_dic, bmp_info_dic, upsert=True) return True
def depression_capacity(maindb, landuse_file, slope_file, soil_texture_file, depression_file, imper_perc=0.3): """Initialize depression capacity according to landuse, soil, and slope. Args: maindb: main MongoDatabase landuse_file: landuse raster file slope_file: slope raster file soil_texture_file: soil texture file depression_file: resulted depression raster file imper_perc: impervious percent in urban cell, 0.3 as default """ # read landuselookup table from MongoDB st_fields = ["DSC_ST%d" % (i, ) for i in range(1, 13)] query_result = maindb['LANDUSELOOKUP'].find() if query_result is None: raise RuntimeError( "LanduseLoop Collection is not existed or empty!") dep_sd0 = dict() for row in query_result: tmpid = row.get('LANDUSE_ID') dep_sd0[tmpid] = [float(row.get(item)) for item in st_fields] landu_r = RasterUtilClass.read_raster(landuse_file) landu_data = landu_r.data geotrans = landu_r.geotrans srs = landu_r.srs xsize = landu_r.nCols ysize = landu_r.nRows landu_nodata = landu_r.noDataValue slo_data = RasterUtilClass.read_raster(slope_file).data soil_texture_array = RasterUtilClass.read_raster( soil_texture_file).data id_omited = [] def cal_dep(landu, soil_texture, slp): """Calculate depression""" last_stid = 0 if abs(landu - landu_nodata) < UTIL_ZERO: return DEFAULT_NODATA landu_id = int(landu) if landu_id not in dep_sd0: if landu_id not in id_omited: print('The landuse ID: %d does not exist.' % (landu_id, )) id_omited.append(landu_id) stid = int(soil_texture) - 1 try: depression_grid0 = dep_sd0[landu_id][stid] last_stid = stid except Exception: depression_grid0 = dep_sd0[landu_id][last_stid] depression_grid = exp( numpy.log(depression_grid0 + 0.0001) + slp * (-9.5)) # TODO, check if it is (landu_id >= 98)? By LJ if landu_id == 106 or landu_id == 107 or landu_id == 105: return 0.5 * imper_perc + (1. - imper_perc) * depression_grid else: return depression_grid cal_dep_numpy = numpy.frompyfunc(cal_dep, 3, 1) dep_storage_cap = cal_dep_numpy(landu_data, soil_texture_array, slo_data) RasterUtilClass.write_gtiff_file(depression_file, ysize, xsize, dep_storage_cap, geotrans, srs, DEFAULT_NODATA, GDT_Float32)