def main(): geofloodHomeDir,projectName,DEM_name,chunk_status,input_fn,output_fn,hr_status = cfg_finder() geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) in_shp = Name_path+ "_channelNetwork.shp" out_shp = Name_path+ "_channelSegment.shp" split_distance = 1000 network_split(in_shp, out_shp, split_distance)
def main(): geofloodHomeDir,projectName,DEM_name,chunk_status,input_fn,output_fn,hr_status = cfg_finder() geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) fdrfn = Name_path + '_fdr.tif' segshp = Name_path+ "_channelSegment.shp" segcatfn = Name_path + '_segmentCatchment.tif' segment_catchment_delineation(fdrfn, segshp, segcatfn)
def main(): geofloodHomeDir,projectName,DEM_name,chunk_status,input_fn,output_fn,hr_status = cfg_finder() geofloodInputDir = os.path.join(geofloodHomeDir, input_fn, "GIS", projectName) flowline_shp = os.path.join(geofloodInputDir, "Flowline.shp") demfn = os.path.join(geofloodInputDir, DEM_name+".tif") geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) nhdfn = Name_path + '_nhdflowline.tif' allofn = Name_path + '_Allocation.tif' negahandfn = Name_path + "_NegaHand.tif" demArray = raster2array(demfn) getnodata(demfn) vector2raster(flowline_shp, demfn, nhdfn) pathArray = raster2array(nhdfn) allocationArray, relaHeightArray = negative_height_identification(demArray, pathArray) array2raster(allofn,demfn,allocationArray,gdal.GDT_Float32) array2raster(negahandfn,demfn,relaHeightArray,gdal.GDT_Byte)
def main(): geofloodHomeDir,projectName,DEM_name,chunk_status,input_fn,output_fn,hr_status = cfg_finder() Name_path = os.path.join(geofloodHomeDir, output_fn, "Hydraulics", projectName) hydropropotxt = os.path.join(Name_path, "hydroprop-basetable.csv") manning_n = os.path.join(geofloodHomeDir, input_fn, "Hydraulics", projectName, "COMID_Roughness.csv") handpropotxt = os.path.join(Name_path, "hydroprop-fulltable.csv") geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) networkmaptxt = Name_path + "_networkMapping.csv" df_result = pd.read_csv(hydropropotxt) df_network = pd.read_csv(networkmaptxt) if manning_n.isdigit(): df_result['Roughness'] = manning_n else: df_n = pd.read_csv(manning_n) df_network = pd.merge(df_network, df_n, on='COMID') df_result = pd.merge(df_result, df_network, left_on='CatchId', right_on='HYDROID') df_result = df_result.drop('HYDROID', axis=1).rename(columns=lambda x: x.strip(" ")) df_result['TopWidth (m)'] = df_result['SurfaceArea (m2)']/df_result['LENGTHKM']/1000 df_result['WettedPerimeter (m)'] = df_result['BedArea (m2)']/df_result['LENGTHKM']/1000 df_result['WetArea (m2)'] = df_result['Volume (m3)']/df_result['LENGTHKM']/1000 df_result['HydraulicRadius (m)'] = df_result['WetArea (m2)']/df_result['WettedPerimeter (m)'] df_result['HydraulicRadius (m)'].fillna(0, inplace=True) df_result['Discharge (m3s-1)'] = df_result['WetArea (m2)']* \ pow(df_result['HydraulicRadius (m)'],2.0/3)* \ pow(df_result['SLOPE'],0.5)/df_result['Roughness'] df_result['FloodAreaRatio'] = df_result['SurfaceArea (m2)']/df_result['AREASQKM']/1000000 if df_result['Discharge (m3s-1)'].isna().sum() == len(df_result): print('Empty DataFrame, check hydroprop basetable and make sure COMID is in \ the COMID_Roughness csv') df_result.to_csv(handpropotxt,index=False)
# global variables comids = None # COMID list from NWM forecast table Qs = None # Q forecast list (discharge) from NWM h = None # hash table for Q forecast lookup, indexed by COMID (station id) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('nwm', help='Path to NWM input NetCDF', type=str) args = parser.parse_args() if (args.nwm[-3:] != '.nc'): print('Not a valid NetCDF file or missing .nc extension') else: nwmfn = args.nwm geofloodHomeDir, projectName, DEM_name, chunk_status, input_fn, output_fn, hr_status = cfg_finder( ) Name_path_hydro = os.path.join(geofloodHomeDir, output_fn, "Hydraulics", projectName) hp_input = os.path.join(Name_path_hydro, "hydroprop-fulltable.csv") Name_path = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) netmap_table = os.path.join(Name_path, DEM_name) + "_networkMapping.csv" df_netmap = pd.read_csv(netmap_table) Name_path = os.path.join(geofloodHomeDir, output_fn, "NWM", projectName) nwm_current_folder = Name_path if not os.path.exists(nwm_current_folder): os.mkdir(nwm_current_folder) stage_output = os.path.join(nwm_current_folder, os.path.basename(nwmfn)) tobj = readForecast(nwmfn, df_netmap) # read forecast, set up hash table timestr = tobj['timestamp'] init_timestr = tobj['init_timestamp'] forecastH(init_timestr, timestr, hp_input, stage_output)
def main(): geofloodHomeDir, projectName, DEM_name, chunk_status, input_fn, output_fn, hr_status = cfg_finder( ) geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) # Read in parameters that could potentially be used in cost function. Name_path = os.path.join(geofloodResultsDir, DEM_name) flowline_csv = Name_path + '_endPoints.csv' curvaturefn = Name_path + '_curvature.tif' facfn = Name_path + '_fac.tif' hr_fn = Name_path + '_NHD_HR.tif' handfn = Name_path + '_NegaHand.tif' flowlinefn = Name_path + '_channelNetwork.shp' costsurfacefn = Name_path + '_cost.tif' pathfn = Name_path + '_path.tif' streamcell_csv = Name_path + '_streamcell.csv' src_fac = rasterio.open(facfn) src_curv = rasterio.open(curvaturefn) src_neghand = rasterio.open(handfn) if (os.path.exists(hr_fn)) and ( hr_status == 1): # HR raster from pygeonet_hr_raster.py script src_hr = rasterio.open(hr_fn) dem_shape = src_curv.shape print(f'Chunk status: {chunk_status}') dem_bytes = src_curv.read(1).nbytes dem_limit = 1_500_000_000 print(f'DEM Size: {round(np.float(dem_bytes)/10**9,3)} GB') if (dem_bytes >= dem_limit) and (chunk_status == 1): # Default setting print("Chunking DEM") tot_chunks = math.ceil(np.float(dem_bytes) / (10**9)) print(f'Number of Chunks: {tot_chunks}') elif (dem_bytes < dem_limit) and (chunk_status == 1): print("DEM not big enough to chunk") tot_chunks = 1 else: print("Not attempting to chunk DEM") tot_chunks = 1 row_iter = 1 sample_rows = math.ceil(src_fac.shape[0] / tot_chunks) # Chunk by the row sample_cols = src_fac.shape[1] costsurfaceArray = np.zeros(shape=src_fac.shape) cost_list = [] # Chunking of rasters. If tot_chunks = 1, the entire raster will be processed at one time. for row_iter in range(1, tot_chunks + 1): facArray = src_fac.read( 1, window=Window(0, (sample_rows * row_iter - sample_rows), sample_cols, sample_rows)) curvatureArray = src_curv.read( 1, window=Window(0, (sample_rows * row_iter - sample_rows), sample_cols, sample_rows)) if (os.path.exists(hr_fn)) and (hr_status == 1): hr_Array = src_hr.read( 1, window=Window(0, (sample_rows * row_iter - sample_rows), sample_cols, sample_rows)) hr_Array = hr_Array.astype(np.uint8) if (row_iter == 1): print('Found HR_Flowline Raster') if (os.path.exists(handfn)): handArray = src_neghand.read( 1, window=Window(0, (sample_rows * row_iter - sample_rows), sample_cols, sample_rows)) # FAC Calculations facArray = np.log(facArray) facArray = normalize(facArray) facArray = facArray.astype(np.float32) flowMean = np.mean(facArray[~np.isnan(facArray[:])]) # Curvature curvatureArray = np.where( (curvatureArray < -10) | (curvatureArray > 10), np.nan, curvatureArray) curvatureArray = normalize(curvatureArray) # The np where operation above is one last check/method to remove likely nan pixels # that have not been flagged as nan. Typical curvature values are between -2 to 2 (Geometric) or -5 to 5 (laplacian) # If the hr_extraction is found in the outputs folder, include in the cost function. if (os.path.exists(hr_fn)) and (hr_status == 1): if (row_iter == 1): print('Calculating cost with NHD HR raster as a parameter.') cost = 1 / (curvatureArray * flowMean + facArray + 0.75 * handArray + hr_Array) elif (not os.path.exists(hr_fn)) and (hr_status == 1): if (row_iter == 1): print("Couldn't find HR raster. Calculating Cost without it.") cost = 1 / (curvatureArray * flowMean + facArray + 0.75 * handArray) elif (os.path.exists(hr_fn)) and (hr_status == 0): if (row_iter == 1): print( "Found HR raster, but 'hr_flowline' variable in project cfg is set to 0. \ Calculate cost without HR. (Set to 1 to use HR)") cost = 1 / (curvatureArray * flowMean + facArray + 0.75 * handArray) elif (not os.path.exists(hr_fn)) and (hr_status == 0): if (row_iter == 1): print('Not using NHD HR Raster in cost function') cost = 1 / (curvatureArray * flowMean + facArray + 0.75 * handArray) elif (hr_status == -1): if (row_iter == 1): print('NOT using any NHD products in Cost Function') cost = 1 / (curvatureArray * flowMean + facArray) cost_list.append(cost) row_iter += 1 del facArray, curvatureArray, cost, handArray if (os.path.exists(hr_fn)) and (hr_status == 1): del hr_Array costsurfaceArray = cost_list del cost_list gc.collect() costsurfaceArray = np.concatenate(costsurfaceArray, axis=0).astype(np.float32) print(f'Cost shape: {costsurfaceArray.shape}') assert costsurfaceArray.shape[0] == dem_shape[0] assert costsurfaceArray.shape[1] == dem_shape[1] costQuantile = np.quantile( costsurfaceArray[~np.isnan(costsurfaceArray[:])], .025) array2raster(costsurfacefn, facfn, costsurfaceArray, gdal.GDT_Float32) costsurfaceArray[np.isnan(costsurfaceArray)] = 100000 # Threshold cost # costsurfaceArray = np.where(costsurfaceArray < costQuantile, costsurfaceArray, 100000) # # # Check memory usage get_raster_info(costsurfacefn) df_flowline = pd.read_csv(flowline_csv) pathArray = np.zeros_like(costsurfaceArray, dtype=np.uint8) costsurfaceArray = costsurfaceArray.astype(np.float32) route_path(costsurfaceArray, df_flowline, pathArray, streamcell_csv, flowlinefn, facfn, curvaturefn) array2raster(pathfn, costsurfacefn, pathArray, gdal.GDT_Byte)
def main(): geofloodHomeDir, projectName, DEM_name, chunk_status, input_fn, output_fn, hr_status = cfg_finder( ) geofloodInputDir = os.path.join(geofloodHomeDir, input_fn, "GIS", projectName) flowline_shp = os.path.join(geofloodInputDir, "Flowline.shp") geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) node_csv = Name_path + '_endPoints.csv' network_node_reading(flowline_shp, node_csv)
def main(): geofloodHomeDir, projectName, DEM_name, chunk_status, input_fn, output_fn, hr_status = cfg_finder( ) geofloodInputDir = os.path.join(geofloodHomeDir, input_fn, "GIS", projectName) cat_shp = os.path.join(geofloodInputDir, "Catchment.shp") geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) seg_shp = Name_path + "_channelSegment.shp" map_csv = Name_path + "_networkMapping.csv" network_mapping(cat_shp, seg_shp, map_csv)
def main(): geofloodHomeDir, projectName, DEM_name, chunk_status, input_fn, output_fn, hr_status = cfg_finder( ) geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn, "GIS", projectName) Name_path = os.path.join(geofloodResultsDir, DEM_name) segment_shp = Name_path + "_channelSegment.shp" segcatfn = Name_path + "_segmentCatchment.tif" segcat_shp = Name_path + "_segmentCatchment.shp" hydro_folder = os.path.join(geofloodHomeDir, output_fn, "Hydraulics", projectName) if not os.path.exists(hydro_folder): os.mkdir(hydro_folder) attribute_txt = os.path.join(hydro_folder, DEM_name + "_River_Attribute.txt") demfn = os.path.join(geofloodHomeDir, input_fn, "GIS", projectName, DEM_name + ".tif") river_attribute_estimation(segment_shp, segcatfn, segcat_shp, demfn, attribute_txt)