def main(folder_path): csv_paths = [ os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(".csv") ] csvs = {os.path.basename(f).split('.')[0]: csv_dict(f) for f in csv_paths} project, realization = create_project(csvs['Project'][0], folder_path) make_reaches(folder_path, csvs['Reach']) make_structures(folder_path, csvs['Structure']) make_photopoints(folder_path, csvs['Photo'], csvs['PhotoPoint']) intermediate_node = project.XMLBuilder.add_sub_element( realization, 'Intermediates') output_node = project.XMLBuilder.add_sub_element(realization, 'Outputs') for csv in csv_paths: relpath = os.path.relpath(csv, folder_path) name = os.path.basename(relpath).split('.')[0] nodeid = 'CSV_' + name.upper() project.add_project_vector(intermediate_node, RSLayer(name, nodeid, 'CSV', relpath)) project.add_project_vector(output_node, LayerTypes['Photos']) project.add_project_vector(output_node, LayerTypes['Reaches']) project.add_project_vector(output_node, LayerTypes['Structures']) print("done")
def process_lst(lst_xml_folder): """This is a slightly hack-y script to create some XMLS for the land_surface_temp script It's a bit of an afterthought so it just plunks down the XMLS all alone in a folder Args: lst_xml_folder ([type]): [description] """ log = Logger("Generate XMLS for LST") hucs = [str(1700 + x) for x in range(1, 13)] for huc in hucs: hucdir = os.path.join(lst_xml_folder, huc) xml_file = os.path.join(hucdir, 'project.rs.xml') safe_makedirs(hucdir) if os.path.exists(xml_file): safe_remove_file(xml_file) project_name = f'Land Surface Temperature for HUC {huc}' project = RSProject(cfg, xml_file) project.create(project_name, 'LST') project.add_metadata({ 'ModelVersion': cfg.version, 'HUC': huc, 'dateCreated': datetime.datetime.now().isoformat(), 'HUC{}'.format(len(huc)): huc }) realizations = project.XMLBuilder.add_sub_element( project.XMLBuilder.root, 'Realizations') realization = project.XMLBuilder.add_sub_element( realizations, 'LST', None, { 'id': 'LST1', 'dateCreated': datetime.datetime.now().isoformat(), 'guid': str(uuid.uuid4()), 'productVersion': cfg.version }) project.XMLBuilder.add_sub_element(realization, 'Name', project_name) output_node = project.XMLBuilder.add_sub_element( realization, 'Outputs') zipfile_node = project.add_dataset(output_node, f'{huc}.zip', RSLayer(f'LST Result for {huc}', 'LST_ZIP', 'ZipFile', '1706.zip'), 'ZipFile', replace=True, rel_path=True) project.XMLBuilder.write() log.info('done')
Path = str initGDALOGRErrors() gdal.UseExceptions() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/Confinement.xsd', __version__) LayerTypes = { # key: (name, id, tag, relpath)] 'INPUTS': RSLayer( 'Inputs', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', { 'FLOWLINES': RSLayer('Flowlines', 'FLOWLINES', 'Vector', 'Flowlines'), 'CONFINING_POLYGON': RSLayer('Confining Polygon', 'CONFINING_POLYGON', 'Vector', 'ConfiningPolygon'), }), 'INTERMEDIATES': RSLayer( 'Intermediates', 'INTERMEDIATES', 'Geopackage', 'intermediates/confinement_intermediates.gpkg', { 'SPLIT_POINTS': RSLayer('Split Points', 'SPLIT_POINTS', 'Vector', 'Split_Points'), 'FLOWLINE_SEGMENTS': RSLayer('Flowline Segments', 'FLOWLINE_SEGMENTS', 'Vector', 'Flowline_Segments'), 'ERROR_POLYLINES': RSLayer('Error Polylines', 'ERROR_POLYLINES', 'Vector', 'Error_Polylines'),
from rscommons.util import parse_metadata from rscommons.build_network import build_network from rscommons.database import create_database, SQLiteCon from sqlbrat.utils.vegetation_summary import vegetation_summary from sqlbrat.utils.reach_geometry import reach_geometry from sqlbrat.utils.conflict_attributes import conflict_attributes from sqlbrat.__version__ import __version__ Path = str initGDALOGRErrors() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/BRAT.xsd', __version__) LayerTypes = { 'DEM': RSLayer('NED 10m DEM', 'DEM', 'DEM', 'inputs/dem.tif'), 'SLOPE': RSLayer('Slope Raster', 'SLOPE', 'Raster', 'inputs/slope.tif'), 'HILLSHADE': RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster', 'inputs/dem_hillshade.tif'), 'EXVEG': RSLayer('Existing Vegetation', 'EXVEG', 'Raster', 'inputs/existing_veg.tif'), 'HISTVEG': RSLayer('Historic Vegetation', 'HISTVEG', 'Raster', 'inputs/historic_veg.tif'), 'INPUTS': RSLayer('Confinement', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', { 'FLOWLINES': RSLayer('Segmented Flowlines', 'FLOWLINES', 'Vector', 'flowlines'), 'FLOW_AREA': RSLayer('NHD Flow Area', 'FLOW_AREA', 'Vector', 'flowareas'), 'WATERBODIES': RSLayer('NHD Waterbody', 'WATERBODIES', 'Vector', 'waterbodies'), 'VALLEY_BOTTOM': RSLayer('Valley Bottom', 'VALLEY_BOTTOM', 'Vector', 'valley_bottom'), 'ROADS': RSLayer('Roads', 'ROADS', 'Vector', 'roads'), 'RAIL': RSLayer('Rail', 'RAIL', 'Vector', 'rail'), 'CANALS': RSLayer('Canals', 'CANALS', 'Vector', 'canals') }), 'INTERMEDIATES': RSLayer('Intermediates', 'INTERMEDIATES', 'Geopackage', 'intermediates/intermediates.gpkg', {}), 'OUTPUTS': RSLayer('BRAT', 'OUTPUTS', 'Geopackage', 'outputs/brat.gpkg', {
from osgeo import ogr from osgeo import gdal from rscommons import Logger, RSProject, RSLayer, ModelConfig, dotenv, initGDALOGRErrors from rscommons.util import safe_makedirs, safe_remove_dir from gnat.__version__ import __version__ initGDALOGRErrors() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/GNAT.xsd', __version__) LayerTypes = { # key: (name, id, tag, relpath) 'DEM': RSLayer('NED 10m DEM', 'DEM', 'DEM', 'topography/dem.tif') } def gnat(huc, output_folder): """[summary] Args: huc ([type]): [description] Raises: Exception: [description] Exception: [description] Exception: [description] Exception: [description]
import datetime import uuid from rscommons import Logger, ModelConfig, RSProject, RSLayer, dotenv from rscommons.shapefile import create_field from rscommons.util import safe_makedirs from osgeo import gdal, ogr, osr import csv from sqlbrat.__version__ import __version__ cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/fmLTPBR.xsd', __version__) # Don't forgbet to pip install dbfread. I didn't add it to requirements.txt LayerTypes = { # key: (name, id, tag, relpath) 'Photos': RSLayer('Photos', 'PHOTO', 'Vector', 'Photos.shp'), 'Reaches': RSLayer('Reaches', 'REACH', 'Vector', 'Reaches.shp'), 'Structures': RSLayer('Structures', 'STRUCT', 'Vector', 'Structures.shp'), } REACH_FIELDS = [ # csv_name, shp_name, type, length ["pk", "pk", ogr.OFTString], ["fk_Project", "fkProj", ogr.OFTString], ["Name", "Name", ogr.OFTString], ["StreamName", "StreamName", ogr.OFTString], ["Type", "Type", ogr.OFTString], ["ObjectivePrimary", "ObjPrimary", ogr.OFTString], ["ObjectiveSecondary", "ObjSecond", ogr.OFTString], ["ActiveFloodplainCurrent", "AFPCurrent", ogr.OFTInteger], ["ActiveFloodplainTarget", "AFPTarget", ogr.OFTInteger],
def rs_context(huc, existing_veg, historic_veg, ownership, fair_market, ecoregions, prism_folder, output_folder, download_folder, scratch_dir, parallel, force_download, meta: Dict[str, str]): """ Download riverscapes context layers for the specified HUC and organize them as a Riverscapes project :param huc: Eight, 10 or 12 digit HUC identification number :param existing_veg: Path to the existing vegetation conditions raster :param historic_veg: Path to the historical vegetation conditions raster :param ownership: Path to the national land ownership Shapefile :param output_folder: Output location for the riverscapes context project :param download_folder: Temporary folder where downloads are cached. This can be shared between rs_context processes :param force_download: If false then downloads can be skipped if the files already exist :param prism_folder: folder containing PRISM rasters in *.bil format :param meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs :return: """ log = Logger("RS Context") log.info('Starting RSContext v.{}'.format(cfg.version)) try: int(huc) except ValueError: raise Exception( 'Invalid HUC identifier "{}". Must be an integer'.format(huc)) if not (len(huc) in [4, 8, 10, 12]): raise Exception( 'Invalid HUC identifier. Must be 4, 8, 10 or 12 digit integer') safe_makedirs(output_folder) safe_makedirs(download_folder) # We need a temporary folder for slope rasters, Stitching inputs, intermeditary products, etc. scratch_dem_folder = os.path.join(scratch_dir, 'rs_context', huc) safe_makedirs(scratch_dem_folder) project, realization = create_project(huc, output_folder) hydrology_gpkg_path = os.path.join(output_folder, LayerTypes['HYDROLOGY'].rel_path) dem_node, dem_raster = project.add_project_raster(realization, LayerTypes['DEM']) _node, hill_raster = project.add_project_raster(realization, LayerTypes['HILLSHADE']) _node, flow_accum = project.add_project_raster(realization, LayerTypes['FA']) _node, drain_area = project.add_project_raster(realization, LayerTypes['DA']) hand_node, hand_raster = project.add_project_raster( realization, LayerTypes['HAND']) _node, slope_raster = project.add_project_raster(realization, LayerTypes['SLOPE']) _node, existing_clip = project.add_project_raster(realization, LayerTypes['EXVEG']) _node, historic_clip = project.add_project_raster(realization, LayerTypes['HISTVEG']) _node, fair_market_clip = project.add_project_raster( realization, LayerTypes['FAIR_MARKET']) # Download the four digit NHD archive containing the flow lines and watershed boundaries log.info('Processing NHD') # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) nhd_download_folder = os.path.join(download_folder, 'nhd', huc[:4]) nhd_unzip_folder = os.path.join(scratch_dir, 'nhd', huc[:4]) nhd, db_path, huc_name, nhd_url = clean_nhd_data( huc, nhd_download_folder, nhd_unzip_folder, os.path.join(output_folder, 'hydrology'), cfg.OUTPUT_EPSG, False) # Clean up the unzipped files. We won't need them again if parallel: safe_remove_dir(nhd_unzip_folder) project.add_metadata({'Watershed': huc_name}) boundary = 'WBDHU{}'.format(len(huc)) # For coarser rasters than the DEM we need to buffer our clip polygon to include enough pixels # This shouldn't be too much more data because these are usually integer rasters that are much lower res. buffered_clip_path100 = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP100'].rel_path) copy_feature_class(nhd[boundary], buffered_clip_path100, epsg=cfg.OUTPUT_EPSG, buffer=100) buffered_clip_path500 = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path) copy_feature_class(nhd[boundary], buffered_clip_path500, epsg=cfg.OUTPUT_EPSG, buffer=500) # PRISM climate rasters mean_annual_precip = None bil_files = glob.glob(os.path.join(prism_folder, '**', '*.bil')) if (len(bil_files) == 0): raise Exception('Could not find any .bil files in the prism folder') for ptype in PrismTypes: try: # Next should always be guarded source_raster_path = next( x for x in bil_files if ptype.lower() in os.path.basename(x).lower()) except StopIteration: raise Exception( 'Could not find .bil file corresponding to "{}"'.format(ptype)) _node, project_raster_path = project.add_project_raster( realization, LayerTypes[ptype]) raster_warp(source_raster_path, project_raster_path, cfg.OUTPUT_EPSG, buffered_clip_path500, {"cutlineBlend": 1}) # Use the mean annual precipitation to calculate bankfull width if ptype.lower() == 'ppt': polygon = get_geometry_unary_union(nhd[boundary], epsg=cfg.OUTPUT_EPSG) mean_annual_precip = raster_buffer_stats2( {1: polygon}, project_raster_path)[1]['Mean'] log.info('Mean annual precipitation for HUC {} is {} mm'.format( huc, mean_annual_precip)) project.add_metadata( {'mean_annual_precipitation_mm': str(mean_annual_precip)}) calculate_bankfull_width(nhd['NHDFlowline'], mean_annual_precip) # Add the DB record to the Project XML db_lyr = RSLayer('NHD Tables', 'NHDTABLES', 'SQLiteDB', os.path.relpath(db_path, output_folder)) sqlite_el = project.add_dataset(realization, db_path, db_lyr, 'SQLiteDB') project.add_metadata({'origin_url': nhd_url}, sqlite_el) # Add any results to project XML for name, file_path in nhd.items(): lyr_obj = RSLayer(name, name, 'Vector', os.path.relpath(file_path, output_folder)) vector_nod, _fpath = project.add_project_vector(realization, lyr_obj) project.add_metadata({'origin_url': nhd_url}, vector_nod) states = get_nhd_states(nhd[boundary]) # Download the NTD archive containing roads and rail log.info('Processing NTD') ntd_raw = {} ntd_unzip_folders = [] ntd_urls = get_ntd_urls(states) for state, ntd_url in ntd_urls.items(): ntd_download_folder = os.path.join(download_folder, 'ntd', state.lower()) ntd_unzip_folder = os.path.join( scratch_dir, 'ntd', state.lower(), 'unzipped' ) # a little awkward but I need a folder for this and this was the best name I could find ntd_raw[state] = download_shapefile_collection(ntd_url, ntd_download_folder, ntd_unzip_folder, force_download) ntd_unzip_folders.append(ntd_unzip_folder) ntd_clean = clean_ntd_data(ntd_raw, nhd['NHDFlowline'], nhd[boundary], os.path.join(output_folder, 'transportation'), cfg.OUTPUT_EPSG) # clean up the NTD Unzip folder. We won't need it again if parallel: for unzip_path in ntd_unzip_folders: safe_remove_dir(unzip_path) # Write transportation layers to project file log.info('Write transportation layers to project file') # Add any results to project XML for name, file_path in ntd_clean.items(): lyr_obj = RSLayer(name, name, 'Vector', os.path.relpath(file_path, output_folder)) ntd_node, _fpath = project.add_project_vector(realization, lyr_obj) project.add_metadata({**ntd_urls}, ntd_node) # Download the HAND raster huc6 = huc[0:6] hand_download_folder = os.path.join(download_folder, 'hand') _hpath, hand_url = download_hand(huc6, cfg.OUTPUT_EPSG, hand_download_folder, nhd[boundary], hand_raster, warp_options={"cutlineBlend": 1}) project.add_metadata({'origin_url': hand_url}, hand_node) # download contributing DEM rasters, mosaic and reproject into compressed GeoTIF ned_download_folder = os.path.join(download_folder, 'ned') ned_unzip_folder = os.path.join(scratch_dir, 'ned') dem_rasters, urls = download_dem(nhd[boundary], cfg.OUTPUT_EPSG, 0.01, ned_download_folder, ned_unzip_folder, force_download) need_dem_rebuild = force_download or not os.path.exists(dem_raster) if need_dem_rebuild: raster_vrt_stitch(dem_rasters, dem_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], warp_options={"cutlineBlend": 1}) verify_areas(dem_raster, nhd[boundary]) # Calculate slope rasters seperately and then stitch them slope_parts = [] hillshade_parts = [] need_slope_build = need_dem_rebuild or not os.path.isfile(slope_raster) need_hs_build = need_dem_rebuild or not os.path.isfile(hill_raster) project.add_metadata( { 'num_rasters': str(len(urls)), 'origin_urls': json.dumps(urls) }, dem_node) for dem_r in dem_rasters: slope_part_path = os.path.join( scratch_dem_folder, 'SLOPE__' + os.path.basename(dem_r).split('.')[0] + '.tif') hs_part_path = os.path.join( scratch_dem_folder, 'HS__' + os.path.basename(dem_r).split('.')[0] + '.tif') slope_parts.append(slope_part_path) hillshade_parts.append(hs_part_path) if force_download or need_dem_rebuild or not os.path.exists( slope_part_path): gdal_dem_geographic(dem_r, slope_part_path, 'slope') need_slope_build = True if force_download or need_dem_rebuild or not os.path.exists( hs_part_path): gdal_dem_geographic(dem_r, hs_part_path, 'hillshade') need_hs_build = True if need_slope_build: raster_vrt_stitch(slope_parts, slope_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], clean=parallel, warp_options={"cutlineBlend": 1}) verify_areas(slope_raster, nhd[boundary]) else: log.info('Skipping slope build because nothing has changed.') if need_hs_build: raster_vrt_stitch(hillshade_parts, hill_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], clean=parallel, warp_options={"cutlineBlend": 1}) verify_areas(hill_raster, nhd[boundary]) else: log.info('Skipping hillshade build because nothing has changed.') # Remove the unzipped rasters. We won't need them anymore if parallel: safe_remove_dir(ned_unzip_folder) # Calculate flow accumulation raster based on the DEM log.info('Running flow accumulation and converting to drainage area.') flow_accumulation(dem_raster, flow_accum, dinfinity=False, pitfill=True) flow_accum_to_drainage_area(flow_accum, drain_area) # Clip and re-project the existing and historic vegetation log.info('Processing existing and historic vegetation rasters.') clip_vegetation(buffered_clip_path100, existing_veg, existing_clip, historic_veg, historic_clip, cfg.OUTPUT_EPSG) log.info('Process the Fair Market Value Raster.') raster_warp(fair_market, fair_market_clip, cfg.OUTPUT_EPSG, clip=buffered_clip_path500, warp_options={"cutlineBlend": 1}) # Clip the landownership Shapefile to a 10km buffer around the watershed boundary own_path = os.path.join(output_folder, LayerTypes['OWNERSHIP'].rel_path) project.add_dataset(realization, own_path, LayerTypes['OWNERSHIP'], 'Vector') clip_ownership(nhd[boundary], ownership, own_path, cfg.OUTPUT_EPSG, 10000) ####################################################### # Segmentation ####################################################### # For now let's just make a copy of the NHD FLowlines tmr = Timer() rs_segmentation(nhd['NHDFlowline'], ntd_clean['Roads'], ntd_clean['Rail'], own_path, hydrology_gpkg_path, SEGMENTATION['Max'], SEGMENTATION['Min'], huc) log.debug('Segmentation done in {:.1f} seconds'.format(tmr.ellapsed())) project.add_project_geopackage(realization, LayerTypes['HYDROLOGY']) # Add Bankfull Buffer Polygons bankfull_path = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BANKFULL_CHANNEL'].rel_path) bankfull_buffer( os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['NETWORK'].rel_path), cfg.OUTPUT_EPSG, bankfull_path, ) # TODO Add nhd/bankfull union when merge feature classes in vector.ops works with Geopackage layers # bankfull_nhd_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path) # clip_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path) # bankfull_nhd_area(bankfull_path, nhd['NHDArea'], clip_path, cfg.OUTPUT_EPSG, hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path) # Filter the ecoregions Shapefile to only include attributes that intersect with our HUC eco_path = os.path.join(output_folder, 'ecoregions', 'ecoregions.shp') project.add_dataset(realization, eco_path, LayerTypes['ECOREGIONS'], 'Vector') filter_ecoregions(nhd[boundary], ecoregions, eco_path, cfg.OUTPUT_EPSG, 10000) report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path) project.add_report(realization, LayerTypes['REPORT'], replace=True) report = RSContextReport(report_path, project, output_folder) report.write() log.info('Process completed successfully.') return { 'DEM': dem_raster, 'Slope': slope_raster, 'ExistingVeg': existing_veg, 'HistoricVeg': historic_veg, 'NHD': nhd }
Instructuions: 1. create a file called .env in the root and add the following line (without the quotes): "PROJECTFILE=C:/path/to/my/folder/that/will/have/project.rs.xml" 2. Run `BuildXML` from VSCode's Run dropdown menu """ # Set up a class to track thigns like default EPSG, xsd validation and tool version cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/Inundation.xsd', __version__) # Define the types of layers we're going to use up top so we can re-use them later # We use our handy RSLayer() class here. LayerTypes = { # RSLayer(name, id, tag, rel_path) 'AP_01': RSLayer('2019 August', 'APORTHO_01', 'Raster', '01_Inputs/01_Imagery/AP_01/orthomosaic.tif'), 'APSTR_01': RSLayer('2019 August Flowlines', 'APSTR_01', 'Vector', '01_Inputs/01_Imagery/AP_01/flowlines.shp'), } def build_xml(projectpath): """Here's an example of how to build a project.rs.xml file Args: projectpath ([type]): [description] """ # Create the top-level nodes log = Logger('build_xml') log.info('Starting the build of the XML')
from rscontext.vegetation import clip_vegetation from rscontext.bankfull import bankfull_buffer, bankfull_nhd_area from rscontext.__version__ import __version__ initGDALOGRErrors() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/RSContext.xsd', __version__) # These are the Prism BIL types we expect PrismTypes = ['PPT', 'TMEAN', 'TMIN', 'TMAX', 'TDMEAN', 'VPDMIN', 'VPDMAX'] LayerTypes = { # key: (name, id, tag, relpath) 'DEM': RSLayer('NED 10m DEM', 'DEM', 'DEM', 'topography/dem.tif'), 'FA': RSLayer('Flow Accumulation', 'FA', 'Raster', 'topography/flow_accum.tif'), 'DA': RSLayer('Drainage Area', 'DA', 'Raster', 'topography/drainarea_sqkm.tif'), 'HILLSHADE': RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster', 'topography/dem_hillshade.tif'), 'SLOPE': RSLayer('Slope', 'SLOPE', 'Raster', 'topography/slope.tif'), 'HAND': RSLayer('Height above nearest drainage', 'HAND', 'Raster', 'topography/hand.tif'), # Veg Layers 'EXVEG': RSLayer('Existing Vegetation', 'EXVEG', 'Raster',
def vbet(huc, flowlines_orig, flowareas_orig, orig_slope, json_transforms, orig_dem, hillshade, max_hand, min_hole_area_m, project_folder, reach_codes: List[str], meta: Dict[str, str]): """[summary] Args: huc ([type]): [description] flowlines_orig ([type]): [description] flowareas_orig ([type]): [description] orig_slope ([type]): [description] json_transforms ([type]): [description] orig_dem ([type]): [description] hillshade ([type]): [description] max_hand ([type]): [description] min_hole_area_m ([type]): [description] project_folder ([type]): [description] reach_codes (List[int]): NHD reach codes for features to include in outputs meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs """ log = Logger('VBET') log.info('Starting VBET v.{}'.format(cfg.version)) project, _realization, proj_nodes = create_project(huc, project_folder) # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) # Copy the inp _proj_slope_node, proj_slope = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['SLOPE_RASTER'], orig_slope) _proj_dem_node, proj_dem = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['DEM'], orig_dem) _hillshade_node, hillshade = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade) # Copy input shapes to a geopackage inputs_gpkg_path = os.path.join(project_folder, LayerTypes['INPUTS'].rel_path) intermediates_gpkg_path = os.path.join( project_folder, LayerTypes['INTERMEDIATES'].rel_path) flowlines_path = os.path.join( inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path) flowareas_path = os.path.join( inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path) # Make sure we're starting with a fresh slate of new geopackages GeopackageLayer.delete(inputs_gpkg_path) GeopackageLayer.delete(intermediates_gpkg_path) copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG) copy_feature_class(flowareas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG) project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS']) # Create a copy of the flow lines with just the perennial and also connectors inside flow areas network_path = os.path.join( intermediates_gpkg_path, LayerTypes['INTERMEDIATES'].sub_layers['VBET_NETWORK'].rel_path) vbet_network(flowlines_path, flowareas_path, network_path, cfg.OUTPUT_EPSG, reach_codes) # Generate HAND from dem and vbet_network # TODO make a place for this temporary folder. it can be removed after hand is generated. temp_hand_dir = os.path.join(project_folder, "intermediates", "hand_processing") safe_makedirs(temp_hand_dir) hand_raster = os.path.join(project_folder, LayerTypes['HAND_RASTER'].rel_path) create_hand_raster(proj_dem, network_path, temp_hand_dir, hand_raster) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['HAND_RASTER']) # Build Transformation Tables with sqlite3.connect(intermediates_gpkg_path) as conn: cursor = conn.cursor() # Build tables with open( os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'vbet_schema.sql')) as sqlfile: sql_commands = sqlfile.read() cursor.executescript(sql_commands) conn.commit() # Load tables for sqldata in glob.glob(os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'data', '**', '*.sql'), recursive=True): with open(sqldata) as sqlfile: sql_commands = sqlfile.read() cursor.executescript(sql_commands) conn.commit() # Load transforms from table transforms = load_transform_functions(json_transforms, intermediates_gpkg_path) # Get raster resolution as min buffer and apply bankfull width buffer to reaches with rasterio.open(proj_slope) as raster: t = raster.transform min_buffer = (t[0] + abs(t[4])) / 2 log.info("Buffering Polyine by bankfull width buffers") network_path_buffered = os.path.join( intermediates_gpkg_path, LayerTypes['INTERMEDIATES']. sub_layers['VBET_NETWORK_BUFFERED'].rel_path) buffer_by_field(network_path, network_path_buffered, "BFwidth", cfg.OUTPUT_EPSG, min_buffer) # Rasterize the channel polygon and write to raster log.info('Writing channel raster using slope as a template') flow_area_raster = os.path.join(project_folder, LayerTypes['FLOW_AREA_RASTER'].rel_path) channel_buffer_raster = os.path.join( project_folder, LayerTypes['CHANNEL_BUFFER_RASTER'].rel_path) rasterize(network_path_buffered, channel_buffer_raster, proj_slope) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['CHANNEL_BUFFER_RASTER']) rasterize(flowareas_path, flow_area_raster, proj_slope) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['FLOW_AREA_RASTER']) channel_dist_raster = os.path.join(project_folder, LayerTypes['CHANNEL_DISTANCE'].rel_path) fa_dist_raster = os.path.join(project_folder, LayerTypes['FLOW_AREA_DISTANCE'].rel_path) proximity_raster(channel_buffer_raster, channel_dist_raster) proximity_raster(flow_area_raster, fa_dist_raster) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['CHANNEL_DISTANCE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['FLOW_AREA_DISTANCE']) slope_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_SLOPE'].rel_path) hand_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_HAND'].rel_path) chan_dist_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_CHANNEL_DISTANCE'].rel_path) fa_dist_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'].rel_path) topo_evidence_raster = os.path.join(project_folder, LayerTypes['EVIDENCE_TOPO'].rel_path) channel_evidence_raster = os.path.join( project_folder, LayerTypes['EVIDENCE_CHANNEL'].rel_path) evidence_raster = os.path.join(project_folder, LayerTypes['VBET_EVIDENCE'].rel_path) # Open evidence rasters concurrently. We're looping over windows so this shouldn't affect # memory consumption too much with rasterio.open(proj_slope) as slp_src, \ rasterio.open(hand_raster) as hand_src, \ rasterio.open(channel_dist_raster) as cdist_src, \ rasterio.open(fa_dist_raster) as fadist_src: # All 3 rasters should have the same extent and properties. They differ only in dtype out_meta = slp_src.meta # Rasterio can't write back to a VRT so rest the driver and number of bands for the output out_meta['driver'] = 'GTiff' out_meta['count'] = 1 out_meta['compress'] = 'deflate' # out_meta['dtype'] = rasterio.uint8 # We use this to buffer the output cell_size = abs(slp_src.get_transform()[1]) with rasterio.open(evidence_raster, 'w', **out_meta) as dest_evidence, \ rasterio.open(topo_evidence_raster, "w", **out_meta) as dest, \ rasterio.open(channel_evidence_raster, 'w', **out_meta) as dest_channel, \ rasterio.open(slope_transform_raster, "w", **out_meta) as slope_ev_out, \ rasterio.open(hand_transform_raster, 'w', **out_meta) as hand_ev_out, \ rasterio.open(chan_dist_transform_raster, 'w', **out_meta) as chan_dist_ev_out, \ rasterio.open(fa_dist_transform_raster, 'w', **out_meta) as fa_dist_ev_out: progbar = ProgressBar(len(list(slp_src.block_windows(1))), 50, "Calculating evidence layer") counter = 0 # Again, these rasters should be orthogonal so their windows should also line up for _ji, window in slp_src.block_windows(1): progbar.update(counter) counter += 1 slope_data = slp_src.read(1, window=window, masked=True) hand_data = hand_src.read(1, window=window, masked=True) cdist_data = cdist_src.read(1, window=window, masked=True) fadist_data = fadist_src.read(1, window=window, masked=True) slope_transform = np.ma.MaskedArray(transforms["Slope"]( slope_data.data), mask=slope_data.mask) hand_transform = np.ma.MaskedArray(transforms["HAND"]( hand_data.data), mask=hand_data.mask) channel_dist_transform = np.ma.MaskedArray( transforms["Channel"](cdist_data.data), mask=cdist_data.mask) fa_dist_transform = np.ma.MaskedArray(transforms["Flow Areas"]( fadist_data.data), mask=fadist_data.mask) fvals_topo = slope_transform * hand_transform fvals_channel = np.maximum(channel_dist_transform, fa_dist_transform) fvals_evidence = np.maximum(fvals_topo, fvals_channel) # Fill the masked values with the appropriate nodata vals # Unthresholded in the base band (mostly for debugging) dest.write(np.ma.filled(np.float32(fvals_topo), out_meta['nodata']), window=window, indexes=1) slope_ev_out.write(slope_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) hand_ev_out.write(hand_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) chan_dist_ev_out.write( channel_dist_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) fa_dist_ev_out.write( fa_dist_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) dest_channel.write(np.ma.filled(np.float32(fvals_channel), out_meta['nodata']), window=window, indexes=1) dest_evidence.write(np.ma.filled(np.float32(fvals_evidence), out_meta['nodata']), window=window, indexes=1) progbar.finish() # The remaining rasters get added to the project project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_SLOPE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_HAND']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_CHANNEL_DISTANCE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_FLOWAREA_DISTANCE']) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['EVIDENCE_TOPO']) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['EVIDENCE_CHANNEL']) project.add_project_raster(proj_nodes['Outputs'], LayerTypes['VBET_EVIDENCE']) # Get the length of a meter (roughly) degree_factor = GeopackageLayer.rough_convert_metres_to_raster_units( proj_slope, 1) buff_dist = cell_size min_hole_degrees = min_hole_area_m * (degree_factor**2) # Get the full paths to the geopackages intermed_gpkg_path = os.path.join(project_folder, LayerTypes['INTERMEDIATES'].rel_path) vbet_path = os.path.join(project_folder, LayerTypes['VBET_OUTPUTS'].rel_path) for str_val, thr_val in thresh_vals.items(): plgnize_id = 'THRESH_{}'.format(str_val) with TempRaster('vbet_raw_thresh_{}'.format(plgnize_id)) as tmp_raw_thresh, \ TempRaster('vbet_cleaned_thresh_{}'.format(plgnize_id)) as tmp_cleaned_thresh: log.debug('Temporary threshold raster: {}'.format( tmp_raw_thresh.filepath)) threshold(evidence_raster, thr_val, tmp_raw_thresh.filepath) raster_clean(tmp_raw_thresh.filepath, tmp_cleaned_thresh.filepath, buffer_pixels=1) plgnize_lyr = RSLayer('Raw Threshold at {}%'.format(str_val), plgnize_id, 'Vector', plgnize_id.lower()) # Add a project node for this thresholded vector LayerTypes['INTERMEDIATES'].add_sub_layer(plgnize_id, plgnize_lyr) vbet_id = 'VBET_{}'.format(str_val) vbet_lyr = RSLayer('Threshold at {}%'.format(str_val), vbet_id, 'Vector', vbet_id.lower()) # Add a project node for this thresholded vector LayerTypes['VBET_OUTPUTS'].add_sub_layer(vbet_id, vbet_lyr) # Now polygonize the raster log.info('Polygonizing') polygonize( tmp_cleaned_thresh.filepath, 1, '{}/{}'.format(intermed_gpkg_path, plgnize_lyr.rel_path), cfg.OUTPUT_EPSG) log.info('Done') # Now the final sanitization sanitize(str_val, '{}/{}'.format(intermed_gpkg_path, plgnize_lyr.rel_path), '{}/{}'.format(vbet_path, vbet_lyr.rel_path), buff_dist, network_path) log.info('Completed thresholding at {}'.format(thr_val)) # Now add our Geopackages to the project XML project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['VBET_OUTPUTS']) report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path) project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True) report = VBETReport(report_path, project) report.write() log.info('VBET Completed Successfully')
from vbet.vbet_network import vbet_network from vbet.vbet_report import VBETReport from vbet.vbet_raster_ops import rasterize, proximity_raster, translate, raster_clean from vbet.vbet_outputs import threshold, sanitize from vbet.__version__ import __version__ initGDALOGRErrors() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/VBET.xsd', __version__) thresh_vals = {"50": 0.5, "60": 0.6, "70": 0.7, "80": 0.8, "90": 0.9, "100": 1} LayerTypes = { 'DEM': RSLayer('DEM', 'DEM', 'Raster', 'inputs/dem.tif'), 'SLOPE_RASTER': RSLayer('Slope Raster', 'SLOPE_RASTER', 'Raster', 'inputs/slope.tif'), 'HILLSHADE': RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster', 'inputs/dem_hillshade.tif'), 'INPUTS': RSLayer( 'Inputs', 'INPUTS', 'Geopackage', 'inputs/vbet_inputs.gpkg', { 'FLOWLINES': RSLayer('NHD Flowlines', 'FLOWLINES', 'Vector', 'flowlines'), 'FLOW_AREA': RSLayer('NHD Flow Areas', 'FLOW_AREA', 'Vector', 'flow_areas'), }), 'CHANNEL_BUFFER_RASTER': RSLayer('Channel Buffer Raster', 'CHANNEL_BUFFER_RASTER', 'Raster',
__version__) # Dictionary of fields that this process outputs, keyed by ShapeFile data type output_fields = { ogr.OFTInteger: ['RiskID', 'LimitationID', 'OpportunityID'], ogr.OFTReal: [ 'iVeg100EX', 'iVeg_30EX', 'iVeg100HPE', 'iVeg_30HPE', 'iPC_LU', 'iPC_VLowLU', 'iPC_LowLU', 'iPC_ModLU', 'iPC_HighLU', 'iHyd_QLow', 'iHyd_Q2', 'iHyd_SPLow', 'iHyd_SP2', 'oVC_HPE', 'oVC_EX', 'oCC_HPE', 'mCC_HPE_CT', 'oCC_EX', 'mCC_EX_CT', 'mCC_HisDep' ] } LayerTypes = { 'EXVEG_SUIT': RSLayer('Existing Vegetation', 'EXVEG_SUIT', 'Raster', 'intermediates/existing_veg_suitability.tif'), 'HISTVEG_SUIT': RSLayer('Historic Vegetation', 'HISTVEG_SUIT', 'Raster', 'intermediates/historic_veg_suitability.tif'), 'BRAT_RUN_REPORT': RSLayer('BRAT Report', 'BRAT_RUN_REPORT', 'HTMLFile', 'outputs/brat.html') } Epochs = [ # (epoch, prefix, LayerType, OrigId) ('Existing', 'EX', 'EXVEG_SUIT', 'EXVEG'), ('Historic', 'HPE', 'HISTVEG_SUIT', 'HISTVEG') ] def brat_run(project_root, csv_dir):
from rscommons.thiessen.shapes import centerline_points, clip_polygons from rvd.rvd_report import RVDReport from rvd.lib.load_vegetation import load_vegetation_raster from rvd.lib.classify_conversions import classify_conversions from rvd.__version__ import __version__ Path = str initGDALOGRErrors() cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/RVD.xsd', __version__) LayerTypes = { # key: (name, id, tag, relpath) 'EXVEG': RSLayer('Existing Vegetation', 'EXVEG', 'Raster', 'inputs/existing_veg.tif'), 'HISTVEG': RSLayer('Historic Vegetation', 'HISTVEG', 'Raster', 'inputs/historic_veg.tif'), 'INPUTS': RSLayer('Inputs', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', { 'FLOWLINES': RSLayer('Segmented Flowlines', 'FLOWLINES', 'Vector', 'flowlines'), 'FLOW_AREA': RSLayer('NHD Flow Area', 'FLOW_AREA', 'Vector', 'flowareas'), 'WATERBODIES': RSLayer('NHD Waterbody', 'WATERBODIES', 'Vector', 'waterbodies'), 'VALLEY_BOTTOM': RSLayer('Valley Bottom', 'VALLEY_BOTTOM', 'Vector', 'valley_bottom'), }), 'INTERMEDIATES': RSLayer('Intermediates', 'INTERMEDIATES', 'Geopackage', 'intermediates/intermediates.gpkg', { 'THIESSEN': RSLayer('Thiessen Polygons', 'THIESSEN', 'Vector', 'Thiessen'), 'THIESSEN_POINTS': RSLayer('Thiessen Points', 'THIESSEN_POINTS', 'Vector', 'Thiessen_Points'), 'THIESSEN_DISSOLVED': RSLayer('Thiessen Dissolved', 'THIESSEN_DISSOLVED', 'Vector', 'ThiessenPolygonsDissolved'), 'THIESSEN_RAW': RSLayer('Thiessen Raw', 'THIESSEN_RAW', 'Vector', 'ThiessenPolygonsRaw') }), 'VEGETATION_CONVERSION': RSLayer('Vegetation Conversion', 'VEGETATION_CONVERSION', 'Raster', 'intermediates/Conversion_Raster.tif'), 'EXISTING_CONVERSION': RSLayer('Existing Vegetation Conversion', 'EXISTING_CONVERSION', 'Raster', 'intermediates/EXISTING_CONVERSION.tif'),