def main(folder_path):
csv_paths = [
os.path.join(folder_path, f) for f in os.listdir(folder_path)
if f.endswith(".csv")
]
csvs = {os.path.basename(f).split('.')[0]: csv_dict(f) for f in csv_paths}
project, realization = create_project(csvs['Project'][0], folder_path)
make_reaches(folder_path, csvs['Reach'])
make_structures(folder_path, csvs['Structure'])
make_photopoints(folder_path, csvs['Photo'], csvs['PhotoPoint'])
intermediate_node = project.XMLBuilder.add_sub_element(
realization, 'Intermediates')
output_node = project.XMLBuilder.add_sub_element(realization, 'Outputs')
for csv in csv_paths:
relpath = os.path.relpath(csv, folder_path)
name = os.path.basename(relpath).split('.')[0]
nodeid = 'CSV_' + name.upper()
project.add_project_vector(intermediate_node,
RSLayer(name, nodeid, 'CSV', relpath))
project.add_project_vector(output_node, LayerTypes['Photos'])
project.add_project_vector(output_node, LayerTypes['Reaches'])
project.add_project_vector(output_node, LayerTypes['Structures'])
print("done")
def process_lst(lst_xml_folder):
"""This is a slightly hack-y script to create some XMLS for the land_surface_temp script
It's a bit of an afterthought so it just plunks down the XMLS all alone in a folder
Args:
lst_xml_folder ([type]): [description]
"""
log = Logger("Generate XMLS for LST")
hucs = [str(1700 + x) for x in range(1, 13)]
for huc in hucs:
hucdir = os.path.join(lst_xml_folder, huc)
xml_file = os.path.join(hucdir, 'project.rs.xml')
safe_makedirs(hucdir)
if os.path.exists(xml_file):
safe_remove_file(xml_file)
project_name = f'Land Surface Temperature for HUC {huc}'
project = RSProject(cfg, xml_file)
project.create(project_name, 'LST')
project.add_metadata({
'ModelVersion': cfg.version,
'HUC': huc,
'dateCreated': datetime.datetime.now().isoformat(),
'HUC{}'.format(len(huc)): huc
})
realizations = project.XMLBuilder.add_sub_element(
project.XMLBuilder.root, 'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'LST', None, {
'id': 'LST1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
output_node = project.XMLBuilder.add_sub_element(
realization, 'Outputs')
zipfile_node = project.add_dataset(output_node,
f'{huc}.zip',
RSLayer(f'LST Result for {huc}',
'LST_ZIP', 'ZipFile',
'1706.zip'),
'ZipFile',
replace=True,
rel_path=True)
project.XMLBuilder.write()
log.info('done')
Path = str
initGDALOGRErrors()
gdal.UseExceptions()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/Confinement.xsd',
__version__)
LayerTypes = {
# key: (name, id, tag, relpath)]
'INPUTS':
RSLayer(
'Inputs', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', {
'FLOWLINES':
RSLayer('Flowlines', 'FLOWLINES', 'Vector', 'Flowlines'),
'CONFINING_POLYGON':
RSLayer('Confining Polygon', 'CONFINING_POLYGON', 'Vector',
'ConfiningPolygon'),
}),
'INTERMEDIATES':
RSLayer(
'Intermediates', 'INTERMEDIATES', 'Geopackage',
'intermediates/confinement_intermediates.gpkg', {
'SPLIT_POINTS':
RSLayer('Split Points', 'SPLIT_POINTS', 'Vector', 'Split_Points'),
'FLOWLINE_SEGMENTS':
RSLayer('Flowline Segments', 'FLOWLINE_SEGMENTS', 'Vector',
'Flowline_Segments'),
'ERROR_POLYLINES':
RSLayer('Error Polylines', 'ERROR_POLYLINES', 'Vector',
'Error_Polylines'),
from rscommons.util import parse_metadata
from rscommons.build_network import build_network
from rscommons.database import create_database, SQLiteCon
from sqlbrat.utils.vegetation_summary import vegetation_summary
from sqlbrat.utils.reach_geometry import reach_geometry
from sqlbrat.utils.conflict_attributes import conflict_attributes
from sqlbrat.__version__ import __version__
Path = str
initGDALOGRErrors()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/BRAT.xsd', __version__)
LayerTypes = {
'DEM': RSLayer('NED 10m DEM', 'DEM', 'DEM', 'inputs/dem.tif'),
'SLOPE': RSLayer('Slope Raster', 'SLOPE', 'Raster', 'inputs/slope.tif'),
'HILLSHADE': RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster', 'inputs/dem_hillshade.tif'),
'EXVEG': RSLayer('Existing Vegetation', 'EXVEG', 'Raster', 'inputs/existing_veg.tif'),
'HISTVEG': RSLayer('Historic Vegetation', 'HISTVEG', 'Raster', 'inputs/historic_veg.tif'),
'INPUTS': RSLayer('Confinement', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', {
'FLOWLINES': RSLayer('Segmented Flowlines', 'FLOWLINES', 'Vector', 'flowlines'),
'FLOW_AREA': RSLayer('NHD Flow Area', 'FLOW_AREA', 'Vector', 'flowareas'),
'WATERBODIES': RSLayer('NHD Waterbody', 'WATERBODIES', 'Vector', 'waterbodies'),
'VALLEY_BOTTOM': RSLayer('Valley Bottom', 'VALLEY_BOTTOM', 'Vector', 'valley_bottom'),
'ROADS': RSLayer('Roads', 'ROADS', 'Vector', 'roads'),
'RAIL': RSLayer('Rail', 'RAIL', 'Vector', 'rail'),
'CANALS': RSLayer('Canals', 'CANALS', 'Vector', 'canals')
}),
'INTERMEDIATES': RSLayer('Intermediates', 'INTERMEDIATES', 'Geopackage', 'intermediates/intermediates.gpkg', {}),
'OUTPUTS': RSLayer('BRAT', 'OUTPUTS', 'Geopackage', 'outputs/brat.gpkg', {
from osgeo import ogr
from osgeo import gdal
from rscommons import Logger, RSProject, RSLayer, ModelConfig, dotenv, initGDALOGRErrors
from rscommons.util import safe_makedirs, safe_remove_dir
from gnat.__version__ import __version__
initGDALOGRErrors()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/GNAT.xsd',
__version__)
LayerTypes = {
# key: (name, id, tag, relpath)
'DEM': RSLayer('NED 10m DEM', 'DEM', 'DEM', 'topography/dem.tif')
}
def gnat(huc, output_folder):
"""[summary]
Args:
huc ([type]): [description]
Raises:
Exception: [description]
Exception: [description]
Exception: [description]
Exception: [description]
import datetime
import uuid
from rscommons import Logger, ModelConfig, RSProject, RSLayer, dotenv
from rscommons.shapefile import create_field
from rscommons.util import safe_makedirs
from osgeo import gdal, ogr, osr
import csv
from sqlbrat.__version__ import __version__
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/fmLTPBR.xsd',
__version__)
# Don't forgbet to pip install dbfread. I didn't add it to requirements.txt
LayerTypes = {
# key: (name, id, tag, relpath)
'Photos': RSLayer('Photos', 'PHOTO', 'Vector', 'Photos.shp'),
'Reaches': RSLayer('Reaches', 'REACH', 'Vector', 'Reaches.shp'),
'Structures': RSLayer('Structures', 'STRUCT', 'Vector', 'Structures.shp'),
}
REACH_FIELDS = [
# csv_name, shp_name, type, length
["pk", "pk", ogr.OFTString],
["fk_Project", "fkProj", ogr.OFTString],
["Name", "Name", ogr.OFTString],
["StreamName", "StreamName", ogr.OFTString],
["Type", "Type", ogr.OFTString],
["ObjectivePrimary", "ObjPrimary", ogr.OFTString],
["ObjectiveSecondary", "ObjSecond", ogr.OFTString],
["ActiveFloodplainCurrent", "AFPCurrent", ogr.OFTInteger],
["ActiveFloodplainTarget", "AFPTarget", ogr.OFTInteger],
def rs_context(huc, existing_veg, historic_veg, ownership, fair_market,
ecoregions, prism_folder, output_folder, download_folder,
scratch_dir, parallel, force_download, meta: Dict[str, str]):
"""
Download riverscapes context layers for the specified HUC and organize them as a Riverscapes project
:param huc: Eight, 10 or 12 digit HUC identification number
:param existing_veg: Path to the existing vegetation conditions raster
:param historic_veg: Path to the historical vegetation conditions raster
:param ownership: Path to the national land ownership Shapefile
:param output_folder: Output location for the riverscapes context project
:param download_folder: Temporary folder where downloads are cached. This can be shared between rs_context processes
:param force_download: If false then downloads can be skipped if the files already exist
:param prism_folder: folder containing PRISM rasters in *.bil format
:param meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
:return:
"""
log = Logger("RS Context")
log.info('Starting RSContext v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) in [4, 8, 10, 12]):
raise Exception(
'Invalid HUC identifier. Must be 4, 8, 10 or 12 digit integer')
safe_makedirs(output_folder)
safe_makedirs(download_folder)
# We need a temporary folder for slope rasters, Stitching inputs, intermeditary products, etc.
scratch_dem_folder = os.path.join(scratch_dir, 'rs_context', huc)
safe_makedirs(scratch_dem_folder)
project, realization = create_project(huc, output_folder)
hydrology_gpkg_path = os.path.join(output_folder,
LayerTypes['HYDROLOGY'].rel_path)
dem_node, dem_raster = project.add_project_raster(realization,
LayerTypes['DEM'])
_node, hill_raster = project.add_project_raster(realization,
LayerTypes['HILLSHADE'])
_node, flow_accum = project.add_project_raster(realization,
LayerTypes['FA'])
_node, drain_area = project.add_project_raster(realization,
LayerTypes['DA'])
hand_node, hand_raster = project.add_project_raster(
realization, LayerTypes['HAND'])
_node, slope_raster = project.add_project_raster(realization,
LayerTypes['SLOPE'])
_node, existing_clip = project.add_project_raster(realization,
LayerTypes['EXVEG'])
_node, historic_clip = project.add_project_raster(realization,
LayerTypes['HISTVEG'])
_node, fair_market_clip = project.add_project_raster(
realization, LayerTypes['FAIR_MARKET'])
# Download the four digit NHD archive containing the flow lines and watershed boundaries
log.info('Processing NHD')
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
nhd_download_folder = os.path.join(download_folder, 'nhd', huc[:4])
nhd_unzip_folder = os.path.join(scratch_dir, 'nhd', huc[:4])
nhd, db_path, huc_name, nhd_url = clean_nhd_data(
huc, nhd_download_folder, nhd_unzip_folder,
os.path.join(output_folder, 'hydrology'), cfg.OUTPUT_EPSG, False)
# Clean up the unzipped files. We won't need them again
if parallel:
safe_remove_dir(nhd_unzip_folder)
project.add_metadata({'Watershed': huc_name})
boundary = 'WBDHU{}'.format(len(huc))
# For coarser rasters than the DEM we need to buffer our clip polygon to include enough pixels
# This shouldn't be too much more data because these are usually integer rasters that are much lower res.
buffered_clip_path100 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP100'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path100,
epsg=cfg.OUTPUT_EPSG,
buffer=100)
buffered_clip_path500 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path500,
epsg=cfg.OUTPUT_EPSG,
buffer=500)
# PRISM climate rasters
mean_annual_precip = None
bil_files = glob.glob(os.path.join(prism_folder, '**', '*.bil'))
if (len(bil_files) == 0):
raise Exception('Could not find any .bil files in the prism folder')
for ptype in PrismTypes:
try:
# Next should always be guarded
source_raster_path = next(
x for x in bil_files
if ptype.lower() in os.path.basename(x).lower())
except StopIteration:
raise Exception(
'Could not find .bil file corresponding to "{}"'.format(ptype))
_node, project_raster_path = project.add_project_raster(
realization, LayerTypes[ptype])
raster_warp(source_raster_path, project_raster_path, cfg.OUTPUT_EPSG,
buffered_clip_path500, {"cutlineBlend": 1})
# Use the mean annual precipitation to calculate bankfull width
if ptype.lower() == 'ppt':
polygon = get_geometry_unary_union(nhd[boundary],
epsg=cfg.OUTPUT_EPSG)
mean_annual_precip = raster_buffer_stats2(
{1: polygon}, project_raster_path)[1]['Mean']
log.info('Mean annual precipitation for HUC {} is {} mm'.format(
huc, mean_annual_precip))
project.add_metadata(
{'mean_annual_precipitation_mm': str(mean_annual_precip)})
calculate_bankfull_width(nhd['NHDFlowline'], mean_annual_precip)
# Add the DB record to the Project XML
db_lyr = RSLayer('NHD Tables', 'NHDTABLES', 'SQLiteDB',
os.path.relpath(db_path, output_folder))
sqlite_el = project.add_dataset(realization, db_path, db_lyr, 'SQLiteDB')
project.add_metadata({'origin_url': nhd_url}, sqlite_el)
# Add any results to project XML
for name, file_path in nhd.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
vector_nod, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({'origin_url': nhd_url}, vector_nod)
states = get_nhd_states(nhd[boundary])
# Download the NTD archive containing roads and rail
log.info('Processing NTD')
ntd_raw = {}
ntd_unzip_folders = []
ntd_urls = get_ntd_urls(states)
for state, ntd_url in ntd_urls.items():
ntd_download_folder = os.path.join(download_folder, 'ntd',
state.lower())
ntd_unzip_folder = os.path.join(
scratch_dir, 'ntd', state.lower(), 'unzipped'
) # a little awkward but I need a folder for this and this was the best name I could find
ntd_raw[state] = download_shapefile_collection(ntd_url,
ntd_download_folder,
ntd_unzip_folder,
force_download)
ntd_unzip_folders.append(ntd_unzip_folder)
ntd_clean = clean_ntd_data(ntd_raw, nhd['NHDFlowline'], nhd[boundary],
os.path.join(output_folder, 'transportation'),
cfg.OUTPUT_EPSG)
# clean up the NTD Unzip folder. We won't need it again
if parallel:
for unzip_path in ntd_unzip_folders:
safe_remove_dir(unzip_path)
# Write transportation layers to project file
log.info('Write transportation layers to project file')
# Add any results to project XML
for name, file_path in ntd_clean.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
ntd_node, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({**ntd_urls}, ntd_node)
# Download the HAND raster
huc6 = huc[0:6]
hand_download_folder = os.path.join(download_folder, 'hand')
_hpath, hand_url = download_hand(huc6,
cfg.OUTPUT_EPSG,
hand_download_folder,
nhd[boundary],
hand_raster,
warp_options={"cutlineBlend": 1})
project.add_metadata({'origin_url': hand_url}, hand_node)
# download contributing DEM rasters, mosaic and reproject into compressed GeoTIF
ned_download_folder = os.path.join(download_folder, 'ned')
ned_unzip_folder = os.path.join(scratch_dir, 'ned')
dem_rasters, urls = download_dem(nhd[boundary], cfg.OUTPUT_EPSG, 0.01,
ned_download_folder, ned_unzip_folder,
force_download)
need_dem_rebuild = force_download or not os.path.exists(dem_raster)
if need_dem_rebuild:
raster_vrt_stitch(dem_rasters,
dem_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
warp_options={"cutlineBlend": 1})
verify_areas(dem_raster, nhd[boundary])
# Calculate slope rasters seperately and then stitch them
slope_parts = []
hillshade_parts = []
need_slope_build = need_dem_rebuild or not os.path.isfile(slope_raster)
need_hs_build = need_dem_rebuild or not os.path.isfile(hill_raster)
project.add_metadata(
{
'num_rasters': str(len(urls)),
'origin_urls': json.dumps(urls)
}, dem_node)
for dem_r in dem_rasters:
slope_part_path = os.path.join(
scratch_dem_folder,
'SLOPE__' + os.path.basename(dem_r).split('.')[0] + '.tif')
hs_part_path = os.path.join(
scratch_dem_folder,
'HS__' + os.path.basename(dem_r).split('.')[0] + '.tif')
slope_parts.append(slope_part_path)
hillshade_parts.append(hs_part_path)
if force_download or need_dem_rebuild or not os.path.exists(
slope_part_path):
gdal_dem_geographic(dem_r, slope_part_path, 'slope')
need_slope_build = True
if force_download or need_dem_rebuild or not os.path.exists(
hs_part_path):
gdal_dem_geographic(dem_r, hs_part_path, 'hillshade')
need_hs_build = True
if need_slope_build:
raster_vrt_stitch(slope_parts,
slope_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(slope_raster, nhd[boundary])
else:
log.info('Skipping slope build because nothing has changed.')
if need_hs_build:
raster_vrt_stitch(hillshade_parts,
hill_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(hill_raster, nhd[boundary])
else:
log.info('Skipping hillshade build because nothing has changed.')
# Remove the unzipped rasters. We won't need them anymore
if parallel:
safe_remove_dir(ned_unzip_folder)
# Calculate flow accumulation raster based on the DEM
log.info('Running flow accumulation and converting to drainage area.')
flow_accumulation(dem_raster, flow_accum, dinfinity=False, pitfill=True)
flow_accum_to_drainage_area(flow_accum, drain_area)
# Clip and re-project the existing and historic vegetation
log.info('Processing existing and historic vegetation rasters.')
clip_vegetation(buffered_clip_path100, existing_veg, existing_clip,
historic_veg, historic_clip, cfg.OUTPUT_EPSG)
log.info('Process the Fair Market Value Raster.')
raster_warp(fair_market,
fair_market_clip,
cfg.OUTPUT_EPSG,
clip=buffered_clip_path500,
warp_options={"cutlineBlend": 1})
# Clip the landownership Shapefile to a 10km buffer around the watershed boundary
own_path = os.path.join(output_folder, LayerTypes['OWNERSHIP'].rel_path)
project.add_dataset(realization, own_path, LayerTypes['OWNERSHIP'],
'Vector')
clip_ownership(nhd[boundary], ownership, own_path, cfg.OUTPUT_EPSG, 10000)
#######################################################
# Segmentation
#######################################################
# For now let's just make a copy of the NHD FLowlines
tmr = Timer()
rs_segmentation(nhd['NHDFlowline'], ntd_clean['Roads'], ntd_clean['Rail'],
own_path, hydrology_gpkg_path, SEGMENTATION['Max'],
SEGMENTATION['Min'], huc)
log.debug('Segmentation done in {:.1f} seconds'.format(tmr.ellapsed()))
project.add_project_geopackage(realization, LayerTypes['HYDROLOGY'])
# Add Bankfull Buffer Polygons
bankfull_path = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BANKFULL_CHANNEL'].rel_path)
bankfull_buffer(
os.path.join(hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['NETWORK'].rel_path),
cfg.OUTPUT_EPSG,
bankfull_path,
)
# TODO Add nhd/bankfull union when merge feature classes in vector.ops works with Geopackage layers
# bankfull_nhd_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# clip_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
# bankfull_nhd_area(bankfull_path, nhd['NHDArea'], clip_path, cfg.OUTPUT_EPSG, hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# Filter the ecoregions Shapefile to only include attributes that intersect with our HUC
eco_path = os.path.join(output_folder, 'ecoregions', 'ecoregions.shp')
project.add_dataset(realization, eco_path, LayerTypes['ECOREGIONS'],
'Vector')
filter_ecoregions(nhd[boundary], ecoregions, eco_path, cfg.OUTPUT_EPSG,
10000)
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(realization, LayerTypes['REPORT'], replace=True)
report = RSContextReport(report_path, project, output_folder)
report.write()
log.info('Process completed successfully.')
return {
'DEM': dem_raster,
'Slope': slope_raster,
'ExistingVeg': existing_veg,
'HistoricVeg': historic_veg,
'NHD': nhd
}
Instructuions:
1. create a file called .env in the root and add the following line (without the quotes):
"PROJECTFILE=C:/path/to/my/folder/that/will/have/project.rs.xml"
2. Run `BuildXML` from VSCode's Run dropdown menu
"""
# Set up a class to track thigns like default EPSG, xsd validation and tool version
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/Inundation.xsd',
__version__)
# Define the types of layers we're going to use up top so we can re-use them later
# We use our handy RSLayer() class here.
LayerTypes = {
# RSLayer(name, id, tag, rel_path)
'AP_01':
RSLayer('2019 August', 'APORTHO_01', 'Raster',
'01_Inputs/01_Imagery/AP_01/orthomosaic.tif'),
'APSTR_01':
RSLayer('2019 August Flowlines', 'APSTR_01', 'Vector',
'01_Inputs/01_Imagery/AP_01/flowlines.shp'),
}
def build_xml(projectpath):
"""Here's an example of how to build a project.rs.xml file
Args:
projectpath ([type]): [description]
"""
# Create the top-level nodes
log = Logger('build_xml')
log.info('Starting the build of the XML')
from rscontext.vegetation import clip_vegetation
from rscontext.bankfull import bankfull_buffer, bankfull_nhd_area
from rscontext.__version__ import __version__
initGDALOGRErrors()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/RSContext.xsd',
__version__)
# These are the Prism BIL types we expect
PrismTypes = ['PPT', 'TMEAN', 'TMIN', 'TMAX', 'TDMEAN', 'VPDMIN', 'VPDMAX']
LayerTypes = {
# key: (name, id, tag, relpath)
'DEM':
RSLayer('NED 10m DEM', 'DEM', 'DEM', 'topography/dem.tif'),
'FA':
RSLayer('Flow Accumulation', 'FA', 'Raster', 'topography/flow_accum.tif'),
'DA':
RSLayer('Drainage Area', 'DA', 'Raster', 'topography/drainarea_sqkm.tif'),
'HILLSHADE':
RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster',
'topography/dem_hillshade.tif'),
'SLOPE':
RSLayer('Slope', 'SLOPE', 'Raster', 'topography/slope.tif'),
'HAND':
RSLayer('Height above nearest drainage', 'HAND', 'Raster',
'topography/hand.tif'),
# Veg Layers
'EXVEG':
RSLayer('Existing Vegetation', 'EXVEG', 'Raster',
def vbet(huc, flowlines_orig, flowareas_orig, orig_slope, json_transforms,
orig_dem, hillshade, max_hand, min_hole_area_m, project_folder,
reach_codes: List[str], meta: Dict[str, str]):
"""[summary]
Args:
huc ([type]): [description]
flowlines_orig ([type]): [description]
flowareas_orig ([type]): [description]
orig_slope ([type]): [description]
json_transforms ([type]): [description]
orig_dem ([type]): [description]
hillshade ([type]): [description]
max_hand ([type]): [description]
min_hole_area_m ([type]): [description]
project_folder ([type]): [description]
reach_codes (List[int]): NHD reach codes for features to include in outputs
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger('VBET')
log.info('Starting VBET v.{}'.format(cfg.version))
project, _realization, proj_nodes = create_project(huc, project_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
# Copy the inp
_proj_slope_node, proj_slope = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['SLOPE_RASTER'], orig_slope)
_proj_dem_node, proj_dem = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['DEM'], orig_dem)
_hillshade_node, hillshade = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade)
# Copy input shapes to a geopackage
inputs_gpkg_path = os.path.join(project_folder,
LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(
project_folder, LayerTypes['INTERMEDIATES'].rel_path)
flowlines_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
flowareas_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
# Make sure we're starting with a fresh slate of new geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(flowareas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Create a copy of the flow lines with just the perennial and also connectors inside flow areas
network_path = os.path.join(
intermediates_gpkg_path,
LayerTypes['INTERMEDIATES'].sub_layers['VBET_NETWORK'].rel_path)
vbet_network(flowlines_path, flowareas_path, network_path, cfg.OUTPUT_EPSG,
reach_codes)
# Generate HAND from dem and vbet_network
# TODO make a place for this temporary folder. it can be removed after hand is generated.
temp_hand_dir = os.path.join(project_folder, "intermediates",
"hand_processing")
safe_makedirs(temp_hand_dir)
hand_raster = os.path.join(project_folder,
LayerTypes['HAND_RASTER'].rel_path)
create_hand_raster(proj_dem, network_path, temp_hand_dir, hand_raster)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['HAND_RASTER'])
# Build Transformation Tables
with sqlite3.connect(intermediates_gpkg_path) as conn:
cursor = conn.cursor()
# Build tables
with open(
os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'database', 'vbet_schema.sql')) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load tables
for sqldata in glob.glob(os.path.join(
os.path.abspath(os.path.dirname(__file__)), '..', 'database',
'data', '**', '*.sql'),
recursive=True):
with open(sqldata) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load transforms from table
transforms = load_transform_functions(json_transforms,
intermediates_gpkg_path)
# Get raster resolution as min buffer and apply bankfull width buffer to reaches
with rasterio.open(proj_slope) as raster:
t = raster.transform
min_buffer = (t[0] + abs(t[4])) / 2
log.info("Buffering Polyine by bankfull width buffers")
network_path_buffered = os.path.join(
intermediates_gpkg_path, LayerTypes['INTERMEDIATES'].
sub_layers['VBET_NETWORK_BUFFERED'].rel_path)
buffer_by_field(network_path, network_path_buffered, "BFwidth",
cfg.OUTPUT_EPSG, min_buffer)
# Rasterize the channel polygon and write to raster
log.info('Writing channel raster using slope as a template')
flow_area_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_RASTER'].rel_path)
channel_buffer_raster = os.path.join(
project_folder, LayerTypes['CHANNEL_BUFFER_RASTER'].rel_path)
rasterize(network_path_buffered, channel_buffer_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['CHANNEL_BUFFER_RASTER'])
rasterize(flowareas_path, flow_area_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['FLOW_AREA_RASTER'])
channel_dist_raster = os.path.join(project_folder,
LayerTypes['CHANNEL_DISTANCE'].rel_path)
fa_dist_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_DISTANCE'].rel_path)
proximity_raster(channel_buffer_raster, channel_dist_raster)
proximity_raster(flow_area_raster, fa_dist_raster)
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['FLOW_AREA_DISTANCE'])
slope_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_SLOPE'].rel_path)
hand_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_HAND'].rel_path)
chan_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_CHANNEL_DISTANCE'].rel_path)
fa_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'].rel_path)
topo_evidence_raster = os.path.join(project_folder,
LayerTypes['EVIDENCE_TOPO'].rel_path)
channel_evidence_raster = os.path.join(
project_folder, LayerTypes['EVIDENCE_CHANNEL'].rel_path)
evidence_raster = os.path.join(project_folder,
LayerTypes['VBET_EVIDENCE'].rel_path)
# Open evidence rasters concurrently. We're looping over windows so this shouldn't affect
# memory consumption too much
with rasterio.open(proj_slope) as slp_src, \
rasterio.open(hand_raster) as hand_src, \
rasterio.open(channel_dist_raster) as cdist_src, \
rasterio.open(fa_dist_raster) as fadist_src:
# All 3 rasters should have the same extent and properties. They differ only in dtype
out_meta = slp_src.meta
# Rasterio can't write back to a VRT so rest the driver and number of bands for the output
out_meta['driver'] = 'GTiff'
out_meta['count'] = 1
out_meta['compress'] = 'deflate'
# out_meta['dtype'] = rasterio.uint8
# We use this to buffer the output
cell_size = abs(slp_src.get_transform()[1])
with rasterio.open(evidence_raster, 'w', **out_meta) as dest_evidence, \
rasterio.open(topo_evidence_raster, "w", **out_meta) as dest, \
rasterio.open(channel_evidence_raster, 'w', **out_meta) as dest_channel, \
rasterio.open(slope_transform_raster, "w", **out_meta) as slope_ev_out, \
rasterio.open(hand_transform_raster, 'w', **out_meta) as hand_ev_out, \
rasterio.open(chan_dist_transform_raster, 'w', **out_meta) as chan_dist_ev_out, \
rasterio.open(fa_dist_transform_raster, 'w', **out_meta) as fa_dist_ev_out:
progbar = ProgressBar(len(list(slp_src.block_windows(1))), 50,
"Calculating evidence layer")
counter = 0
# Again, these rasters should be orthogonal so their windows should also line up
for _ji, window in slp_src.block_windows(1):
progbar.update(counter)
counter += 1
slope_data = slp_src.read(1, window=window, masked=True)
hand_data = hand_src.read(1, window=window, masked=True)
cdist_data = cdist_src.read(1, window=window, masked=True)
fadist_data = fadist_src.read(1, window=window, masked=True)
slope_transform = np.ma.MaskedArray(transforms["Slope"](
slope_data.data),
mask=slope_data.mask)
hand_transform = np.ma.MaskedArray(transforms["HAND"](
hand_data.data),
mask=hand_data.mask)
channel_dist_transform = np.ma.MaskedArray(
transforms["Channel"](cdist_data.data),
mask=cdist_data.mask)
fa_dist_transform = np.ma.MaskedArray(transforms["Flow Areas"](
fadist_data.data),
mask=fadist_data.mask)
fvals_topo = slope_transform * hand_transform
fvals_channel = np.maximum(channel_dist_transform,
fa_dist_transform)
fvals_evidence = np.maximum(fvals_topo, fvals_channel)
# Fill the masked values with the appropriate nodata vals
# Unthresholded in the base band (mostly for debugging)
dest.write(np.ma.filled(np.float32(fvals_topo),
out_meta['nodata']),
window=window,
indexes=1)
slope_ev_out.write(slope_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
hand_ev_out.write(hand_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
chan_dist_ev_out.write(
channel_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
fa_dist_ev_out.write(
fa_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
dest_channel.write(np.ma.filled(np.float32(fvals_channel),
out_meta['nodata']),
window=window,
indexes=1)
dest_evidence.write(np.ma.filled(np.float32(fvals_evidence),
out_meta['nodata']),
window=window,
indexes=1)
progbar.finish()
# The remaining rasters get added to the project
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_SLOPE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_HAND'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_TOPO'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_CHANNEL'])
project.add_project_raster(proj_nodes['Outputs'],
LayerTypes['VBET_EVIDENCE'])
# Get the length of a meter (roughly)
degree_factor = GeopackageLayer.rough_convert_metres_to_raster_units(
proj_slope, 1)
buff_dist = cell_size
min_hole_degrees = min_hole_area_m * (degree_factor**2)
# Get the full paths to the geopackages
intermed_gpkg_path = os.path.join(project_folder,
LayerTypes['INTERMEDIATES'].rel_path)
vbet_path = os.path.join(project_folder,
LayerTypes['VBET_OUTPUTS'].rel_path)
for str_val, thr_val in thresh_vals.items():
plgnize_id = 'THRESH_{}'.format(str_val)
with TempRaster('vbet_raw_thresh_{}'.format(plgnize_id)) as tmp_raw_thresh, \
TempRaster('vbet_cleaned_thresh_{}'.format(plgnize_id)) as tmp_cleaned_thresh:
log.debug('Temporary threshold raster: {}'.format(
tmp_raw_thresh.filepath))
threshold(evidence_raster, thr_val, tmp_raw_thresh.filepath)
raster_clean(tmp_raw_thresh.filepath,
tmp_cleaned_thresh.filepath,
buffer_pixels=1)
plgnize_lyr = RSLayer('Raw Threshold at {}%'.format(str_val),
plgnize_id, 'Vector', plgnize_id.lower())
# Add a project node for this thresholded vector
LayerTypes['INTERMEDIATES'].add_sub_layer(plgnize_id, plgnize_lyr)
vbet_id = 'VBET_{}'.format(str_val)
vbet_lyr = RSLayer('Threshold at {}%'.format(str_val), vbet_id,
'Vector', vbet_id.lower())
# Add a project node for this thresholded vector
LayerTypes['VBET_OUTPUTS'].add_sub_layer(vbet_id, vbet_lyr)
# Now polygonize the raster
log.info('Polygonizing')
polygonize(
tmp_cleaned_thresh.filepath, 1,
'{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path), cfg.OUTPUT_EPSG)
log.info('Done')
# Now the final sanitization
sanitize(str_val, '{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path),
'{}/{}'.format(vbet_path,
vbet_lyr.rel_path), buff_dist, network_path)
log.info('Completed thresholding at {}'.format(thr_val))
# Now add our Geopackages to the project XML
project.add_project_geopackage(proj_nodes['Intermediates'],
LayerTypes['INTERMEDIATES'])
project.add_project_geopackage(proj_nodes['Outputs'],
LayerTypes['VBET_OUTPUTS'])
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'],
LayerTypes['REPORT'],
replace=True)
report = VBETReport(report_path, project)
report.write()
log.info('VBET Completed Successfully')
from vbet.vbet_network import vbet_network
from vbet.vbet_report import VBETReport
from vbet.vbet_raster_ops import rasterize, proximity_raster, translate, raster_clean
from vbet.vbet_outputs import threshold, sanitize
from vbet.__version__ import __version__
initGDALOGRErrors()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/VBET.xsd',
__version__)
thresh_vals = {"50": 0.5, "60": 0.6, "70": 0.7, "80": 0.8, "90": 0.9, "100": 1}
LayerTypes = {
'DEM':
RSLayer('DEM', 'DEM', 'Raster', 'inputs/dem.tif'),
'SLOPE_RASTER':
RSLayer('Slope Raster', 'SLOPE_RASTER', 'Raster', 'inputs/slope.tif'),
'HILLSHADE':
RSLayer('DEM Hillshade', 'HILLSHADE', 'Raster',
'inputs/dem_hillshade.tif'),
'INPUTS':
RSLayer(
'Inputs', 'INPUTS', 'Geopackage', 'inputs/vbet_inputs.gpkg', {
'FLOWLINES':
RSLayer('NHD Flowlines', 'FLOWLINES', 'Vector', 'flowlines'),
'FLOW_AREA':
RSLayer('NHD Flow Areas', 'FLOW_AREA', 'Vector', 'flow_areas'),
}),
'CHANNEL_BUFFER_RASTER':
RSLayer('Channel Buffer Raster', 'CHANNEL_BUFFER_RASTER', 'Raster',
__version__)
# Dictionary of fields that this process outputs, keyed by ShapeFile data type
output_fields = {
ogr.OFTInteger: ['RiskID', 'LimitationID', 'OpportunityID'],
ogr.OFTReal: [
'iVeg100EX', 'iVeg_30EX', 'iVeg100HPE', 'iVeg_30HPE', 'iPC_LU',
'iPC_VLowLU', 'iPC_LowLU', 'iPC_ModLU', 'iPC_HighLU', 'iHyd_QLow',
'iHyd_Q2', 'iHyd_SPLow', 'iHyd_SP2', 'oVC_HPE', 'oVC_EX', 'oCC_HPE',
'mCC_HPE_CT', 'oCC_EX', 'mCC_EX_CT', 'mCC_HisDep'
]
}
LayerTypes = {
'EXVEG_SUIT':
RSLayer('Existing Vegetation', 'EXVEG_SUIT', 'Raster',
'intermediates/existing_veg_suitability.tif'),
'HISTVEG_SUIT':
RSLayer('Historic Vegetation', 'HISTVEG_SUIT', 'Raster',
'intermediates/historic_veg_suitability.tif'),
'BRAT_RUN_REPORT':
RSLayer('BRAT Report', 'BRAT_RUN_REPORT', 'HTMLFile', 'outputs/brat.html')
}
Epochs = [
# (epoch, prefix, LayerType, OrigId)
('Existing', 'EX', 'EXVEG_SUIT', 'EXVEG'),
('Historic', 'HPE', 'HISTVEG_SUIT', 'HISTVEG')
]
def brat_run(project_root, csv_dir):
from rscommons.thiessen.shapes import centerline_points, clip_polygons
from rvd.rvd_report import RVDReport
from rvd.lib.load_vegetation import load_vegetation_raster
from rvd.lib.classify_conversions import classify_conversions
from rvd.__version__ import __version__
Path = str
initGDALOGRErrors()
cfg = ModelConfig('http://xml.riverscapes.xyz/Projects/XSD/V1/RVD.xsd', __version__)
LayerTypes = {
# key: (name, id, tag, relpath)
'EXVEG': RSLayer('Existing Vegetation', 'EXVEG', 'Raster', 'inputs/existing_veg.tif'),
'HISTVEG': RSLayer('Historic Vegetation', 'HISTVEG', 'Raster', 'inputs/historic_veg.tif'),
'INPUTS': RSLayer('Inputs', 'INPUTS', 'Geopackage', 'inputs/inputs.gpkg', {
'FLOWLINES': RSLayer('Segmented Flowlines', 'FLOWLINES', 'Vector', 'flowlines'),
'FLOW_AREA': RSLayer('NHD Flow Area', 'FLOW_AREA', 'Vector', 'flowareas'),
'WATERBODIES': RSLayer('NHD Waterbody', 'WATERBODIES', 'Vector', 'waterbodies'),
'VALLEY_BOTTOM': RSLayer('Valley Bottom', 'VALLEY_BOTTOM', 'Vector', 'valley_bottom'),
}),
'INTERMEDIATES': RSLayer('Intermediates', 'INTERMEDIATES', 'Geopackage', 'intermediates/intermediates.gpkg', {
'THIESSEN': RSLayer('Thiessen Polygons', 'THIESSEN', 'Vector', 'Thiessen'),
'THIESSEN_POINTS': RSLayer('Thiessen Points', 'THIESSEN_POINTS', 'Vector', 'Thiessen_Points'),
'THIESSEN_DISSOLVED': RSLayer('Thiessen Dissolved', 'THIESSEN_DISSOLVED', 'Vector', 'ThiessenPolygonsDissolved'),
'THIESSEN_RAW': RSLayer('Thiessen Raw', 'THIESSEN_RAW', 'Vector', 'ThiessenPolygonsRaw')
}),
'VEGETATION_CONVERSION': RSLayer('Vegetation Conversion', 'VEGETATION_CONVERSION', 'Raster', 'intermediates/Conversion_Raster.tif'),
'EXISTING_CONVERSION': RSLayer('Existing Vegetation Conversion', 'EXISTING_CONVERSION', 'Raster', 'intermediates/EXISTING_CONVERSION.tif'),