def __init__(self, database, report_path, rs_project):
super().__init__(rs_project, report_path)
self.log = Logger('GNAT Report')
self.database = database
self.images_dir = os.path.join(os.path.dirname(report_path), 'images')
safe_makedirs(self.images_dir)
def download_unzip_nhd(huc, download_folder, unzip_folder, force_download):
try:
nhd_url = get_nhdhr_url(huc[:4])
except Exception:
# Fallback to guess at an address
nhd_url = 'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_GDB.zip'.format(
huc[:4])
safe_makedirs(download_folder)
safe_makedirs(unzip_folder)
nhd_unzip_folder = download_unzip(nhd_url, download_folder, unzip_folder,
force_download)
# get the gdb folder
def matchGdb(fname):
fpath = os.path.join(nhd_unzip_folder, fname)
return os.path.isdir(fpath) and re.match(r"^.*\.gdb", fpath)
try:
filegdb = os.path.join(
nhd_unzip_folder,
next(filter(matchGdb, os.listdir(nhd_unzip_folder))))
except Exception:
raise Exception("Could not find the GDB folder inside: {}".format(
nhd_unzip_folder))
return filegdb, nhd_url
def _create_ds(self):
"""Note: this wipes any existing Datasets (files). It also creates and
opens the new dataset. This makes heavy use of the DatasetRegistry
Raises:
VectorBaseException: [description]
"""
# Make a folder if we need to
ds_dir = os.path.dirname(self.filepath)
if not os.path.isdir(ds_dir):
self.log.debug('Creating directory: {}'.format(ds_dir))
safe_makedirs(ds_dir)
# Wipe the existing dataset if it exists.
if os.path.exists(self.filepath):
self.log.info('Deleting existing dataset: {}'.format(
self.filepath))
self.__ds_reg.delete_dataset(self.filepath, self.driver)
else:
self.log.info('Dataset not found. Creating: {}'.format(
self.filepath))
self.allow_write = True
self.ogr_ds = self.__ds_reg.create(self.filepath, self.ogr_layer_name,
self.driver)
self.log.debug('Dataset created: {}'.format(self.filepath))
def gnat(huc, output_folder):
"""[summary]
Args:
huc ([type]): [description]
Raises:
Exception: [description]
Exception: [description]
Exception: [description]
Exception: [description]
Returns:
[type]: [description]
"""
log = Logger("GNAT")
log.info('GNAT v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
safe_makedirs(output_folder)
def process_lst(lst_xml_folder):
"""This is a slightly hack-y script to create some XMLS for the land_surface_temp script
It's a bit of an afterthought so it just plunks down the XMLS all alone in a folder
Args:
lst_xml_folder ([type]): [description]
"""
log = Logger("Generate XMLS for LST")
hucs = [str(1700 + x) for x in range(1, 13)]
for huc in hucs:
hucdir = os.path.join(lst_xml_folder, huc)
xml_file = os.path.join(hucdir, 'project.rs.xml')
safe_makedirs(hucdir)
if os.path.exists(xml_file):
safe_remove_file(xml_file)
project_name = f'Land Surface Temperature for HUC {huc}'
project = RSProject(cfg, xml_file)
project.create(project_name, 'LST')
project.add_metadata({
'ModelVersion': cfg.version,
'HUC': huc,
'dateCreated': datetime.datetime.now().isoformat(),
'HUC{}'.format(len(huc)): huc
})
realizations = project.XMLBuilder.add_sub_element(
project.XMLBuilder.root, 'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'LST', None, {
'id': 'LST1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
output_node = project.XMLBuilder.add_sub_element(
realization, 'Outputs')
zipfile_node = project.add_dataset(output_node,
f'{huc}.zip',
RSLayer(f'LST Result for {huc}',
'LST_ZIP', 'ZipFile',
'1706.zip'),
'ZipFile',
replace=True,
rel_path=True)
project.XMLBuilder.write()
log.info('done')
def extract_canals(flowlines, epsg, boundary, outpath):
log = Logger('Canals')
driver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = driver.Open(flowlines, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
inLayer.SetAttributeFilter("FCode IN ({0})".format(','.join(
[str(fcode) for fcode in canal_fcodes])))
log.info('{:,} canal features identified in NHD flow lines.'.format(
inLayer.GetFeatureCount()))
if os.path.isfile(outpath):
log.info(
'Skipping extracting canals from NHD flow lines because file exists.'
)
return outpath
extract_path = os.path.dirname(outpath)
safe_makedirs(extract_path)
# Create the output shapefile
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg)
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('network',
outSpatialRef,
geom_type=ogr.wkbMultiLineString)
outLayerDefn = outLayer.GetLayerDefn()
for feature in inLayer:
geom = feature.GetGeometryRef()
geom.Transform(transform)
outFeature = ogr.Feature(outLayerDefn)
outFeature.SetGeometry(geom)
outLayer.CreateFeature(outFeature)
outFeature = None
feature = None
log.info('{:,} features written to canals shapefile.'.format(
outLayer.GetFeatureCount()))
inDataSource = None
outDataSource = None
return outpath
def __init__(self, database, report_path, rs_project):
super().__init__(rs_project, report_path)
self.log = Logger('Confinement Report')
self.database = database
self.images_dir = os.path.join(os.path.dirname(report_path), 'images')
safe_makedirs(self.images_dir)
self.report_intro('Confinement')
# reach_attribute_summary(database, images_dir, inner_div)
self.raw_confinement()
[
self.confinement_ratio(field, field)
for field in ['Confinement_Ratio', 'Constriction_Ratio']
]
def main():
# TODO Add transportation networks to vbet inputs
# TODO Prepare clipped NHD Catchments as vbet polygons input
parser = argparse.ArgumentParser(
description='Floodplain Connectivity (BETA)',
# epilog="This is an epilog"
)
parser.add_argument('vbet_network', help='Vector line network', type=str)
parser.add_argument('vbet_polygon', help='Vector polygon layer', type=str)
parser.add_argument('roads', help='Vector line network', type=str)
parser.add_argument('railroads', help='Vector line network', type=str)
parser.add_argument('output_dir',
help='Folder where output project will be created',
type=str)
parser.add_argument('--debug_gpkg', help='Debug geopackage', type=str)
parser.add_argument('--verbose',
help='(optional) a little extra logging ',
action='store_true',
default=False)
args = dotenv.parse_args_env(parser)
# make sure the output folder exists
safe_makedirs(args.output_dir)
# Initiate the log file
log = Logger('FLOOD_CONN')
log.setup(logPath=os.path.join(args.output_dir,
'floodplain_connectivity.log'),
verbose=args.verbose)
log.title('Floodplain Connectivity (BETA)')
try:
floodplain_connectivity(args.vbet_network, args.vbet_polygon,
args.roads, args.railroads, args.output_dir,
args.debug_gpkg)
except Exception as e:
log.error(e)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def download_hand(huc6,
epsg,
download_folder,
boundary,
raster_path,
force_download=False,
warp_options: dict = {}):
"""
Identify rasters within HUC8, download them and mosaic into single GeoTIF
:param vector_path: Path to bounding polygon ShapeFile
:param epsg: Output spatial reference
:param buffer_dist: Distance in DEGREES to buffer the bounding polygon
:param output_folder: Temporary folder where downloaded rasters will be saved
:param force_download: The download will always be performed if this is true.
:param warp_options: Extra options to pass to raster warp
:return:
"""
log = Logger('HAND')
file_path = os.path.join(download_folder, huc6 + 'hand.tif')
raster_url = 'unknown'
# First check the cache of 6 digit HUC rasters (if it exists)
if os.path.isfile(file_path):
temp_path = file_path
log.info('Hand download file found. Skipping download: {}'.format(
file_path))
else:
safe_makedirs(download_folder)
# Download the HAND raster
raster_url = '/'.join(
s.strip('/') for s in [base_url, huc6, huc6 + 'hand.tif'])
log.info('HAND URL {}'.format(raster_url))
temp_path = download_file(raster_url, download_folder)
# Reproject to the desired SRS and clip to the watershed boundary
raster_warp(temp_path,
raster_path,
epsg,
clip=boundary,
warp_options=warp_options)
return raster_path, raster_url
def __init__(self, database, report_path, rs_project):
# Need to call the constructor of the inherited class:
super().__init__(rs_project, report_path)
self.log = Logger('BratReport')
self.database = database
# The report has a core CSS file but we can extend it with our own if we want:
css_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'brat_report.css')
self.add_css(css_path)
self.images_dir = os.path.join(os.path.dirname(report_path), 'images')
safe_makedirs(self.images_dir)
# Now we just need to write the sections we want in the report in order
self.report_intro()
self.reach_attribute_summary()
self.dam_capacity()
self.hydrology_plots()
self.ownership()
self.vegetation()
self.conservation()
def download_unzip_national(download_folder, unzip_folder, force_download):
nhd_url = 'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip'
safe_makedirs(download_folder)
safe_makedirs(unzip_folder)
nhd_unzip_folder = download_unzip(nhd_url, download_folder, unzip_folder,
force_download)
# get the gdb folder
def matchGdb(fname):
fpath = os.path.join(nhd_unzip_folder, fname)
return os.path.isdir(fpath) and re.match(r"^.*\.gdb", fpath)
try:
filegdb = os.path.join(
nhd_unzip_folder,
next(filter(matchGdb, os.listdir(nhd_unzip_folder))))
except Exception:
raise Exception("Could not find the GDB folder inside: {}".format(
nhd_unzip_folder))
return filegdb
def create_project(huc, output_dir, realization_meta):
project_name = 'Confinement for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'Confinement')
project.add_metadata({'HUC{}'.format(len(huc)): str(huc)})
project.add_metadata({'HUC': str(huc)})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'Confinement', None, {
'id': 'Confinement1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
project.add_metadata(realization_meta)
proj_nodes = {
'Inputs':
project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates':
project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs':
project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
proj_dir = os.path.dirname(project.xml_path)
safe_makedirs(os.path.join(proj_dir, 'inputs'))
safe_makedirs(os.path.join(proj_dir, 'intermediates'))
safe_makedirs(os.path.join(proj_dir, 'outputs'))
report_path = os.path.join(project.project_dir,
LayerTypes['CONFINEMENT_RUN_REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'],
LayerTypes['CONFINEMENT_RUN_REPORT'],
replace=True)
project.XMLBuilder.write()
return project, realization, proj_nodes, report_path
def create_project(huc, output_dir):
project_name = 'VBET for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'VBET')
project.add_metadata({
'HUC{}'.format(len(huc)): str(huc),
'HUC': str(huc),
'VBETVersion': cfg.version,
'VBETTimestamp': str(int(time.time()))
})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'VBET', None, {
'id': 'VBET',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
proj_nodes = {
'Inputs':
project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates':
project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs':
project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
# Make sure we have these folders
proj_dir = os.path.dirname(project.xml_path)
safe_makedirs(os.path.join(proj_dir, 'inputs'))
safe_makedirs(os.path.join(proj_dir, 'intermediates'))
safe_makedirs(os.path.join(proj_dir, 'outputs'))
project.XMLBuilder.write()
return project, realization, proj_nodes
def unzip(file_path, destination_folder, force_overwrite=False, retries=3):
"""[summary]
Args:
file_path: Full path to an existing zip archive
destination_folder: Path where the zip archive will be unzipped
force_overwrite (bool, optional): Force overwrite of a file if it's already there. Defaults to False.
retries (int, optional): Number of retries on a single file. Defaults to 3.
Raises:
Exception: [description]
Exception: [description]
Exception: [description]
"""
log = Logger('Unzipper')
if not os.path.isfile(file_path):
raise Exception('Unzip error: file not found: {}'.format(file_path))
try:
log.info('Attempting unzip: {} ==> {}'.format(file_path, destination_folder))
zip_ref = zipfile.ZipFile(file_path, 'r')
# only unzip files we don't already have
safe_makedirs(destination_folder)
log.info('Extracting: {}'.format(file_path))
# Only unzip things we haven't already unzipped
for fitem in zip_ref.filelist:
uz_success = False
uz_retry = 0
while not uz_success and uz_retry < retries:
try:
outfile = os.path.join(destination_folder, fitem.filename)
if fitem.is_dir():
if not os.path.isdir(outfile):
zip_ref.extract(fitem, destination_folder)
log.debug(' (creating) {}'.format(fitem.filename))
else:
log.debug(' (skipping) {}'.format(fitem.filename))
else:
if force_overwrite or (fitem.file_size > 0 and not os.path.isfile(outfile)) or (os.path.getsize(outfile) / fitem.file_size) < 0.99999:
log.debug(' (unzipping) {}'.format(fitem.filename))
zip_ref.extract(fitem, destination_folder)
else:
log.debug(' (skipping) {}'.format(fitem.filename))
uz_success = True
except Exception as e:
log.debug(e)
log.warning('unzipping file failed. waiting 3 seconds and retrying...')
time.sleep(3)
uz_retry += 1
if (not uz_success):
raise Exception('Unzipping of file {} failed after {} attempts'.format(fitem.filename, retries))
zip_ref.close()
log.info('Done')
except zipfile.BadZipFile as e:
# If the zip file is bad then we have to remove it.
log.error('BadZipFile. Cleaning up zip file and output folder')
safe_remove_file(file_path)
safe_remove_dir(destination_folder)
raise Exception('Unzip error: BadZipFile')
except Exception as e:
log.error('Error unzipping. Cleaning up output folder')
safe_remove_dir(destination_folder)
raise Exception('Unzip error: file could not be unzipped')
def cleaner(inpath, outpath):
driver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = driver.Open(inpath, 0)
inLayer = inDataSource.GetLayer()
geom_type = inLayer.GetGeomType()
inSpatialRef = inLayer.GetSpatialRef()
safe_makedirs(os.path.dirname(outpath))
if os.path.exists(outpath) and os.path.isfile(outpath):
driver.DeleteDataSource(outpath)
# Create the output shapefile
outSpatialRef = osr.SpatialReference()
outSpatialRef.ImportFromWkt(inSpatialRef.ExportToWkt())
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('network',
outSpatialRef,
geom_type=geom_type)
outLayerDefn = outLayer.GetLayerDefn()
# Create new fields in the output shp and get a list of field names for feature creation
fieldNames = []
for i in range(inLayer.GetLayerDefn().GetFieldCount()):
fieldDefn = inLayer.GetLayerDefn().GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
fieldNames.append(fieldDefn.name)
total = inLayer.GetFeatureCount()
counter = 0
nongeom = 0
progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Unioning features")
for feature in inLayer:
progbar.update(counter)
ingeom = feature.GetGeometryRef()
fieldVals = [] # make list of field values for feature
for f in fieldNames:
fieldVals.append(feature.GetField(f))
if ingeom:
outFeature = ogr.Feature(outLayerDefn)
sys.stdout.write(
"\nProcessing feature: {}/{} {}: nongeometrics found. ".
format(counter, total, nongeom))
# sys.stdout.flush()
# Buffer by 0 cleans everything up and makes it nice
sys.stdout.write(' BUFFERING...')
geomBuffer = ingeom.Buffer(0)
sys.stdout.write('SETTING...')
outFeature.SetGeometry(geomBuffer)
sys.stdout.write('SETTING...')
for v, val in enumerate(
fieldVals): # Set output feature attributes
sys.stdout.write('FIELDS...')
outFeature.SetField(fieldNames[v], val)
sys.stdout.write('WRITING...')
outLayer.CreateFeature(outFeature)
outFeature = None
sys.stdout.write('DONE!\n')
else:
# DISCARD features with no geometry
nongeom += 1
counter += 1
progbar.finish()
inDataSource = None
outDataSource = None
def intersect_geometry_with_feature_class(geometry, feature_class, epsg,
out_path, output_geom_type):
if output_geom_type not in [ogr.wkbMultiPoint, ogr.wkbMultiLineString]:
raise Exception('Unsupported ogr type: "{}"'.format(output_geom_type))
# Remove output shapefile if it already exists
driver = ogr.GetDriverByName("ESRI Shapefile")
if os.path.exists(out_path):
driver.DeleteDataSource(out_path)
else:
# Make sure the output folder exists
safe_makedirs(os.path.dirname(out_path))
# Create the output shapefile
data_source = driver.CreateDataSource(out_path)
spatial_ref = osr.SpatialReference()
spatial_ref.ImportFromEPSG(epsg)
layer = data_source.CreateLayer('intersection',
spatial_ref,
geom_type=output_geom_type)
geom_union = get_geometry_unary_union(feature_class, epsg)
# Nothing to do if there were no features in the feature class
if not geom_union:
data_source = None
return
geom_inter = geometry.intersection(geom_union)
# Nothing to do if the intersection is empty
if geom_inter.is_empty:
data_source = None
return
# Single features and collections need to be converted into Multi-features
if output_geom_type == ogr.wkbMultiPoint and not isinstance(
geom_inter, MultiPoint):
if isinstance(geom_inter, Point):
geom_inter = MultiPoint([(geom_inter)])
elif isinstance(geom_inter, LineString):
# Break this linestring down into vertices as points
geom_inter = MultiPoint(list(geom_inter.coords))
elif isinstance(geom_inter, MultiLineString):
# Break this linestring down into vertices as points
geom_inter = MultiPoint(
reduce(lambda acc, ls: acc + list(ls.coords),
list(geom_inter.geoms), []))
elif isinstance(geom_inter, GeometryCollection):
geom_inter = MultiPoint(
[geom for geom in geom_inter.geoms if isinstance(geom, Point)])
elif output_geom_type == ogr.wkbMultiLineString and not isinstance(
geom_inter, MultiLineString):
if isinstance(geom_inter, LineString):
geom_inter = MultiLineString([(geom_inter)])
else:
raise Exception(
'Unsupported ogr type: "{}" does not match shapely type of "{}"'
.format(output_geom_type, geom_inter.type))
out_layer_def = layer.GetLayerDefn()
feature = ogr.Feature(out_layer_def)
feature.SetGeometry(ogr.CreateGeometryFromWkb(geom_inter.wkb))
layer.CreateFeature(feature)
data_source = None
def download_file(s3_url, download_folder, force_download=False):
"""
Download a file given a HTTPS URL that points to a file on S3
:param s3_url: HTTPS URL for a file on S3
:param download_folder: Folder where the file will be downloaded.
:param force_download:
:return: Local file path where the file was downloaded
"""
log = Logger('Download')
safe_makedirs(download_folder)
# Retrieve the S3 bucket and path from the HTTPS URL
result = re.match(r'https://([^.]+)[^/]+/(.*)', s3_url)
# If file already exists and forcing download then ensure unique file name
file_path = os.path.join(download_folder, os.path.basename(result.group(2)))
file_path_pending = os.path.join(download_folder, os.path.basename(result.group(2)) + '.pending')
if os.path.isfile(file_path) and force_download:
safe_remove_file(file_path)
# If there is a pending path and the pending path is fairly new
# then wait for it.
while pending_check(file_path_pending, PENDING_TIMEOUT):
log.debug('Waiting for .pending file. Another process is working on this.')
time.sleep(30)
log.info('Waiting done. Proceeding.')
# Skip the download if the file exists
if os.path.isfile(file_path) and os.path.getsize(file_path) > 0:
log.info('Skipping download because file exists.')
else:
_file, tmpfilepath = tempfile.mkstemp(suffix=".temp", prefix="rstools_download")
# Write our pending file. No matter what we must clean this file up!!!
def refresh_pending(init=False):
with open(file_path_pending, 'w') as f:
f.write(str(datetime.datetime.now()))
# Cleaning up the commone areas is really important
def download_cleanup():
os.close(_file)
safe_remove_file(tmpfilepath)
safe_remove_file(file_path_pending)
refresh_pending()
pending_timer = Timer()
log.info('Downloading {}'.format(s3_url))
# Actual file download
for download_retries in range(MAX_ATTEMPTS):
if download_retries > 0:
log.warning('Download file retry: {}'.format(download_retries))
try:
dl = 0
_file, tmpfilepath = tempfile.mkstemp(suffix=".temp", prefix="rstools_download")
with requests.get(s3_url, stream=True) as r:
r.raise_for_status()
byte_total = int(r.headers.get('content-length'))
progbar = ProgressBar(byte_total, 50, s3_url, byteFormat=True)
# Binary write to file
with open(tmpfilepath, 'wb') as tempf:
for chunk in r.iter_content(chunk_size=8192):
# Periodically refreshing our .pending file
# so other processes will be aware we are still working on it.
if pending_timer.ellapsed() > 10:
refresh_pending()
if chunk: # filter out keep-alive new chunks
dl += len(chunk)
tempf.write(chunk)
progbar.update(dl)
# Close the temporary file. It will be removed
if (not os.path.isfile(tmpfilepath)):
raise Exception('Error writing to temporary file: {}'.format(tmpfilepath))
progbar.finish()
break
except Exception as e:
log.debug('Error downloading file from s3 {}: \n{}'.format(s3_url, str(e)))
# if this is our last chance then the function must fail [0,1,2]
if download_retries == MAX_ATTEMPTS - 1:
download_cleanup() # Always clean up
raise e
# Now copy the temporary file (retry 3 times)
for copy_retries in range(MAX_ATTEMPTS):
if copy_retries > 0:
log.warning('Copy file retry: {}'.format(copy_retries))
try:
shutil.copy(tmpfilepath, file_path)
# Make sure to clean up so the next process doesn't encounter a broken file
if not file_compare(file_path, tmpfilepath):
raise Exception('Error copying temporary download to final path')
break
except Exception as e:
log.debug('Error copying file from temporary location {}: \n{}'.format(tmpfilepath, str(e)))
# if this is our last chance then the function must fail [0,1,2]
if copy_retries == MAX_ATTEMPTS - 1:
download_cleanup() # Always clean up
raise e
download_cleanup() # Always clean up
return file_path
def make_photopoints(folder_path, photos, photo_pts):
log = Logger('Shapefile')
outpath = os.path.join(folder_path, LayerTypes['Photos'].rel_path)
safe_makedirs(os.path.dirname(outpath))
driver = ogr.GetDriverByName("ESRI Shapefile")
if os.path.exists(outpath):
driver.DeleteDataSource(outpath)
# Create the output shapefile
outSpatialRef = osr.SpatialReference()
outSpatialRef.ImportFromEPSG(int(cfg.OUTPUT_EPSG))
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('photos',
outSpatialRef,
geom_type=ogr.wkbPoint)
# Add the fields we're interested in
for field in PHOTO_FIELDS:
csv_name = field[0]
shp_name = field[1]
ogr_type = field[2]
fld = ogr.FieldDefn(shp_name, ogr_type)
if ogr_type == ogr.OFTString and len(field) > 3:
fld.SetWidth(field[3])
outLayer.CreateField(fld)
for p in photos:
photo_point = [
phpt for phpt in photo_pts if phpt['pk'] == p['fk_PhotoPoint']
][0].copy()
# Pull some fields from the other record
photo_point['pk'] = p['pk']
photo_point['PhotoDate'] = p['PhotoDate']
photo_point['PhotoNotes'] = p['PhotoNotes']
photo_point['Path'] = 'Photo/{}.jpg'.format(photo_point['pk'])
if not os.path.isfile(os.path.join(folder_path, photo_point['Path'])):
raise Exception('file not found: {}'.format(photo_point['Path']))
feature = ogr.Feature(outLayer.GetLayerDefn())
for field in PHOTO_FIELDS:
if photo_point[csv_name]:
# Set the attributes using the values from the delimited text file
if ogr_type == ogr.OFTInteger:
feature.SetField(shp_name, int(photo_point[csv_name]))
elif ogr_type == ogr.OFTReal:
feature.SetField(shp_name, float(photo_point[csv_name]))
elif ogr_type == ogr.OFTString:
feature.SetField(shp_name, photo_point[csv_name])
# create the WKT for the feature using Python string formatting
pt = ogr.Geometry(ogr.wkbPoint)
pt.AddPoint(float(photo_point['Longitude']),
float(photo_point['Latitude']))
# Set the feature geometry using the point
feature.SetGeometry(pt)
# Create the feature in the layer (shapefile)
outLayer.CreateFeature(feature)
# Dereference the feature
feature = None
huc4 = row[0][0:4]
if huc4 not in hucs:
hucs[huc4] = {}
hucs[huc4][row[0]] = {}
locations = []
with open(usu_csv, 'r') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
huc8 = row['HUC8Dir'].split('_')[1]
huc4 = huc8[0:4]
if huc4 in hucs:
if huc8 in hucs[huc4]:
hucs[huc4][huc8]['USU'] = float(row['PRECIP_IN']) * 25.4
safe_makedirs(temp_folder)
driver = ogr.GetDriverByName("OpenFileGDB")
chart_values = []
for huc4, huc8s in hucs.items():
print('Processing', huc4)
unzip_folder = os.path.join(temp_folder, huc4)
try:
nhd_url = get_nhdhr_url(huc4)
nhd_unzip_folder = download_unzip(nhd_url, temp_folder, unzip_folder, False)
except Exception as e:
print('WARNING: Failed to download', huc4)
continue
folder_name = os.path.basename(nhd_url).split('.')[0]
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('projectpath',
help='NHD flow line ShapeFile path',
type=str)
parser.add_argument('--verbose',
help='(optional) a little extra logging ',
action='store_true',
default=False)
args = dotenv.parse_args_env(parser)
if args.projectpath is None or len(args.projectpath) < 10:
raise Exception('projectpath has invalid value')
safe_makedirs(args.projectpath)
# Initiate the log file
log = Logger('Inundation XML')
log.setup(logPath=os.path.join(args.projectpath, 'Inundation.log'),
verbose=args.verbose)
try:
log.info('Starting')
build_xml(args.projectpath)
edit_xml(args.projectpath)
log.info('Exiting')
except Exception as e:
log.error(e)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def vbet(huc, flowlines_orig, flowareas_orig, orig_slope, json_transforms,
orig_dem, hillshade, max_hand, min_hole_area_m, project_folder,
reach_codes: List[str], meta: Dict[str, str]):
"""[summary]
Args:
huc ([type]): [description]
flowlines_orig ([type]): [description]
flowareas_orig ([type]): [description]
orig_slope ([type]): [description]
json_transforms ([type]): [description]
orig_dem ([type]): [description]
hillshade ([type]): [description]
max_hand ([type]): [description]
min_hole_area_m ([type]): [description]
project_folder ([type]): [description]
reach_codes (List[int]): NHD reach codes for features to include in outputs
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger('VBET')
log.info('Starting VBET v.{}'.format(cfg.version))
project, _realization, proj_nodes = create_project(huc, project_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
# Copy the inp
_proj_slope_node, proj_slope = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['SLOPE_RASTER'], orig_slope)
_proj_dem_node, proj_dem = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['DEM'], orig_dem)
_hillshade_node, hillshade = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade)
# Copy input shapes to a geopackage
inputs_gpkg_path = os.path.join(project_folder,
LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(
project_folder, LayerTypes['INTERMEDIATES'].rel_path)
flowlines_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
flowareas_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
# Make sure we're starting with a fresh slate of new geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(flowareas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Create a copy of the flow lines with just the perennial and also connectors inside flow areas
network_path = os.path.join(
intermediates_gpkg_path,
LayerTypes['INTERMEDIATES'].sub_layers['VBET_NETWORK'].rel_path)
vbet_network(flowlines_path, flowareas_path, network_path, cfg.OUTPUT_EPSG,
reach_codes)
# Generate HAND from dem and vbet_network
# TODO make a place for this temporary folder. it can be removed after hand is generated.
temp_hand_dir = os.path.join(project_folder, "intermediates",
"hand_processing")
safe_makedirs(temp_hand_dir)
hand_raster = os.path.join(project_folder,
LayerTypes['HAND_RASTER'].rel_path)
create_hand_raster(proj_dem, network_path, temp_hand_dir, hand_raster)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['HAND_RASTER'])
# Build Transformation Tables
with sqlite3.connect(intermediates_gpkg_path) as conn:
cursor = conn.cursor()
# Build tables
with open(
os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'database', 'vbet_schema.sql')) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load tables
for sqldata in glob.glob(os.path.join(
os.path.abspath(os.path.dirname(__file__)), '..', 'database',
'data', '**', '*.sql'),
recursive=True):
with open(sqldata) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load transforms from table
transforms = load_transform_functions(json_transforms,
intermediates_gpkg_path)
# Get raster resolution as min buffer and apply bankfull width buffer to reaches
with rasterio.open(proj_slope) as raster:
t = raster.transform
min_buffer = (t[0] + abs(t[4])) / 2
log.info("Buffering Polyine by bankfull width buffers")
network_path_buffered = os.path.join(
intermediates_gpkg_path, LayerTypes['INTERMEDIATES'].
sub_layers['VBET_NETWORK_BUFFERED'].rel_path)
buffer_by_field(network_path, network_path_buffered, "BFwidth",
cfg.OUTPUT_EPSG, min_buffer)
# Rasterize the channel polygon and write to raster
log.info('Writing channel raster using slope as a template')
flow_area_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_RASTER'].rel_path)
channel_buffer_raster = os.path.join(
project_folder, LayerTypes['CHANNEL_BUFFER_RASTER'].rel_path)
rasterize(network_path_buffered, channel_buffer_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['CHANNEL_BUFFER_RASTER'])
rasterize(flowareas_path, flow_area_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['FLOW_AREA_RASTER'])
channel_dist_raster = os.path.join(project_folder,
LayerTypes['CHANNEL_DISTANCE'].rel_path)
fa_dist_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_DISTANCE'].rel_path)
proximity_raster(channel_buffer_raster, channel_dist_raster)
proximity_raster(flow_area_raster, fa_dist_raster)
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['FLOW_AREA_DISTANCE'])
slope_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_SLOPE'].rel_path)
hand_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_HAND'].rel_path)
chan_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_CHANNEL_DISTANCE'].rel_path)
fa_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'].rel_path)
topo_evidence_raster = os.path.join(project_folder,
LayerTypes['EVIDENCE_TOPO'].rel_path)
channel_evidence_raster = os.path.join(
project_folder, LayerTypes['EVIDENCE_CHANNEL'].rel_path)
evidence_raster = os.path.join(project_folder,
LayerTypes['VBET_EVIDENCE'].rel_path)
# Open evidence rasters concurrently. We're looping over windows so this shouldn't affect
# memory consumption too much
with rasterio.open(proj_slope) as slp_src, \
rasterio.open(hand_raster) as hand_src, \
rasterio.open(channel_dist_raster) as cdist_src, \
rasterio.open(fa_dist_raster) as fadist_src:
# All 3 rasters should have the same extent and properties. They differ only in dtype
out_meta = slp_src.meta
# Rasterio can't write back to a VRT so rest the driver and number of bands for the output
out_meta['driver'] = 'GTiff'
out_meta['count'] = 1
out_meta['compress'] = 'deflate'
# out_meta['dtype'] = rasterio.uint8
# We use this to buffer the output
cell_size = abs(slp_src.get_transform()[1])
with rasterio.open(evidence_raster, 'w', **out_meta) as dest_evidence, \
rasterio.open(topo_evidence_raster, "w", **out_meta) as dest, \
rasterio.open(channel_evidence_raster, 'w', **out_meta) as dest_channel, \
rasterio.open(slope_transform_raster, "w", **out_meta) as slope_ev_out, \
rasterio.open(hand_transform_raster, 'w', **out_meta) as hand_ev_out, \
rasterio.open(chan_dist_transform_raster, 'w', **out_meta) as chan_dist_ev_out, \
rasterio.open(fa_dist_transform_raster, 'w', **out_meta) as fa_dist_ev_out:
progbar = ProgressBar(len(list(slp_src.block_windows(1))), 50,
"Calculating evidence layer")
counter = 0
# Again, these rasters should be orthogonal so their windows should also line up
for _ji, window in slp_src.block_windows(1):
progbar.update(counter)
counter += 1
slope_data = slp_src.read(1, window=window, masked=True)
hand_data = hand_src.read(1, window=window, masked=True)
cdist_data = cdist_src.read(1, window=window, masked=True)
fadist_data = fadist_src.read(1, window=window, masked=True)
slope_transform = np.ma.MaskedArray(transforms["Slope"](
slope_data.data),
mask=slope_data.mask)
hand_transform = np.ma.MaskedArray(transforms["HAND"](
hand_data.data),
mask=hand_data.mask)
channel_dist_transform = np.ma.MaskedArray(
transforms["Channel"](cdist_data.data),
mask=cdist_data.mask)
fa_dist_transform = np.ma.MaskedArray(transforms["Flow Areas"](
fadist_data.data),
mask=fadist_data.mask)
fvals_topo = slope_transform * hand_transform
fvals_channel = np.maximum(channel_dist_transform,
fa_dist_transform)
fvals_evidence = np.maximum(fvals_topo, fvals_channel)
# Fill the masked values with the appropriate nodata vals
# Unthresholded in the base band (mostly for debugging)
dest.write(np.ma.filled(np.float32(fvals_topo),
out_meta['nodata']),
window=window,
indexes=1)
slope_ev_out.write(slope_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
hand_ev_out.write(hand_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
chan_dist_ev_out.write(
channel_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
fa_dist_ev_out.write(
fa_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
dest_channel.write(np.ma.filled(np.float32(fvals_channel),
out_meta['nodata']),
window=window,
indexes=1)
dest_evidence.write(np.ma.filled(np.float32(fvals_evidence),
out_meta['nodata']),
window=window,
indexes=1)
progbar.finish()
# The remaining rasters get added to the project
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_SLOPE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_HAND'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_TOPO'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_CHANNEL'])
project.add_project_raster(proj_nodes['Outputs'],
LayerTypes['VBET_EVIDENCE'])
# Get the length of a meter (roughly)
degree_factor = GeopackageLayer.rough_convert_metres_to_raster_units(
proj_slope, 1)
buff_dist = cell_size
min_hole_degrees = min_hole_area_m * (degree_factor**2)
# Get the full paths to the geopackages
intermed_gpkg_path = os.path.join(project_folder,
LayerTypes['INTERMEDIATES'].rel_path)
vbet_path = os.path.join(project_folder,
LayerTypes['VBET_OUTPUTS'].rel_path)
for str_val, thr_val in thresh_vals.items():
plgnize_id = 'THRESH_{}'.format(str_val)
with TempRaster('vbet_raw_thresh_{}'.format(plgnize_id)) as tmp_raw_thresh, \
TempRaster('vbet_cleaned_thresh_{}'.format(plgnize_id)) as tmp_cleaned_thresh:
log.debug('Temporary threshold raster: {}'.format(
tmp_raw_thresh.filepath))
threshold(evidence_raster, thr_val, tmp_raw_thresh.filepath)
raster_clean(tmp_raw_thresh.filepath,
tmp_cleaned_thresh.filepath,
buffer_pixels=1)
plgnize_lyr = RSLayer('Raw Threshold at {}%'.format(str_val),
plgnize_id, 'Vector', plgnize_id.lower())
# Add a project node for this thresholded vector
LayerTypes['INTERMEDIATES'].add_sub_layer(plgnize_id, plgnize_lyr)
vbet_id = 'VBET_{}'.format(str_val)
vbet_lyr = RSLayer('Threshold at {}%'.format(str_val), vbet_id,
'Vector', vbet_id.lower())
# Add a project node for this thresholded vector
LayerTypes['VBET_OUTPUTS'].add_sub_layer(vbet_id, vbet_lyr)
# Now polygonize the raster
log.info('Polygonizing')
polygonize(
tmp_cleaned_thresh.filepath, 1,
'{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path), cfg.OUTPUT_EPSG)
log.info('Done')
# Now the final sanitization
sanitize(str_val, '{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path),
'{}/{}'.format(vbet_path,
vbet_lyr.rel_path), buff_dist, network_path)
log.info('Completed thresholding at {}'.format(thr_val))
# Now add our Geopackages to the project XML
project.add_project_geopackage(proj_nodes['Intermediates'],
LayerTypes['INTERMEDIATES'])
project.add_project_geopackage(proj_nodes['Outputs'],
LayerTypes['VBET_OUTPUTS'])
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'],
LayerTypes['REPORT'],
replace=True)
report = VBETReport(report_path, project)
report.write()
log.info('VBET Completed Successfully')
def main():
parser = argparse.ArgumentParser(description='Riverscapes Context Tool',
# epilog="This is an epilog"
)
parser.add_argument('huc', help='HUC identifier', type=str)
parser.add_argument('existing',
help='National existing vegetation raster',
type=str)
parser.add_argument('historic',
help='National historic vegetation raster',
type=str)
parser.add_argument('ownership',
help='National land ownership shapefile',
type=str)
parser.add_argument('fairmarket',
help='National fair market value raster',
type=str)
parser.add_argument('ecoregions',
help='National EcoRegions shapefile',
type=str)
parser.add_argument('prism',
help='Folder containing PRISM rasters in BIL format',
type=str)
parser.add_argument('output', help='Path to the output folder', type=str)
parser.add_argument(
'download',
help=
'Temporary folder for downloading data. Different HUCs may share this',
type=str)
parser.add_argument('--force',
help='(optional) download existing files ',
action='store_true',
default=False)
parser.add_argument(
'--parallel',
help=
'(optional) for running multiple instances of this at the same time',
action='store_true',
default=False)
parser.add_argument('--temp_folder',
help='(optional) cache folder for downloading files ',
type=str)
parser.add_argument(
'--meta',
help='riverscapes project metadata as comma separated key=value pairs',
type=str)
parser.add_argument('--verbose',
help='(optional) a little extra logging ',
action='store_true',
default=False)
parser.add_argument(
'--debug',
help=
'(optional) more output about things like memory usage. There is a performance cost',
action='store_true',
default=False)
args = dotenv.parse_args_env(parser)
# Initiate the log file
log = Logger("RS Context")
log.setup(logPath=os.path.join(args.output, "rs_context.log"),
verbose=args.verbose)
log.title('Riverscapes Context For HUC: {}'.format(args.huc))
log.info('HUC: {}'.format(args.huc))
log.info('EPSG: {}'.format(cfg.OUTPUT_EPSG))
log.info('Existing veg: {}'.format(args.existing))
log.info('Historical veg: {}'.format(args.historic))
log.info('Ownership: {}'.format(args.ownership))
log.info('Fair Market Value Raster: {}'.format(args.fairmarket))
log.info('Output folder: {}'.format(args.output))
log.info('Download folder: {}'.format(args.download))
log.info('Force download: {}'.format(args.force))
# This is a general place for unzipping downloaded files and other temporary work.
# We use GUIDS to make it specific to a particular run of the tool to avoid unzip collisions
parallel_code = "-" + str(uuid.uuid4()) if args.parallel is True else ""
scratch_dir = args.temp_folder if args.temp_folder else os.path.join(
args.download, 'scratch', 'rs_context{}'.format(parallel_code))
safe_makedirs(scratch_dir)
meta = parse_metadata(args.meta)
try:
if args.debug is True:
from rscommons.debug import ThreadRun
memfile = os.path.join(args.output, 'rs_context_memusage.log')
retcode, max_obj = ThreadRun(
rs_context, memfile, args.huc, args.existing, args.historic,
args.ownership, args.fairmarket, args.ecoregions, args.prism,
args.output, args.download, scratch_dir, args.parallel,
args.force, meta)
log.debug('Return code: {}, [Max process usage] {}'.format(
retcode, max_obj))
else:
rs_context(args.huc, args.existing, args.historic, args.ownership,
args.fairmarket, args.ecoregions, args.prism,
args.output, args.download, scratch_dir, args.parallel,
args.force, meta)
except Exception as e:
log.error(e)
traceback.print_exc(file=sys.stdout)
# Cleaning up the scratch folder is essential
safe_remove_dir(scratch_dir)
sys.exit(1)
# Cleaning up the scratch folder is essential
safe_remove_dir(scratch_dir)
sys.exit(0)
def merge_feature_classes(feature_classes, epsg, boundary, outpath):
log = Logger('Shapefile')
if os.path.isfile(outpath):
log.info('Skipping merging feature classes because file exists.')
return
safe_makedirs(os.path.dirname(outpath))
log.info('Merging {} feature classes.'.format(len(feature_classes)))
driver = ogr.GetDriverByName("ESRI Shapefile")
# Create the output shapefile
outDataSource = driver.CreateDataSource(outpath)
outLayer = None
outSpatialRef = None
transform = None
fccount = 0
for inpath in feature_classes:
fccount += 1
log.info("Merging feature class {}/{}".format(fccount,
len(feature_classes)))
inDataSource = driver.Open(inpath, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
inLayer.SetSpatialFilter(ogr.CreateGeometryFromWkb(boundary.wkb))
# First input spatial ref sets the SRS for the output file
outSpatialRefTmp, transform = get_transform_from_epsg(
inSpatialRef, epsg)
if outLayer is None:
outSpatialRef = outSpatialRefTmp
outLayer = outDataSource.CreateLayer(
'network', outSpatialRef, geom_type=ogr.wkbMultiLineString)
outLayerDefn = outLayer.GetLayerDefn()
# Transfer fields over
inLayerDef = inLayer.GetLayerDefn()
for i in range(inLayerDef.GetFieldCount()):
inFieldDef = inLayerDef.GetFieldDefn(i)
# Only create fields if we really don't have them
# NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE
if outLayerDefn.GetFieldIndex(inFieldDef.GetName()) == -1:
outLayer.CreateField(inFieldDef)
progbar = ProgressBar(inLayer.GetFeatureCount(), 50,
"Processing features")
outLayerDefn = outLayer.GetLayerDefn()
counter = 0
for feature in inLayer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feature.GetFID()))
continue
geom.Transform(transform)
# get a Shapely representation of the line
# featobj = json.loads(geom.ExportToJson())
# polyline = shape(featobj)
# if boundary.intersects(polyline):
# clipped = boundary.intersection(polyline)
outFeature = ogr.Feature(outLayerDefn)
for i in range(inLayerDef.GetFieldCount()):
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
feature.GetField(i))
outFeature.SetGeometry(geom)
outLayer.CreateFeature(outFeature)
feature = None
outFeature = None
progbar.finish()
inDataSource = None
outDataSource = None
log.info('Merge complete.')
def rs_context(huc, existing_veg, historic_veg, ownership, fair_market,
ecoregions, prism_folder, output_folder, download_folder,
scratch_dir, parallel, force_download, meta: Dict[str, str]):
"""
Download riverscapes context layers for the specified HUC and organize them as a Riverscapes project
:param huc: Eight, 10 or 12 digit HUC identification number
:param existing_veg: Path to the existing vegetation conditions raster
:param historic_veg: Path to the historical vegetation conditions raster
:param ownership: Path to the national land ownership Shapefile
:param output_folder: Output location for the riverscapes context project
:param download_folder: Temporary folder where downloads are cached. This can be shared between rs_context processes
:param force_download: If false then downloads can be skipped if the files already exist
:param prism_folder: folder containing PRISM rasters in *.bil format
:param meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
:return:
"""
log = Logger("RS Context")
log.info('Starting RSContext v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) in [4, 8, 10, 12]):
raise Exception(
'Invalid HUC identifier. Must be 4, 8, 10 or 12 digit integer')
safe_makedirs(output_folder)
safe_makedirs(download_folder)
# We need a temporary folder for slope rasters, Stitching inputs, intermeditary products, etc.
scratch_dem_folder = os.path.join(scratch_dir, 'rs_context', huc)
safe_makedirs(scratch_dem_folder)
project, realization = create_project(huc, output_folder)
hydrology_gpkg_path = os.path.join(output_folder,
LayerTypes['HYDROLOGY'].rel_path)
dem_node, dem_raster = project.add_project_raster(realization,
LayerTypes['DEM'])
_node, hill_raster = project.add_project_raster(realization,
LayerTypes['HILLSHADE'])
_node, flow_accum = project.add_project_raster(realization,
LayerTypes['FA'])
_node, drain_area = project.add_project_raster(realization,
LayerTypes['DA'])
hand_node, hand_raster = project.add_project_raster(
realization, LayerTypes['HAND'])
_node, slope_raster = project.add_project_raster(realization,
LayerTypes['SLOPE'])
_node, existing_clip = project.add_project_raster(realization,
LayerTypes['EXVEG'])
_node, historic_clip = project.add_project_raster(realization,
LayerTypes['HISTVEG'])
_node, fair_market_clip = project.add_project_raster(
realization, LayerTypes['FAIR_MARKET'])
# Download the four digit NHD archive containing the flow lines and watershed boundaries
log.info('Processing NHD')
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
nhd_download_folder = os.path.join(download_folder, 'nhd', huc[:4])
nhd_unzip_folder = os.path.join(scratch_dir, 'nhd', huc[:4])
nhd, db_path, huc_name, nhd_url = clean_nhd_data(
huc, nhd_download_folder, nhd_unzip_folder,
os.path.join(output_folder, 'hydrology'), cfg.OUTPUT_EPSG, False)
# Clean up the unzipped files. We won't need them again
if parallel:
safe_remove_dir(nhd_unzip_folder)
project.add_metadata({'Watershed': huc_name})
boundary = 'WBDHU{}'.format(len(huc))
# For coarser rasters than the DEM we need to buffer our clip polygon to include enough pixels
# This shouldn't be too much more data because these are usually integer rasters that are much lower res.
buffered_clip_path100 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP100'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path100,
epsg=cfg.OUTPUT_EPSG,
buffer=100)
buffered_clip_path500 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path500,
epsg=cfg.OUTPUT_EPSG,
buffer=500)
# PRISM climate rasters
mean_annual_precip = None
bil_files = glob.glob(os.path.join(prism_folder, '**', '*.bil'))
if (len(bil_files) == 0):
raise Exception('Could not find any .bil files in the prism folder')
for ptype in PrismTypes:
try:
# Next should always be guarded
source_raster_path = next(
x for x in bil_files
if ptype.lower() in os.path.basename(x).lower())
except StopIteration:
raise Exception(
'Could not find .bil file corresponding to "{}"'.format(ptype))
_node, project_raster_path = project.add_project_raster(
realization, LayerTypes[ptype])
raster_warp(source_raster_path, project_raster_path, cfg.OUTPUT_EPSG,
buffered_clip_path500, {"cutlineBlend": 1})
# Use the mean annual precipitation to calculate bankfull width
if ptype.lower() == 'ppt':
polygon = get_geometry_unary_union(nhd[boundary],
epsg=cfg.OUTPUT_EPSG)
mean_annual_precip = raster_buffer_stats2(
{1: polygon}, project_raster_path)[1]['Mean']
log.info('Mean annual precipitation for HUC {} is {} mm'.format(
huc, mean_annual_precip))
project.add_metadata(
{'mean_annual_precipitation_mm': str(mean_annual_precip)})
calculate_bankfull_width(nhd['NHDFlowline'], mean_annual_precip)
# Add the DB record to the Project XML
db_lyr = RSLayer('NHD Tables', 'NHDTABLES', 'SQLiteDB',
os.path.relpath(db_path, output_folder))
sqlite_el = project.add_dataset(realization, db_path, db_lyr, 'SQLiteDB')
project.add_metadata({'origin_url': nhd_url}, sqlite_el)
# Add any results to project XML
for name, file_path in nhd.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
vector_nod, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({'origin_url': nhd_url}, vector_nod)
states = get_nhd_states(nhd[boundary])
# Download the NTD archive containing roads and rail
log.info('Processing NTD')
ntd_raw = {}
ntd_unzip_folders = []
ntd_urls = get_ntd_urls(states)
for state, ntd_url in ntd_urls.items():
ntd_download_folder = os.path.join(download_folder, 'ntd',
state.lower())
ntd_unzip_folder = os.path.join(
scratch_dir, 'ntd', state.lower(), 'unzipped'
) # a little awkward but I need a folder for this and this was the best name I could find
ntd_raw[state] = download_shapefile_collection(ntd_url,
ntd_download_folder,
ntd_unzip_folder,
force_download)
ntd_unzip_folders.append(ntd_unzip_folder)
ntd_clean = clean_ntd_data(ntd_raw, nhd['NHDFlowline'], nhd[boundary],
os.path.join(output_folder, 'transportation'),
cfg.OUTPUT_EPSG)
# clean up the NTD Unzip folder. We won't need it again
if parallel:
for unzip_path in ntd_unzip_folders:
safe_remove_dir(unzip_path)
# Write transportation layers to project file
log.info('Write transportation layers to project file')
# Add any results to project XML
for name, file_path in ntd_clean.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
ntd_node, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({**ntd_urls}, ntd_node)
# Download the HAND raster
huc6 = huc[0:6]
hand_download_folder = os.path.join(download_folder, 'hand')
_hpath, hand_url = download_hand(huc6,
cfg.OUTPUT_EPSG,
hand_download_folder,
nhd[boundary],
hand_raster,
warp_options={"cutlineBlend": 1})
project.add_metadata({'origin_url': hand_url}, hand_node)
# download contributing DEM rasters, mosaic and reproject into compressed GeoTIF
ned_download_folder = os.path.join(download_folder, 'ned')
ned_unzip_folder = os.path.join(scratch_dir, 'ned')
dem_rasters, urls = download_dem(nhd[boundary], cfg.OUTPUT_EPSG, 0.01,
ned_download_folder, ned_unzip_folder,
force_download)
need_dem_rebuild = force_download or not os.path.exists(dem_raster)
if need_dem_rebuild:
raster_vrt_stitch(dem_rasters,
dem_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
warp_options={"cutlineBlend": 1})
verify_areas(dem_raster, nhd[boundary])
# Calculate slope rasters seperately and then stitch them
slope_parts = []
hillshade_parts = []
need_slope_build = need_dem_rebuild or not os.path.isfile(slope_raster)
need_hs_build = need_dem_rebuild or not os.path.isfile(hill_raster)
project.add_metadata(
{
'num_rasters': str(len(urls)),
'origin_urls': json.dumps(urls)
}, dem_node)
for dem_r in dem_rasters:
slope_part_path = os.path.join(
scratch_dem_folder,
'SLOPE__' + os.path.basename(dem_r).split('.')[0] + '.tif')
hs_part_path = os.path.join(
scratch_dem_folder,
'HS__' + os.path.basename(dem_r).split('.')[0] + '.tif')
slope_parts.append(slope_part_path)
hillshade_parts.append(hs_part_path)
if force_download or need_dem_rebuild or not os.path.exists(
slope_part_path):
gdal_dem_geographic(dem_r, slope_part_path, 'slope')
need_slope_build = True
if force_download or need_dem_rebuild or not os.path.exists(
hs_part_path):
gdal_dem_geographic(dem_r, hs_part_path, 'hillshade')
need_hs_build = True
if need_slope_build:
raster_vrt_stitch(slope_parts,
slope_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(slope_raster, nhd[boundary])
else:
log.info('Skipping slope build because nothing has changed.')
if need_hs_build:
raster_vrt_stitch(hillshade_parts,
hill_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(hill_raster, nhd[boundary])
else:
log.info('Skipping hillshade build because nothing has changed.')
# Remove the unzipped rasters. We won't need them anymore
if parallel:
safe_remove_dir(ned_unzip_folder)
# Calculate flow accumulation raster based on the DEM
log.info('Running flow accumulation and converting to drainage area.')
flow_accumulation(dem_raster, flow_accum, dinfinity=False, pitfill=True)
flow_accum_to_drainage_area(flow_accum, drain_area)
# Clip and re-project the existing and historic vegetation
log.info('Processing existing and historic vegetation rasters.')
clip_vegetation(buffered_clip_path100, existing_veg, existing_clip,
historic_veg, historic_clip, cfg.OUTPUT_EPSG)
log.info('Process the Fair Market Value Raster.')
raster_warp(fair_market,
fair_market_clip,
cfg.OUTPUT_EPSG,
clip=buffered_clip_path500,
warp_options={"cutlineBlend": 1})
# Clip the landownership Shapefile to a 10km buffer around the watershed boundary
own_path = os.path.join(output_folder, LayerTypes['OWNERSHIP'].rel_path)
project.add_dataset(realization, own_path, LayerTypes['OWNERSHIP'],
'Vector')
clip_ownership(nhd[boundary], ownership, own_path, cfg.OUTPUT_EPSG, 10000)
#######################################################
# Segmentation
#######################################################
# For now let's just make a copy of the NHD FLowlines
tmr = Timer()
rs_segmentation(nhd['NHDFlowline'], ntd_clean['Roads'], ntd_clean['Rail'],
own_path, hydrology_gpkg_path, SEGMENTATION['Max'],
SEGMENTATION['Min'], huc)
log.debug('Segmentation done in {:.1f} seconds'.format(tmr.ellapsed()))
project.add_project_geopackage(realization, LayerTypes['HYDROLOGY'])
# Add Bankfull Buffer Polygons
bankfull_path = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BANKFULL_CHANNEL'].rel_path)
bankfull_buffer(
os.path.join(hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['NETWORK'].rel_path),
cfg.OUTPUT_EPSG,
bankfull_path,
)
# TODO Add nhd/bankfull union when merge feature classes in vector.ops works with Geopackage layers
# bankfull_nhd_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# clip_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
# bankfull_nhd_area(bankfull_path, nhd['NHDArea'], clip_path, cfg.OUTPUT_EPSG, hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# Filter the ecoregions Shapefile to only include attributes that intersect with our HUC
eco_path = os.path.join(output_folder, 'ecoregions', 'ecoregions.shp')
project.add_dataset(realization, eco_path, LayerTypes['ECOREGIONS'],
'Vector')
filter_ecoregions(nhd[boundary], ecoregions, eco_path, cfg.OUTPUT_EPSG,
10000)
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(realization, LayerTypes['REPORT'], replace=True)
report = RSContextReport(report_path, project, output_folder)
report.write()
log.info('Process completed successfully.')
return {
'DEM': dem_raster,
'Slope': slope_raster,
'ExistingVeg': existing_veg,
'HistoricVeg': historic_veg,
'NHD': nhd
}
def main():
parser = argparse.ArgumentParser(description='Riverscapes VBET Tool',
# epilog="This is an epilog"
)
parser.add_argument('huc', help='NHD flow line ShapeFile path', type=str)
parser.add_argument('flowlines',
help='NHD flow line ShapeFile path',
type=str)
parser.add_argument('flowareas',
help='NHD flow areas ShapeFile path',
type=str)
parser.add_argument('slope', help='Slope raster path', type=str)
parser.add_argument('dem', help='DEM raster path', type=str)
parser.add_argument('hillshade', help='Hillshade raster path', type=str)
parser.add_argument(
'output_dir',
help='Folder where output VBET project will be created',
type=str)
parser.add_argument(
'--reach_codes',
help=
'Comma delimited reach codes (FCode) to retain when filtering features. Omitting this option retains all features.',
type=str)
parser.add_argument('--max_slope',
help='Maximum slope to be considered',
type=float,
default=12)
parser.add_argument('--max_hand',
help='Maximum HAND to be considered',
type=float,
default=50)
parser.add_argument('--min_hole_area',
help='Minimum hole retained in valley bottom (sq m)',
type=float,
default=50000)
parser.add_argument(
'--meta',
help='riverscapes project metadata as comma separated key=value pairs',
type=str)
parser.add_argument('--verbose',
help='(optional) a little extra logging ',
action='store_true',
default=False)
parser.add_argument(
'--debug',
help=
'Add debug tools for tracing things like memory usage at a performance cost.',
action='store_true',
default=False)
args = dotenv.parse_args_env(parser)
# make sure the output folder exists
safe_makedirs(args.output_dir)
# Initiate the log file
log = Logger('VBET')
log.setup(logPath=os.path.join(args.output_dir, 'vbet.log'),
verbose=args.verbose)
log.title('Riverscapes VBET For HUC: {}'.format(args.huc))
meta = parse_metadata(args.meta)
json_transform = json.dumps({
"Slope": 1,
"HAND": 2,
"Channel": 3,
"Flow Areas": 4
})
reach_codes = args.reach_codes.split(',') if args.reach_codes else None
try:
if args.debug is True:
from rscommons.debug import ThreadRun
memfile = os.path.join(args.output_dir, 'vbet_mem.log')
retcode, max_obj = ThreadRun(vbet, memfile, args.huc,
args.flowlines, args.flowareas,
args.slope, json_transform, args.dem,
args.hillshade, args.max_hand,
args.min_hole_area, args.output_dir,
reach_codes, meta)
log.debug('Return code: {}, [Max process usage] {}'.format(
retcode, max_obj))
else:
vbet(args.huc, args.flowlines, args.flowareas, args.slope,
json_transform, args.dem, args.hillshade, args.max_hand,
args.min_hole_area, args.output_dir, reach_codes, meta)
except Exception as e:
log.error(e)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def load_vegetation_raster(rasterpath: str, gpkg: str, existing=False, output_folder=None):
"""[summary]
Args:
rasterpath ([type]): [description]
existing (bool, optional): [description]. Defaults to False.
output_folder ([type], optional): [description]. Defaults to None.
Returns:
[type]: [description]
"""
conversion_lookup = {
"Open Water": 500,
"Riparian": 100,
"Hardwood": 100,
"Grassland": 50,
"Shrubland": 50,
"Non-vegetated": 40,
"Snow-Ice": 40,
"Sparsely Vegetated": 40,
"Barren": 40,
"Hardwood-Conifer": 20, # New for LANDFIRE 200
"Conifer-Hardwood": 20,
"Conifer": 20,
"Developed": 2,
"Developed-Low Intensity": 2,
"Developed-Medium Intensity": 2,
"Developed-High Intensity": 2,
"Developed-Roads": 2,
"Quarries-Strip Mines-Gravel Pits-Well and Wind Pads": 2, # Updated for LANDFIRE 200
"Exotic Tree-Shrub": 3,
"Exotic Herbaceous": 3,
"Ruderal Wet Meadow and Marsh": 3, # New for LANDFIRE 200
"Agricultural": 1
} if existing else {
"Open Water": 500,
"Riparian": 100,
"Hardwood": 100,
"Shrubland": 50,
"Grassland": 50,
"Perennial Ice/Snow": 40,
"Barren-Rock/Sand/Clay": 40,
"Sparse": 40,
"Conifer": 20,
"Hardwood-Conifer": 20,
"Conifer-Hardwood": 20}
vegetated_classes = [
"Riparian",
"Hardwood",
"Hardwood-Conifer",
"Grassland",
"Shrubland",
"Sparsely Vegetated",
"Conifer-Hardwood",
"Conifer",
"Ruderal Wet Meadow and Marsh",
"Agricultural"
] if existing else [
"Riparian",
"Hardwood",
"Conifer",
"Shrubland",
"Hardwood-Conifer",
"Conifer-Hardwood",
"Grassland"]
lui_lookup = { # TODO check for Landfire 200 updates
"Agricultural-Aquaculture": 0.66,
"Agricultural-Bush fruit and berries": 0.66,
"Agricultural-Close Grown Crop": 0.66,
"Agricultural-Fallow/Idle Cropland": 0.33,
"Agricultural-Orchard": 0.66,
"Agricultural-Pasture and Hayland": 0.33,
"Agricultural-Row Crop": 0.66,
"Agricultural-Row Crop-Close Grown Crop": 0.66,
"Agricultural-Vineyard": 0.66,
"Agricultural-Wheat": 0.66,
"Developed-High Intensity": 1.0,
"Developed-Medium Intensity": 1.0,
"Developed-Low Intensity": 1.0,
"Developed-Roads": 1.0,
"Developed-Upland Deciduous Forest": 1.0,
"Developed-Upland Evergreen Forest": 1.0,
"Developed-Upland Herbaceous": 1.0,
"Developed-Upland Mixed Forest": 1.0,
"Developed-Upland Shrubland": 1.0,
"Managed Tree Plantation - Northern and Central Hardwood and Conifer Plantation Group": 0.66,
"Managed Tree Plantation - Southeast Conifer and Hardwood Plantation Group": 0.66,
"Quarries-Strip Mines-Gravel Pits": 1.0}
# Read xml for reclass - arcgis tends to overwrite this file. use csv instead and make sure to ship with rasters
# root = ET.parse(f"{rasterpath}.aux.xml").getroot()
# ifield_value = int(root.findall(".//FieldDefn/[Name='VALUE']")[0].attrib['index'])
# ifield_conversion_source = int(root.findall(".//FieldDefn/[Name='EVT_PHYS']")[0].attrib['index']) if existing else int(root.findall(".//FieldDefn/[Name='GROUPVEG']")[0].attrib['index'])
# ifield_group_name = int(root.findall(".//FieldDefn/[Name='EVT_GP_N']")[0].attrib['index']) if existing else int(root.findall(".//FieldDefn/[Name='GROUPNAME']")[0].attrib['index'])
# conversion_values = {int(n[ifield_value].text): conversion_lookup.setdefault(n[ifield_conversion_source].text, 0) for n in root.findall(".//Row")}
# riparian_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text == "Riparian" else 0 for n in root.findall(".//Row")}
# native_riparian_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text == "Riparian" and not ("Introduced" in n[ifield_group_name].text) else 0 for n in root.findall(".//Row")}
# vegetation_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text in vegetated_classes else 0 for n in root.findall(".//Row")}
# lui_values = {int(n[ifield_value].text): lui_lookup.setdefault(n[ifield_group_name].text, 0) for n in root.findall(".//Row")} if existing is True else {}
# Load reclass values
with sqlite3.connect(gpkg) as conn:
c = conn.cursor()
valid_values = [v[0] for v in c.execute("SELECT VegetationID FROM VegetationTypes").fetchall()]
conversion_values = {row[0]: conversion_lookup.setdefault(row[1], 0) for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()}
riparian_values = {row[0]: 1 if row[1] == "Riparian" else 0 for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()}
native_riparian_values = {row[0]: 1 if row[1] == "Riparian" and not("Introduced" in row[2]) else 0 for row in c.execute('SELECT VegetationID, Physiognomy, LandUseGroup FROM VegetationTypes').fetchall()}
vegetation_values = {row[0]: 1 if row[1] in vegetated_classes else 0 for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()}
lui_values = {row[0]: lui_lookup.setdefault(row[1], 0) for row in c.execute('SELECT VegetationID, LandUseGroup FROM VegetationTypes').fetchall()} if existing else {}
# Read array
with rasterio.open(rasterpath) as raster:
no_data = int(raster.nodatavals[0])
conversion_values[no_data] = 0
riparian_values[no_data] = 0
native_riparian_values[no_data] = 0
vegetation_values[no_data] = 0
if existing:
lui_values[no_data] = 0.0
valid_values.append(no_data)
raw_array = raster.read(1, masked=True)
for value in np.unique(raw_array):
if not isinstance(value, type(np.ma.masked)) and value not in valid_values:
raise Exception(f"Vegetation raster value {value} not found in current data dictionary")
# Reclass array https://stackoverflow.com/questions/16992713/translate-every-element-in-numpy-array-according-to-key
riparian_array = np.vectorize(riparian_values.get)(raw_array)
native_riparian_array = np.vectorize(native_riparian_values.get)(raw_array)
vegetated_array = np.vectorize(vegetation_values.get)(raw_array)
conversion_array = np.vectorize(conversion_values.get)(raw_array)
lui_array = np.vectorize(lui_values.get)(raw_array) if existing else None
output = {"RAW": raw_array,
"RIPARIAN": riparian_array,
"NATIVE_RIPARIAN": native_riparian_array,
"VEGETATED": vegetated_array,
"CONVERSION": conversion_array,
"LUI": lui_array}
if output_folder:
for raster_name, raster_array in output.items():
if raster_array is not None:
safe_makedirs(output_folder)
with rasterio.open(os.path.join(output_folder, f"{'EXISTING' if existing else 'HISTORIC'}_{raster_name}.tif"),
'w',
driver='GTiff',
height=raster.height,
width=raster.width,
count=1,
dtype=np.int16,
crs=raster.crs,
transform=raster.transform) as dataset:
dataset.write(raster_array.astype(np.int16), 1)
return output
def export_feature_class(filegdb, featureclass, output_dir, output_epsg,
retained_fields, attribute_filter, spatial_filter):
log = Logger('Export FC')
log.info('Exporting geodatabase feature class {}'.format(featureclass))
# Get the input layer
in_driver = ogr.GetDriverByName("OpenFileGDB")
in_datasource = in_driver.Open(filegdb, 0)
in_layer = in_datasource.GetLayer(featureclass)
inSpatialRef = in_layer.GetSpatialRef()
if attribute_filter:
log.info('Export attribute filter: {}'.format(attribute_filter))
in_layer.SetAttributeFilter(attribute_filter)
if spatial_filter:
log.info('Export spatial filter area: {}'.format(spatial_filter.area))
in_layer.SetSpatialFilter(
ogr.CreateGeometryFromJson(json.dumps(mapping(spatial_filter))))
safe_makedirs(output_dir)
# Create the output layer
out_shapefile = os.path.join(output_dir, featureclass + '.shp')
out_driver = ogr.GetDriverByName("ESRI Shapefile")
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef,
output_epsg)
# Remove output shapefile if it already exists
if os.path.exists(out_shapefile):
out_driver.DeleteDataSource(out_shapefile)
# Create the output shapefile
out_datasource = out_driver.CreateDataSource(out_shapefile)
out_layer = out_datasource.CreateLayer(featureclass,
outSpatialRef,
geom_type=in_layer.GetGeomType())
# Add input Layer Fields to the output Layer if it is the one we want
in_layer_def = in_layer.GetLayerDefn()
for i in range(0, in_layer_def.GetFieldCount()):
field_def = in_layer_def.GetFieldDefn(i)
field_name = field_def.GetName()
if retained_fields and field_name not in retained_fields:
continue
fieldTypeCode = field_def.GetType()
if field_name.lower() == 'nhdplusid' and fieldTypeCode == ogr.OFTReal:
field_def.SetWidth(32)
field_def.SetPrecision(0)
out_layer.CreateField(field_def)
# Get the output Layer's Feature Definition
out_layer_def = out_layer.GetLayerDefn()
# Add features to the ouput Layer
progbar = ProgressBar(in_layer.GetFeatureCount(), 50,
"Adding features to output layer")
counter = 0
for in_feature in in_layer:
counter += 1
progbar.update(counter)
# Create output Feature
out_feature = ogr.Feature(out_layer_def)
geom = in_feature.GetGeometryRef()
geom.Transform(transform)
# Add field values from input Layer
for i in range(0, out_layer_def.GetFieldCount()):
field_def = out_layer_def.GetFieldDefn(i)
field_name = field_def.GetName()
if retained_fields and field_name not in retained_fields:
continue
out_feature.SetField(
out_layer_def.GetFieldDefn(i).GetNameRef(),
in_feature.GetField(i))
# Set geometry as centroid
out_feature.SetGeometry(geom)
# Add new feature to output Layer
out_layer.CreateFeature(out_feature)
out_feature = None
progbar.finish()
# Save and close DataSources
in_datasource = None
out_datasource = None
return out_shapefile
def copy_feature_class(inpath,
epsg,
outpath,
clip_shape=None,
attribute_filter=None):
"""Copy a Shapefile from one location to another
This method is capable of reprojecting the geometries as they are copied.
It is also possible to filter features by both attributes and also clip the
features to another geometryNone
Arguments:
inpath {str} -- File path to input Shapefile that will be copied.
epsg {int} -- Output coordinate system
outpath {str} -- File path where the output Shapefile will be generated.
Keyword Arguments:
clip_shape {shape} -- Shapely polygon geometry in the output EPSG used to clip the input geometries (default: {None})
attribute_filter {str} -- Attribute filter used to limit the input features that will be copied. (default: {None})
"""
log = Logger('Shapefile')
# if os.path.isfile(outpath):
# log.info('Skipping copy of feature classes because output file exists.')
# return
driver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = driver.Open(inpath, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
geom_type = inLayer.GetGeomType()
# Optionally limit which features are copied by using an attribute filter
if attribute_filter:
inLayer.SetAttributeFilter(attribute_filter)
# If there's a clip geometry provided then limit the features copied to
# those that intersect (partially or entirely) by this clip feature.
# Note that this makes the subsequent intersection process a lot more
# performant because the SetSaptialFilter() uses the ShapeFile's spatial
# index which is much faster than manually checking if all pairs of features intersect.
clip_geom = None
if clip_shape:
clip_geom = ogr.CreateGeometryFromWkb(clip_shape.wkb)
inLayer.SetSpatialFilter(clip_geom)
outpath_dir = os.path.dirname(outpath)
safe_makedirs(outpath_dir)
# Create the output shapefile
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg)
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('network',
outSpatialRef,
geom_type=geom_type)
outLayerDefn = outLayer.GetLayerDefn()
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Copying features")
counter = 0
for feature in inLayer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(
feature.GetFID()))
continue
geom.Transform(transform)
# if clip_shape:
# raw = shape(json.loads(geom.ExportToJson()))
# try:
# clip = raw.intersection(clip_shape)
# geom = ogr.CreateGeometryFromJson(json.dumps(mapping(clip)))
# except Exception as e:
# progbar.erase() # get around the progressbar
# log.warning('Invalid shape with FID={} cannot be intersected'.format(feature.GetFID()))
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
outFeature.SetGeometry(geom)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
outLayer.CreateFeature(outFeature)
outFeature = None
progbar.finish()
inDataSource = None
outDataSource = None
def distance_from_features(polygons, tmp_folder, bounds, cell_size_meters,
cell_size_degrees, output, features, statistic,
field):
"""[summary]
Feature class rasterization
https://gdal.org/programs/gdal_rasterize.html
Euclidean distance documentation
https://www.pydoc.io/pypi/pygeoprocessing-1.0.0/autoapi/geoprocessing/index.html#geoprocessing.distance_transform_edt
Arguments:
polygons {[type]} -- [description]
features_shapefile {[type]} -- [description]
temp_folder {[type]} -- [description]
"""
log = Logger('Conflict')
if features is None:
log.warning(
'Skipping distance calculation for {} because feature class does not exist.'
.format(field))
return
with get_shp_or_gpkg(features) as lyr:
if lyr.ogr_layer.GetFeatureCount() < 1:
log.warning(
'Skipping distance calculation for {} because feature class is empty.'
.format(field))
return
safe_makedirs(tmp_folder)
root_path = os.path.join(tmp_folder,
os.path.splitext(os.path.basename(features))[0])
features_raster = root_path + '_features.tif'
distance_raster = root_path + '_euclidean.tif'
if os.path.isfile(features_raster):
rasterio.shutil.delete(features_raster)
if os.path.isfile(distance_raster):
rasterio.shutil.delete(distance_raster)
progbar = ProgressBar(100, 50, "Rasterizing ")
def poly_progress(progress, _msg, _data):
progbar.update(int(progress * 100))
# Rasterize the features (roads, rail etc) and calculate a raster of Euclidean distance from these features
log.info(
'Rasterizing {:,} features at {}m cell size for generating {} field using {} distance.'
.format(len(polygons), cell_size_meters, field, statistic))
progbar.update(0)
gdal.Rasterize(features_raster,
os.path.dirname(features),
layers=os.path.basename(features),
xRes=cell_size_degrees,
yRes=cell_size_degrees,
burnValues=1,
outputType=gdal.GDT_Int16,
creationOptions=['COMPRESS=LZW'],
outputBounds=bounds,
callback=poly_progress)
progbar.finish()
log.info('Calculating Euclidean distance for {}'.format(field))
geoprocessing.distance_transform_edt((features_raster, 1), distance_raster)
# Calculate the Euclidean distance statistics (mean, min, max etc) for each polygon
values = raster_buffer_stats2(polygons, distance_raster)
# Retrieve the desired statistic and store it as the specified output field
log.info('Extracting {} statistic for {}.'.format(statistic, field))
progbar = ProgressBar(len(values), 50, "Extracting Statistics")
counter = 0
for reach_id, statistics in values.items():
counter += 1
progbar.update(counter)
if reach_id not in output:
output[reach_id] = {}
output[reach_id][field] = round(
statistics[statistic] * cell_size_meters, 0)
progbar.finish()
# Cleanup the temporary datasets
rasterio.shutil.delete(features_raster)
rasterio.shutil.delete(distance_raster)
log.info('{} distance calculation complete'.format(field))
def rvd(huc: int, flowlines_orig: Path, existing_veg_orig: Path, historic_veg_orig: Path,
valley_bottom_orig: Path, output_folder: Path, reach_codes: List[str], flow_areas_orig: Path, waterbodies_orig: Path, meta=None):
"""[Generate segmented reaches on flowline network and calculate RVD from historic and existing vegetation rasters
Args:
huc (integer): Watershed ID
flowlines_orig (Path): Segmented flowlines feature layer
existing_veg_orig (Path): LANDFIRE version 2.00 evt raster, with adjacent xml metadata file
historic_veg_orig (Path): LANDFIRE version 2.00 bps raster, with adjacent xml metadata file
valley_bottom_orig (Path): Vbet polygon feature layer
output_folder (Path): destination folder for project output
reach_codes (List[int]): NHD reach codes for features to include in outputs
flow_areas_orig (Path): NHD flow area polygon feature layer
waterbodies (Path): NHD waterbodies polygon feature layer
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("RVD")
log.info('RVD v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception('Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
safe_makedirs(output_folder)
project, _realization, proj_nodes = create_project(huc, output_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
log.info('Adding inputs to project')
_prj_existing_path_node, prj_existing_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['EXVEG'], existing_veg_orig)
_prj_historic_path_node, prj_historic_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HISTVEG'], historic_veg_orig)
# TODO: Don't forget the att_filter
# _prj_flowlines_node, prj_flowlines = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'], flowlines, att_filter="\"ReachCode\" Like '{}%'".format(huc))
# Copy in the vectors we need
inputs_gpkg_path = os.path.join(output_folder, LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path)
outputs_gpkg_path = os.path.join(output_folder, LayerTypes['OUTPUTS'].rel_path)
# Make sure we're starting with empty/fresh geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
GeopackageLayer.delete(outputs_gpkg_path)
# Copy our input layers and also find the difference in the geometry for the valley bottom
flowlines_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
vbottom_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['VALLEY_BOTTOM'].rel_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(valley_bottom_orig, vbottom_path, epsg=cfg.OUTPUT_EPSG)
with GeopackageLayer(flowlines_path) as flow_lyr:
# Set the output spatial ref as this for the whole project
out_srs = flow_lyr.spatial_ref
meter_conversion = flow_lyr.rough_convert_metres_to_vector_units(1)
distance_buffer = flow_lyr.rough_convert_metres_to_vector_units(1)
# Transform issues reading 102003 as espg id. Using sr wkt seems to work, however arcgis has problems loading feature classes with this method...
raster_srs = ogr.osr.SpatialReference()
ds = gdal.Open(prj_existing_path, 0)
raster_srs.ImportFromWkt(ds.GetProjectionRef())
raster_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
transform_shp_to_raster = VectorBase.get_transform(out_srs, raster_srs)
gt = ds.GetGeoTransform()
cell_area = ((gt[1] / meter_conversion) * (-gt[5] / meter_conversion))
# Create the output feature class fields
with GeopackageLayer(outputs_gpkg_path, layer_name='ReachGeometry', delete_dataset=True) as out_lyr:
out_lyr.create_layer(ogr.wkbMultiLineString, spatial_ref=out_srs, options=['FID=ReachID'], fields={
'GNIS_NAME': ogr.OFTString,
'ReachCode': ogr.OFTString,
'TotDASqKm': ogr.OFTReal,
'NHDPlusID': ogr.OFTReal,
'WatershedID': ogr.OFTInteger
})
metadata = {
'RVD_DateTime': datetime.datetime.now().isoformat(),
'Reach_Codes': reach_codes
}
# Execute the SQL to create the lookup tables in the RVD geopackage SQLite database
watershed_name = create_database(huc, outputs_gpkg_path, metadata, cfg.OUTPUT_EPSG, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'rvd_schema.sql'))
project.add_metadata({'Watershed': watershed_name})
geom_vbottom = get_geometry_unary_union(vbottom_path, spatial_ref=raster_srs)
flowareas_path = None
if flow_areas_orig:
flowareas_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
copy_feature_class(flow_areas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
geom_flow_areas = get_geometry_unary_union(flowareas_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_flow_areas)
else:
del LayerTypes['INPUTS'].sub_layers['FLOW_AREA']
waterbodies_path = None
if waterbodies_orig:
waterbodies_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['WATERBODIES'].rel_path)
copy_feature_class(waterbodies_orig, waterbodies_path, epsg=cfg.OUTPUT_EPSG)
geom_waterbodies = get_geometry_unary_union(waterbodies_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_waterbodies)
else:
del LayerTypes['INPUTS'].sub_layers['WATERBODIES']
# Add the inputs to the XML
_nd, _in_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Filter the flow lines to just the required features and then segment to desired length
# TODO: These are brat methods that need to be refactored to use VectorBase layers
cleaned_path = os.path.join(outputs_gpkg_path, 'ReachGeometry')
build_network(flowlines_path, flowareas_path, cleaned_path, waterbodies_path=waterbodies_path, epsg=cfg.OUTPUT_EPSG, reach_codes=reach_codes, create_layer=False)
# Generate Voroni polygons
log.info("Calculating Voronoi Polygons...")
# Add all the points (including islands) to the list
flowline_thiessen_points_groups = centerline_points(cleaned_path, distance_buffer, transform_shp_to_raster)
flowline_thiessen_points = [pt for group in flowline_thiessen_points_groups.values() for pt in group]
simple_save([pt.point for pt in flowline_thiessen_points], ogr.wkbPoint, raster_srs, "Thiessen_Points", intermediates_gpkg_path)
# Exterior is the shell and there is only ever 1
myVorL = NARVoronoi(flowline_thiessen_points)
# Generate Thiessen Polys
myVorL.createshapes()
# Dissolve by flowlines
log.info("Dissolving Thiessen Polygons")
dissolved_polys = myVorL.dissolve_by_property('fid')
# Clip Thiessen Polys
log.info("Clipping Thiessen Polygons to Valley Bottom")
clipped_thiessen = clip_polygons(geom_vbottom, dissolved_polys)
# Save Intermediates
simple_save(clipped_thiessen.values(), ogr.wkbPolygon, raster_srs, "Thiessen", intermediates_gpkg_path)
simple_save(dissolved_polys.values(), ogr.wkbPolygon, raster_srs, "ThiessenPolygonsDissolved", intermediates_gpkg_path)
simple_save(myVorL.polys, ogr.wkbPolygon, raster_srs, "ThiessenPolygonsRaw", intermediates_gpkg_path)
_nd, _inter_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES'])
# OLD METHOD FOR AUDIT
# dissolved_polys2 = dissolve_by_points(flowline_thiessen_points_groups, myVorL.polys)
# simple_save(dissolved_polys2.values(), ogr.wkbPolygon, out_srs, "ThiessenPolygonsDissolved_OLD", intermediates_gpkg_path)
# Load Vegetation Rasters
log.info(f"Loading Existing and Historic Vegetation Rasters")
vegetation = {}
vegetation["EXISTING"] = load_vegetation_raster(prj_existing_path, outputs_gpkg_path, True, output_folder=os.path.join(output_folder, 'Intermediates'))
vegetation["HISTORIC"] = load_vegetation_raster(prj_historic_path, outputs_gpkg_path, False, output_folder=os.path.join(output_folder, 'Intermediates'))
for epoch in vegetation.keys():
for name in vegetation[epoch].keys():
if not f"{epoch}_{name}" == "HISTORIC_LUI":
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[f"{epoch}_{name}"])
if vegetation["EXISTING"]["RAW"].shape != vegetation["HISTORIC"]["RAW"].shape:
raise Exception('Vegetation raster shapes are not equal Existing={} Historic={}. Cannot continue'.format(vegetation["EXISTING"]["RAW"].shape, vegetation["HISTORIC"]["RAW"].shape))
# Vegetation zone calculations
riparian_zone_arrays = {}
riparian_zone_arrays["RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["RIPARIAN"] + vegetation["HISTORIC"]["RIPARIAN"]) > 0) * 1
riparian_zone_arrays["NATIVE_RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["NATIVE_RIPARIAN"] + vegetation["HISTORIC"]["NATIVE_RIPARIAN"]) > 0) * 1
riparian_zone_arrays["VEGETATION_ZONES"] = ((vegetation["EXISTING"]["VEGETATED"] + vegetation["HISTORIC"]["VEGETATED"]) > 0) * 1
# Save Intermediate Rasters
for name, raster in riparian_zone_arrays.items():
save_intarr_to_geotiff(raster, os.path.join(output_folder, "Intermediates", f"{name}.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[name])
# Calculate Riparian Departure per Reach
riparian_arrays = {f"{epoch.capitalize()}{(name.capitalize()).replace('Native_riparian', 'NativeRiparian')}Mean": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name in ["RIPARIAN", "NATIVE_RIPARIAN"]}
# Vegetation Cell Counts
raw_arrays = {f"{epoch}": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name == "RAW"}
# Generate Vegetation Conversions
vegetation_change = (vegetation["HISTORIC"]["CONVERSION"] - vegetation["EXISTING"]["CONVERSION"])
save_intarr_to_geotiff(vegetation_change, os.path.join(output_folder, "Intermediates", "Conversion_Raster.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['VEGETATION_CONVERSION'])
# load conversion types dictionary from database
conn = sqlite3.connect(outputs_gpkg_path)
conn.row_factory = dict_factory
curs = conn.cursor()
curs.execute('SELECT * FROM ConversionTypes')
conversion_classifications = curs.fetchall()
curs.execute('SELECT * FROM vwConversions')
conversion_ids = curs.fetchall()
# Split vegetation change classes into binary arrays
vegetation_change_arrays = {
c['FieldName']: (vegetation_change == int(c["TypeValue"])) * 1 if int(c["TypeValue"]) in np.unique(vegetation_change) else None
for c in conversion_classifications
}
# Calcuate average and unique cell counts per reach
progbar = ProgressBar(len(clipped_thiessen.keys()), 50, "Extracting array values by reach...")
counter = 0
discarded = 0
with rasterio.open(prj_existing_path) as dataset:
unique_vegetation_counts = {}
reach_average_riparian = {}
reach_average_change = {}
for reachid, poly in clipped_thiessen.items():
counter += 1
progbar.update(counter)
# we can discount a lot of shapes here.
if not poly.is_valid or poly.is_empty or poly.area == 0 or poly.geom_type not in ["Polygon", "MultiPolygon"]:
discarded += 1
continue
raw_values_unique = {}
change_values_mean = {}
riparian_values_mean = {}
reach_raster = np.ma.masked_invalid(
features.rasterize(
[poly],
out_shape=dataset.shape,
transform=dataset.transform,
all_touched=True,
fill=np.nan))
for raster_name, raster in raw_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
raw_values_unique[raster_name] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True)
else:
raw_values_unique[raster_name] = []
for raster_name, raster in riparian_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
riparian_values_mean[raster_name] = np.ma.mean(current_raster)
else:
riparian_values_mean[raster_name] = 0.0
for raster_name, raster in vegetation_change_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
change_values_mean[raster_name] = np.ma.mean(current_raster)
else:
change_values_mean[raster_name] = 0.0
unique_vegetation_counts[reachid] = raw_values_unique
reach_average_riparian[reachid] = riparian_values_mean
reach_average_change[reachid] = change_values_mean
progbar.finish()
with SQLiteCon(outputs_gpkg_path) as gpkg:
# Ensure all reaches are present in the ReachAttributes table before storing RVD output values
gpkg.curs.execute('INSERT INTO ReachAttributes (ReachID) SELECT ReachID FROM ReachGeometry;')
errs = 0
for reachid, epochs in unique_vegetation_counts.items():
for epoch in epochs.values():
insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
try:
gpkg.curs.executemany('''INSERT INTO ReachVegetation (
ReachID,
VegetationID,
Area,
CellCount)
VALUES (?,?,?,?)''', insert_values)
# Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
except sqlite3.IntegrityError as err:
# THis is likely a constraint error.
errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
log.error(errstr)
errs += 1
except sqlite3.Error as err:
# This is any other kind of error
errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
log.error(errstr)
errs += 1
if errs > 0:
raise Exception('Errors were found inserting records into the database. Cannot continue.')
gpkg.conn.commit()
# load RVD departure levels from DepartureLevels database table
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxRVD FROM DepartureLevels ORDER BY MaxRVD ASC')
departure_levels = gpkg.curs.fetchall()
# Calcuate Average Departure for Riparian and Native Riparian
riparian_departure_values = riparian_departure(reach_average_riparian, departure_levels)
write_db_attributes(outputs_gpkg_path, riparian_departure_values, departure_type_columns)
# Add Conversion Code, Type to Vegetation Conversion
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxValue, NAME FROM ConversionLevels ORDER BY MaxValue ASC')
conversion_levels = gpkg.curs.fetchall()
reach_values_with_conversion_codes = classify_conversions(reach_average_change, conversion_ids, conversion_levels)
write_db_attributes(outputs_gpkg_path, reach_values_with_conversion_codes, rvd_columns)
# # Write Output to GPKG table
# log.info('Insert values to GPKG tables')
# # TODO move this to write_attirubtes method
# with get_shp_or_gpkg(outputs_gpkg_path, layer_name='ReachAttributes', write=True, ) as in_layer:
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "FromConifer": ogr.OFTReal,
# "FromDevegetated": ogr.OFTReal,
# "FromGrassShrubland": ogr.OFTReal,
# "FromDeciduous": ogr.OFTReal,
# "NoChange": ogr.OFTReal,
# "Deciduous": ogr.OFTReal,
# "GrassShrubland": ogr.OFTReal,
# "Devegetation": ogr.OFTReal,
# "Conifer": ogr.OFTReal,
# "Invasive": ogr.OFTReal,
# "Development": ogr.OFTReal,
# "Agriculture": ogr.OFTReal,
# "ConversionCode": ogr.OFTInteger,
# "ConversionType": ogr.OFTString}.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in reach_values_with_conversion_codes:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in reach_values_with_conversion_codes[reach]:
# if not reach_values_with_conversion_codes[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, reach_values_with_conversion_codes[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "EXISTING_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_RIPARIAN_MEAN": ogr.OFTReal,
# "RIPARIAN_DEPARTURE": ogr.OFTReal,
# "EXISTING_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "NATIVE_RIPARIAN_DEPARTURE": ogr.OFTReal, }.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in riparian_departure_values:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in riparian_departure_values[reach]:
# if not riparian_departure_values[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, riparian_departure_values[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# with sqlite3.connect(outputs_gpkg_path) as conn:
# cursor = conn.cursor()
# errs = 0
# for reachid, epochs in unique_vegetation_counts.items():
# for epoch in epochs.values():
# insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
# try:
# cursor.executemany('''INSERT INTO ReachVegetation (
# ReachID,
# VegetationID,
# Area,
# CellCount)
# VALUES (?,?,?,?)''', insert_values)
# # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
# except sqlite3.IntegrityError as err:
# # THis is likely a constraint error.
# errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
# log.error(errstr)
# errs += 1
# except sqlite3.Error as err:
# # This is any other kind of error
# errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
# log.error(errstr)
# errs += 1
# if errs > 0:
# raise Exception('Errors were found inserting records into the database. Cannot continue.')
# conn.commit()
# Add intermediates and the report to the XML
# project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) already
# added above
project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['OUTPUTS'])
# Add the report to the XML
report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True)
report = RVDReport(report_path, project)
report.write()
log.info('RVD complete')
