def get_riverpoints(inpath, epsg, attribute_filter=None):
"""[summary]
Args:
inpath ([type]): Path to a ShapeFile
epsg ([type]): Desired output spatial reference
attribute_filter ([type], optional): [description]. Defaults to None.
Returns:
[type]: List of RiverPoint objects
"""
log = Logger('get_riverpoints')
points = []
with get_shp_or_gpkg(inpath) as in_lyr:
_out_spatial_ref, transform = get_transform_from_epsg(
in_lyr.spatial_ref, epsg)
for feat, _counter, progbar in in_lyr.iterate_features(
'Getting points for use in Thiessen',
attribute_filter=attribute_filter):
new_geom = feat.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feat.GetFID()))
continue
new_geom.Transform(transform)
new_shape = VectorBase.ogr2shapely(new_geom)
if new_shape.type == 'Polygon':
new_shape = MultiPolygon([new_shape])
for poly in new_shape:
# Exterior is the shell and there is only ever 1
for pt in list(poly.exterior.coords):
points.append(RiverPoint(pt, interior=False))
# Now we consider interiors. NB: Interiors are only qualifying islands in this case
for idx, island in enumerate(poly.interiors):
for pt in list(island.coords):
points.append(RiverPoint(pt, interior=True,
island=idx))
return points
def extract_canals(flowlines, epsg, boundary, outpath):
log = Logger('Canals')
driver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = driver.Open(flowlines, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
inLayer.SetAttributeFilter("FCode IN ({0})".format(','.join(
[str(fcode) for fcode in canal_fcodes])))
log.info('{:,} canal features identified in NHD flow lines.'.format(
inLayer.GetFeatureCount()))
if os.path.isfile(outpath):
log.info(
'Skipping extracting canals from NHD flow lines because file exists.'
)
return outpath
extract_path = os.path.dirname(outpath)
safe_makedirs(extract_path)
# Create the output shapefile
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg)
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('network',
outSpatialRef,
geom_type=ogr.wkbMultiLineString)
outLayerDefn = outLayer.GetLayerDefn()
for feature in inLayer:
geom = feature.GetGeometryRef()
geom.Transform(transform)
outFeature = ogr.Feature(outLayerDefn)
outFeature.SetGeometry(geom)
outLayer.CreateFeature(outFeature)
outFeature = None
feature = None
log.info('{:,} features written to canals shapefile.'.format(
outLayer.GetFeatureCount()))
inDataSource = None
outDataSource = None
return outpath
def load_mean_annual_data(gpkg, input_csv, attribute_name, epsg):
"""
Loads PRISM attribute csv file into PRISM geopackage as mean annual value
Args:
gpkg (str): Geopackage of PRISM Locations
input_csv (str): name of PRISM csv attribute file to load
attribute_name (str): Name of new field to add to geopackage
"""
# Load GPKG
driver = ogr.GetDriverByName("GPKG")
source = driver.Open(gpkg, 1)
layer = source.GetLayerByName("PRISM_Data")
in_spatial_ref = layer.GetSpatialRef()
_out_spatial_ref, transform = get_transform_from_epsg(in_spatial_ref, epsg)
# Read through CSV Lines and write to layer features
layer.StartTransaction()
with open(input_csv, 'r') as csvfile:
next(csvfile)
reader = csv.reader(csvfile)
for row in reader:
# Retrieve just the valid integer values
valint = [int(val) for val in row[1:] if val.isdigit()]
# Calculate how many years these data points represent
years = float(len(valint)) / 12.0
# Convert values into annual precip in mm
tot = (float(sum(valint)) / 100.0) / years
feature = layer.GetFeature(int(row[0]))
new_geom = feature.GetGeometryRef()
new_geom.Transform(transform)
feature.SetGeometry(new_geom)
feature.SetField(attribute_name, tot)
layer.SetFeature(feature)
feature = None
layer.CommitTransaction()
layer = None
source = None
return
def calc_max_drainage(huc_search, precip_raster, wbd, bankfull):
""" Temporary script to calculate the BRAT maximum drainage area threshold
Takes HUC8 watershed boundary polygons and finds the mean annual
precipitation then uses the inverted Beechi and Imaki formula
to derive drainage area at the specified constant bankfull width.
Args:
huc_search (str): feature layer attribute filter string (e.g. '17%' for hydro region 17)
precip_raster (str): path to the national PRISM annual precipitation raster
wbd (str): file path to the national watershed boundary dataset (WBD) file geodatabase
bankfull (float): bankfull width at which drainage area threshold is calculated
"""
# open watershed boundary file geodatabase
driver = ogr.GetDriverByName('OpenFileGDB')
data_source = driver.Open(wbd, 0)
wbd_layer = data_source.GetLayer('WBDHU8')
wbd_layer.SetAttributeFilter('HUC8 LIKE \'{}\''.format(huc_search))
# Need to convert watersheds to the PESG:4269 used by the PRISM raster
_srs, transform = get_transform_from_epsg(wbd_layer.GetSpatialRef(), 4269)
watersheds = {}
for feature in wbd_layer:
huc = feature.GetField('HUC8')
states = feature.GetField('states')
if 'cn' not in states.lower():
watersheds[huc] = VectorBase.ogr2shapely(feature, transform)
stats = raster_buffer_stats2(watersheds, precip_raster)
for huc, stat in stats.items():
# PRISM precipitation is in mm but Beechie and Imaki require it in cm
mean_precip_cm = stat['Mean'] / 10.0
max_drain = pow(bankfull / (0.177) / (pow(mean_precip_cm, 0.453)),
1 / 0.397)
print(
"UPDATE watersheds SET max_drainage = {} WHERE watershed_id = '{}' AND ((max_drainage IS NULL) OR (max_drainage = 0)); -- {}"
.format(int(max_drain), huc, mean_precip_cm))
def load_idaho(shapefile, database):
conn = sqlite3.connect(database)
# Clear the database first
conn.execute('DELETE FROM Reaches')
conn.commit()
lookup = {
'oPBRC_CR': load_lookup(database, 'DamOpportunities', 'OpportunityID'),
'oPBRC_UI': load_lookup(database, 'DamRisks', 'RiskID'),
'oPBRC_UD': load_lookup(database, 'DamLimitations', 'LimitationID'),
'ADMIN_AGEN': load_lookup(database, 'Agencies', 'AgencyID', 'Abbreviation')
}
db_fields = []
for field in fields:
if field == 'HUC_ID':
db_fields.append('WatershedID')
elif field == 'FCode':
db_fields.append('ReachCode')
elif field == 'ADMIN_AGEN':
db_fields.append('AgencyID')
elif field == 'oPBRC_CR':
db_fields.append('OpportunityID')
elif field == 'oPBRC_UI':
db_fields.append('RiskID')
elif field == 'oPBRC_UD':
db_fields.append('LimitationID')
else:
db_fields.append(field)
driver = ogr.GetDriverByName('ESRI Shapefile')
dataset = driver.Open(shapefile, 0)
layer = dataset.GetLayer()
sr_idaho = layer.GetSpatialRef()
out_spatial_ref, transform = get_transform_from_epsg(sr_idaho, 4326)
progbar = ProgressBar(layer.GetFeatureCount(), 50, "Loading features")
counter = 0
for feature in layer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
geom.Transform(transform)
reach_values = [geom.ExportToJson()]
for field in fields:
if field in lookup:
value = feature.GetField(field)
if field == 'oPBRC_UI' and str.isdigit(value[0]):
value = value[1:]
elif value == '4PVT':
value = 'PVT'
reach_values.append(lookup[field][value])
else:
reach_values.append(feature.GetField(field))
conn.execute('INSERT INTO Reaches (Geometry, {}) Values (?, {})'.format(','.join(db_fields), ','.join('?' * len(db_fields))), reach_values)
conn.commit()
progbar.finish()
print('Process complete')
def huc_precipitation(watersheds, database):
"""Loop over each eight digit HUC watershed and
calculate the mean annual precipitation of all
PRISM locations that exist within the watershed
boundary.
Arguments:
watersheds {str} -- Path to the national watershed
boundary file geodatabase.
database {str} -- Path to BRAT SQLite database that
contains the PRISM locations and mean precipiation
at each location.
Raises:
Exception: [description]
Exception: [description]
"""
# National HUC8 watershed boundary dataset
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(watersheds, 0)
layer = datasource.GetLayer('WBDHU8')
vector_srs = layer.GetSpatialRef()
feature_count = layer.GetFeatureCount()
# Transformation ensuring watershed polygons are in geographic coords
out_spatial_ref, transform = get_transform_from_epsg(vector_srs, 4296)
# BRAT SQLite database
conn = sqlite3.connect(database)
curs = conn.cursor()
processing = 0
for feature in layer:
processing += 1
huc = feature.GetField('HUC8')
print('Processing {}, {:,} of {:,}'.format(huc, processing, feature_count))
geom = feature.GetGeometryRef()
geom.Transform(transform)
envelope = geom.GetEnvelope()
polygon = shape(json.loads(geom.ExportToJson()))
try:
# Select PRISM locations with the watershed boundary envelope
curs.execute('SELECT Longitude, Latitude, MeanPrecip FROM Precipitation'
' WHERE (Latitude >= ?) AND (Latitude <= ?) AND (Longitude >= ?) AND (Longitude <= ?)',
[envelope[2], envelope[3], envelope[0], envelope[1]])
# Filter to just those PRISM locations that are actually within the HUC boundary polygon
monthly_precip = []
for row in curs.fetchall():
point = Point(row[0], row[1])
if polygon.contains(point):
monthly_precip.append(row[2])
if len(monthly_precip) < 1:
raise Exception('No monthly precipitation values found.')
if sum(monthly_precip) < 1:
raise Exception('Zero monthly precipitation.')
curs.execute('UPDATE HUCs SET MeanPrecip = ? WHERE HUC = ?', [mean(monthly_precip), huc])
conn.commit()
except Exception as e:
print('Error processing {}'.format(huc))
def import_shapefile(shapefile, host, port, user_name, password, database):
log = Logger('Import')
conn = psycopg2.connect(user=user_name, password=password, host=host, port=port, database=database)
curs = conn.cursor()
driver = ogr.GetDriverByName('ESRI ShapeFile')
data_source = driver.Open(shapefile)
layer = data_source.GetLayer()
total_features = layer.GetFeatureCount()
_spatial_ref, transform = get_transform_from_epsg( layer.GetSpatialRef(), 4326)
curs.execute('INSERT INTO uploads (added_by, file_name, remarks) VALUES (%s, %s, %s) RETURNING id',
[getpass.getuser(), os.path.basename(shapefile), 'Python Script Import'])
upload_id = curs.fetchone()[0]
curs.execute('SELECT name, id FROM observation_types')
obs_types = {row[0].replace('Dam', '').replace(' ', '').lower(): row[1] for row in curs.fetchall()}
certainties = {'Low': 1, 'Medium': 2, 'High': 3}
# Reach statistics for each reach in our batch
progbar = ProgressBar(total_features, 50, "Importing features")
try:
observations = []
for feature in layer:
geom = feature.GetGeometryRef()
geom.Transform(transform)
geom.FlattenTo2D()
metadata = {field: feature.GetField(field) for field in [
'Feature_Ty', 'Certainty', 'Year', 'Dam_Type', 'CreationDa',
'Creator', 'EditDate', 'Editor', 'Snapped', 'Imagery_Ye'] }
# print(metadata)
clean_type = feature.GetField('Feature_Ty').replace('_', '').replace('Dam', '').lower() if feature.GetField('Feature_Ty') else 'Unknown'.lower()
obs_date = datetime.datetime.strptime(feature.GetField('CreationDa'), '%Y/%m/%d')
year = int(feature.GetField('Year')) if feature.GetField('Year') else None
certainty = certainties[feature.GetField('Certainty')] if feature.GetField('Certainty') in certainties else 0
observations.append((
upload_id,
geom.ExportToWkb(),
obs_date,
feature.GetField('Creator'),
year,
obs_types[clean_type],
True,
certainty,
json.dumps(metadata)
))
progbar.update(len(observations))
progbar.finish()
curs.executemany("""
INSERT INTO observations (
upload_id,
geom,
obs_date,
observer,
obs_year,
obs_type_id,
is_public,
confidence,
metadata
) VALUES (%s, ST_GeomFromWKB(%s, 4326), %s, %s, %s, %s, %s, %s, %s)""", observations)
conn.commit()
except Exception as ex:
conn.rollback()
log.error(ex)
log.info('Shapefile import complete')
def export_feature_class(filegdb, featureclass, output_dir, output_epsg,
retained_fields, attribute_filter, spatial_filter):
log = Logger('Export FC')
log.info('Exporting geodatabase feature class {}'.format(featureclass))
# Get the input layer
in_driver = ogr.GetDriverByName("OpenFileGDB")
in_datasource = in_driver.Open(filegdb, 0)
in_layer = in_datasource.GetLayer(featureclass)
inSpatialRef = in_layer.GetSpatialRef()
if attribute_filter:
log.info('Export attribute filter: {}'.format(attribute_filter))
in_layer.SetAttributeFilter(attribute_filter)
if spatial_filter:
log.info('Export spatial filter area: {}'.format(spatial_filter.area))
in_layer.SetSpatialFilter(
ogr.CreateGeometryFromJson(json.dumps(mapping(spatial_filter))))
safe_makedirs(output_dir)
# Create the output layer
out_shapefile = os.path.join(output_dir, featureclass + '.shp')
out_driver = ogr.GetDriverByName("ESRI Shapefile")
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef,
output_epsg)
# Remove output shapefile if it already exists
if os.path.exists(out_shapefile):
out_driver.DeleteDataSource(out_shapefile)
# Create the output shapefile
out_datasource = out_driver.CreateDataSource(out_shapefile)
out_layer = out_datasource.CreateLayer(featureclass,
outSpatialRef,
geom_type=in_layer.GetGeomType())
# Add input Layer Fields to the output Layer if it is the one we want
in_layer_def = in_layer.GetLayerDefn()
for i in range(0, in_layer_def.GetFieldCount()):
field_def = in_layer_def.GetFieldDefn(i)
field_name = field_def.GetName()
if retained_fields and field_name not in retained_fields:
continue
fieldTypeCode = field_def.GetType()
if field_name.lower() == 'nhdplusid' and fieldTypeCode == ogr.OFTReal:
field_def.SetWidth(32)
field_def.SetPrecision(0)
out_layer.CreateField(field_def)
# Get the output Layer's Feature Definition
out_layer_def = out_layer.GetLayerDefn()
# Add features to the ouput Layer
progbar = ProgressBar(in_layer.GetFeatureCount(), 50,
"Adding features to output layer")
counter = 0
for in_feature in in_layer:
counter += 1
progbar.update(counter)
# Create output Feature
out_feature = ogr.Feature(out_layer_def)
geom = in_feature.GetGeometryRef()
geom.Transform(transform)
# Add field values from input Layer
for i in range(0, out_layer_def.GetFieldCount()):
field_def = out_layer_def.GetFieldDefn(i)
field_name = field_def.GetName()
if retained_fields and field_name not in retained_fields:
continue
out_feature.SetField(
out_layer_def.GetFieldDefn(i).GetNameRef(),
in_feature.GetField(i))
# Set geometry as centroid
out_feature.SetGeometry(geom)
# Add new feature to output Layer
out_layer.CreateFeature(out_feature)
out_feature = None
progbar.finish()
# Save and close DataSources
in_datasource = None
out_datasource = None
return out_shapefile
from shapely.geometry import shape, mapping
from rscommons.shapefile import get_transform_from_epsg
# TODO: Paths need to be reset
raise Exception('PATHS NEED TO BE RESET')
watersheds = '/SOMEPATH/GISData/WatershedBoundaries/WBD_National_GDB/WBD_National_GDB.gdb'
brat = '/SOMEPATH/code/beaver/pyBRAT4/database/brat_template.sqlite'
# Load National Watershed boundary dataset
driver = ogr.GetDriverByName('OpenFileGDB')
ds = driver.Open(watersheds, 0)
layer = ds.GetLayer('WBDHU8')
in_spatial_ref = layer.GetSpatialRef()
out_spatial_ref, transform = get_transform_from_epsg(in_spatial_ref, 4326)
hucs = []
for f in layer:
# geom = f.GetGeometryRef()
# if transform:
# geom.Transform(transform)
# geojson = geom.ExportToJson()
hucs.append(
(f.GetField('HUC8'), f.GetField('NAME'), f.GetField('STATES'),
round(f.GetField('AREASQKM'), 2) if f.GetField('AREASQKM') else None))
# geojson))
# Update the BRAT