def getCatchmentFeaturesForStreamflowGage(
config, outputDir, catchmentFilename, reachcode, measure, format=OGR_SHAPEFILE_DRIVER_NAME
):
""" Query NHDPlus V2 web service for features (in WGS 84) for
the drainage area associated with a given NHD (National
Hydrography Dataset) streamflow gage identified by a reach
code and measure.
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative
path of the directory into which output rasters should be
written
@param format String representing OGR driver to use
@param catchmentFilename String representing name of file to
save catchment features to. The appropriate extension will be added to the file name
@param reachcode String representing NHD streamflow gage
@param measure Float representing the measure along reach
where Stream Gage is located in percent from downstream
end of the one or more NHDFlowline features that are
assigned to the ReachCode (see NHDPlusV21 GageLoc table)
@return Tuple(String representing the name of the dataset in outputDir created to hold
the features, URL of the request)
@raise IOError(errno.EACCESS) if OGR binary is not executable
@raise IOError(errno.ENOTDIR) if outputDir is not a directory
@raise IOError(errno.EACCESS) if outputDir is not writable
@raise Exception if output format is not known
@raise WebserviceError if an error occurred calling the web service
"""
ogrCmdPath = config.get("GDAL/OGR", "PATH_OF_OGR2OGR")
if not os.access(ogrCmdPath, os.X_OK):
raise IOError(errno.EACCES, "The ogr2ogr binary at %s is not executable" % ogrCmdPath)
ogrCmdPath = os.path.abspath(ogrCmdPath)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
if not format in OGR_DRIVERS.keys():
raise Exception("Output format '%s' is not known" % (format,))
catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep, OGR_DRIVERS[format])
catchmentFilepath = os.path.join(outputDir, catchmentFilename)
url = URL_PROTO_CATCHMENT.format(reachcode=reachcode, measure=str(measure))
urlFetched = "http://%s%s" % (HOST, url)
conn = httplib.HTTPConnection(HOST)
try:
conn.request("GET", url)
res = conn.getresponse(buffering=True)
except socket.error as e:
raise WebserviceError(urlFetched, str(e))
if 200 != res.status:
error = "%d %s" % (res.status, res.reason)
raise WebserviceError(urlFetched, error)
contentType = res.getheader("Content-Type")
if contentType.find(CONTENT_TYPE_ERROR) != -1:
error = res.read()
raise WebserviceError(urlFetched, error)
elif contentType.find(CONTENT_TYPE) != -1:
# The data returned were of the type expected, read the data
tmpdir = tempfile.mkdtemp()
failure = False
data = res.read(_BUFF_LEN)
if data:
tmpfile = os.path.join(tmpdir, "catchment.geojson")
f = open(tmpfile, "wb")
while data:
f.write(data)
data = res.read(_BUFF_LEN)
f.close()
# Convert GeoJSON to ESRI Shapfile using OGR
ogrCommand = "%s -s_srs EPSG:4326 -t_srs EPSG:4326 -f '%s' %s %s" % (
ogrCmdPath,
format,
catchmentFilepath,
tmpfile,
)
os.system(ogrCommand)
if not os.path.exists(catchmentFilepath):
failure = False
shutil.rmtree(tmpdir)
if failure:
raise WebserviceError(urlFetched, "Failed to store catchment features in file %s" % (catchmentFilepath,))
else:
error = "Recieved content type %s but expected type %s" % (contentType, CONTENT_TYPE)
raise WebserviceError(urlFetched, error)
return (catchmentFilename, urlFetched)
def getCatchmentFeaturesForReaches(config,
outputDir,
catchmentFilename,
reaches,
format=OGR_SHAPEFILE_DRIVER_NAME):
""" Get features (in WGS 84) for the drainage area associated with a
set of NHD (National Hydrography Dataset) stream reaches.
@param config A Python ConfigParser containing the following
sections and options:
'PATH_OF_NHDPLUS2_CATCHMENT' (absolute path to
NHD catchment shapefile)
@param outputDir String representing the absolute/relative
path of the directory into which output rasters should be
written
@param catchmentFilename String representing name of file to
save catchment features to. The appropriate extension will be added to the file name
@param reaches List representing catchment features to be output
@param format String representing OGR driver to use
@return String representing the name of the dataset in outputDir created to hold
the features
@raise ConfigParser.NoSectionError
@raise ConfigParser.NoOptionError
@raise IOError(errno.ENOTDIR) if outputDir is not a directory
@raise IOError(errno.EACCESS) if outputDir is not writable
@raise Exception if output format is not known
@todo Detect and fix non-closed geometries, e.g.
kalisti:archive miles$ ./GetCatchmentsForComidsSP.py -p test -c 10462287
Traceback (most recent call last):
File "./GetCatchmentsForComidsSP.py", line 29, in <module>
catchmentFilename, comid)
File "/Users/miles/Dropbox/EarthCube-Multilayered/RHESSys-workflow/eclipse/EcohydroWorkflowLib/ecohydrolib/nhdplus2/networkanalysis.py", line 506, in getCatchmentFeaturesForComid
outGeom = outGeom.Union( inGeom )
File "/usr/local/Cellar/python/2.7.5/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/osgeo/ogr.py", line 4065, in Union
return _ogr.Geometry_Union(self, *args)
RuntimeError: TopologyException: found non-noded intersection between LINESTRING (-77.9145 37.0768, -77.9147 37.0768) and LINESTRING (-77.9147 37.0768, -77.9145 37.0768) at -77.914621661942761 37.076822779115943
"""
catchmentFeatureDBPath = config.get('NHDPLUS2',
'PATH_OF_NHDPLUS2_CATCHMENT')
if not os.access(catchmentFeatureDBPath, os.R_OK):
raise IOError(
errno.EACCES, "The catchment feature DB at %s is not readable" %
catchmentFeatureDBPath)
catchmentFeatureDBPath = os.path.abspath(catchmentFeatureDBPath)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR,
"Output directory %s is not a directory" % (outputDir, ))
if not os.access(outputDir, os.W_OK):
raise IOError(
errno.EACCES,
"Not allowed to write to output directory %s" % (outputDir, ))
outputDir = os.path.abspath(outputDir)
if not format in list(OGR_DRIVERS.keys()):
raise Exception("Output format '%s' is not known" % (format, ))
catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep,
OGR_DRIVERS[format])
catchmentFilepath = os.path.join(outputDir, catchmentFilename)
# Open input layer
ogr.UseExceptions()
poDS = ogr.Open(catchmentFeatureDBPath, OGR_UPDATE_MODE)
if not poDS:
raise Exception("Unable to open catchment feature database %s" (
catchmentFeatureDBPath, ))
assert (poDS.GetLayerCount() > 0)
poLayer = poDS.GetLayer(0)
assert (poLayer)
# Create output data source
poDriver = ogr.GetDriverByName(format)
assert (poDriver)
poODS = poDriver.CreateDataSource(catchmentFilepath)
assert (poODS != None)
# poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), poLayer.GetGeomType())
poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(),
ogr.wkbMultiPolygon)
# poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbPolygon )
# Create fields in output layer
layerDefn = poLayer.GetLayerDefn()
i = 0
fieldCount = layerDefn.GetFieldCount()
while i < fieldCount:
fieldDefn = layerDefn.GetFieldDefn(i)
poOLayer.CreateField(fieldDefn)
i = i + 1
# Create single geometry to hold catchment polygon in output shapefile
outGeom = ogr.Geometry(poOLayer.GetGeomType())
# polygon = Polygon()
# Copy features, unioning them as we go
numReaches = len(reaches)
# Copy features in batches of UPSTREAM_SEARCH_THRESHOLD to overcome limit in
# OGR driver for input layer
start = 0
end = UPSTREAM_SEARCH_THRESHOLD
while end < numReaches:
whereFilter = "featureid=%s" % (reaches[start], )
for reach in reaches[start + 1:end]:
whereFilter = whereFilter + " OR featureid=%s" % (reach, )
# Copy features
assert (poLayer.SetAttributeFilter(whereFilter) == 0)
inFeature = poLayer.GetNextFeature()
# Union geometry of input feature to output feature
while inFeature:
# inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001)
inGeom = inFeature.GetGeometryRef()
outGeom = outGeom.Union(inGeom)
# polygon = polygon.union( loads( inGeom.ExportToWkb() ) )
# polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] )
inFeature.Destroy()
inFeature = poLayer.GetNextFeature()
start = end
end = end + UPSTREAM_SEARCH_THRESHOLD
# Copy remaining features
whereFilter = "featureid=%s" % (reaches[start], )
for reach in reaches[start + 1:end]:
whereFilter = whereFilter + " OR featureid=%s" % (reach, )
# Copy features
poLayer.SetAttributeFilter(whereFilter)
assert (poLayer.SetAttributeFilter(whereFilter) == 0)
inFeature = poLayer.GetNextFeature()
while inFeature:
# inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001)
inGeom = inFeature.GetGeometryRef()
outGeom = outGeom.Union(inGeom)
# polygon = polygon.union( loads( inGeom.ExportToWkb() ) )
# polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] )
inFeature.Destroy()
inFeature = poLayer.GetNextFeature()
# Create a new polygon that only contains exterior points
outGeom = ogr.ForceToPolygon(outGeom)
polygon = loads(outGeom.ExportToWkb())
if polygon.exterior:
coords = polygon.exterior.coords
newPolygon = Polygon(coords)
else:
newPolygon = Polygon()
# Write new feature to output feature data source
outFeat = ogr.Feature(poOLayer.GetLayerDefn())
outFeat.SetGeometry(ogr.CreateGeometryFromWkb(dumps(newPolygon)))
poOLayer.CreateFeature(outFeat)
return catchmentFilename
def getCatchmentFeaturesForReaches(config, outputDir,
catchmentFilename, reaches,
format=OGR_SHAPEFILE_DRIVER_NAME):
""" Get features (in WGS 84) for the drainage area associated with a
set of NHD (National Hydrography Dataset) stream reaches.
@param config A Python ConfigParser containing the following
sections and options:
'PATH_OF_NHDPLUS2_CATCHMENT' (absolute path to
NHD catchment shapefile)
@param outputDir String representing the absolute/relative
path of the directory into which output rasters should be
written
@param catchmentFilename String representing name of file to
save catchment features to. The appropriate extension will be added to the file name
@param reaches List representing catchment features to be output
@param format String representing OGR driver to use
@return String representing the name of the dataset in outputDir created to hold
the features
@raise ConfigParser.NoSectionError
@raise ConfigParser.NoOptionError
@raise IOError(errno.ENOTDIR) if outputDir is not a directory
@raise IOError(errno.EACCESS) if outputDir is not writable
@raise Exception if output format is not known
@todo Detect and fix non-closed geometries, e.g.
kalisti:archive miles$ ./GetCatchmentsForComidsSP.py -p test -c 10462287
Traceback (most recent call last):
File "./GetCatchmentsForComidsSP.py", line 29, in <module>
catchmentFilename, comid)
File "/Users/miles/Dropbox/EarthCube-Multilayered/RHESSys-workflow/eclipse/EcohydroWorkflowLib/ecohydrolib/nhdplus2/networkanalysis.py", line 506, in getCatchmentFeaturesForComid
outGeom = outGeom.Union( inGeom )
File "/usr/local/Cellar/python/2.7.5/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/osgeo/ogr.py", line 4065, in Union
return _ogr.Geometry_Union(self, *args)
RuntimeError: TopologyException: found non-noded intersection between LINESTRING (-77.9145 37.0768, -77.9147 37.0768) and LINESTRING (-77.9147 37.0768, -77.9145 37.0768) at -77.914621661942761 37.076822779115943
"""
catchmentFeatureDBPath = config.get('NHDPLUS2', 'PATH_OF_NHDPLUS2_CATCHMENT')
if not os.access(catchmentFeatureDBPath, os.R_OK):
raise IOError(errno.EACCES, "The catchment feature DB at %s is not readable" %
catchmentFeatureDBPath)
catchmentFeatureDBPath = os.path.abspath(catchmentFeatureDBPath)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
if not format in OGR_DRIVERS.keys():
raise Exception("Output format '%s' is not known" % (format,) )
catchmentFilename ="%s%s%s" % ( catchmentFilename, os.extsep, OGR_DRIVERS[format] )
catchmentFilepath = os.path.join(outputDir, catchmentFilename)
# Open input layer
ogr.UseExceptions()
poDS = ogr.Open(catchmentFeatureDBPath, OGR_UPDATE_MODE)
if not poDS:
raise Exception("Unable to open catchment feature database %s" (catchmentFeatureDBPath,))
assert(poDS.GetLayerCount() > 0)
poLayer = poDS.GetLayer(0)
assert(poLayer)
# Create output data source
poDriver = ogr.GetDriverByName(format)
assert(poDriver)
poODS = poDriver.CreateDataSource(catchmentFilepath)
assert(poODS != None)
# poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), poLayer.GetGeomType())
poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbMultiPolygon )
# poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbPolygon )
# Create fields in output layer
layerDefn = poLayer.GetLayerDefn()
i = 0
fieldCount = layerDefn.GetFieldCount()
while i < fieldCount:
fieldDefn = layerDefn.GetFieldDefn(i)
poOLayer.CreateField(fieldDefn)
i = i + 1
# Create single geometry to hold catchment polygon in output shapefile
outGeom = ogr.Geometry( poOLayer.GetGeomType() )
# polygon = Polygon()
# Copy features, unioning them as we go
numReaches = len(reaches)
# Copy features in batches of UPSTREAM_SEARCH_THRESHOLD to overcome limit in
# OGR driver for input layer
start = 0
end = UPSTREAM_SEARCH_THRESHOLD
while end < numReaches:
whereFilter = "featureid=%s" % (reaches[start],)
for reach in reaches[start+1:end]:
whereFilter = whereFilter + " OR featureid=%s" % (reach,)
# Copy features
assert(poLayer.SetAttributeFilter(whereFilter) == 0)
inFeature = poLayer.GetNextFeature()
# Union geometry of input feature to output feature
while inFeature:
# inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001)
inGeom = inFeature.GetGeometryRef()
outGeom = outGeom.Union( inGeom )
# polygon = polygon.union( loads( inGeom.ExportToWkb() ) )
# polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] )
inFeature.Destroy()
inFeature = poLayer.GetNextFeature()
start = end
end = end + UPSTREAM_SEARCH_THRESHOLD
# Copy remaining features
whereFilter = "featureid=%s" % (reaches[start],)
for reach in reaches[start+1:end]:
whereFilter = whereFilter + " OR featureid=%s" % (reach,)
# Copy features
poLayer.SetAttributeFilter(whereFilter)
assert(poLayer.SetAttributeFilter(whereFilter) == 0)
inFeature = poLayer.GetNextFeature()
while inFeature:
# inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001)
inGeom = inFeature.GetGeometryRef()
outGeom = outGeom.Union( inGeom )
# polygon = polygon.union( loads( inGeom.ExportToWkb() ) )
# polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] )
inFeature.Destroy()
inFeature = poLayer.GetNextFeature()
# Create a new polygon that only contains exterior points
outGeom = ogr.ForceToPolygon( outGeom )
polygon = loads( outGeom.ExportToWkb() )
if polygon.exterior:
coords = polygon.exterior.coords
newPolygon = Polygon(coords)
else:
newPolygon = Polygon()
# Write new feature to output feature data source
outFeat = ogr.Feature( poOLayer.GetLayerDefn() )
outFeat.SetGeometry( ogr.CreateGeometryFromWkb( dumps(newPolygon) ) )
poOLayer.CreateFeature(outFeat)
return catchmentFilename
def getCatchmentFeaturesForStreamflowGage(config,
outputDir,
catchmentFilename,
reachcode,
measure,
format=OGR_SHAPEFILE_DRIVER_NAME):
""" Query NHDPlus V2 web service for features (in WGS 84) for
the drainage area associated with a given NHD (National
Hydrography Dataset) streamflow gage identified by a reach
code and measure.
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative
path of the directory into which output rasters should be
written
@param format String representing OGR driver to use
@param catchmentFilename String representing name of file to
save catchment features to. The appropriate extension will be added to the file name
@param reachcode String representing NHD streamflow gage
@param measure Float representing the measure along reach
where Stream Gage is located in percent from downstream
end of the one or more NHDFlowline features that are
assigned to the ReachCode (see NHDPlusV21 GageLoc table)
@return Tuple(String representing the name of the dataset in outputDir created to hold
the features, URL of the request)
@raise IOError(errno.EACCESS) if OGR binary is not executable
@raise IOError(errno.ENOTDIR) if outputDir is not a directory
@raise IOError(errno.EACCESS) if outputDir is not writable
@raise Exception if output format is not known
@raise WebserviceError if an error occurred calling the web service
"""
ogrCmdPath = config.get('GDAL/OGR', 'PATH_OF_OGR2OGR')
if not os.access(ogrCmdPath, os.X_OK):
raise IOError(
errno.EACCES,
"The ogr2ogr binary at %s is not executable" % ogrCmdPath)
ogrCmdPath = os.path.abspath(ogrCmdPath)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR,
"Output directory %s is not a directory" % (outputDir, ))
if not os.access(outputDir, os.W_OK):
raise IOError(
errno.EACCES,
"Not allowed to write to output directory %s" % (outputDir, ))
outputDir = os.path.abspath(outputDir)
if not format in list(OGR_DRIVERS.keys()):
raise Exception("Output format '%s' is not known" % (format, ))
catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep,
OGR_DRIVERS[format])
catchmentFilepath = os.path.join(outputDir, catchmentFilename)
url = URL_PROTO_CATCHMENT.format(reachcode=reachcode, measure=str(measure))
urlFetched = "http://%s%s" % (HOST, url)
conn = http.client.HTTPConnection(HOST)
try:
conn.request('GET', url)
res = conn.getresponse(buffering=True)
except socket.error as e:
raise WebserviceError(urlFetched, str(e))
if 200 != res.status:
error = "%d %s" % (res.status, res.reason)
raise WebserviceError(urlFetched, error)
contentType = res.getheader('Content-Type')
if contentType.find(CONTENT_TYPE_ERROR) != -1:
error = res.read()
raise WebserviceError(urlFetched, error)
elif contentType.find(CONTENT_TYPE) != -1:
# The data returned were of the type expected, read the data
tmpdir = tempfile.mkdtemp()
failure = False
data = res.read(_BUFF_LEN)
if data:
tmpfile = os.path.join(tmpdir, 'catchment.geojson')
f = open(tmpfile, 'wb')
while data:
f.write(data)
data = res.read(_BUFF_LEN)
f.close()
# Convert GeoJSON to ESRI Shapfile using OGR
ogrCommand = "%s -s_srs EPSG:4326 -t_srs EPSG:4326 -f '%s' %s %s" % (
ogrCmdPath, format, catchmentFilepath, tmpfile)
os.system(ogrCommand)
if not os.path.exists(catchmentFilepath):
failure = False
shutil.rmtree(tmpdir)
if failure:
raise WebserviceError(
urlFetched, "Failed to store catchment features in file %s" %
(catchmentFilepath, ))
else:
error = "Recieved content type %s but expected type %s" % (
contentType, CONTENT_TYPE)
raise WebserviceError(urlFetched, error)
return (catchmentFilename, urlFetched)