def getCatchmentFeaturesForStreamflowGage( config, outputDir, catchmentFilename, reachcode, measure, format=OGR_SHAPEFILE_DRIVER_NAME ): """ Query NHDPlus V2 web service for features (in WGS 84) for the drainage area associated with a given NHD (National Hydrography Dataset) streamflow gage identified by a reach code and measure. @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output rasters should be written @param format String representing OGR driver to use @param catchmentFilename String representing name of file to save catchment features to. The appropriate extension will be added to the file name @param reachcode String representing NHD streamflow gage @param measure Float representing the measure along reach where Stream Gage is located in percent from downstream end of the one or more NHDFlowline features that are assigned to the ReachCode (see NHDPlusV21 GageLoc table) @return Tuple(String representing the name of the dataset in outputDir created to hold the features, URL of the request) @raise IOError(errno.EACCESS) if OGR binary is not executable @raise IOError(errno.ENOTDIR) if outputDir is not a directory @raise IOError(errno.EACCESS) if outputDir is not writable @raise Exception if output format is not known @raise WebserviceError if an error occurred calling the web service """ ogrCmdPath = config.get("GDAL/OGR", "PATH_OF_OGR2OGR") if not os.access(ogrCmdPath, os.X_OK): raise IOError(errno.EACCES, "The ogr2ogr binary at %s is not executable" % ogrCmdPath) ogrCmdPath = os.path.abspath(ogrCmdPath) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,)) if not os.access(outputDir, os.W_OK): raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,)) outputDir = os.path.abspath(outputDir) if not format in OGR_DRIVERS.keys(): raise Exception("Output format '%s' is not known" % (format,)) catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep, OGR_DRIVERS[format]) catchmentFilepath = os.path.join(outputDir, catchmentFilename) url = URL_PROTO_CATCHMENT.format(reachcode=reachcode, measure=str(measure)) urlFetched = "http://%s%s" % (HOST, url) conn = httplib.HTTPConnection(HOST) try: conn.request("GET", url) res = conn.getresponse(buffering=True) except socket.error as e: raise WebserviceError(urlFetched, str(e)) if 200 != res.status: error = "%d %s" % (res.status, res.reason) raise WebserviceError(urlFetched, error) contentType = res.getheader("Content-Type") if contentType.find(CONTENT_TYPE_ERROR) != -1: error = res.read() raise WebserviceError(urlFetched, error) elif contentType.find(CONTENT_TYPE) != -1: # The data returned were of the type expected, read the data tmpdir = tempfile.mkdtemp() failure = False data = res.read(_BUFF_LEN) if data: tmpfile = os.path.join(tmpdir, "catchment.geojson") f = open(tmpfile, "wb") while data: f.write(data) data = res.read(_BUFF_LEN) f.close() # Convert GeoJSON to ESRI Shapfile using OGR ogrCommand = "%s -s_srs EPSG:4326 -t_srs EPSG:4326 -f '%s' %s %s" % ( ogrCmdPath, format, catchmentFilepath, tmpfile, ) os.system(ogrCommand) if not os.path.exists(catchmentFilepath): failure = False shutil.rmtree(tmpdir) if failure: raise WebserviceError(urlFetched, "Failed to store catchment features in file %s" % (catchmentFilepath,)) else: error = "Recieved content type %s but expected type %s" % (contentType, CONTENT_TYPE) raise WebserviceError(urlFetched, error) return (catchmentFilename, urlFetched)
def getCatchmentFeaturesForReaches(config, outputDir, catchmentFilename, reaches, format=OGR_SHAPEFILE_DRIVER_NAME): """ Get features (in WGS 84) for the drainage area associated with a set of NHD (National Hydrography Dataset) stream reaches. @param config A Python ConfigParser containing the following sections and options: 'PATH_OF_NHDPLUS2_CATCHMENT' (absolute path to NHD catchment shapefile) @param outputDir String representing the absolute/relative path of the directory into which output rasters should be written @param catchmentFilename String representing name of file to save catchment features to. The appropriate extension will be added to the file name @param reaches List representing catchment features to be output @param format String representing OGR driver to use @return String representing the name of the dataset in outputDir created to hold the features @raise ConfigParser.NoSectionError @raise ConfigParser.NoOptionError @raise IOError(errno.ENOTDIR) if outputDir is not a directory @raise IOError(errno.EACCESS) if outputDir is not writable @raise Exception if output format is not known @todo Detect and fix non-closed geometries, e.g. kalisti:archive miles$ ./GetCatchmentsForComidsSP.py -p test -c 10462287 Traceback (most recent call last): File "./GetCatchmentsForComidsSP.py", line 29, in <module> catchmentFilename, comid) File "/Users/miles/Dropbox/EarthCube-Multilayered/RHESSys-workflow/eclipse/EcohydroWorkflowLib/ecohydrolib/nhdplus2/networkanalysis.py", line 506, in getCatchmentFeaturesForComid outGeom = outGeom.Union( inGeom ) File "/usr/local/Cellar/python/2.7.5/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/osgeo/ogr.py", line 4065, in Union return _ogr.Geometry_Union(self, *args) RuntimeError: TopologyException: found non-noded intersection between LINESTRING (-77.9145 37.0768, -77.9147 37.0768) and LINESTRING (-77.9147 37.0768, -77.9145 37.0768) at -77.914621661942761 37.076822779115943 """ catchmentFeatureDBPath = config.get('NHDPLUS2', 'PATH_OF_NHDPLUS2_CATCHMENT') if not os.access(catchmentFeatureDBPath, os.R_OK): raise IOError( errno.EACCES, "The catchment feature DB at %s is not readable" % catchmentFeatureDBPath) catchmentFeatureDBPath = os.path.abspath(catchmentFeatureDBPath) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir, )) if not os.access(outputDir, os.W_OK): raise IOError( errno.EACCES, "Not allowed to write to output directory %s" % (outputDir, )) outputDir = os.path.abspath(outputDir) if not format in list(OGR_DRIVERS.keys()): raise Exception("Output format '%s' is not known" % (format, )) catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep, OGR_DRIVERS[format]) catchmentFilepath = os.path.join(outputDir, catchmentFilename) # Open input layer ogr.UseExceptions() poDS = ogr.Open(catchmentFeatureDBPath, OGR_UPDATE_MODE) if not poDS: raise Exception("Unable to open catchment feature database %s" ( catchmentFeatureDBPath, )) assert (poDS.GetLayerCount() > 0) poLayer = poDS.GetLayer(0) assert (poLayer) # Create output data source poDriver = ogr.GetDriverByName(format) assert (poDriver) poODS = poDriver.CreateDataSource(catchmentFilepath) assert (poODS != None) # poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), poLayer.GetGeomType()) poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbMultiPolygon) # poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbPolygon ) # Create fields in output layer layerDefn = poLayer.GetLayerDefn() i = 0 fieldCount = layerDefn.GetFieldCount() while i < fieldCount: fieldDefn = layerDefn.GetFieldDefn(i) poOLayer.CreateField(fieldDefn) i = i + 1 # Create single geometry to hold catchment polygon in output shapefile outGeom = ogr.Geometry(poOLayer.GetGeomType()) # polygon = Polygon() # Copy features, unioning them as we go numReaches = len(reaches) # Copy features in batches of UPSTREAM_SEARCH_THRESHOLD to overcome limit in # OGR driver for input layer start = 0 end = UPSTREAM_SEARCH_THRESHOLD while end < numReaches: whereFilter = "featureid=%s" % (reaches[start], ) for reach in reaches[start + 1:end]: whereFilter = whereFilter + " OR featureid=%s" % (reach, ) # Copy features assert (poLayer.SetAttributeFilter(whereFilter) == 0) inFeature = poLayer.GetNextFeature() # Union geometry of input feature to output feature while inFeature: # inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001) inGeom = inFeature.GetGeometryRef() outGeom = outGeom.Union(inGeom) # polygon = polygon.union( loads( inGeom.ExportToWkb() ) ) # polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] ) inFeature.Destroy() inFeature = poLayer.GetNextFeature() start = end end = end + UPSTREAM_SEARCH_THRESHOLD # Copy remaining features whereFilter = "featureid=%s" % (reaches[start], ) for reach in reaches[start + 1:end]: whereFilter = whereFilter + " OR featureid=%s" % (reach, ) # Copy features poLayer.SetAttributeFilter(whereFilter) assert (poLayer.SetAttributeFilter(whereFilter) == 0) inFeature = poLayer.GetNextFeature() while inFeature: # inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001) inGeom = inFeature.GetGeometryRef() outGeom = outGeom.Union(inGeom) # polygon = polygon.union( loads( inGeom.ExportToWkb() ) ) # polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] ) inFeature.Destroy() inFeature = poLayer.GetNextFeature() # Create a new polygon that only contains exterior points outGeom = ogr.ForceToPolygon(outGeom) polygon = loads(outGeom.ExportToWkb()) if polygon.exterior: coords = polygon.exterior.coords newPolygon = Polygon(coords) else: newPolygon = Polygon() # Write new feature to output feature data source outFeat = ogr.Feature(poOLayer.GetLayerDefn()) outFeat.SetGeometry(ogr.CreateGeometryFromWkb(dumps(newPolygon))) poOLayer.CreateFeature(outFeat) return catchmentFilename
def getCatchmentFeaturesForReaches(config, outputDir, catchmentFilename, reaches, format=OGR_SHAPEFILE_DRIVER_NAME): """ Get features (in WGS 84) for the drainage area associated with a set of NHD (National Hydrography Dataset) stream reaches. @param config A Python ConfigParser containing the following sections and options: 'PATH_OF_NHDPLUS2_CATCHMENT' (absolute path to NHD catchment shapefile) @param outputDir String representing the absolute/relative path of the directory into which output rasters should be written @param catchmentFilename String representing name of file to save catchment features to. The appropriate extension will be added to the file name @param reaches List representing catchment features to be output @param format String representing OGR driver to use @return String representing the name of the dataset in outputDir created to hold the features @raise ConfigParser.NoSectionError @raise ConfigParser.NoOptionError @raise IOError(errno.ENOTDIR) if outputDir is not a directory @raise IOError(errno.EACCESS) if outputDir is not writable @raise Exception if output format is not known @todo Detect and fix non-closed geometries, e.g. kalisti:archive miles$ ./GetCatchmentsForComidsSP.py -p test -c 10462287 Traceback (most recent call last): File "./GetCatchmentsForComidsSP.py", line 29, in <module> catchmentFilename, comid) File "/Users/miles/Dropbox/EarthCube-Multilayered/RHESSys-workflow/eclipse/EcohydroWorkflowLib/ecohydrolib/nhdplus2/networkanalysis.py", line 506, in getCatchmentFeaturesForComid outGeom = outGeom.Union( inGeom ) File "/usr/local/Cellar/python/2.7.5/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/osgeo/ogr.py", line 4065, in Union return _ogr.Geometry_Union(self, *args) RuntimeError: TopologyException: found non-noded intersection between LINESTRING (-77.9145 37.0768, -77.9147 37.0768) and LINESTRING (-77.9147 37.0768, -77.9145 37.0768) at -77.914621661942761 37.076822779115943 """ catchmentFeatureDBPath = config.get('NHDPLUS2', 'PATH_OF_NHDPLUS2_CATCHMENT') if not os.access(catchmentFeatureDBPath, os.R_OK): raise IOError(errno.EACCES, "The catchment feature DB at %s is not readable" % catchmentFeatureDBPath) catchmentFeatureDBPath = os.path.abspath(catchmentFeatureDBPath) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,)) if not os.access(outputDir, os.W_OK): raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,)) outputDir = os.path.abspath(outputDir) if not format in OGR_DRIVERS.keys(): raise Exception("Output format '%s' is not known" % (format,) ) catchmentFilename ="%s%s%s" % ( catchmentFilename, os.extsep, OGR_DRIVERS[format] ) catchmentFilepath = os.path.join(outputDir, catchmentFilename) # Open input layer ogr.UseExceptions() poDS = ogr.Open(catchmentFeatureDBPath, OGR_UPDATE_MODE) if not poDS: raise Exception("Unable to open catchment feature database %s" (catchmentFeatureDBPath,)) assert(poDS.GetLayerCount() > 0) poLayer = poDS.GetLayer(0) assert(poLayer) # Create output data source poDriver = ogr.GetDriverByName(format) assert(poDriver) poODS = poDriver.CreateDataSource(catchmentFilepath) assert(poODS != None) # poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), poLayer.GetGeomType()) poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbMultiPolygon ) # poOLayer = poODS.CreateLayer("catchment", poLayer.GetSpatialRef(), ogr.wkbPolygon ) # Create fields in output layer layerDefn = poLayer.GetLayerDefn() i = 0 fieldCount = layerDefn.GetFieldCount() while i < fieldCount: fieldDefn = layerDefn.GetFieldDefn(i) poOLayer.CreateField(fieldDefn) i = i + 1 # Create single geometry to hold catchment polygon in output shapefile outGeom = ogr.Geometry( poOLayer.GetGeomType() ) # polygon = Polygon() # Copy features, unioning them as we go numReaches = len(reaches) # Copy features in batches of UPSTREAM_SEARCH_THRESHOLD to overcome limit in # OGR driver for input layer start = 0 end = UPSTREAM_SEARCH_THRESHOLD while end < numReaches: whereFilter = "featureid=%s" % (reaches[start],) for reach in reaches[start+1:end]: whereFilter = whereFilter + " OR featureid=%s" % (reach,) # Copy features assert(poLayer.SetAttributeFilter(whereFilter) == 0) inFeature = poLayer.GetNextFeature() # Union geometry of input feature to output feature while inFeature: # inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001) inGeom = inFeature.GetGeometryRef() outGeom = outGeom.Union( inGeom ) # polygon = polygon.union( loads( inGeom.ExportToWkb() ) ) # polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] ) inFeature.Destroy() inFeature = poLayer.GetNextFeature() start = end end = end + UPSTREAM_SEARCH_THRESHOLD # Copy remaining features whereFilter = "featureid=%s" % (reaches[start],) for reach in reaches[start+1:end]: whereFilter = whereFilter + " OR featureid=%s" % (reach,) # Copy features poLayer.SetAttributeFilter(whereFilter) assert(poLayer.SetAttributeFilter(whereFilter) == 0) inFeature = poLayer.GetNextFeature() while inFeature: # inGeom = inFeature.GetGeometryRef().SimplifyPreserveTopology(0.0001) inGeom = inFeature.GetGeometryRef() outGeom = outGeom.Union( inGeom ) # polygon = polygon.union( loads( inGeom.ExportToWkb() ) ) # polygon = cascaded_union( [polygon, loads( inGeom.ExportToWkb() )] ) inFeature.Destroy() inFeature = poLayer.GetNextFeature() # Create a new polygon that only contains exterior points outGeom = ogr.ForceToPolygon( outGeom ) polygon = loads( outGeom.ExportToWkb() ) if polygon.exterior: coords = polygon.exterior.coords newPolygon = Polygon(coords) else: newPolygon = Polygon() # Write new feature to output feature data source outFeat = ogr.Feature( poOLayer.GetLayerDefn() ) outFeat.SetGeometry( ogr.CreateGeometryFromWkb( dumps(newPolygon) ) ) poOLayer.CreateFeature(outFeat) return catchmentFilename
def getCatchmentFeaturesForStreamflowGage(config, outputDir, catchmentFilename, reachcode, measure, format=OGR_SHAPEFILE_DRIVER_NAME): """ Query NHDPlus V2 web service for features (in WGS 84) for the drainage area associated with a given NHD (National Hydrography Dataset) streamflow gage identified by a reach code and measure. @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output rasters should be written @param format String representing OGR driver to use @param catchmentFilename String representing name of file to save catchment features to. The appropriate extension will be added to the file name @param reachcode String representing NHD streamflow gage @param measure Float representing the measure along reach where Stream Gage is located in percent from downstream end of the one or more NHDFlowline features that are assigned to the ReachCode (see NHDPlusV21 GageLoc table) @return Tuple(String representing the name of the dataset in outputDir created to hold the features, URL of the request) @raise IOError(errno.EACCESS) if OGR binary is not executable @raise IOError(errno.ENOTDIR) if outputDir is not a directory @raise IOError(errno.EACCESS) if outputDir is not writable @raise Exception if output format is not known @raise WebserviceError if an error occurred calling the web service """ ogrCmdPath = config.get('GDAL/OGR', 'PATH_OF_OGR2OGR') if not os.access(ogrCmdPath, os.X_OK): raise IOError( errno.EACCES, "The ogr2ogr binary at %s is not executable" % ogrCmdPath) ogrCmdPath = os.path.abspath(ogrCmdPath) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir, )) if not os.access(outputDir, os.W_OK): raise IOError( errno.EACCES, "Not allowed to write to output directory %s" % (outputDir, )) outputDir = os.path.abspath(outputDir) if not format in list(OGR_DRIVERS.keys()): raise Exception("Output format '%s' is not known" % (format, )) catchmentFilename = "%s%s%s" % (catchmentFilename, os.extsep, OGR_DRIVERS[format]) catchmentFilepath = os.path.join(outputDir, catchmentFilename) url = URL_PROTO_CATCHMENT.format(reachcode=reachcode, measure=str(measure)) urlFetched = "http://%s%s" % (HOST, url) conn = http.client.HTTPConnection(HOST) try: conn.request('GET', url) res = conn.getresponse(buffering=True) except socket.error as e: raise WebserviceError(urlFetched, str(e)) if 200 != res.status: error = "%d %s" % (res.status, res.reason) raise WebserviceError(urlFetched, error) contentType = res.getheader('Content-Type') if contentType.find(CONTENT_TYPE_ERROR) != -1: error = res.read() raise WebserviceError(urlFetched, error) elif contentType.find(CONTENT_TYPE) != -1: # The data returned were of the type expected, read the data tmpdir = tempfile.mkdtemp() failure = False data = res.read(_BUFF_LEN) if data: tmpfile = os.path.join(tmpdir, 'catchment.geojson') f = open(tmpfile, 'wb') while data: f.write(data) data = res.read(_BUFF_LEN) f.close() # Convert GeoJSON to ESRI Shapfile using OGR ogrCommand = "%s -s_srs EPSG:4326 -t_srs EPSG:4326 -f '%s' %s %s" % ( ogrCmdPath, format, catchmentFilepath, tmpfile) os.system(ogrCommand) if not os.path.exists(catchmentFilepath): failure = False shutil.rmtree(tmpdir) if failure: raise WebserviceError( urlFetched, "Failed to store catchment features in file %s" % (catchmentFilepath, )) else: error = "Recieved content type %s but expected type %s" % ( contentType, CONTENT_TYPE) raise WebserviceError(urlFetched, error) return (catchmentFilename, urlFetched)