def getMapunitFeaturesForBoundingBox(outputDir, bbox, \
mapunitExtended=False, tileBbox=False):
""" Query USDA Soil Data Mart for SSURGO Mapunit features with a given bounding box.
Features will be written to one or more GML files, one file for each bboxTile tile,
stored in the specified output directory. The filename will be returned as a string.
Will fetch SSURGO tabular data (see ssurgolib.attributequery.attributeList for a list
of attributes) and join those data to the features in the GML files(s).
@note Will silently exit if features already exist.
@param outputDir String representing the absolute/relative path of the directory into which features should be written
@param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
@param mapunitExtended True if extended mapunit features should be fetched.
@param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
@return A list of strings representing the name of the GML file(s) to which the mapunit features were saved.
@exception IOError if output directory is not a directory
@exception IOError if output directory is not writable
@exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
"""
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
if mapunitExtended:
typeName = 'MapunitPolyExtended'
else:
typeName = 'MapunitPoly'
if tileBbox:
bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT)
sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
else:
if calculateBoundingBoxAreaSqMeters(bbox) > MAX_SSURGO_EXTENT:
raise Exception("Bounding box area is greater than %f sq. meters" % (MAX_SSURGO_EXTENT,))
bboxes = [bbox]
gmlFiles = []
for bboxTile in bboxes:
minX = bboxTile['minX']; minY = bboxTile['minY']; maxX = bboxTile['maxX']; maxY = bboxTile['maxY']
bboxLabel = str(minX) + "_" + str(minY) + "_" + str(maxX) + "_" + str(maxY)
gmlFilename = "%s_bbox_%s-attr.gml" % (typeName, bboxLabel)
gmlFilepath = os.path.join(outputDir, gmlFilename)
if not os.path.exists(gmlFilepath):
sys.stderr.write("Fetching SSURGO data for sub bboxTile %s\n" % bboxLabel)
wfs = WebFeatureService(WFS_URL, version='1.0.0')
filter = "<Filter><BBOX><PropertyName>Geometry</PropertyName> <Box srsName='EPSG:4326'><coordinates>%f,%f %f,%f</coordinates> </Box></BBOX></Filter>" % (minX, minY, maxX, maxY)
gml = wfs.getfeature(typename=(typeName,), filter=filter, propertyname=None)
# Write intermediate GML to a file
intGmlFilename = "%s_bbox_%s.gml" % (typeName, bboxLabel)
intGmlFilepath = os.path.join(outputDir, intGmlFilename)
out = open(intGmlFilepath, 'w')
out.write(gml.read())
out.close()
# Parse GML to get list of MUKEYs
gmlFile = open(intGmlFilepath, 'r')
ssurgoFeatureHandler = SSURGOFeatureHandler()
xml.sax.parse(gmlFile, ssurgoFeatureHandler)
gmlFile.close()
mukeys = ssurgoFeatureHandler.mukeys
# Get attributes (ksat, texture, %clay, %silt, and %sand) for all components in MUKEYS
attributes = getParentMatKsatTexturePercentClaySiltSandForComponentsInMUKEYs(mukeys)
# Compute weighted average of soil properties across all components in each map unit
avgAttributes = computeWeightedAverageKsatClaySandSilt(attributes)
# Join map unit component-averaged soil properties to attribute table in GML file
gmlFile = open(intGmlFilepath, 'r')
joinedGmlStr = joinSSURGOAttributesToFeaturesByMUKEY(gmlFile, typeName, avgAttributes)
gmlFile.close()
# Write Joined GML to a file
out = open(gmlFilepath, 'w')
out.write(joinedGmlStr)
out.close()
# Delete intermediate GML file
os.unlink(intGmlFilepath)
gmlFiles.append(gmlFilename)
# TODO: join tiled data if tileBbox
return gmlFiles
def getMapunitFeaturesForBoundingBox(config, outputDir, bbox, tileBbox=False, t_srs='EPSG:4326',
tileDivisor=SSURGO_BBOX_TILE_DIVISOR,
keepOriginals=False,
overwrite=True,
nprocesses=None):
""" Query USDA Soil Data Mart for SSURGO MapunitPolyExtended features with a given bounding box.
Features will be written to one or more shapefiles, one file for each bboxTile tile,
stored in the specified output directory. The filename will be returned as a string.
Will fetch SSURGO tabular data (see ssurgolib.attributequery.ATTRIBUTE_LIST for a list
of attributes) and join those data to the features in the final shapefiles(s).
@note Will silently exit if features already exist.
@param config onfigParser containing the section 'GDAL/OGR' and option 'PATH_OF_OGR2OGR'
@param outputDir String representing the absolute/relative path of the directory into which features should be written
@param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
@param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
@param t_srs String representing the spatial reference system of the output shapefiles, of the form 'EPSG:XXXX'
@param tileDivisor Float representing amount by which to divide tile slides.
@param keepOriginals Boolean, if True original feature layers will be retained (otherwise they will be deleted)
@param overwrite Boolean, if True any existing files will be overwritten
@param nprocesses Integer representing number of processes to use for fetching SSURGO tiles in parallel (used only if bounding box needs to be tiled).
if None, multiprocessing.cpu_count() will be used.
@return A list of strings representing the name of the shapefile(s) to which the mapunit features were saved.
@exception IOError if output directory is not a directory
@exception IOError if output directory is not writable
@exception Exception if tileDivisor is not a postive float > 0.0.
@exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
@exception Exception if no MUKEYs were returned
"""
if type(tileDivisor) != float or tileDivisor <= 0.0:
raise Exception("Tile divisor must be a float > 0.0.")
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
typeName = 'MapunitPolyExtended'
if tileBbox:
bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT, t_srs, tileDivisor)
sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
else:
bboxArea = calculateBoundingBoxArea(bbox, t_srs)
if bboxArea > MAX_SSURGO_EXTENT:
raise Exception("Bounding box area %.2f sq. km is greater than %.2f sq. km. You must tile the bounding box." % (bboxArea/1000/1000, MAX_SSURGO_EXTENT/1000/1000,))
bboxes = [bbox]
numTiles = len(bboxes)
assert(numTiles >= 1)
outFiles = []
if numTiles == 1:
# No tiling, fetch SSURGO features for bbox in the current process
geojsonFilename = _getMapunitFeaturesForBoundingBoxTile(config, outputDir, bboxes[0], typeName, 1, numTiles)
outFiles.append(geojsonFilename)
else:
# Fetch SSURGO feature tiles in parallel
if nprocesses is None:
nprocesses = multiprocessing.cpu_count()
assert(type(nprocesses) == int)
assert(nprocesses > 0)
# Start my pool
pool = multiprocessing.Pool( nprocesses )
tasks = []
# Build task list
i = 1
for bboxTile in bboxes:
tasks.append( (config, outputDir, bboxTile, typeName, i, numTiles) )
i += 1
# Send tasks to pool (i.e. fetch SSURGO features for each tile in parallel)
results = [pool.apply_async(_getMapunitFeaturesForBoundingBoxTile, t) for t in tasks]
# Get resulting filenames for each tile
for result in results:
outFiles.append(result.get())
pool.close()
pool.join()
# Join tiled data if necessary
if len(outFiles) > 1:
sys.stderr.write('Merging tiled features to single shapefile...')
sys.stderr.flush()
shpFilepath = mergeFeatureLayers(config, outputDir, outFiles, typeName,
outFormat=OGR_SHAPEFILE_DRIVER_NAME,
keepOriginals=keepOriginals,
t_srs=t_srs,
overwrite=overwrite)
shpFilename = os.path.basename(shpFilepath)
sys.stderr.write('done\n')
else:
# Convert GeoJSON to shapefile
filepath = outFiles[0]
sys.stderr.write('Converting SSURGO features from GeoJSON to shapefile format...')
sys.stderr.flush()
shpFilename = convertGeoJSONToShapefile(config, outputDir, filepath, typeName, t_srs=t_srs)
os.unlink(filepath)
sys.stderr.write('done\n')
return shpFilename
def getMapunitFeaturesForBoundingBox(config, outputDir, bbox, tileBbox=False, t_srs='EPSG:4326'):
""" Query USDA Soil Data Mart for SSURGO MapunitPolyExtended features with a given bounding box.
Features will be written to one or more shapefiles, one file for each bboxTile tile,
stored in the specified output directory. The filename will be returned as a string.
Will fetch SSURGO tabular data (see ssurgolib.attributequery.ATTRIBUTE_LIST for a list
of attributes) and join those data to the features in the final shapefiles(s).
@note Will silently exit if features already exist.
@param config onfigParser containing the section 'GDAL/OGR' and option 'PATH_OF_OGR2OGR'
@param outputDir String representing the absolute/relative path of the directory into which features should be written
@param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
@param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
@param t_srs String representing the spatial reference system of the output shapefiles, of the form 'EPSG:XXXX'
@return A list of strings representing the name of the shapefile(s) to which the mapunit features were saved.
@exception IOError if output directory is not a directory
@exception IOError if output directory is not writable
@exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
@exception Exception if no MUKEYs were returned
"""
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
typeName = 'MapunitPolyExtended'
if tileBbox:
bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT)
sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
else:
if calculateBoundingBoxArea(bbox, t_srs) > MAX_SSURGO_EXTENT:
raise Exception("Bounding box area is greater than %f sq. meters" % (MAX_SSURGO_EXTENT,))
bboxes = [bbox]
outFiles = []
for bboxTile in bboxes:
minX = bboxTile['minX']; minY = bboxTile['minY']; maxX = bboxTile['maxX']; maxY = bboxTile['maxY']
bboxLabel = str(minX) + "_" + str(minY) + "_" + str(maxX) + "_" + str(maxY)
gmlFilename = "%s_bbox_%s-attr.gml" % (typeName, bboxLabel)
gmlFilepath = os.path.join(outputDir, gmlFilename)
if not os.path.exists(gmlFilepath):
sys.stderr.write("Fetching SSURGO data for sub bboxTile %s\n" % bboxLabel)
wfs = WebFeatureService(WFS_URL, version='1.0.0')
filter = "<Filter><BBOX><PropertyName>Geometry</PropertyName> <Box srsName='EPSG:4326'><coordinates>%f,%f %f,%f</coordinates> </Box></BBOX></Filter>" % (minX, minY, maxX, maxY)
gml = wfs.getfeature(typename=(typeName,), filter=filter, propertyname=None)
# Write intermediate GML to a file
intGmlFilename = "%s_bbox_%s.gml" % (typeName, bboxLabel)
intGmlFilepath = os.path.join(outputDir, intGmlFilename)
out = open(intGmlFilepath, 'w')
out.write(gml.read())
out.close()
# Parse GML to get list of MUKEYs
gmlFile = open(intGmlFilepath, 'r')
ssurgoFeatureHandler = SSURGOFeatureHandler()
xml.sax.parse(gmlFile, ssurgoFeatureHandler)
gmlFile.close()
mukeys = ssurgoFeatureHandler.mukeys
if len(mukeys) < 1:
raise Exception("No SSURGO features returned from WFS query. SSURGO GML format may have changed.\nPlease contact the developer.")
# Get attributes (ksat, texture, %clay, %silt, and %sand) for all components in MUKEYS
attributes = getParentMatKsatTexturePercentClaySiltSandForComponentsInMUKEYs(mukeys)
# Compute weighted average of soil properties across all components in each map unit
avgAttributes = computeWeightedAverageKsatClaySandSilt(attributes)
# Convert GML to GeoJSON so that we can add fields easily (GDAL 1.10+ validates GML schema
# and won't let us add fields)
tmpGeoJSONFilename = convertGMLToGeoJSON(config, outputDir, intGmlFilepath, typeName)
tmpGeoJSONFilepath = os.path.join(outputDir, tmpGeoJSONFilename)
# Join map unit component-averaged soil properties to attribute table in GML file
# gmlFile = open(intGmlFilepath, 'r')
# joinedGmlStr = joinSSURGOAttributesToFeaturesByMUKEY(gmlFile, typeName, avgAttributes)
# gmlFile.close()
tmpGeoJSONFile = open(tmpGeoJSONFilepath, 'r')
geojson = json.load(tmpGeoJSONFile)
tmpGeoJSONFile.close()
joinSSURGOAttributesToFeaturesByMUKEY_GeoJSON(geojson, typeName, avgAttributes)
# Write Joined GeoJSON to a file
out = open(tmpGeoJSONFilepath, 'w')
json.dump(geojson, out)
out.close()
# Convert GeoJSON to shapefile
filename = os.path.splitext(intGmlFilename)[0]
shpFilename = convertGeoJSONToShapefile(config, outputDir, tmpGeoJSONFilepath, filename, t_srs=t_srs)
# Delete intermediate files
os.unlink(intGmlFilepath)
os.unlink(tmpGeoJSONFilepath)
outFiles.append(shpFilename)
# TODO: join tiled data if tileBbox
return outFiles
def getMapunitFeaturesForBoundingBox(config, outputDir, bbox, tileBbox=False, t_srs='EPSG:4326',
tileDivisor=SSURGO_BBOX_TILE_DIVISOR,
keepOriginals=False,
overwrite=True,
nprocesses=None):
""" Query USDA Soil Data Mart for SSURGO MapunitPolyExtended features with a given bounding box.
Features will be written to one or more shapefiles, one file for each bboxTile tile,
stored in the specified output directory. The filename will be returned as a string.
Will fetch SSURGO tabular data (see ssurgolib.attributequery.ATTRIBUTE_LIST for a list
of attributes) and join those data to the features in the final shapefiles(s).
@note Will silently exit if features already exist.
@param config onfigParser containing the section 'GDAL/OGR' and option 'PATH_OF_OGR2OGR'
@param outputDir String representing the absolute/relative path of the directory into which features should be written
@param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
@param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
@param t_srs String representing the spatial reference system of the output shapefiles, of the form 'EPSG:XXXX'
@param tileDivisor Float representing amount by which to divide tile slides.
@param keepOriginals Boolean, if True original feature layers will be retained (otherwise they will be deleted)
@param overwrite Boolean, if True any existing files will be overwritten
@param nprocesses Integer representing number of processes to use for fetching SSURGO tiles in parallel (used only if bounding box needs to be tiled).
if None, multiprocessing.cpu_count() will be used.
@return A list of strings representing the name of the shapefile(s) to which the mapunit features were saved.
@exception IOError if output directory is not a directory
@exception IOError if output directory is not writable
@exception Exception if tileDivisor is not a postive float > 0.0.
@exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
@exception Exception if no MUKEYs were returned
"""
if type(tileDivisor) != float or tileDivisor <= 0.0:
raise Exception("Tile divisor must be a float > 0.0.")
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
typeName = 'MapunitPolyExtended'
if tileBbox:
bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT, t_srs, tileDivisor)
sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
else:
bboxArea = calculateBoundingBoxArea(bbox, t_srs)
if bboxArea > MAX_SSURGO_EXTENT:
raise Exception("Bounding box area %.2f sq. km is greater than %.2f sq. km. You must tile the bounding box." % (bboxArea/1000/1000, MAX_SSURGO_EXTENT/1000/1000,))
bboxes = [bbox]
numTiles = len(bboxes)
assert(numTiles >= 1)
outFiles = []
if numTiles == 1:
# No tiling, fetch SSURGO features for bbox in the current process
geojsonFilename = _getMapunitFeaturesForBoundingBoxTile(config, outputDir, bboxes[0], typeName, 1, numTiles)
outFiles.append(geojsonFilename)
else:
# Fetch SSURGO feature tiles in parallel
if nprocesses is None:
nprocesses = multiprocessing.cpu_count()
assert(type(nprocesses) == int)
assert(nprocesses > 0)
# Start my pool
pool = multiprocessing.Pool( nprocesses )
tasks = []
# Build task list
i = 1
for bboxTile in bboxes:
tasks.append( (config, outputDir, bboxTile, typeName, i, numTiles) )
i += 1
# Send tasks to pool (i.e. fetch SSURGO features for each tile in parallel)
results = [pool.apply_async(_getMapunitFeaturesForBoundingBoxTile, t) for t in tasks]
# Get resulting filenames for each tile
for result in results:
outFiles.append(result.get())
pool.close()
pool.join()
# Join tiled data if necessary
if len(outFiles) > 1:
sys.stderr.write('Merging tiled features to single shapefile...')
sys.stderr.flush()
shpFilepath = mergeFeatureLayers(config, outputDir, outFiles, typeName,
outFormat=OGR_SHAPEFILE_DRIVER_NAME,
keepOriginals=keepOriginals,
t_srs=t_srs,
overwrite=overwrite)
shpFilename = os.path.basename(shpFilepath)
sys.stderr.write('done\n')
else:
# Convert GeoJSON to shapefile
filepath = outFiles[0]
sys.stderr.write('Converting SSURGO features from GeoJSON to shapefile format...')
sys.stderr.flush()
shpFilename = convertGeoJSONToShapefile(config, outputDir, filepath, typeName, t_srs=t_srs)
os.unlink(filepath)
sys.stderr.write('done\n')
return shpFilename
