예제 #1
0
def getMapunitFeaturesForBoundingBox(config, outputDir, bbox, tileBbox=False, t_srs='EPSG:4326'):
    """ Query USDA Soil Data Mart for SSURGO MapunitPolyExtended features with a given bounding box.
        Features will be written to one or more shapefiles, one file for each bboxTile tile,
        stored in the specified output directory. The filename will be returned as a string.
        Will fetch SSURGO tabular data (see ssurgolib.attributequery.ATTRIBUTE_LIST for a list
        of attributes) and join those data to the features in the final shapefiles(s).
    
        @note Will silently exit if features already exist.
    
        @param config onfigParser containing the section 'GDAL/OGR' and option 'PATH_OF_OGR2OGR'
        @param outputDir String representing the absolute/relative path of the directory into which features should be written
        @param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
        @param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
        @param t_srs String representing the spatial reference system of the output shapefiles, of the form 'EPSG:XXXX'
        
        @return A list of strings representing the name of the shapefile(s) to which the mapunit features were saved.
        
        @exception IOError if output directory is not a directory
        @exception IOError if output directory is not writable
        @exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
        @exception Exception if no MUKEYs were returned
    """
    if not os.path.isdir(outputDir):
        raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
    if not os.access(outputDir, os.W_OK):
        raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
    outputDir = os.path.abspath(outputDir)

    typeName = 'MapunitPolyExtended'

    if tileBbox:
        bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT)
        sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
    else:
        if calculateBoundingBoxArea(bbox, t_srs) > MAX_SSURGO_EXTENT:
            raise Exception("Bounding box area is greater than %f sq. meters" % (MAX_SSURGO_EXTENT,))
        bboxes = [bbox]
    
    outFiles = []
    
    for bboxTile in bboxes:
        minX = bboxTile['minX']; minY = bboxTile['minY']; maxX = bboxTile['maxX']; maxY = bboxTile['maxY']
        bboxLabel = str(minX) + "_" + str(minY) + "_" + str(maxX) + "_" + str(maxY)
    
        gmlFilename = "%s_bbox_%s-attr.gml" % (typeName, bboxLabel)
        gmlFilepath = os.path.join(outputDir, gmlFilename)
    
        if not os.path.exists(gmlFilepath):
            sys.stderr.write("Fetching SSURGO data for sub bboxTile %s\n" % bboxLabel)
        
            wfs = WebFeatureService(WFS_URL, version='1.0.0')
            filter = "<Filter><BBOX><PropertyName>Geometry</PropertyName> <Box srsName='EPSG:4326'><coordinates>%f,%f %f,%f</coordinates> </Box></BBOX></Filter>" % (minX, minY, maxX, maxY)
            gml = wfs.getfeature(typename=(typeName,), filter=filter, propertyname=None)
    
            # Write intermediate GML to a file
            intGmlFilename = "%s_bbox_%s.gml" % (typeName, bboxLabel)
            intGmlFilepath = os.path.join(outputDir, intGmlFilename)
            out = open(intGmlFilepath, 'w')
            out.write(gml.read())
            out.close()
            
            # Parse GML to get list of MUKEYs
            gmlFile = open(intGmlFilepath, 'r')
            ssurgoFeatureHandler = SSURGOFeatureHandler()
            xml.sax.parse(gmlFile, ssurgoFeatureHandler)
            gmlFile.close()
            mukeys = ssurgoFeatureHandler.mukeys
            
            if len(mukeys) < 1:
                raise Exception("No SSURGO features returned from WFS query.  SSURGO GML format may have changed.\nPlease contact the developer.")
            
            # Get attributes (ksat, texture, %clay, %silt, and %sand) for all components in MUKEYS
            attributes = getParentMatKsatTexturePercentClaySiltSandForComponentsInMUKEYs(mukeys)
            
            # Compute weighted average of soil properties across all components in each map unit
            avgAttributes = computeWeightedAverageKsatClaySandSilt(attributes)
            
            # Convert GML to GeoJSON so that we can add fields easily (GDAL 1.10+ validates GML schema 
            #   and won't let us add fields)
            tmpGeoJSONFilename = convertGMLToGeoJSON(config, outputDir, intGmlFilepath, typeName)
            tmpGeoJSONFilepath = os.path.join(outputDir, tmpGeoJSONFilename)
            
            # Join map unit component-averaged soil properties to attribute table in GML file
#             gmlFile = open(intGmlFilepath, 'r')
#             joinedGmlStr = joinSSURGOAttributesToFeaturesByMUKEY(gmlFile, typeName, avgAttributes)
#             gmlFile.close()
            tmpGeoJSONFile = open(tmpGeoJSONFilepath, 'r')
            geojson = json.load(tmpGeoJSONFile)
            tmpGeoJSONFile.close()
            joinSSURGOAttributesToFeaturesByMUKEY_GeoJSON(geojson, typeName, avgAttributes)
            
            # Write Joined GeoJSON to a file
            out = open(tmpGeoJSONFilepath, 'w')
            json.dump(geojson, out)
            out.close()
            
            # Convert GeoJSON to shapefile
            filename = os.path.splitext(intGmlFilename)[0]
            shpFilename = convertGeoJSONToShapefile(config, outputDir, tmpGeoJSONFilepath, filename, t_srs=t_srs)
            
            # Delete intermediate files
            os.unlink(intGmlFilepath)
            os.unlink(tmpGeoJSONFilepath)
        
        outFiles.append(shpFilename)
    
    # TODO: join tiled data if tileBbox
        
    return outFiles
    
    
예제 #2
0
def getMapunitFeaturesForBoundingBox(outputDir, bbox, \
                                     mapunitExtended=False, tileBbox=False):
    """ Query USDA Soil Data Mart for SSURGO Mapunit features with a given bounding box.
        Features will be written to one or more GML files, one file for each bboxTile tile,
        stored in the specified output directory. The filename will be returned as a string.
        Will fetch SSURGO tabular data (see ssurgolib.attributequery.attributeList for a list
        of attributes) and join those data to the features in the GML files(s).
    
        @note Will silently exit if features already exist.
    
        @param outputDir String representing the absolute/relative path of the directory into which features should be written
        @param bbox A dict containing keys: minX, minY, maxX, maxY, srs, where srs='EPSG:4326'
        @param mapunitExtended True if extended mapunit features should be fetched.
        @param tileBoundingBox True if bounding box should be tiled if extent exceeds featurequery.MAX_SSURGO_EXTENT
        
        @return A list of strings representing the name of the GML file(s) to which the mapunit features were saved.
        
        @exception IOError if output directory is not a directory
        @exception IOError if output directory is not writable
        @exception Exception if bounding box area is greater than MAX_SSURGO_EXTENT
    """
    if not os.path.isdir(outputDir):
        raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
    if not os.access(outputDir, os.W_OK):
        raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
    outputDir = os.path.abspath(outputDir)

    if mapunitExtended:
        typeName = 'MapunitPolyExtended'
    else:
        typeName = 'MapunitPoly'

    if tileBbox:
        bboxes = tileBoundingBox(bbox, MAX_SSURGO_EXTENT)
        sys.stderr.write("Dividing bounding box %s into %d tiles\n" % (str(bbox), len(bboxes)))
    else:
        if calculateBoundingBoxAreaSqMeters(bbox) > MAX_SSURGO_EXTENT:
            raise Exception("Bounding box area is greater than %f sq. meters" % (MAX_SSURGO_EXTENT,))
        bboxes = [bbox]
    
    gmlFiles = []
    
    for bboxTile in bboxes:
        minX = bboxTile['minX']; minY = bboxTile['minY']; maxX = bboxTile['maxX']; maxY = bboxTile['maxY']
        bboxLabel = str(minX) + "_" + str(minY) + "_" + str(maxX) + "_" + str(maxY)
    
        gmlFilename = "%s_bbox_%s-attr.gml" % (typeName, bboxLabel)
        gmlFilepath = os.path.join(outputDir, gmlFilename)
    
        if not os.path.exists(gmlFilepath):
            sys.stderr.write("Fetching SSURGO data for sub bboxTile %s\n" % bboxLabel)
        
            wfs = WebFeatureService(WFS_URL, version='1.0.0')
            filter = "<Filter><BBOX><PropertyName>Geometry</PropertyName> <Box srsName='EPSG:4326'><coordinates>%f,%f %f,%f</coordinates> </Box></BBOX></Filter>" % (minX, minY, maxX, maxY)
            gml = wfs.getfeature(typename=(typeName,), filter=filter, propertyname=None)
    
            # Write intermediate GML to a file
            intGmlFilename = "%s_bbox_%s.gml" % (typeName, bboxLabel)
            intGmlFilepath = os.path.join(outputDir, intGmlFilename)
            out = open(intGmlFilepath, 'w')
            out.write(gml.read())
            out.close()
            
            # Parse GML to get list of MUKEYs
            gmlFile = open(intGmlFilepath, 'r')
            ssurgoFeatureHandler = SSURGOFeatureHandler()
            xml.sax.parse(gmlFile, ssurgoFeatureHandler)
            gmlFile.close()
            mukeys = ssurgoFeatureHandler.mukeys
            
            # Get attributes (ksat, texture, %clay, %silt, and %sand) for all components in MUKEYS
            attributes = getParentMatKsatTexturePercentClaySiltSandForComponentsInMUKEYs(mukeys)
            
            # Compute weighted average of soil properties across all components in each map unit
            avgAttributes = computeWeightedAverageKsatClaySandSilt(attributes)
            
            # Join map unit component-averaged soil properties to attribute table in GML file
            gmlFile = open(intGmlFilepath, 'r')
            joinedGmlStr = joinSSURGOAttributesToFeaturesByMUKEY(gmlFile, typeName, avgAttributes)
            gmlFile.close()
            
            # Write Joined GML to a file
            out = open(gmlFilepath, 'w')
            out.write(joinedGmlStr)
            out.close()
            
            # Delete intermediate GML file
            os.unlink(intGmlFilepath)
        
        gmlFiles.append(gmlFilename)
    
    # TODO: join tiled data if tileBbox
        
    return gmlFiles
    
    
예제 #3
0
def _getMapunitFeaturesForBoundingBoxTile(config, outputDir, bboxTile, typeName, currTile, numTiles):
    minX = bboxTile['minX']; minY = bboxTile['minY']; maxX = bboxTile['maxX']; maxY = bboxTile['maxY']
    bboxLabel = str(minX) + "_" + str(minY) + "_" + str(maxX) + "_" + str(maxY)

    gmlFilename = "%s_bbox_%s-attr.gml" % (typeName, bboxLabel)
    gmlFilepath = os.path.join(outputDir, gmlFilename)
    geoJSONLayername = "%s_bbox_%s-attr" % (typeName, bboxLabel)

    if not os.path.exists(gmlFilepath):
        sys.stderr.write("Fetching SSURGO data for tile %s of %s, bbox: %s\n" % (currTile, numTiles, bboxLabel))
        sys.stderr.flush()
    
        wfs = WebFeatureService(WFS_URL, version='1.1.0', timeout=SSURGO_WFS_TIMEOUT_SEC)
        filter = "<Filter><BBOX><PropertyName>Geometry</PropertyName> <Box srsName='EPSG:4326'><coordinates>%f,%f %f,%f</coordinates> </Box></BBOX></Filter>" % (minX, minY, maxX, maxY)
        
        intGmlFilename = "%s_bbox_%s.gml" % (typeName, bboxLabel)
        intGmlFilepath = os.path.join(outputDir, intGmlFilename)
        ssurgoFeatureHandler = SSURGOFeatureHandler()
        
        downloadComplete = False
        downloadAttempts = 0
        while not downloadComplete:
            try:
                gml = wfs.getfeature(typename=typeName, filter=filter, propertyname=None)
        
                # Write intermediate GML to a file
                out = open(intGmlFilepath, 'w')
                out.write(gml.read())
                out.close()
                
                # Parse GML to get list of MUKEYs
                gmlFile = open(intGmlFilepath, 'r')
                xml.sax.parse(gmlFile, ssurgoFeatureHandler)
                gmlFile.close()
                downloadComplete = True
            except xml.sax.SAXParseException as e:
                # Try to re-download
                downloadAttempts += 1
                if downloadAttempts > SSURGO_GML_MAX_DOWNLOAD_ATTEMPTS:
                    raise Exception("Giving up on downloading tile {0} of {1} after {2} attempts.  There may be something wrong with the web service.  Try again later.".format(currTile, numTiles, downloadAttempts))
                else:
                    sys.stderr.write("Initial download of tile {0} of {1} possibly incomplete, error: {0}.  Retrying...".format(currTile, numTiles, str(e)))
                    sys.stderr.flush()
                    
        mukeys = ssurgoFeatureHandler.mukeys
        
        if len(mukeys) < 1:
            raise Exception("No SSURGO features returned from WFS query.  SSURGO GML format may have changed.\nPlease contact the developer.")
        
        # Get attributes (ksat, texture, %clay, %silt, and %sand) for all components in MUKEYS
        attributes = getParentMatKsatTexturePercentClaySiltSandForComponentsInMUKEYs(mukeys)
        
        # Compute weighted average of soil properties across all components in each map unit
        avgAttributes = computeWeightedAverageKsatClaySandSilt(attributes)
        
        # Convert GML to GeoJSON so that we can add fields easily (GDAL 1.10+ validates GML schema 
        #   and won't let us add fields)
        tmpGeoJSONFilename = convertGMLToGeoJSON(config, outputDir, intGmlFilepath, geoJSONLayername,
                                                 flip_gml_coords=True)
        tmpGeoJSONFilepath = os.path.join(outputDir, tmpGeoJSONFilename)
        
        # Join map unit component-averaged soil properties to attribute table in GeoJSON file
        tmpGeoJSONFile = open(tmpGeoJSONFilepath, 'r')
        geojson = json.load(tmpGeoJSONFile)
        tmpGeoJSONFile.close()
        joinSSURGOAttributesToFeaturesByMUKEY_GeoJSON(geojson, typeName, avgAttributes)
        
        # Write joined GeoJSON to a file
        out = open(tmpGeoJSONFilepath, 'w')
        json.dump(geojson, out)
        out.close()
        
        # Delete intermediate files
        os.unlink(intGmlFilepath)
        
        return tmpGeoJSONFilepath