Beispiel #1
0
def getDEMForBoundingBox(config, outputDir, outFilename, bbox, srs, coverage=DEFAULT_COVERAGE, 
                         resx=None, resy=None, interpolation=DEFAULT_RASTER_RESAMPLE_METHOD, scale=1.0, overwrite=True,
                         verbose=False, outfp=sys.stdout):
    """ Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query.
    
        @note Adapted from code provided by [email protected].
    
        @param config A Python ConfigParser (not currently used)
        @param outputDir String representing the absolute/relative path of the directory into which output raster should be written
        @param outFilename String representing the name of the raster file to be written
        @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
            for which the raster is to be extracted.  The following keys must be specified: minX, minY, maxX, maxY, srs.
        @param srs String representing the spatial reference of the raster to be returned.
        @param coverage String representing the raster source from which to get the raster coverage.  Must be one of: NHDPlus_hydroDEM, NED
        @param resx Float representing the X resolution of the raster(s) to be returned
        @param resy Float representing the Y resolution of the raster(s) to be returned
        @param interpolation String representing interpolation method.  Must be one of RASTER_RESAMPLE_METHOD.  Defaults to DEFAULT_RASTER_RESAMPLE_METHOD.
        @param scale Float representing factor by which to scale elevation data.  Defaults to 1.0.
        @param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten.
            If False and filename exists, IOError exception will be thrown with errno.EEXIST
        @param verbose Boolean True if detailed output information should be printed to outfp
        @param outfp File-like object to which verbose output should be printed
    
        @raise IOError if outputDir is not a writable directory
        @raise IOError if outFilename already exists and overwrite is False (see above)
        @raise Exception if there was an error making the WCS request
    
        @return Tuple(True if raster data were fetched and False if not, URL of raster fetched)
    """
    dataFetched = False
    assert(coverage in COVERAGES.keys())
    assert('minX' in bbox)
    assert('minY' in bbox)
    assert('maxX' in bbox)
    assert('maxY' in bbox)
    assert('srs' in bbox)
    assert(scale > 0.0)
    
    if not os.path.isdir(outputDir):
        raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
    if not os.access(outputDir, os.W_OK):
        raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
    outputDir = os.path.abspath(outputDir)
    
    outFilepath = os.path.join(outputDir, outFilename)
    
    deleteOldfile = False
    if os.path.exists(outFilepath):
        if overwrite: 
            deleteOldfile = True
        else:
            raise IOError(errno.EEXIST, "Raster file %s already exists" % outFilepath)
     
    cov = COVERAGES[coverage]
    grid_origin = cov['grid_origin']
    grid_offset = cov['grid_offset']
    grid_extent = cov['grid_extent']
    s_srs = cov['srs']
    
    if resx is None:
        resx = abs(grid_offset[0])
    if resy is None:
        resy = abs(grid_offset[1])
    t_srs = srs

    # For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin
    grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0
    grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0
    
    p = Proj(init=DEFAULT_SRS)
    (x1, y1) = p(bbox['minX'], bbox['minY'])
    (x2, y2) = p(bbox['maxX'], bbox['maxY'])
    # Pad the width of the bounding box as the Albers transform results in regions of interest
    # that are a bit narrower than I would like, which risks watershed boundaries lying beyond
    # the DEM boundary.
    len_x = x2 - x1
    del_x = len_x * 0.30
    x1 = x1 - del_x
    x2 = x2 + del_x
    bbox_srs = DEFAULT_SRS
 
    # Find the number of grid cells from the grid origin to each edge of the request.
    # Multiply by the grid_offset and add the grid origin to get to the request location.
    xi1 = floor((x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
    xi2 = ceil((x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
    yi1 = floor((y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
    yi2 = ceil((y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
 
    # coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation
    url = URL_PROTO.format(coverage=coverage, x1=xi1, y1=yi1, x2=xi2, y2=yi2, bbox_srs=bbox_srs,
                           xoffset=grid_offset[0], yoffset=grid_offset[1])
    #ORG urlFetched = "http://%s%s" % (HOST, url)
    urlFetched = "https://%s%s" % (HOST, url)

    if verbose:
        outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched))

    # Make initial request, which will return the URL of our clipped coverage
    r = requests.get(urlFetched)
    if r.status_code != 200:
        raise Exception("Error fetching {url}, HTTP status code was {code} {reason}".format(urlFetched,
                                                                                            r.status_code,
                                                                                            r.reason))
    usgs_dem_coverage_handler = USGSDEMCoverageHandler()
    xml.sax.parseString(r.text, usgs_dem_coverage_handler)
    coverage_url = usgs_dem_coverage_handler.coverage_url
    if coverage_url is None:
        raise Exception("Unable to deteremine coverage URL from WCS server response.  Response text was: {0}".format(r.text))
    parsed_coverage_url = urlparse.urlparse(coverage_url)
    
    if verbose:
        outfp.write("Downloading DEM coverage from {0} ...\n".format(coverage_url))
    
    # Download coverage to tempfile
    tmp_dir = tempfile.mkdtemp()
    tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp')
    tmp_out = open(tmp_cov_name, mode='w+b')
    
    #ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc)
    conn = httplib.HTTPSConnection(parsed_coverage_url.netloc)
    try:
        conn.request('GET', parsed_coverage_url.path)
        res = conn.getresponse(buffering=True)
    except socket.error as e:
        msg = "Encountered the following error when trying to read raster from %s. Error: %s.  Please try again later or contact the developer." % \
            (urlFetched, str(e) )
        raise Exception(msg)
  
    if 200 != res.status:
        msg = "HTTP response %d %s encountered when querying %s.  Please try again later or contact the developer." % \
            (res.status, res.reason, urlFetched)
        raise Exception(msg) 
     
    contentType = res.getheader('Content-Type')
    
    mimeType = 'image/tiff'
    if contentType.startswith(mimeType):
        # The data returned were of the type expected, read the data
        data = res.read(_BUFF_LEN)
        if data: 
            dataFetched = True
            while data:
                tmp_out.write(data)
                data = res.read(_BUFF_LEN)
            tmp_out.close()
    elif contentType.startswith(CONTENT_TYPE_ERRORS):
        # Read the error and print to stderr
        msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\
            (coverage_url, )
        data = res.read(_BUFF_LEN)
        while data:
            msg += data 
        raise Exception(msg)
    else:
        msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \
              " Operation failed." % \
            (coverage_url, contentType, mimeType)
        raise Exception(msg)

    # Rescale raster values if requested        
    if scale != 1.0:
        # Rescale values in raster
        if verbose:
            outfp.write("Rescaling raster values by factor {0}".format(scale))
        rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name))
        rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale)
        tmp_cov_name = os.path.join(tmp_dir, rescale_out)
    
    if deleteOldfile:
        deleteGeoTiff(outFilepath)
    
    if verbose:
        msg = "Resampling raster from {s_srs} to {t_srs} " + \
              "with X resolution {resx} and Y resolution {resy}\n"
        outfp.write(msg.format(s_srs=s_srs,
                               t_srs=t_srs,
                               resx=resx,
                               resy=resy))
    
    # Re-sample to target spatial reference and resolution
    resampleRaster(config, outputDir, tmp_cov_name,  outFilename,
                   s_srs, t_srs, resx, resy, interpolation)
    
    # Delete temp directory
    shutil.rmtree(tmp_dir)
        
    return ( dataFetched, urlFetched )
Beispiel #2
0
def getDEMForBoundingBox(config,
                         outputDir,
                         outFilename,
                         bbox,
                         srs,
                         coverage=DEFAULT_COVERAGE,
                         resx=None,
                         resy=None,
                         interpolation=DEFAULT_RASTER_RESAMPLE_METHOD,
                         scale=1.0,
                         overwrite=True,
                         verbose=False,
                         outfp=sys.stdout):
    """ Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query.
    
        @note Adapted from code provided by [email protected].
    
        @param config A Python ConfigParser (not currently used)
        @param outputDir String representing the absolute/relative path of the directory into which output raster should be written
        @param outFilename String representing the name of the raster file to be written
        @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
            for which the raster is to be extracted.  The following keys must be specified: minX, minY, maxX, maxY, srs.
        @param srs String representing the spatial reference of the raster to be returned.
        @param coverage String representing the raster source from which to get the raster coverage.  Must be one of: NHDPlus_hydroDEM, NED
        @param resx Float representing the X resolution of the raster(s) to be returned
        @param resy Float representing the Y resolution of the raster(s) to be returned
        @param interpolation String representing interpolation method.  Must be one of RASTER_RESAMPLE_METHOD.  Defaults to DEFAULT_RASTER_RESAMPLE_METHOD.
        @param scale Float representing factor by which to scale elevation data.  Defaults to 1.0.
        @param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten.
            If False and filename exists, IOError exception will be thrown with errno.EEXIST
        @param verbose Boolean True if detailed output information should be printed to outfp
        @param outfp File-like object to which verbose output should be printed
    
        @raise IOError if outputDir is not a writable directory
        @raise IOError if outFilename already exists and overwrite is False (see above)
        @raise Exception if there was an error making the WCS request
    
        @return Tuple(True if raster data were fetched and False if not, URL of raster fetched)
    """
    dataFetched = False
    assert (coverage in list(COVERAGES.keys()))
    assert ('minX' in bbox)
    assert ('minY' in bbox)
    assert ('maxX' in bbox)
    assert ('maxY' in bbox)
    assert ('srs' in bbox)
    assert (scale > 0.0)

    if not os.path.isdir(outputDir):
        raise IOError(errno.ENOTDIR,
                      "Output directory %s is not a directory" % (outputDir, ))
    if not os.access(outputDir, os.W_OK):
        raise IOError(
            errno.EACCES,
            "Not allowed to write to output directory %s" % (outputDir, ))
    outputDir = os.path.abspath(outputDir)

    outFilepath = os.path.join(outputDir, outFilename)

    deleteOldfile = False
    if os.path.exists(outFilepath):
        if overwrite:
            deleteOldfile = True
        else:
            raise IOError(errno.EEXIST,
                          "Raster file %s already exists" % outFilepath)

    cov = COVERAGES[coverage]
    grid_origin = cov['grid_origin']
    grid_offset = cov['grid_offset']
    grid_extent = cov['grid_extent']
    s_srs = cov['srs']

    if resx is None:
        resx = abs(grid_offset[0])
    if resy is None:
        resy = abs(grid_offset[1])
    t_srs = srs

    # For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin
    grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0
    grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0

    p = Proj(init=DEFAULT_SRS)
    (x1, y1) = p(bbox['minX'], bbox['minY'])
    (x2, y2) = p(bbox['maxX'], bbox['maxY'])
    # Pad the width of the bounding box as the Albers transform results in regions of interest
    # that are a bit narrower than I would like, which risks watershed boundaries lying beyond
    # the DEM boundary.
    len_x = x2 - x1
    del_x = len_x * 0.30
    x1 = x1 - del_x
    x2 = x2 + del_x
    bbox_srs = DEFAULT_SRS

    # Find the number of grid cells from the grid origin to each edge of the request.
    # Multiply by the grid_offset and add the grid origin to get to the request location.
    xi1 = floor(
        (x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
    xi2 = ceil(
        (x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
    yi1 = floor(
        (y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
    yi2 = ceil(
        (y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1

    # coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation
    url = URL_PROTO.format(coverage=coverage,
                           x1=xi1,
                           y1=yi1,
                           x2=xi2,
                           y2=yi2,
                           bbox_srs=bbox_srs,
                           xoffset=grid_offset[0],
                           yoffset=grid_offset[1])
    #ORG urlFetched = "http://%s%s" % (HOST, url)
    urlFetched = "https://%s%s" % (HOST, url)

    if verbose:
        outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched))

    # Make initial request, which will return the URL of our clipped coverage
    r = requests.get(urlFetched)
    if r.status_code != 200:
        raise Exception(
            "Error fetching {url}, HTTP status code was {code} {reason}".
            format(urlFetched, r.status_code, r.reason))
    usgs_dem_coverage_handler = USGSDEMCoverageHandler()
    xml.sax.parseString(r.text, usgs_dem_coverage_handler)
    coverage_url = usgs_dem_coverage_handler.coverage_url
    if coverage_url is None:
        raise Exception(
            "Unable to deteremine coverage URL from WCS server response.  Response text was: {0}"
            .format(r.text))
    parsed_coverage_url = urllib.parse.urlparse(coverage_url)

    if verbose:
        outfp.write(
            "Downloading DEM coverage from {0} ...\n".format(coverage_url))

    # Download coverage to tempfile
    tmp_dir = tempfile.mkdtemp()
    tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp')
    tmp_out = open(tmp_cov_name, mode='w+b')

    #ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc)
    conn = http.client.HTTPSConnection(parsed_coverage_url.netloc)
    try:
        conn.request('GET', parsed_coverage_url.path)
        res = conn.getresponse(buffering=True)
    except socket.error as e:
        msg = "Encountered the following error when trying to read raster from %s. Error: %s.  Please try again later or contact the developer." % \
            (urlFetched, str(e) )
        raise Exception(msg)

    if 200 != res.status:
        msg = "HTTP response %d %s encountered when querying %s.  Please try again later or contact the developer." % \
            (res.status, res.reason, urlFetched)
        raise Exception(msg)

    contentType = res.getheader('Content-Type')

    mimeType = 'image/tiff'
    if contentType.startswith(mimeType):
        # The data returned were of the type expected, read the data
        data = res.read(_BUFF_LEN)
        if data:
            dataFetched = True
            while data:
                tmp_out.write(data)
                data = res.read(_BUFF_LEN)
            tmp_out.close()
    elif contentType.startswith(CONTENT_TYPE_ERRORS):
        # Read the error and print to stderr
        msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\
            (coverage_url, )
        data = res.read(_BUFF_LEN)
        while data:
            msg += data
        raise Exception(msg)
    else:
        msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \
              " Operation failed." % \
            (coverage_url, contentType, mimeType)
        raise Exception(msg)

    # Rescale raster values if requested
    if scale != 1.0:
        # Rescale values in raster
        if verbose:
            outfp.write("Rescaling raster values by factor {0}".format(scale))
        rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name))
        rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale)
        tmp_cov_name = os.path.join(tmp_dir, rescale_out)

    if deleteOldfile:
        deleteGeoTiff(outFilepath)

    if verbose:
        msg = "Resampling raster from {s_srs} to {t_srs} " + \
              "with X resolution {resx} and Y resolution {resy}\n"
        outfp.write(msg.format(s_srs=s_srs, t_srs=t_srs, resx=resx, resy=resy))

    # Re-sample to target spatial reference and resolution
    resampleRaster(config, outputDir, tmp_cov_name, outFilename, s_srs, t_srs,
                   resx, resy, interpolation)

    # Delete temp directory
    shutil.rmtree(tmp_dir)

    return (dataFetched, urlFetched)
if args.demResolution:
    demResolutionX = args.demResolution[0]
    demResolutionY = args.demResolution[1]
    
    if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY:
        resample = True
else:
    demResolutionX = demSrs[0]
    demResolutionY = demSrs[1]

# Resample DEM to target srs and resolution
if resample:
    sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." % (demResolutionX, demResolutionY) )
    sys.stdout.flush()
    resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \
                   s_srs=t_srs, t_srs=t_srs, \
                   trX=demResolutionX, trY=demResolutionY)
    sys.stdout.write('done\n')
else:
    shutil.move(tmpDEMFilepath, demFilepath)

# Write metadata
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX)
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY)
GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs)

# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
Beispiel #4
0
def getSoilsRasterDataForBoundingBox(config,
                                     outputDir,
                                     bbox,
                                     srs='EPSG:4326',
                                     resx=0.000277777777778,
                                     resy=0.000277777777778,
                                     interpolation='bilinear',
                                     fmt=FORMAT_GEOTIFF,
                                     overwrite=False,
                                     verbose=False,
                                     outfp=sys.stdout):
    """
        Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/
        For each property, rasters for the first 1-m of the soil profile will be downloaded
        from which the depth-weighted mean of the property will be calculated and stored in outpufDir
    
        @param config A Python ConfigParser (not currently used)
        @param outputDir String representing the absolute/relative path of the directory into which output raster should be written
        @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
            for which the raster is to be extracted.  The following keys must be specified: minX, minY, maxX, maxY, srs.
        @param srs String representing the spatial reference of the raster to be returned.
        @param resx Float representing the X resolution of the raster(s) to be returned
        @param resy Float representing the Y resolution of the raster(s) to be returned
        @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD.
        @param fmt String representing format of raster file.  Must be one of FORMATS.
        @param overwrite Boolean True if existing data should be overwritten
        @param verbose Boolean True if detailed output information should be printed to outfp
        @param outfp File-like object to which verbose output should be printed
    
        @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e.
            dict[soilPropertyName] = (soilPropertyFilePath, WCS URL)
    
        @exception Exception if interpolation method is not known
        @exception Exception if fmt is not a known format
        @exception Exception if output already exists by overwrite is False
        @exception Exception if a gdal_calc.py command fails
    """
    if interpolation not in RASTER_RESAMPLE_METHOD:
        raise Exception(
            "Interpolation method {0} is not of a known method {1}".format(
                interpolation, RASTER_RESAMPLE_METHOD))
    if fmt not in FORMATS:
        raise Exception("Format {0} is not of a known format {1}".format(
            fmt, str(FORMATS)))
    if verbose:
        outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER))

    soilPropertyRasters = {}

    #import logging
    #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
    #owslib_log = logging.getLogger('owslib')
    # Add formatting and handlers as needed
    #owslib_log.setLevel(logging.DEBUG)

    # Set-up gdal_calc.py command
    gdalBase = None
    try:
        gdalBase = config.get('GDAL/OGR', 'GDAL_BASE')
    except configparser.NoOptionError:
        gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP'))

    gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py')
    if not os.access(gdalCmdPath, os.X_OK):
        raise IOError(
            errno.EACCES,
            "The gdal_calc.py binary at %s is not executable" % gdalCmdPath)
    gdalCmdPath = os.path.abspath(gdalCmdPath)

    tmpdir = tempfile.mkdtemp()
    #print(tmpdir)

    bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']]

    # For each soil variable, download desired depth layers
    for v in list(VARIABLE.keys()):
        variable = VARIABLE[v]

        soilPropertyName = "soil_raster_pct{var}".format(var=v)
        soilPropertyFilename = "{name}.tif".format(name=soilPropertyName)
        soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename)
        soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename)

        if verbose:
            outfp.write("Getting attribute {0} ...\n".format(soilPropertyName))

        delete = False
        if os.path.exists(soilPropertyFilepath):
            if not overwrite:
                raise Exception(
                    "File {0} already exists, and overwrite is false".format(
                        soilPropertyFilepath))
            else:
                delete = True

        url = URL_BASE.format(variable=variable)

        wcs = WebCoverageService(url, version='1.0.0')
        (coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(
            wcs, variable)

        outfiles = []
        weights = []
        for c in list(coverages.keys()):
            coverage = coverages[c]
            weights.append(weights_abs[c])
            #coverage = c.format(variable=variable)
            wcsfp = wcs.getCoverage(
                identifier=coverage,
                bbox=bbox,
                crs='EPSG:4326',
                resx=resx,  # their WCS seems to accept resx, resy in meters
                resy=resy,
                format=fmt)
            filename = os.path.join(tmpdir,
                                    "{coverage}.tif".format(coverage=c))
            outfiles.append(filename)
            f = open(filename, 'wb')
            f.write(wcsfp.read())
            f.close()

        # Compute depth-length weighted-average for each coverage using gdal_calc.py
        assert (len(outfiles) == len(COVERAGES))
        gdalCommand = gdalCmdPath

        calcStr = '0'  # Identity element for addition
        for (i, outfile) in enumerate(outfiles):
            ord = i + 1
            var_label = ordinalToAlpha(ord)
            gdalCommand += " -{var} {outfile}".format(var=var_label,
                                                      outfile=outfile)
            calcStr += "+({weight}*{var})".format(weight=weights[i],
                                                  var=var_label)

        gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(
            calc=calcStr, outfile=soilPropertyFilepathTmp)
        #print("GDAL command:\n{0}".format(gdalCommand))
        process = Popen(gdalCommand,
                        cwd=outputDir,
                        shell=True,
                        stdout=PIPE,
                        stderr=PIPE)
        (process_stdout, process_stderr) = process.communicate()
        if process.returncode != 0:
            raise Exception(
                "GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n."
                .format(gdalCommand, process.returncode, process_stdout,
                        process_stderr))
        if verbose:
            outfp.write(process_stdout)
            outfp.write(process_stderr)

        # Resample raster
        if delete:
            os.unlink(soilPropertyFilepath)
        resampleRaster(config,
                       outputDir,
                       soilPropertyFilepathTmp,
                       soilPropertyFilename,
                       'EPSG:4326',
                       srs,
                       resx,
                       resy,
                       resampleMethod=interpolation)

        soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url)

    # Clean-up
    shutil.rmtree(tmpdir)

    return soilPropertyRasters
    demResolutionX = args.demResolution[0]
    demResolutionY = args.demResolution[1]

    if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY:
        resample = True
else:
    demResolutionX = demSrs[0]
    demResolutionY = demSrs[1]

# Resample DEM to target srs and resolution
if resample:
    sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." %
                     (demResolutionX, demResolutionY))
    sys.stdout.flush()
    resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \
                   s_srs=t_srs, t_srs=t_srs, \
                   trX=demResolutionX, trY=demResolutionY)
    sys.stdout.write('done\n')
else:
    shutil.move(tmpDEMFilepath, demFilepath)

# Write metadata
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX)
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY)
GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs)

# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
Beispiel #6
0
def getSoilsRasterDataForBoundingBox(config, outputDir, bbox, 
                                     srs='EPSG:4326',
                                     resx=0.000277777777778,
                                     resy=0.000277777777778,
                                     interpolation='bilinear',
                                     fmt=FORMAT_GEOTIFF, 
                                     overwrite=False,
                                     verbose=False,
                                     outfp=sys.stdout):
    """
        Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/
        For each property, rasters for the first 1-m of the soil profile will be downloaded
        from which the depth-weighted mean of the property will be calculated and stored in outpufDir
    
        @param config A Python ConfigParser (not currently used)
        @param outputDir String representing the absolute/relative path of the directory into which output raster should be written
        @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
            for which the raster is to be extracted.  The following keys must be specified: minX, minY, maxX, maxY, srs.
        @param srs String representing the spatial reference of the raster to be returned.
        @param resx Float representing the X resolution of the raster(s) to be returned
        @param resy Float representing the Y resolution of the raster(s) to be returned
        @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD.
        @param fmt String representing format of raster file.  Must be one of FORMATS.
        @param overwrite Boolean True if existing data should be overwritten
        @param verbose Boolean True if detailed output information should be printed to outfp
        @param outfp File-like object to which verbose output should be printed
    
        @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e.
            dict[soilPropertyName] = (soilPropertyFilePath, WCS URL)
    
        @exception Exception if interpolation method is not known
        @exception Exception if fmt is not a known format
        @exception Exception if output already exists by overwrite is False
        @exception Exception if a gdal_calc.py command fails
    """
    if interpolation not in RASTER_RESAMPLE_METHOD:
        raise Exception("Interpolation method {0} is not of a known method {1}".format(interpolation,
                                                                                       RASTER_RESAMPLE_METHOD))
    if fmt not in FORMATS:
        raise Exception("Format {0} is not of a known format {1}".format(fmt, str(FORMATS)))
    if verbose:
        outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER))
    
    soilPropertyRasters = {}
    
    #import logging
    #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
    #owslib_log = logging.getLogger('owslib')
    # Add formatting and handlers as needed
    #owslib_log.setLevel(logging.DEBUG)
    
    # Set-up gdal_calc.py command
    gdalBase = None
    try:
        gdalBase = config.get('GDAL/OGR', 'GDAL_BASE')
    except ConfigParser.NoOptionError:
        gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP'))
    
    gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py')
    if not os.access(gdalCmdPath, os.X_OK):
        raise IOError(errno.EACCES, "The gdal_calc.py binary at %s is not executable" %
                      gdalCmdPath)
    gdalCmdPath = os.path.abspath(gdalCmdPath)
    
    tmpdir = tempfile.mkdtemp()
    #print(tmpdir)
    
    bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']]
    
    # For each soil variable, download desired depth layers
    for v in VARIABLE.keys():
        variable = VARIABLE[v]
        
        soilPropertyName = "soil_raster_pct{var}".format(var=v)
        soilPropertyFilename = "{name}.tif".format(name=soilPropertyName)
        soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename)
        soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename)
        
        if verbose:
            outfp.write("Getting attribute {0} ...\n".format(soilPropertyName))
        
        delete = False
        if os.path.exists(soilPropertyFilepath):
            if not overwrite:
                raise Exception("File {0} already exists, and overwrite is false".format(soilPropertyFilepath))
            else:
                delete = True
        
        url = URL_BASE.format(variable=variable)

        wcs = WebCoverageService(url, version='1.0.0')
        (coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(wcs, variable)
        
        outfiles = []
        weights = []
        for c in coverages.keys():
            coverage = coverages[c]
            weights.append(weights_abs[c])
            #coverage = c.format(variable=variable)
            wcsfp = wcs.getCoverage(identifier=coverage, bbox=bbox,
                                    crs='EPSG:4326',
                                    resx=resx, # their WCS seems to accept resx, resy in meters
                                    resy=resy,
                                    format=fmt)
            filename = os.path.join(tmpdir, "{coverage}.tif".format(coverage=c))
            outfiles.append(filename)
            f = open(filename, 'wb')
            f.write(wcsfp.read())
            f.close()
        
        # Compute depth-length weighted-average for each coverage using gdal_calc.py
        assert(len(outfiles) == len(COVERAGES))
        gdalCommand = gdalCmdPath
        
        calcStr = '0' # Identity element for addition
        for (i, outfile) in enumerate(outfiles):
            ord = i + 1
            var_label = ordinalToAlpha(ord)
            gdalCommand += " -{var} {outfile}".format(var=var_label, outfile=outfile)
            calcStr += "+({weight}*{var})".format(weight=weights[i],
                                                  var=var_label)
            
        gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(calc=calcStr,
                                                                                                                         outfile=soilPropertyFilepathTmp)     
        #print("GDAL command:\n{0}".format(gdalCommand))
        process = Popen(gdalCommand, cwd=outputDir, shell=True,
                        stdout=PIPE, stderr=PIPE)
        (process_stdout, process_stderr) = process.communicate()
        if process.returncode != 0:
            raise Exception("GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n.".format(gdalCommand, 
                                                                                                           process.returncode,
                                                                                                           process_stdout,
                                                                                                           process_stderr))
        if verbose:
            outfp.write(process_stdout)
            outfp.write(process_stderr)
    
        # Resample raster
        if delete:
            os.unlink(soilPropertyFilepath)
        resampleRaster(config, outputDir, soilPropertyFilepathTmp, soilPropertyFilename,
                       'EPSG:4326', srs, resx, resy, resampleMethod=interpolation)
    
        soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url)
    
    # Clean-up
    shutil.rmtree(tmpdir)
    
    return soilPropertyRasters
        
        
Beispiel #7
0
    manifest = GenericMetadata.readManifestEntries(context)
    demFilename = manifest['dem']
    demFilepath = os.path.join(context.projectDir, demFilename)
    demFilepath = os.path.abspath(demFilepath)
    extractTileFromRasterByRasterExtent(context.config, context.projectDir,
                                        demFilepath, inRasterPath,
                                        rasterFilepath, args.resampleMethod)
else:
    if resample:
        # Reproject raster, copying into project directory in the process
        processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \
            (args.type, rasterSrs, srs, rasterX, rasterX,
             demResolutionX, demResolutionY)
        sys.stdout.write(textwrap.fill("%s..." % (processingNotes, )))
        resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \
                       s_srs=None, t_srs=srs, \
                       trX=demResolutionX, trY=demResolutionY, \
                       resampleMethod=args.resampleMethod)
    else:
        # Copy the raster in to the project directory
        processingNotes = "Importing %s raster from %s without resampling" % (
            args.type, inRasterPath)
        sys.stdout.write(textwrap.fill("%s..." % (processingNotes, )))
        sys.stdout.flush()
        copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath,
                            rasterFilename)
sys.stdout.write('done\n')

# Make sure extent of resampled raster is the same as the extent of the DEM
newRasterMetadata = getDimensionsForRaster(rasterFilepath)
if (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows):
    if args.type == GenericMetadata.RASTER_TYPE_STREAM_BURNED_DEM:
Beispiel #8
0
    sys.stdout.write("Importing DEM...")
    sys.stdout.flush()
    if args.scale:
        copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath,
                            demFilenameTemp)
    else:
        copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath,
                            demFilename)
else:
    sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \
                     (s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1],
                      demResolutionX, demResolutionY) )
    sys.stdout.flush()
    if args.scale:
        resampleRaster(context.config, context.projectDir, inDEMPath,
                       demFilenameTemp, s_srs, t_srs, demResolutionX,
                       demResolutionY, args.resampleMethod)
    else:
        resampleRaster(context.config, context.projectDir, inDEMPath,
                       demFilename, s_srs, t_srs, demResolutionX,
                       demResolutionY, args.resampleMethod)

if args.scale:
    rescaleRaster(context.config, context.projectDir, demFilepathTemp,
                  demFilename, args.scale)
    os.unlink(demFilepathTemp)

sys.stdout.write('done\n')
# Get the bounding box for the DEM
bbox = getBoundingBoxForRaster(demFilepath)
# Write a shapefile for the bounding box
# Copy the raster in to the project directory (reprojecting if need be)
if not resample:
    sys.stdout.write("Importing DEM...")
    sys.stdout.flush()
    if args.scale:
        copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilenameTemp)
    else:
        copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilename)
else:
    sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \
                     (s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1],
                      demResolutionX, demResolutionY) )
    sys.stdout.flush()
    if args.scale:
        resampleRaster(context.config, context.projectDir, inDEMPath, demFilenameTemp,
                       s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod)
    else:
        resampleRaster(context.config, context.projectDir, inDEMPath, demFilename,
                       s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod)

if args.scale:
    rescaleRaster(context.config, context.projectDir, demFilepathTemp, demFilename, args.scale)
    os.unlink(demFilepathTemp)

sys.stdout.write('done\n')
# Get the bounding box for the DEM
bbox = getBoundingBoxForRaster(demFilepath)
# Write a shapefile for the bounding box
shpFilename = writeBboxPolygonToShapefile(bbox, context.projectDir, "studyarea")

# Write metadata
Beispiel #10
0
rasterSrs = rasterMetadata[5]
rasterX = float(rasterMetadata[0])
rasterY = float(rasterMetadata[1])
if (rasterSrs != srs):
    resample = True
elif (not args.noresample) and ( (rasterX != demResolutionX) or (rasterY != demResolutionY) ):
    resample = True
    
if resample:
    # Reproject raster, copying into project directory in the process
    processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \
        (args.type, rasterSrs, srs, rasterX, rasterX,
         demResolutionX, demResolutionY) 
    sys.stdout.write("%s..." % (processingNotes,) )
    resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \
                   s_srs=rasterSrs, t_srs=srs, \
                   trX=demResolutionX, trY=demResolutionY, \
                   resampleMethod=args.resampleMethod)
else:
    # Copy the raster in to the project directory
    processingNotes = "Importing %s raster from %s without resampling" % (args.type, inRasterPath)
    sys.stdout.write("%s..." % (processingNotes,) )
    sys.stdout.flush()
    copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath, rasterFilename)
sys.stdout.write('done\n')

# Make sure extent of resampled raster is the same as the extent of the DEM
newRasterMetadata = getDimensionsForRaster(rasterFilepath)
if (not force) and ( (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows) ):
    # Extents to not match, roll back and bail out
    os.unlink(rasterFilepath)
    sys.exit(textwrap.fill("ERROR: Extent of raster dataset %s does not match extent of DEM in project directory %s. Use --force to override.") %