def getDEMForBoundingBox(config, outputDir, outFilename, bbox, srs, coverage=DEFAULT_COVERAGE, resx=None, resy=None, interpolation=DEFAULT_RASTER_RESAMPLE_METHOD, scale=1.0, overwrite=True, verbose=False, outfp=sys.stdout): """ Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query. @note Adapted from code provided by [email protected]. @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param outFilename String representing the name of the raster file to be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param coverage String representing the raster source from which to get the raster coverage. Must be one of: NHDPlus_hydroDEM, NED @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing interpolation method. Must be one of RASTER_RESAMPLE_METHOD. Defaults to DEFAULT_RASTER_RESAMPLE_METHOD. @param scale Float representing factor by which to scale elevation data. Defaults to 1.0. @param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten. If False and filename exists, IOError exception will be thrown with errno.EEXIST @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @raise IOError if outputDir is not a writable directory @raise IOError if outFilename already exists and overwrite is False (see above) @raise Exception if there was an error making the WCS request @return Tuple(True if raster data were fetched and False if not, URL of raster fetched) """ dataFetched = False assert(coverage in COVERAGES.keys()) assert('minX' in bbox) assert('minY' in bbox) assert('maxX' in bbox) assert('maxY' in bbox) assert('srs' in bbox) assert(scale > 0.0) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,)) if not os.access(outputDir, os.W_OK): raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,)) outputDir = os.path.abspath(outputDir) outFilepath = os.path.join(outputDir, outFilename) deleteOldfile = False if os.path.exists(outFilepath): if overwrite: deleteOldfile = True else: raise IOError(errno.EEXIST, "Raster file %s already exists" % outFilepath) cov = COVERAGES[coverage] grid_origin = cov['grid_origin'] grid_offset = cov['grid_offset'] grid_extent = cov['grid_extent'] s_srs = cov['srs'] if resx is None: resx = abs(grid_offset[0]) if resy is None: resy = abs(grid_offset[1]) t_srs = srs # For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0 grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0 p = Proj(init=DEFAULT_SRS) (x1, y1) = p(bbox['minX'], bbox['minY']) (x2, y2) = p(bbox['maxX'], bbox['maxY']) # Pad the width of the bounding box as the Albers transform results in regions of interest # that are a bit narrower than I would like, which risks watershed boundaries lying beyond # the DEM boundary. len_x = x2 - x1 del_x = len_x * 0.30 x1 = x1 - del_x x2 = x2 + del_x bbox_srs = DEFAULT_SRS # Find the number of grid cells from the grid origin to each edge of the request. # Multiply by the grid_offset and add the grid origin to get to the request location. xi1 = floor((x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0 xi2 = ceil((x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0 yi1 = floor((y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1 yi2 = ceil((y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1 # coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation url = URL_PROTO.format(coverage=coverage, x1=xi1, y1=yi1, x2=xi2, y2=yi2, bbox_srs=bbox_srs, xoffset=grid_offset[0], yoffset=grid_offset[1]) #ORG urlFetched = "http://%s%s" % (HOST, url) urlFetched = "https://%s%s" % (HOST, url) if verbose: outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched)) # Make initial request, which will return the URL of our clipped coverage r = requests.get(urlFetched) if r.status_code != 200: raise Exception("Error fetching {url}, HTTP status code was {code} {reason}".format(urlFetched, r.status_code, r.reason)) usgs_dem_coverage_handler = USGSDEMCoverageHandler() xml.sax.parseString(r.text, usgs_dem_coverage_handler) coverage_url = usgs_dem_coverage_handler.coverage_url if coverage_url is None: raise Exception("Unable to deteremine coverage URL from WCS server response. Response text was: {0}".format(r.text)) parsed_coverage_url = urlparse.urlparse(coverage_url) if verbose: outfp.write("Downloading DEM coverage from {0} ...\n".format(coverage_url)) # Download coverage to tempfile tmp_dir = tempfile.mkdtemp() tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp') tmp_out = open(tmp_cov_name, mode='w+b') #ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc) conn = httplib.HTTPSConnection(parsed_coverage_url.netloc) try: conn.request('GET', parsed_coverage_url.path) res = conn.getresponse(buffering=True) except socket.error as e: msg = "Encountered the following error when trying to read raster from %s. Error: %s. Please try again later or contact the developer." % \ (urlFetched, str(e) ) raise Exception(msg) if 200 != res.status: msg = "HTTP response %d %s encountered when querying %s. Please try again later or contact the developer." % \ (res.status, res.reason, urlFetched) raise Exception(msg) contentType = res.getheader('Content-Type') mimeType = 'image/tiff' if contentType.startswith(mimeType): # The data returned were of the type expected, read the data data = res.read(_BUFF_LEN) if data: dataFetched = True while data: tmp_out.write(data) data = res.read(_BUFF_LEN) tmp_out.close() elif contentType.startswith(CONTENT_TYPE_ERRORS): # Read the error and print to stderr msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\ (coverage_url, ) data = res.read(_BUFF_LEN) while data: msg += data raise Exception(msg) else: msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \ " Operation failed." % \ (coverage_url, contentType, mimeType) raise Exception(msg) # Rescale raster values if requested if scale != 1.0: # Rescale values in raster if verbose: outfp.write("Rescaling raster values by factor {0}".format(scale)) rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name)) rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale) tmp_cov_name = os.path.join(tmp_dir, rescale_out) if deleteOldfile: deleteGeoTiff(outFilepath) if verbose: msg = "Resampling raster from {s_srs} to {t_srs} " + \ "with X resolution {resx} and Y resolution {resy}\n" outfp.write(msg.format(s_srs=s_srs, t_srs=t_srs, resx=resx, resy=resy)) # Re-sample to target spatial reference and resolution resampleRaster(config, outputDir, tmp_cov_name, outFilename, s_srs, t_srs, resx, resy, interpolation) # Delete temp directory shutil.rmtree(tmp_dir) return ( dataFetched, urlFetched )
def getDEMForBoundingBox(config, outputDir, outFilename, bbox, srs, coverage=DEFAULT_COVERAGE, resx=None, resy=None, interpolation=DEFAULT_RASTER_RESAMPLE_METHOD, scale=1.0, overwrite=True, verbose=False, outfp=sys.stdout): """ Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query. @note Adapted from code provided by [email protected]. @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param outFilename String representing the name of the raster file to be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param coverage String representing the raster source from which to get the raster coverage. Must be one of: NHDPlus_hydroDEM, NED @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing interpolation method. Must be one of RASTER_RESAMPLE_METHOD. Defaults to DEFAULT_RASTER_RESAMPLE_METHOD. @param scale Float representing factor by which to scale elevation data. Defaults to 1.0. @param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten. If False and filename exists, IOError exception will be thrown with errno.EEXIST @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @raise IOError if outputDir is not a writable directory @raise IOError if outFilename already exists and overwrite is False (see above) @raise Exception if there was an error making the WCS request @return Tuple(True if raster data were fetched and False if not, URL of raster fetched) """ dataFetched = False assert (coverage in list(COVERAGES.keys())) assert ('minX' in bbox) assert ('minY' in bbox) assert ('maxX' in bbox) assert ('maxY' in bbox) assert ('srs' in bbox) assert (scale > 0.0) if not os.path.isdir(outputDir): raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir, )) if not os.access(outputDir, os.W_OK): raise IOError( errno.EACCES, "Not allowed to write to output directory %s" % (outputDir, )) outputDir = os.path.abspath(outputDir) outFilepath = os.path.join(outputDir, outFilename) deleteOldfile = False if os.path.exists(outFilepath): if overwrite: deleteOldfile = True else: raise IOError(errno.EEXIST, "Raster file %s already exists" % outFilepath) cov = COVERAGES[coverage] grid_origin = cov['grid_origin'] grid_offset = cov['grid_offset'] grid_extent = cov['grid_extent'] s_srs = cov['srs'] if resx is None: resx = abs(grid_offset[0]) if resy is None: resy = abs(grid_offset[1]) t_srs = srs # For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0 grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0 p = Proj(init=DEFAULT_SRS) (x1, y1) = p(bbox['minX'], bbox['minY']) (x2, y2) = p(bbox['maxX'], bbox['maxY']) # Pad the width of the bounding box as the Albers transform results in regions of interest # that are a bit narrower than I would like, which risks watershed boundaries lying beyond # the DEM boundary. len_x = x2 - x1 del_x = len_x * 0.30 x1 = x1 - del_x x2 = x2 + del_x bbox_srs = DEFAULT_SRS # Find the number of grid cells from the grid origin to each edge of the request. # Multiply by the grid_offset and add the grid origin to get to the request location. xi1 = floor( (x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0 xi2 = ceil( (x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0 yi1 = floor( (y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1 yi2 = ceil( (y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1 # coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation url = URL_PROTO.format(coverage=coverage, x1=xi1, y1=yi1, x2=xi2, y2=yi2, bbox_srs=bbox_srs, xoffset=grid_offset[0], yoffset=grid_offset[1]) #ORG urlFetched = "http://%s%s" % (HOST, url) urlFetched = "https://%s%s" % (HOST, url) if verbose: outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched)) # Make initial request, which will return the URL of our clipped coverage r = requests.get(urlFetched) if r.status_code != 200: raise Exception( "Error fetching {url}, HTTP status code was {code} {reason}". format(urlFetched, r.status_code, r.reason)) usgs_dem_coverage_handler = USGSDEMCoverageHandler() xml.sax.parseString(r.text, usgs_dem_coverage_handler) coverage_url = usgs_dem_coverage_handler.coverage_url if coverage_url is None: raise Exception( "Unable to deteremine coverage URL from WCS server response. Response text was: {0}" .format(r.text)) parsed_coverage_url = urllib.parse.urlparse(coverage_url) if verbose: outfp.write( "Downloading DEM coverage from {0} ...\n".format(coverage_url)) # Download coverage to tempfile tmp_dir = tempfile.mkdtemp() tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp') tmp_out = open(tmp_cov_name, mode='w+b') #ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc) conn = http.client.HTTPSConnection(parsed_coverage_url.netloc) try: conn.request('GET', parsed_coverage_url.path) res = conn.getresponse(buffering=True) except socket.error as e: msg = "Encountered the following error when trying to read raster from %s. Error: %s. Please try again later or contact the developer." % \ (urlFetched, str(e) ) raise Exception(msg) if 200 != res.status: msg = "HTTP response %d %s encountered when querying %s. Please try again later or contact the developer." % \ (res.status, res.reason, urlFetched) raise Exception(msg) contentType = res.getheader('Content-Type') mimeType = 'image/tiff' if contentType.startswith(mimeType): # The data returned were of the type expected, read the data data = res.read(_BUFF_LEN) if data: dataFetched = True while data: tmp_out.write(data) data = res.read(_BUFF_LEN) tmp_out.close() elif contentType.startswith(CONTENT_TYPE_ERRORS): # Read the error and print to stderr msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\ (coverage_url, ) data = res.read(_BUFF_LEN) while data: msg += data raise Exception(msg) else: msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \ " Operation failed." % \ (coverage_url, contentType, mimeType) raise Exception(msg) # Rescale raster values if requested if scale != 1.0: # Rescale values in raster if verbose: outfp.write("Rescaling raster values by factor {0}".format(scale)) rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name)) rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale) tmp_cov_name = os.path.join(tmp_dir, rescale_out) if deleteOldfile: deleteGeoTiff(outFilepath) if verbose: msg = "Resampling raster from {s_srs} to {t_srs} " + \ "with X resolution {resx} and Y resolution {resy}\n" outfp.write(msg.format(s_srs=s_srs, t_srs=t_srs, resx=resx, resy=resy)) # Re-sample to target spatial reference and resolution resampleRaster(config, outputDir, tmp_cov_name, outFilename, s_srs, t_srs, resx, resy, interpolation) # Delete temp directory shutil.rmtree(tmp_dir) return (dataFetched, urlFetched)
if args.demResolution: demResolutionX = args.demResolution[0] demResolutionY = args.demResolution[1] if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY: resample = True else: demResolutionX = demSrs[0] demResolutionY = demSrs[1] # Resample DEM to target srs and resolution if resample: sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." % (demResolutionX, demResolutionY) ) sys.stdout.flush() resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \ s_srs=t_srs, t_srs=t_srs, \ trX=demResolutionX, trY=demResolutionY) sys.stdout.write('done\n') else: shutil.move(tmpDEMFilepath, demFilepath) # Write metadata GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX) GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY) GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs) # Get rows and columns for DEM (columns, rows) = getDimensionsForRaster(demFilepath) GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns) GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
def getSoilsRasterDataForBoundingBox(config, outputDir, bbox, srs='EPSG:4326', resx=0.000277777777778, resy=0.000277777777778, interpolation='bilinear', fmt=FORMAT_GEOTIFF, overwrite=False, verbose=False, outfp=sys.stdout): """ Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/ For each property, rasters for the first 1-m of the soil profile will be downloaded from which the depth-weighted mean of the property will be calculated and stored in outpufDir @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD. @param fmt String representing format of raster file. Must be one of FORMATS. @param overwrite Boolean True if existing data should be overwritten @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e. dict[soilPropertyName] = (soilPropertyFilePath, WCS URL) @exception Exception if interpolation method is not known @exception Exception if fmt is not a known format @exception Exception if output already exists by overwrite is False @exception Exception if a gdal_calc.py command fails """ if interpolation not in RASTER_RESAMPLE_METHOD: raise Exception( "Interpolation method {0} is not of a known method {1}".format( interpolation, RASTER_RESAMPLE_METHOD)) if fmt not in FORMATS: raise Exception("Format {0} is not of a known format {1}".format( fmt, str(FORMATS))) if verbose: outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER)) soilPropertyRasters = {} #import logging #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) #owslib_log = logging.getLogger('owslib') # Add formatting and handlers as needed #owslib_log.setLevel(logging.DEBUG) # Set-up gdal_calc.py command gdalBase = None try: gdalBase = config.get('GDAL/OGR', 'GDAL_BASE') except configparser.NoOptionError: gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP')) gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py') if not os.access(gdalCmdPath, os.X_OK): raise IOError( errno.EACCES, "The gdal_calc.py binary at %s is not executable" % gdalCmdPath) gdalCmdPath = os.path.abspath(gdalCmdPath) tmpdir = tempfile.mkdtemp() #print(tmpdir) bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']] # For each soil variable, download desired depth layers for v in list(VARIABLE.keys()): variable = VARIABLE[v] soilPropertyName = "soil_raster_pct{var}".format(var=v) soilPropertyFilename = "{name}.tif".format(name=soilPropertyName) soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename) soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename) if verbose: outfp.write("Getting attribute {0} ...\n".format(soilPropertyName)) delete = False if os.path.exists(soilPropertyFilepath): if not overwrite: raise Exception( "File {0} already exists, and overwrite is false".format( soilPropertyFilepath)) else: delete = True url = URL_BASE.format(variable=variable) wcs = WebCoverageService(url, version='1.0.0') (coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle( wcs, variable) outfiles = [] weights = [] for c in list(coverages.keys()): coverage = coverages[c] weights.append(weights_abs[c]) #coverage = c.format(variable=variable) wcsfp = wcs.getCoverage( identifier=coverage, bbox=bbox, crs='EPSG:4326', resx=resx, # their WCS seems to accept resx, resy in meters resy=resy, format=fmt) filename = os.path.join(tmpdir, "{coverage}.tif".format(coverage=c)) outfiles.append(filename) f = open(filename, 'wb') f.write(wcsfp.read()) f.close() # Compute depth-length weighted-average for each coverage using gdal_calc.py assert (len(outfiles) == len(COVERAGES)) gdalCommand = gdalCmdPath calcStr = '0' # Identity element for addition for (i, outfile) in enumerate(outfiles): ord = i + 1 var_label = ordinalToAlpha(ord) gdalCommand += " -{var} {outfile}".format(var=var_label, outfile=outfile) calcStr += "+({weight}*{var})".format(weight=weights[i], var=var_label) gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format( calc=calcStr, outfile=soilPropertyFilepathTmp) #print("GDAL command:\n{0}".format(gdalCommand)) process = Popen(gdalCommand, cwd=outputDir, shell=True, stdout=PIPE, stderr=PIPE) (process_stdout, process_stderr) = process.communicate() if process.returncode != 0: raise Exception( "GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n." .format(gdalCommand, process.returncode, process_stdout, process_stderr)) if verbose: outfp.write(process_stdout) outfp.write(process_stderr) # Resample raster if delete: os.unlink(soilPropertyFilepath) resampleRaster(config, outputDir, soilPropertyFilepathTmp, soilPropertyFilename, 'EPSG:4326', srs, resx, resy, resampleMethod=interpolation) soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url) # Clean-up shutil.rmtree(tmpdir) return soilPropertyRasters
demResolutionX = args.demResolution[0] demResolutionY = args.demResolution[1] if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY: resample = True else: demResolutionX = demSrs[0] demResolutionY = demSrs[1] # Resample DEM to target srs and resolution if resample: sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." % (demResolutionX, demResolutionY)) sys.stdout.flush() resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \ s_srs=t_srs, t_srs=t_srs, \ trX=demResolutionX, trY=demResolutionY) sys.stdout.write('done\n') else: shutil.move(tmpDEMFilepath, demFilepath) # Write metadata GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX) GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY) GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs) # Get rows and columns for DEM (columns, rows) = getDimensionsForRaster(demFilepath) GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns) GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
def getSoilsRasterDataForBoundingBox(config, outputDir, bbox, srs='EPSG:4326', resx=0.000277777777778, resy=0.000277777777778, interpolation='bilinear', fmt=FORMAT_GEOTIFF, overwrite=False, verbose=False, outfp=sys.stdout): """ Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/ For each property, rasters for the first 1-m of the soil profile will be downloaded from which the depth-weighted mean of the property will be calculated and stored in outpufDir @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD. @param fmt String representing format of raster file. Must be one of FORMATS. @param overwrite Boolean True if existing data should be overwritten @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e. dict[soilPropertyName] = (soilPropertyFilePath, WCS URL) @exception Exception if interpolation method is not known @exception Exception if fmt is not a known format @exception Exception if output already exists by overwrite is False @exception Exception if a gdal_calc.py command fails """ if interpolation not in RASTER_RESAMPLE_METHOD: raise Exception("Interpolation method {0} is not of a known method {1}".format(interpolation, RASTER_RESAMPLE_METHOD)) if fmt not in FORMATS: raise Exception("Format {0} is not of a known format {1}".format(fmt, str(FORMATS))) if verbose: outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER)) soilPropertyRasters = {} #import logging #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) #owslib_log = logging.getLogger('owslib') # Add formatting and handlers as needed #owslib_log.setLevel(logging.DEBUG) # Set-up gdal_calc.py command gdalBase = None try: gdalBase = config.get('GDAL/OGR', 'GDAL_BASE') except ConfigParser.NoOptionError: gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP')) gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py') if not os.access(gdalCmdPath, os.X_OK): raise IOError(errno.EACCES, "The gdal_calc.py binary at %s is not executable" % gdalCmdPath) gdalCmdPath = os.path.abspath(gdalCmdPath) tmpdir = tempfile.mkdtemp() #print(tmpdir) bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']] # For each soil variable, download desired depth layers for v in VARIABLE.keys(): variable = VARIABLE[v] soilPropertyName = "soil_raster_pct{var}".format(var=v) soilPropertyFilename = "{name}.tif".format(name=soilPropertyName) soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename) soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename) if verbose: outfp.write("Getting attribute {0} ...\n".format(soilPropertyName)) delete = False if os.path.exists(soilPropertyFilepath): if not overwrite: raise Exception("File {0} already exists, and overwrite is false".format(soilPropertyFilepath)) else: delete = True url = URL_BASE.format(variable=variable) wcs = WebCoverageService(url, version='1.0.0') (coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(wcs, variable) outfiles = [] weights = [] for c in coverages.keys(): coverage = coverages[c] weights.append(weights_abs[c]) #coverage = c.format(variable=variable) wcsfp = wcs.getCoverage(identifier=coverage, bbox=bbox, crs='EPSG:4326', resx=resx, # their WCS seems to accept resx, resy in meters resy=resy, format=fmt) filename = os.path.join(tmpdir, "{coverage}.tif".format(coverage=c)) outfiles.append(filename) f = open(filename, 'wb') f.write(wcsfp.read()) f.close() # Compute depth-length weighted-average for each coverage using gdal_calc.py assert(len(outfiles) == len(COVERAGES)) gdalCommand = gdalCmdPath calcStr = '0' # Identity element for addition for (i, outfile) in enumerate(outfiles): ord = i + 1 var_label = ordinalToAlpha(ord) gdalCommand += " -{var} {outfile}".format(var=var_label, outfile=outfile) calcStr += "+({weight}*{var})".format(weight=weights[i], var=var_label) gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(calc=calcStr, outfile=soilPropertyFilepathTmp) #print("GDAL command:\n{0}".format(gdalCommand)) process = Popen(gdalCommand, cwd=outputDir, shell=True, stdout=PIPE, stderr=PIPE) (process_stdout, process_stderr) = process.communicate() if process.returncode != 0: raise Exception("GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n.".format(gdalCommand, process.returncode, process_stdout, process_stderr)) if verbose: outfp.write(process_stdout) outfp.write(process_stderr) # Resample raster if delete: os.unlink(soilPropertyFilepath) resampleRaster(config, outputDir, soilPropertyFilepathTmp, soilPropertyFilename, 'EPSG:4326', srs, resx, resy, resampleMethod=interpolation) soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url) # Clean-up shutil.rmtree(tmpdir) return soilPropertyRasters
manifest = GenericMetadata.readManifestEntries(context) demFilename = manifest['dem'] demFilepath = os.path.join(context.projectDir, demFilename) demFilepath = os.path.abspath(demFilepath) extractTileFromRasterByRasterExtent(context.config, context.projectDir, demFilepath, inRasterPath, rasterFilepath, args.resampleMethod) else: if resample: # Reproject raster, copying into project directory in the process processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \ (args.type, rasterSrs, srs, rasterX, rasterX, demResolutionX, demResolutionY) sys.stdout.write(textwrap.fill("%s..." % (processingNotes, ))) resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \ s_srs=None, t_srs=srs, \ trX=demResolutionX, trY=demResolutionY, \ resampleMethod=args.resampleMethod) else: # Copy the raster in to the project directory processingNotes = "Importing %s raster from %s without resampling" % ( args.type, inRasterPath) sys.stdout.write(textwrap.fill("%s..." % (processingNotes, ))) sys.stdout.flush() copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath, rasterFilename) sys.stdout.write('done\n') # Make sure extent of resampled raster is the same as the extent of the DEM newRasterMetadata = getDimensionsForRaster(rasterFilepath) if (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows): if args.type == GenericMetadata.RASTER_TYPE_STREAM_BURNED_DEM:
sys.stdout.write("Importing DEM...") sys.stdout.flush() if args.scale: copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilenameTemp) else: copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilename) else: sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \ (s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1], demResolutionX, demResolutionY) ) sys.stdout.flush() if args.scale: resampleRaster(context.config, context.projectDir, inDEMPath, demFilenameTemp, s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod) else: resampleRaster(context.config, context.projectDir, inDEMPath, demFilename, s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod) if args.scale: rescaleRaster(context.config, context.projectDir, demFilepathTemp, demFilename, args.scale) os.unlink(demFilepathTemp) sys.stdout.write('done\n') # Get the bounding box for the DEM bbox = getBoundingBoxForRaster(demFilepath) # Write a shapefile for the bounding box
# Copy the raster in to the project directory (reprojecting if need be) if not resample: sys.stdout.write("Importing DEM...") sys.stdout.flush() if args.scale: copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilenameTemp) else: copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilename) else: sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \ (s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1], demResolutionX, demResolutionY) ) sys.stdout.flush() if args.scale: resampleRaster(context.config, context.projectDir, inDEMPath, demFilenameTemp, s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod) else: resampleRaster(context.config, context.projectDir, inDEMPath, demFilename, s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod) if args.scale: rescaleRaster(context.config, context.projectDir, demFilepathTemp, demFilename, args.scale) os.unlink(demFilepathTemp) sys.stdout.write('done\n') # Get the bounding box for the DEM bbox = getBoundingBoxForRaster(demFilepath) # Write a shapefile for the bounding box shpFilename = writeBboxPolygonToShapefile(bbox, context.projectDir, "studyarea") # Write metadata
rasterSrs = rasterMetadata[5] rasterX = float(rasterMetadata[0]) rasterY = float(rasterMetadata[1]) if (rasterSrs != srs): resample = True elif (not args.noresample) and ( (rasterX != demResolutionX) or (rasterY != demResolutionY) ): resample = True if resample: # Reproject raster, copying into project directory in the process processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \ (args.type, rasterSrs, srs, rasterX, rasterX, demResolutionX, demResolutionY) sys.stdout.write("%s..." % (processingNotes,) ) resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \ s_srs=rasterSrs, t_srs=srs, \ trX=demResolutionX, trY=demResolutionY, \ resampleMethod=args.resampleMethod) else: # Copy the raster in to the project directory processingNotes = "Importing %s raster from %s without resampling" % (args.type, inRasterPath) sys.stdout.write("%s..." % (processingNotes,) ) sys.stdout.flush() copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath, rasterFilename) sys.stdout.write('done\n') # Make sure extent of resampled raster is the same as the extent of the DEM newRasterMetadata = getDimensionsForRaster(rasterFilepath) if (not force) and ( (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows) ): # Extents to not match, roll back and bail out os.unlink(rasterFilepath) sys.exit(textwrap.fill("ERROR: Extent of raster dataset %s does not match extent of DEM in project directory %s. Use --force to override.") %