def getDEMForBoundingBox(config, outputDir, outFilename, bbox, srs, coverage=DEFAULT_COVERAGE,
resx=None, resy=None, interpolation=DEFAULT_RASTER_RESAMPLE_METHOD, scale=1.0, overwrite=True,
verbose=False, outfp=sys.stdout):
""" Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query.
@note Adapted from code provided by [email protected].
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative path of the directory into which output raster should be written
@param outFilename String representing the name of the raster file to be written
@param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs.
@param srs String representing the spatial reference of the raster to be returned.
@param coverage String representing the raster source from which to get the raster coverage. Must be one of: NHDPlus_hydroDEM, NED
@param resx Float representing the X resolution of the raster(s) to be returned
@param resy Float representing the Y resolution of the raster(s) to be returned
@param interpolation String representing interpolation method. Must be one of RASTER_RESAMPLE_METHOD. Defaults to DEFAULT_RASTER_RESAMPLE_METHOD.
@param scale Float representing factor by which to scale elevation data. Defaults to 1.0.
@param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten.
If False and filename exists, IOError exception will be thrown with errno.EEXIST
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@raise IOError if outputDir is not a writable directory
@raise IOError if outFilename already exists and overwrite is False (see above)
@raise Exception if there was an error making the WCS request
@return Tuple(True if raster data were fetched and False if not, URL of raster fetched)
"""
dataFetched = False
assert(coverage in COVERAGES.keys())
assert('minX' in bbox)
assert('minY' in bbox)
assert('maxX' in bbox)
assert('maxY' in bbox)
assert('srs' in bbox)
assert(scale > 0.0)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR, "Output directory %s is not a directory" % (outputDir,))
if not os.access(outputDir, os.W_OK):
raise IOError(errno.EACCES, "Not allowed to write to output directory %s" % (outputDir,))
outputDir = os.path.abspath(outputDir)
outFilepath = os.path.join(outputDir, outFilename)
deleteOldfile = False
if os.path.exists(outFilepath):
if overwrite:
deleteOldfile = True
else:
raise IOError(errno.EEXIST, "Raster file %s already exists" % outFilepath)
cov = COVERAGES[coverage]
grid_origin = cov['grid_origin']
grid_offset = cov['grid_offset']
grid_extent = cov['grid_extent']
s_srs = cov['srs']
if resx is None:
resx = abs(grid_offset[0])
if resy is None:
resy = abs(grid_offset[1])
t_srs = srs
# For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin
grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0
grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0
p = Proj(init=DEFAULT_SRS)
(x1, y1) = p(bbox['minX'], bbox['minY'])
(x2, y2) = p(bbox['maxX'], bbox['maxY'])
# Pad the width of the bounding box as the Albers transform results in regions of interest
# that are a bit narrower than I would like, which risks watershed boundaries lying beyond
# the DEM boundary.
len_x = x2 - x1
del_x = len_x * 0.30
x1 = x1 - del_x
x2 = x2 + del_x
bbox_srs = DEFAULT_SRS
# Find the number of grid cells from the grid origin to each edge of the request.
# Multiply by the grid_offset and add the grid origin to get to the request location.
xi1 = floor((x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
xi2 = ceil((x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
yi1 = floor((y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
yi2 = ceil((y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
# coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation
url = URL_PROTO.format(coverage=coverage, x1=xi1, y1=yi1, x2=xi2, y2=yi2, bbox_srs=bbox_srs,
xoffset=grid_offset[0], yoffset=grid_offset[1])
#ORG urlFetched = "http://%s%s" % (HOST, url)
urlFetched = "https://%s%s" % (HOST, url)
if verbose:
outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched))
# Make initial request, which will return the URL of our clipped coverage
r = requests.get(urlFetched)
if r.status_code != 200:
raise Exception("Error fetching {url}, HTTP status code was {code} {reason}".format(urlFetched,
r.status_code,
r.reason))
usgs_dem_coverage_handler = USGSDEMCoverageHandler()
xml.sax.parseString(r.text, usgs_dem_coverage_handler)
coverage_url = usgs_dem_coverage_handler.coverage_url
if coverage_url is None:
raise Exception("Unable to deteremine coverage URL from WCS server response. Response text was: {0}".format(r.text))
parsed_coverage_url = urlparse.urlparse(coverage_url)
if verbose:
outfp.write("Downloading DEM coverage from {0} ...\n".format(coverage_url))
# Download coverage to tempfile
tmp_dir = tempfile.mkdtemp()
tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp')
tmp_out = open(tmp_cov_name, mode='w+b')
#ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc)
conn = httplib.HTTPSConnection(parsed_coverage_url.netloc)
try:
conn.request('GET', parsed_coverage_url.path)
res = conn.getresponse(buffering=True)
except socket.error as e:
msg = "Encountered the following error when trying to read raster from %s. Error: %s. Please try again later or contact the developer." % \
(urlFetched, str(e) )
raise Exception(msg)
if 200 != res.status:
msg = "HTTP response %d %s encountered when querying %s. Please try again later or contact the developer." % \
(res.status, res.reason, urlFetched)
raise Exception(msg)
contentType = res.getheader('Content-Type')
mimeType = 'image/tiff'
if contentType.startswith(mimeType):
# The data returned were of the type expected, read the data
data = res.read(_BUFF_LEN)
if data:
dataFetched = True
while data:
tmp_out.write(data)
data = res.read(_BUFF_LEN)
tmp_out.close()
elif contentType.startswith(CONTENT_TYPE_ERRORS):
# Read the error and print to stderr
msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\
(coverage_url, )
data = res.read(_BUFF_LEN)
while data:
msg += data
raise Exception(msg)
else:
msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \
" Operation failed." % \
(coverage_url, contentType, mimeType)
raise Exception(msg)
# Rescale raster values if requested
if scale != 1.0:
# Rescale values in raster
if verbose:
outfp.write("Rescaling raster values by factor {0}".format(scale))
rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name))
rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale)
tmp_cov_name = os.path.join(tmp_dir, rescale_out)
if deleteOldfile:
deleteGeoTiff(outFilepath)
if verbose:
msg = "Resampling raster from {s_srs} to {t_srs} " + \
"with X resolution {resx} and Y resolution {resy}\n"
outfp.write(msg.format(s_srs=s_srs,
t_srs=t_srs,
resx=resx,
resy=resy))
# Re-sample to target spatial reference and resolution
resampleRaster(config, outputDir, tmp_cov_name, outFilename,
s_srs, t_srs, resx, resy, interpolation)
# Delete temp directory
shutil.rmtree(tmp_dir)
return ( dataFetched, urlFetched )
def getDEMForBoundingBox(config,
outputDir,
outFilename,
bbox,
srs,
coverage=DEFAULT_COVERAGE,
resx=None,
resy=None,
interpolation=DEFAULT_RASTER_RESAMPLE_METHOD,
scale=1.0,
overwrite=True,
verbose=False,
outfp=sys.stdout):
""" Fetch U.S. 1/3 arcsecond DEM data hosted by U.S. Geological Survey using OGC WCS 1.1.1 query.
@note Adapted from code provided by [email protected].
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative path of the directory into which output raster should be written
@param outFilename String representing the name of the raster file to be written
@param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs.
@param srs String representing the spatial reference of the raster to be returned.
@param coverage String representing the raster source from which to get the raster coverage. Must be one of: NHDPlus_hydroDEM, NED
@param resx Float representing the X resolution of the raster(s) to be returned
@param resy Float representing the Y resolution of the raster(s) to be returned
@param interpolation String representing interpolation method. Must be one of RASTER_RESAMPLE_METHOD. Defaults to DEFAULT_RASTER_RESAMPLE_METHOD.
@param scale Float representing factor by which to scale elevation data. Defaults to 1.0.
@param overwrite Boolean value indicating whether or not the file indicated by filename should be overwritten.
If False and filename exists, IOError exception will be thrown with errno.EEXIST
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@raise IOError if outputDir is not a writable directory
@raise IOError if outFilename already exists and overwrite is False (see above)
@raise Exception if there was an error making the WCS request
@return Tuple(True if raster data were fetched and False if not, URL of raster fetched)
"""
dataFetched = False
assert (coverage in list(COVERAGES.keys()))
assert ('minX' in bbox)
assert ('minY' in bbox)
assert ('maxX' in bbox)
assert ('maxY' in bbox)
assert ('srs' in bbox)
assert (scale > 0.0)
if not os.path.isdir(outputDir):
raise IOError(errno.ENOTDIR,
"Output directory %s is not a directory" % (outputDir, ))
if not os.access(outputDir, os.W_OK):
raise IOError(
errno.EACCES,
"Not allowed to write to output directory %s" % (outputDir, ))
outputDir = os.path.abspath(outputDir)
outFilepath = os.path.join(outputDir, outFilename)
deleteOldfile = False
if os.path.exists(outFilepath):
if overwrite:
deleteOldfile = True
else:
raise IOError(errno.EEXIST,
"Raster file %s already exists" % outFilepath)
cov = COVERAGES[coverage]
grid_origin = cov['grid_origin']
grid_offset = cov['grid_offset']
grid_extent = cov['grid_extent']
s_srs = cov['srs']
if resx is None:
resx = abs(grid_offset[0])
if resy is None:
resy = abs(grid_offset[1])
t_srs = srs
# For requests, grid cell centers are used. Need to add half the grid_offset to the grid_origin
grid_origin_0 = grid_origin[0] + grid_offset[0] / 2.0
grid_origin_1 = grid_origin[1] + grid_offset[1] / 2.0
p = Proj(init=DEFAULT_SRS)
(x1, y1) = p(bbox['minX'], bbox['minY'])
(x2, y2) = p(bbox['maxX'], bbox['maxY'])
# Pad the width of the bounding box as the Albers transform results in regions of interest
# that are a bit narrower than I would like, which risks watershed boundaries lying beyond
# the DEM boundary.
len_x = x2 - x1
del_x = len_x * 0.30
x1 = x1 - del_x
x2 = x2 + del_x
bbox_srs = DEFAULT_SRS
# Find the number of grid cells from the grid origin to each edge of the request.
# Multiply by the grid_offset and add the grid origin to get to the request location.
xi1 = floor(
(x1 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
xi2 = ceil(
(x2 - grid_origin_0) / grid_offset[0]) * grid_offset[0] + grid_origin_0
yi1 = floor(
(y1 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
yi2 = ceil(
(y2 - grid_origin_1) / grid_offset[1]) * grid_offset[1] + grid_origin_1
# coverage, crs, bbox, format. May have the following fields: response_srs, store, resx, resy, interpolation
url = URL_PROTO.format(coverage=coverage,
x1=xi1,
y1=yi1,
x2=xi2,
y2=yi2,
bbox_srs=bbox_srs,
xoffset=grid_offset[0],
yoffset=grid_offset[1])
#ORG urlFetched = "http://%s%s" % (HOST, url)
urlFetched = "https://%s%s" % (HOST, url)
if verbose:
outfp.write("Acquiring DEM data from {0} ...\n".format(urlFetched))
# Make initial request, which will return the URL of our clipped coverage
r = requests.get(urlFetched)
if r.status_code != 200:
raise Exception(
"Error fetching {url}, HTTP status code was {code} {reason}".
format(urlFetched, r.status_code, r.reason))
usgs_dem_coverage_handler = USGSDEMCoverageHandler()
xml.sax.parseString(r.text, usgs_dem_coverage_handler)
coverage_url = usgs_dem_coverage_handler.coverage_url
if coverage_url is None:
raise Exception(
"Unable to deteremine coverage URL from WCS server response. Response text was: {0}"
.format(r.text))
parsed_coverage_url = urllib.parse.urlparse(coverage_url)
if verbose:
outfp.write(
"Downloading DEM coverage from {0} ...\n".format(coverage_url))
# Download coverage to tempfile
tmp_dir = tempfile.mkdtemp()
tmp_cov_name = os.path.join(tmp_dir, 'usgswcsdemtmp')
tmp_out = open(tmp_cov_name, mode='w+b')
#ORG conn = httplib.HTTPConnection(parsed_coverage_url.netloc)
conn = http.client.HTTPSConnection(parsed_coverage_url.netloc)
try:
conn.request('GET', parsed_coverage_url.path)
res = conn.getresponse(buffering=True)
except socket.error as e:
msg = "Encountered the following error when trying to read raster from %s. Error: %s. Please try again later or contact the developer." % \
(urlFetched, str(e) )
raise Exception(msg)
if 200 != res.status:
msg = "HTTP response %d %s encountered when querying %s. Please try again later or contact the developer." % \
(res.status, res.reason, urlFetched)
raise Exception(msg)
contentType = res.getheader('Content-Type')
mimeType = 'image/tiff'
if contentType.startswith(mimeType):
# The data returned were of the type expected, read the data
data = res.read(_BUFF_LEN)
if data:
dataFetched = True
while data:
tmp_out.write(data)
data = res.read(_BUFF_LEN)
tmp_out.close()
elif contentType.startswith(CONTENT_TYPE_ERRORS):
# Read the error and print to stderr
msg = "The following error was encountered reading WCS coverage URL %s\n\n" %\
(coverage_url, )
data = res.read(_BUFF_LEN)
while data:
msg += data
raise Exception(msg)
else:
msg = "Query for raster from URL %s returned content type %s, was expecting type %s. " + \
" Operation failed." % \
(coverage_url, contentType, mimeType)
raise Exception(msg)
# Rescale raster values if requested
if scale != 1.0:
# Rescale values in raster
if verbose:
outfp.write("Rescaling raster values by factor {0}".format(scale))
rescale_out = os.path.basename("{0}_rescale".format(tmp_cov_name))
rescaleRaster(config, tmp_dir, tmp_cov_name, rescale_out, scale)
tmp_cov_name = os.path.join(tmp_dir, rescale_out)
if deleteOldfile:
deleteGeoTiff(outFilepath)
if verbose:
msg = "Resampling raster from {s_srs} to {t_srs} " + \
"with X resolution {resx} and Y resolution {resy}\n"
outfp.write(msg.format(s_srs=s_srs, t_srs=t_srs, resx=resx, resy=resy))
# Re-sample to target spatial reference and resolution
resampleRaster(config, outputDir, tmp_cov_name, outFilename, s_srs, t_srs,
resx, resy, interpolation)
# Delete temp directory
shutil.rmtree(tmp_dir)
return (dataFetched, urlFetched)
if args.demResolution:
demResolutionX = args.demResolution[0]
demResolutionY = args.demResolution[1]
if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY:
resample = True
else:
demResolutionX = demSrs[0]
demResolutionY = demSrs[1]
# Resample DEM to target srs and resolution
if resample:
sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." % (demResolutionX, demResolutionY) )
sys.stdout.flush()
resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \
s_srs=t_srs, t_srs=t_srs, \
trX=demResolutionX, trY=demResolutionY)
sys.stdout.write('done\n')
else:
shutil.move(tmpDEMFilepath, demFilepath)
# Write metadata
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX)
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY)
GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
def getSoilsRasterDataForBoundingBox(config,
outputDir,
bbox,
srs='EPSG:4326',
resx=0.000277777777778,
resy=0.000277777777778,
interpolation='bilinear',
fmt=FORMAT_GEOTIFF,
overwrite=False,
verbose=False,
outfp=sys.stdout):
"""
Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/
For each property, rasters for the first 1-m of the soil profile will be downloaded
from which the depth-weighted mean of the property will be calculated and stored in outpufDir
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative path of the directory into which output raster should be written
@param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs.
@param srs String representing the spatial reference of the raster to be returned.
@param resx Float representing the X resolution of the raster(s) to be returned
@param resy Float representing the Y resolution of the raster(s) to be returned
@param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD.
@param fmt String representing format of raster file. Must be one of FORMATS.
@param overwrite Boolean True if existing data should be overwritten
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@return A dictionary mapping soil property names to soil property file path and WCS URL, i.e.
dict[soilPropertyName] = (soilPropertyFilePath, WCS URL)
@exception Exception if interpolation method is not known
@exception Exception if fmt is not a known format
@exception Exception if output already exists by overwrite is False
@exception Exception if a gdal_calc.py command fails
"""
if interpolation not in RASTER_RESAMPLE_METHOD:
raise Exception(
"Interpolation method {0} is not of a known method {1}".format(
interpolation, RASTER_RESAMPLE_METHOD))
if fmt not in FORMATS:
raise Exception("Format {0} is not of a known format {1}".format(
fmt, str(FORMATS)))
if verbose:
outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER))
soilPropertyRasters = {}
#import logging
#logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
#owslib_log = logging.getLogger('owslib')
# Add formatting and handlers as needed
#owslib_log.setLevel(logging.DEBUG)
# Set-up gdal_calc.py command
gdalBase = None
try:
gdalBase = config.get('GDAL/OGR', 'GDAL_BASE')
except configparser.NoOptionError:
gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP'))
gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py')
if not os.access(gdalCmdPath, os.X_OK):
raise IOError(
errno.EACCES,
"The gdal_calc.py binary at %s is not executable" % gdalCmdPath)
gdalCmdPath = os.path.abspath(gdalCmdPath)
tmpdir = tempfile.mkdtemp()
#print(tmpdir)
bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']]
# For each soil variable, download desired depth layers
for v in list(VARIABLE.keys()):
variable = VARIABLE[v]
soilPropertyName = "soil_raster_pct{var}".format(var=v)
soilPropertyFilename = "{name}.tif".format(name=soilPropertyName)
soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename)
soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename)
if verbose:
outfp.write("Getting attribute {0} ...\n".format(soilPropertyName))
delete = False
if os.path.exists(soilPropertyFilepath):
if not overwrite:
raise Exception(
"File {0} already exists, and overwrite is false".format(
soilPropertyFilepath))
else:
delete = True
url = URL_BASE.format(variable=variable)
wcs = WebCoverageService(url, version='1.0.0')
(coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(
wcs, variable)
outfiles = []
weights = []
for c in list(coverages.keys()):
coverage = coverages[c]
weights.append(weights_abs[c])
#coverage = c.format(variable=variable)
wcsfp = wcs.getCoverage(
identifier=coverage,
bbox=bbox,
crs='EPSG:4326',
resx=resx, # their WCS seems to accept resx, resy in meters
resy=resy,
format=fmt)
filename = os.path.join(tmpdir,
"{coverage}.tif".format(coverage=c))
outfiles.append(filename)
f = open(filename, 'wb')
f.write(wcsfp.read())
f.close()
# Compute depth-length weighted-average for each coverage using gdal_calc.py
assert (len(outfiles) == len(COVERAGES))
gdalCommand = gdalCmdPath
calcStr = '0' # Identity element for addition
for (i, outfile) in enumerate(outfiles):
ord = i + 1
var_label = ordinalToAlpha(ord)
gdalCommand += " -{var} {outfile}".format(var=var_label,
outfile=outfile)
calcStr += "+({weight}*{var})".format(weight=weights[i],
var=var_label)
gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(
calc=calcStr, outfile=soilPropertyFilepathTmp)
#print("GDAL command:\n{0}".format(gdalCommand))
process = Popen(gdalCommand,
cwd=outputDir,
shell=True,
stdout=PIPE,
stderr=PIPE)
(process_stdout, process_stderr) = process.communicate()
if process.returncode != 0:
raise Exception(
"GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n."
.format(gdalCommand, process.returncode, process_stdout,
process_stderr))
if verbose:
outfp.write(process_stdout)
outfp.write(process_stderr)
# Resample raster
if delete:
os.unlink(soilPropertyFilepath)
resampleRaster(config,
outputDir,
soilPropertyFilepathTmp,
soilPropertyFilename,
'EPSG:4326',
srs,
resx,
resy,
resampleMethod=interpolation)
soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url)
# Clean-up
shutil.rmtree(tmpdir)
return soilPropertyRasters
demResolutionX = args.demResolution[0]
demResolutionY = args.demResolution[1]
if demSrs[0] != demResolutionX or demSrs[1] != demResolutionY:
resample = True
else:
demResolutionX = demSrs[0]
demResolutionY = demSrs[1]
# Resample DEM to target srs and resolution
if resample:
sys.stdout.write("Resampling DEM to resolution %.2f x %.2f..." %
(demResolutionX, demResolutionY))
sys.stdout.flush()
resampleRaster(context.config, context.projectDir, tmpDEMFilepath, demFilename, \
s_srs=t_srs, t_srs=t_srs, \
trX=demResolutionX, trY=demResolutionY)
sys.stdout.write('done\n')
else:
shutil.move(tmpDEMFilepath, demFilepath)
# Write metadata
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_x', demResolutionX)
GenericMetadata.writeStudyAreaEntry(context, 'dem_res_y', demResolutionY)
GenericMetadata.writeStudyAreaEntry(context, 'dem_srs', t_srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(context, 'dem_rows', rows)
def getSoilsRasterDataForBoundingBox(config, outputDir, bbox,
srs='EPSG:4326',
resx=0.000277777777778,
resy=0.000277777777778,
interpolation='bilinear',
fmt=FORMAT_GEOTIFF,
overwrite=False,
verbose=False,
outfp=sys.stdout):
"""
Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/
For each property, rasters for the first 1-m of the soil profile will be downloaded
from which the depth-weighted mean of the property will be calculated and stored in outpufDir
@param config A Python ConfigParser (not currently used)
@param outputDir String representing the absolute/relative path of the directory into which output raster should be written
@param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area
for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs.
@param srs String representing the spatial reference of the raster to be returned.
@param resx Float representing the X resolution of the raster(s) to be returned
@param resy Float representing the Y resolution of the raster(s) to be returned
@param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD.
@param fmt String representing format of raster file. Must be one of FORMATS.
@param overwrite Boolean True if existing data should be overwritten
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@return A dictionary mapping soil property names to soil property file path and WCS URL, i.e.
dict[soilPropertyName] = (soilPropertyFilePath, WCS URL)
@exception Exception if interpolation method is not known
@exception Exception if fmt is not a known format
@exception Exception if output already exists by overwrite is False
@exception Exception if a gdal_calc.py command fails
"""
if interpolation not in RASTER_RESAMPLE_METHOD:
raise Exception("Interpolation method {0} is not of a known method {1}".format(interpolation,
RASTER_RESAMPLE_METHOD))
if fmt not in FORMATS:
raise Exception("Format {0} is not of a known format {1}".format(fmt, str(FORMATS)))
if verbose:
outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER))
soilPropertyRasters = {}
#import logging
#logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
#owslib_log = logging.getLogger('owslib')
# Add formatting and handlers as needed
#owslib_log.setLevel(logging.DEBUG)
# Set-up gdal_calc.py command
gdalBase = None
try:
gdalBase = config.get('GDAL/OGR', 'GDAL_BASE')
except ConfigParser.NoOptionError:
gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP'))
gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py')
if not os.access(gdalCmdPath, os.X_OK):
raise IOError(errno.EACCES, "The gdal_calc.py binary at %s is not executable" %
gdalCmdPath)
gdalCmdPath = os.path.abspath(gdalCmdPath)
tmpdir = tempfile.mkdtemp()
#print(tmpdir)
bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']]
# For each soil variable, download desired depth layers
for v in VARIABLE.keys():
variable = VARIABLE[v]
soilPropertyName = "soil_raster_pct{var}".format(var=v)
soilPropertyFilename = "{name}.tif".format(name=soilPropertyName)
soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename)
soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename)
if verbose:
outfp.write("Getting attribute {0} ...\n".format(soilPropertyName))
delete = False
if os.path.exists(soilPropertyFilepath):
if not overwrite:
raise Exception("File {0} already exists, and overwrite is false".format(soilPropertyFilepath))
else:
delete = True
url = URL_BASE.format(variable=variable)
wcs = WebCoverageService(url, version='1.0.0')
(coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(wcs, variable)
outfiles = []
weights = []
for c in coverages.keys():
coverage = coverages[c]
weights.append(weights_abs[c])
#coverage = c.format(variable=variable)
wcsfp = wcs.getCoverage(identifier=coverage, bbox=bbox,
crs='EPSG:4326',
resx=resx, # their WCS seems to accept resx, resy in meters
resy=resy,
format=fmt)
filename = os.path.join(tmpdir, "{coverage}.tif".format(coverage=c))
outfiles.append(filename)
f = open(filename, 'wb')
f.write(wcsfp.read())
f.close()
# Compute depth-length weighted-average for each coverage using gdal_calc.py
assert(len(outfiles) == len(COVERAGES))
gdalCommand = gdalCmdPath
calcStr = '0' # Identity element for addition
for (i, outfile) in enumerate(outfiles):
ord = i + 1
var_label = ordinalToAlpha(ord)
gdalCommand += " -{var} {outfile}".format(var=var_label, outfile=outfile)
calcStr += "+({weight}*{var})".format(weight=weights[i],
var=var_label)
gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(calc=calcStr,
outfile=soilPropertyFilepathTmp)
#print("GDAL command:\n{0}".format(gdalCommand))
process = Popen(gdalCommand, cwd=outputDir, shell=True,
stdout=PIPE, stderr=PIPE)
(process_stdout, process_stderr) = process.communicate()
if process.returncode != 0:
raise Exception("GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n.".format(gdalCommand,
process.returncode,
process_stdout,
process_stderr))
if verbose:
outfp.write(process_stdout)
outfp.write(process_stderr)
# Resample raster
if delete:
os.unlink(soilPropertyFilepath)
resampleRaster(config, outputDir, soilPropertyFilepathTmp, soilPropertyFilename,
'EPSG:4326', srs, resx, resy, resampleMethod=interpolation)
soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url)
# Clean-up
shutil.rmtree(tmpdir)
return soilPropertyRasters
manifest = GenericMetadata.readManifestEntries(context)
demFilename = manifest['dem']
demFilepath = os.path.join(context.projectDir, demFilename)
demFilepath = os.path.abspath(demFilepath)
extractTileFromRasterByRasterExtent(context.config, context.projectDir,
demFilepath, inRasterPath,
rasterFilepath, args.resampleMethod)
else:
if resample:
# Reproject raster, copying into project directory in the process
processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \
(args.type, rasterSrs, srs, rasterX, rasterX,
demResolutionX, demResolutionY)
sys.stdout.write(textwrap.fill("%s..." % (processingNotes, )))
resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \
s_srs=None, t_srs=srs, \
trX=demResolutionX, trY=demResolutionY, \
resampleMethod=args.resampleMethod)
else:
# Copy the raster in to the project directory
processingNotes = "Importing %s raster from %s without resampling" % (
args.type, inRasterPath)
sys.stdout.write(textwrap.fill("%s..." % (processingNotes, )))
sys.stdout.flush()
copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath,
rasterFilename)
sys.stdout.write('done\n')
# Make sure extent of resampled raster is the same as the extent of the DEM
newRasterMetadata = getDimensionsForRaster(rasterFilepath)
if (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows):
if args.type == GenericMetadata.RASTER_TYPE_STREAM_BURNED_DEM:
sys.stdout.write("Importing DEM...")
sys.stdout.flush()
if args.scale:
copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath,
demFilenameTemp)
else:
copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath,
demFilename)
else:
sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \
(s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1],
demResolutionX, demResolutionY) )
sys.stdout.flush()
if args.scale:
resampleRaster(context.config, context.projectDir, inDEMPath,
demFilenameTemp, s_srs, t_srs, demResolutionX,
demResolutionY, args.resampleMethod)
else:
resampleRaster(context.config, context.projectDir, inDEMPath,
demFilename, s_srs, t_srs, demResolutionX,
demResolutionY, args.resampleMethod)
if args.scale:
rescaleRaster(context.config, context.projectDir, demFilepathTemp,
demFilename, args.scale)
os.unlink(demFilepathTemp)
sys.stdout.write('done\n')
# Get the bounding box for the DEM
bbox = getBoundingBoxForRaster(demFilepath)
# Write a shapefile for the bounding box
# Copy the raster in to the project directory (reprojecting if need be)
if not resample:
sys.stdout.write("Importing DEM...")
sys.stdout.flush()
if args.scale:
copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilenameTemp)
else:
copyRasterToGeoTIFF(context.config, context.projectDir, inDEMPath, demFilename)
else:
sys.stdout.write("Reprojecting DEM from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)..." % \
(s_srs, t_srs, inSpatialMetadata[0], inSpatialMetadata[1],
demResolutionX, demResolutionY) )
sys.stdout.flush()
if args.scale:
resampleRaster(context.config, context.projectDir, inDEMPath, demFilenameTemp,
s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod)
else:
resampleRaster(context.config, context.projectDir, inDEMPath, demFilename,
s_srs, t_srs, demResolutionX, demResolutionY, args.resampleMethod)
if args.scale:
rescaleRaster(context.config, context.projectDir, demFilepathTemp, demFilename, args.scale)
os.unlink(demFilepathTemp)
sys.stdout.write('done\n')
# Get the bounding box for the DEM
bbox = getBoundingBoxForRaster(demFilepath)
# Write a shapefile for the bounding box
shpFilename = writeBboxPolygonToShapefile(bbox, context.projectDir, "studyarea")
# Write metadata
rasterSrs = rasterMetadata[5]
rasterX = float(rasterMetadata[0])
rasterY = float(rasterMetadata[1])
if (rasterSrs != srs):
resample = True
elif (not args.noresample) and ( (rasterX != demResolutionX) or (rasterY != demResolutionY) ):
resample = True
if resample:
# Reproject raster, copying into project directory in the process
processingNotes = "Resampling %s raster from %s to %s, spatial resolution (%.2f, %.2f) to (%.2f, %.2f)" % \
(args.type, rasterSrs, srs, rasterX, rasterX,
demResolutionX, demResolutionY)
sys.stdout.write("%s..." % (processingNotes,) )
resampleRaster(context.config, context.projectDir, inRasterPath, rasterFilepath, \
s_srs=rasterSrs, t_srs=srs, \
trX=demResolutionX, trY=demResolutionY, \
resampleMethod=args.resampleMethod)
else:
# Copy the raster in to the project directory
processingNotes = "Importing %s raster from %s without resampling" % (args.type, inRasterPath)
sys.stdout.write("%s..." % (processingNotes,) )
sys.stdout.flush()
copyRasterToGeoTIFF(context.config, context.projectDir, inRasterPath, rasterFilename)
sys.stdout.write('done\n')
# Make sure extent of resampled raster is the same as the extent of the DEM
newRasterMetadata = getDimensionsForRaster(rasterFilepath)
if (not force) and ( (newRasterMetadata[0] != demColumns) or (newRasterMetadata[1] != demRows) ):
# Extents to not match, roll back and bail out
os.unlink(rasterFilepath)
sys.exit(textwrap.fill("ERROR: Extent of raster dataset %s does not match extent of DEM in project directory %s. Use --force to override.") %