def run(self, *args, **kwargs):
""" Run the command: Acquire Australian soils data.
Arguments:
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
try:
rasters = getSoilsRasterDataForBoundingBox(
self.context.config,
self.context.projectDir,
bbox,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
for attr in list(rasters.keys()):
(filepath, url) = rasters[attr]
filename = os.path.basename(filepath)
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = attr
asset.dcIdentifier = filename
asset.dcSource = url
asset.dcTitle = attr
asset.dcPublisher = soilwcs.DC_PUBLISHER
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire Australian soils data.
Arguments:
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
try:
rasters = getSoilsRasterDataForBoundingBox(self.context.config,
self.context.projectDir,
bbox,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
for attr in rasters.keys():
(filepath, url) = rasters[attr]
filename = os.path.basename(filepath)
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = attr
asset.dcIdentifier = filename
asset.dcSource = url
asset.dcTitle = attr
asset.dcPublisher = soilwcs.DC_PUBLISHER
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def test_provenance_overwrite(self):
""" Test case writing provenance metadata, with overwrite """
asset = AssetProvenance()
asset.section = GenericMetadata.MANIFEST_SECTION
asset.name = "dem"
asset.dcIdentifier = "dem.tif"
asset.dcSource = "http://www.demexplorer.com/..."
asset.dcTitle = "Study area DEM"
asset.dcDate = datetime.strptime("201303", "%Y%m")
asset.dcPublisher = "USGS"
asset.dcDescription = "RegisterDEM.py ..."
asset.writeToMetadata(self.context)
assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0]
self.assertTrue(asset.section == assetProvenance.section)
self.assertTrue(asset.name == assetProvenance.name)
self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier)
self.assertTrue(asset.dcSource == assetProvenance.dcSource)
self.assertTrue(asset.dcTitle == assetProvenance.dcTitle)
self.assertTrue(asset.dcDate == assetProvenance.dcDate)
self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher)
self.assertTrue(asset.dcDescription == assetProvenance.dcDescription)
asset.dcIdentifier = 'foo.img'
asset.dcSource = "http://a.different.url/..."
asset.dcTitle = "A different study area DEM"
asset.dcDate = datetime.strptime("201304", "%Y%m")
asset.dcPublisher = "NASA"
asset.dcDescription = "GetDEMExplorerDEM.py ..."
asset.writeToMetadata(self.context)
assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0]
self.assertTrue(asset.section == assetProvenance.section)
self.assertTrue(asset.name == assetProvenance.name)
self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier)
self.assertTrue(asset.dcSource == assetProvenance.dcSource)
self.assertTrue(asset.dcTitle == assetProvenance.dcTitle)
self.assertTrue(asset.dcDate == assetProvenance.dcDate)
self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher)
self.assertTrue(asset.dcDescription == assetProvenance.dcDescription)
if result:
gage_lat = result[1]
gage_lon = result[0]
else:
gage_lat = gage_lon = "Gage not found"
# Write gage coordinates to a shapefile in the project directory
shpFilename = writeCoordinatePairsToPointShapefile(context.projectDir, "gage",
"gage_id", [args.gageid], [(gage_lon, gage_lat)])
# Write study area metadata
GenericMetadata.writeStudyAreaEntry(context, 'gage_id_attr', 'gage_id')
GenericMetadata.writeStudyAreaEntry(context, 'gage_id', args.gageid)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_reachcode', reachcode)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_measure_pct', measure)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lat_wgs84', gage_lat)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lon_wgs84', gage_lon)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'gage'
asset.dcIdentifier = shpFilename
asset.dcSource = 'http://www.horizon-systems.com/NHDPlus/NHDPlusV2_home.php'
asset.dcTitle = 'Streamflow gage'
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire USGS DEM data.
Arguments:
coverage -- string Source dataset from which DEM tile should be extracted.
outfile -- string The name of the DEM file to be written. File extension ".tif" will be added.
demResolution list<float>[2] Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
srs -- string Target spatial reference system of output, in EPSG:num format.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
outfile = kwargs.get('outfile', None)
demResolution = kwargs.get('demResolution', None)
srs = kwargs.get('srs', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
if not outfile:
outfile = 'DEM'
demFilename = "%s.tif" % (outfile)
demResolutionX = demResolutionY = None
if demResolution:
demResolutionX = demResolution[0]
demResolutionY = demResolution[1]
if srs:
if not isValidSrs(srs):
msg = "ERROR: '%s' is not a valid spatial reference. Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'. For more information, see: http://www.spatialreference.org/" % (srs,)
raise RunException(msg)
else:
# Default for UTM
(centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
(utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
srs = getEPSGStringForUTMZone(utmZone, isNorth)
try:
(dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config,
self.context.projectDir,
demFilename,
bbox,
srs,
coverage=coverage,
resx=demResolutionX,
resy=demResolutionY,
scale=0.01,
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not dataFetched:
raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
# Write metadata
demFilepath = os.path.join(self.context.projectDir, demFilename)
demSrs = getSpatialReferenceForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'dem'
asset.dcIdentifier = demFilename
asset.dcSource = urlFetched
asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
asset.dcPublisher = 'U.S. Geological Survey'
asset.dcDescription = cmdline
asset.processingNotes = "Elevation values rescaled from centimeters to meters. "
asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
asset.processingNotes = asset.processingNotes.format(srs=srs,
xres=demSrs[0],
yres=demSrs[1])
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
if not context.config.has_option('HYDRO1k', 'PATH_OF_HYDRO1K_BAS'):
sys.exit("Config file %s does not define option %s in section %s" % \
(args.configfile, 'HYDRO1k', 'PATH_OF_HYDRO1K_BAS'))
if not context.config.has_option('HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'):
sys.exit("Config file %s does not define option %s in section %s" % \
(args.configfile, 'HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'))
if args.outfile:
outfile = args.outfile
else:
outfile = "catchment"
shapeFilename = "%s.shp" % (outfile)
shapeFilepath = os.path.join(context.projectDir, shapeFilename)
if not os.path.exists(shapeFilepath):
getCatchmentShapefileForHYDRO1kBasins(context.config, context.projectDir,
shapeFilename, args.basins)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'study_area_shapefile'
asset.dcIdentifier = shapeFilename
asset.dcSource = 'http://eros.usgs.gov/#/Find_Data/Products_and_Data_Available/gtopo30/hydro/namerica'
asset.dcTitle = 'Study area shapefile derived from HYDRO1k Basins'
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire USGS DEM data.
Arguments:
coverage -- string Source dataset from which DEM tile should be extracted.
outfile -- string The name of the DEM file to be written. File extension ".tif" will be added.
demResolution list<float>[2] Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
srs -- string Target spatial reference system of output, in EPSG:num format.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
outfile = kwargs.get('outfile', None)
demResolution = kwargs.get('demResolution', None)
srs = kwargs.get('srs', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
if not outfile:
outfile = 'DEM'
demFilename = "%s.tif" % (outfile)
demResolutionX = demResolutionY = None
if demResolution:
demResolutionX = demResolution[0]
demResolutionY = demResolution[1]
if srs:
if not isValidSrs(srs):
msg = "ERROR: '%s' is not a valid spatial reference. Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'. For more information, see: http://www.spatialreference.org/" % (srs,)
raise RunException(msg)
else:
# Default for UTM
(centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
(utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
srs = getEPSGStringForUTMZone(utmZone, isNorth)
try:
(dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config,
self.context.projectDir,
demFilename,
bbox,
srs,
coverage=coverage,
resx=demResolutionX,
resy=demResolutionY,
scale=0.01,
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not dataFetched:
raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
# Write metadata
demFilepath = os.path.join(self.context.projectDir, demFilename)
demSrs = getSpatialReferenceForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'dem'
asset.dcIdentifier = demFilename
asset.dcSource = urlFetched
asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
asset.dcPublisher = 'U.S. Geological Survey'
asset.dcDescription = cmdline
asset.processingNotes = "Elevation values rescaled from centimeters to meters. "
asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
asset.processingNotes = asset.processingNotes.format(srs=srs,
xres=demSrs[0],
yres=demSrs[1])
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire NLCD data from USGS WCS web service.
Arguments:
lctype -- string Source dataset from which NLCD tile should be extracted.
outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
outfile = kwargs.get('outfile', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
msg = "Land cover type {lctype} is not in the list of supported types {types}"
raise CommandException(msg.format(lctype=lctype,
types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))
self.checkMetadata()
demFilename = self.manifest['dem']
demFilepath = os.path.join(self.context.projectDir, demFilename)
demFilepath = os.path.abspath(demFilepath)
bbox = getRasterExtentAsBbox(demFilepath)
if not outfile:
outfile = 'NLCD'
try:
(resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(self.context.config,
self.context.projectDir,
bbox,
coverage=LC_TYPE_TO_COVERAGE[lctype],
filename=outfile,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not resp:
raise RunException("Failed to download NLCD data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type", lctype)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'landcover'
asset.dcIdentifier = fname
asset.dcSource = urlFetched
asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire NLCD data from USGS WCS web service.
Arguments:
lctype -- string Source dataset from which NLCD tile should be extracted.
outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
outfile = kwargs.get('outfile', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
msg = "Land cover type {lctype} is not in the list of supported types {types}"
raise CommandException(
msg.format(lctype=lctype,
types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))
self.checkMetadata()
demFilename = self.manifest['dem']
demFilepath = os.path.join(self.context.projectDir, demFilename)
demFilepath = os.path.abspath(demFilepath)
bbox = getRasterExtentAsBbox(demFilepath)
if not outfile:
outfile = 'NLCD'
try:
(resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(
self.context.config,
self.context.projectDir,
bbox,
coverage=LC_TYPE_TO_COVERAGE[lctype],
filename=outfile,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not resp:
raise RunException(
"Failed to download NLCD data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type",
lctype)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'landcover'
asset.dcIdentifier = fname
asset.dcSource = urlFetched
asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)