def run(self, *args, **kwargs):
""" Run the command: Acquire USGS DEM data.
Arguments:
auth hs_restclient.HydroShareAuth object
title string representing the title of the resource
hydroshare_host string representing DNS name of the HydroShare
server in which to create the resource
hydroshare_port int representing the TCP port of the HydroShare
server
use_https True if HTTPS should be used. Default: False
resource_type string representing the HydroShare resource type
that should be used to create the resource
abstract string representing the abstract of the resource
keywords list of strings representing the keywords to assign
to the resource
create_callback user-defined callable that takes as input a
file size in bytes, and generates a callable to provide feedback
to the user about the progress of the upload of resource_file.
For more information, see:
http://toolbelt.readthedocs.org/en/latest/uploading-data.html#monitoring-your-streaming-multipart-upload
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
auth = kwargs.get('auth', None)
if auth is None:
raise RunException("No HydroShare authentication mechanism was defined.")
title = kwargs.get('title', None)
if title is None:
raise RunException("Title for new HydroShare resource was not specified.")
hydroshare_host = kwargs.get('hydroshare_host', None)
hydroshare_port = kwargs.get('hydroshare_port', None)
use_https = kwargs.get('use_https', False)
resource_type = kwargs.get('resource_type', 'GenericResource')
abstract = kwargs.get('abstract', None)
keywords = kwargs.get('keywords', None)
create_callback = kwargs.get('create_callback', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata(overwrite=overwrite)
resource_id = create_hydroshare_resource(self.context, auth, title,
hydroshare_host=hydroshare_host,
hydroshare_port=hydroshare_port, use_https=use_https,
resource_type=resource_type, abstract=abstract,
keywords=keywords, create_callback=create_callback,
verbose=verbose)
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
# Write metadata
GenericMetadata.writeHydroShareEntry(self.context, 'resource_id', resource_id)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire Australian soils data.
Arguments:
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
try:
rasters = getSoilsRasterDataForBoundingBox(
self.context.config,
self.context.projectDir,
bbox,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
for attr in list(rasters.keys()):
(filepath, url) = rasters[attr]
filename = os.path.basename(filepath)
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = attr
asset.dcIdentifier = filename
asset.dcSource = url
asset.dcTitle = attr
asset.dcPublisher = soilwcs.DC_PUBLISHER
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def test_version_conflict(self):
""" Induce a version conflict """
step1 = "mkdir foo; cd foo"
step2 = "touch README.txt"
GenericMetadata.appendProcessingHistoryItem(self.context, step1)
# For testing purposes only, users should not modify
# GenericMetadata._ecohydrolibVersion
_prevVersion = GenericMetadata._ecohydrolibVersion
GenericMetadata._ecohydrolibVersion = '11'
caughtMetadataVersionError = False
try:
GenericMetadata.appendProcessingHistoryItem(self.context, step2)
except MetadataVersionError:
caughtMetadataVersionError = True
self.assertTrue(caughtMetadataVersionError, "Expected metadata version mismatch, but none found.")
GenericMetadata._ecohydrolibVersion = _prevVersion
def run(self, *args, **kwargs):
""" Run the command: Acquire Australian soils data.
Arguments:
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
try:
rasters = getSoilsRasterDataForBoundingBox(self.context.config,
self.context.projectDir,
bbox,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
for attr in rasters.keys():
(filepath, url) = rasters[attr]
filename = os.path.basename(filepath)
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = attr
asset.dcIdentifier = filename
asset.dcSource = url
asset.dcTitle = attr
asset.dcPublisher = soilwcs.DC_PUBLISHER
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def test_check_metadata_version(self):
""" Test check metadata version """
step1 = "mkdir foo; cd foo"
GenericMetadata.appendProcessingHistoryItem(self.context, step1)
GenericMetadata.checkMetadataVersion(self.context.projectDir)
# For testing purposes only, users should not modify
# GenericMetadata._ecohydrolibVersion
_prevVersion = GenericMetadata._ecohydrolibVersion
GenericMetadata._ecohydrolibVersion = '11'
caughtMetadataVersionError = False
try:
GenericMetadata.checkMetadataVersion(self.context.projectDir)
except MetadataVersionError:
caughtMetadataVersionError = True
self.assertTrue(caughtMetadataVersionError, "Expected metadata version mismatch, but none found.")
GenericMetadata._ecohydrolibVersion = _prevVersion
def test_processing_history(self):
""" Test processing history metadata """
projectDir = "/tmp"
step1 = "mkdir foo; cd foo"
step2 = "touch README.txt"
step3 = "git init"
GenericMetadata.appendProcessingHistoryItem(self.context, step1)
GenericMetadata.appendProcessingHistoryItem(self.context, step2)
GenericMetadata.appendProcessingHistoryItem(self.context, step3)
history = GenericMetadata.getProcessingHistoryList(self.context)
self.assertTrue(len(history) == 3, "Expected history length to be 3, but it is %d" % (len(history),) )
self.assertTrue(history[0] == step1)
self.assertTrue(history[1] == step2)
self.assertTrue(history[2] == step3)
if result:
gage_lat = result[1]
gage_lon = result[0]
else:
gage_lat = gage_lon = "Gage not found"
# Write gage coordinates to a shapefile in the project directory
shpFilename = writeCoordinatePairsToPointShapefile(context.projectDir, "gage",
"gage_id", [args.gageid], [(gage_lon, gage_lat)])
# Write study area metadata
GenericMetadata.writeStudyAreaEntry(context, 'gage_id_attr', 'gage_id')
GenericMetadata.writeStudyAreaEntry(context, 'gage_id', args.gageid)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_reachcode', reachcode)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_measure_pct', measure)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lat_wgs84', gage_lat)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lon_wgs84', gage_lon)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'gage'
asset.dcIdentifier = shpFilename
asset.dcSource = 'http://www.horizon-systems.com/NHDPlus/NHDPlusV2_home.php'
asset.dcTitle = 'Streamflow gage'
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire USGS DEM data.
Arguments:
coverage -- string Source dataset from which DEM tile should be extracted.
outfile -- string The name of the DEM file to be written. File extension ".tif" will be added.
demResolution list<float>[2] Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
srs -- string Target spatial reference system of output, in EPSG:num format.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
outfile = kwargs.get('outfile', None)
demResolution = kwargs.get('demResolution', None)
srs = kwargs.get('srs', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
if not outfile:
outfile = 'DEM'
demFilename = "%s.tif" % (outfile)
demResolutionX = demResolutionY = None
if demResolution:
demResolutionX = demResolution[0]
demResolutionY = demResolution[1]
if srs:
if not isValidSrs(srs):
msg = "ERROR: '%s' is not a valid spatial reference. Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'. For more information, see: http://www.spatialreference.org/" % (srs,)
raise RunException(msg)
else:
# Default for UTM
(centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
(utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
srs = getEPSGStringForUTMZone(utmZone, isNorth)
try:
(dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config,
self.context.projectDir,
demFilename,
bbox,
srs,
coverage=coverage,
resx=demResolutionX,
resy=demResolutionY,
scale=0.01,
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not dataFetched:
raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
# Write metadata
demFilepath = os.path.join(self.context.projectDir, demFilename)
demSrs = getSpatialReferenceForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'dem'
asset.dcIdentifier = demFilename
asset.dcSource = urlFetched
asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
asset.dcPublisher = 'U.S. Geological Survey'
asset.dcDescription = cmdline
asset.processingNotes = "Elevation values rescaled from centimeters to meters. "
asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
asset.processingNotes = asset.processingNotes.format(srs=srs,
xres=demSrs[0],
yres=demSrs[1])
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
if not context.config.has_option('HYDRO1k', 'PATH_OF_HYDRO1K_BAS'):
sys.exit("Config file %s does not define option %s in section %s" % \
(args.configfile, 'HYDRO1k', 'PATH_OF_HYDRO1K_BAS'))
if not context.config.has_option('HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'):
sys.exit("Config file %s does not define option %s in section %s" % \
(args.configfile, 'HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'))
if args.outfile:
outfile = args.outfile
else:
outfile = "catchment"
shapeFilename = "%s.shp" % (outfile)
shapeFilepath = os.path.join(context.projectDir, shapeFilename)
if not os.path.exists(shapeFilepath):
getCatchmentShapefileForHYDRO1kBasins(context.config, context.projectDir,
shapeFilename, args.basins)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'study_area_shapefile'
asset.dcIdentifier = shapeFilename
asset.dcSource = 'http://eros.usgs.gov/#/Find_Data/Products_and_Data_Available/gtopo30/hydro/namerica'
asset.dcTitle = 'Study area shapefile derived from HYDRO1k Basins'
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire USGS DEM data.
Arguments:
coverage -- string Source dataset from which DEM tile should be extracted.
outfile -- string The name of the DEM file to be written. File extension ".tif" will be added.
demResolution list<float>[2] Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
srs -- string Target spatial reference system of output, in EPSG:num format.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
outfile = kwargs.get('outfile', None)
demResolution = kwargs.get('demResolution', None)
srs = kwargs.get('srs', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
self.checkMetadata()
bbox = bboxFromString(self.studyArea['bbox_wgs84'])
if not outfile:
outfile = 'DEM'
demFilename = "%s.tif" % (outfile)
demResolutionX = demResolutionY = None
if demResolution:
demResolutionX = demResolution[0]
demResolutionY = demResolution[1]
if srs:
if not isValidSrs(srs):
msg = "ERROR: '%s' is not a valid spatial reference. Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'. For more information, see: http://www.spatialreference.org/" % (srs,)
raise RunException(msg)
else:
# Default for UTM
(centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
(utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
srs = getEPSGStringForUTMZone(utmZone, isNorth)
try:
(dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config,
self.context.projectDir,
demFilename,
bbox,
srs,
coverage=coverage,
resx=demResolutionX,
resy=demResolutionY,
scale=0.01,
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not dataFetched:
raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
# Write metadata
demFilepath = os.path.join(self.context.projectDir, demFilename)
demSrs = getSpatialReferenceForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
# Get rows and columns for DEM
(columns, rows) = getDimensionsForRaster(demFilepath)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'dem'
asset.dcIdentifier = demFilename
asset.dcSource = urlFetched
asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
asset.dcPublisher = 'U.S. Geological Survey'
asset.dcDescription = cmdline
asset.processingNotes = "Elevation values rescaled from centimeters to meters. "
asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
asset.processingNotes = asset.processingNotes.format(srs=srs,
xres=demSrs[0],
yres=demSrs[1])
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire NLCD data from USGS WCS web service.
Arguments:
lctype -- string Source dataset from which NLCD tile should be extracted.
outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
outfile = kwargs.get('outfile', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
msg = "Land cover type {lctype} is not in the list of supported types {types}"
raise CommandException(msg.format(lctype=lctype,
types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))
self.checkMetadata()
demFilename = self.manifest['dem']
demFilepath = os.path.join(self.context.projectDir, demFilename)
demFilepath = os.path.abspath(demFilepath)
bbox = getRasterExtentAsBbox(demFilepath)
if not outfile:
outfile = 'NLCD'
try:
(resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(self.context.config,
self.context.projectDir,
bbox,
coverage=LC_TYPE_TO_COVERAGE[lctype],
filename=outfile,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not resp:
raise RunException("Failed to download NLCD data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type", lctype)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'landcover'
asset.dcIdentifier = fname
asset.dcSource = urlFetched
asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs):
""" Run the command: Acquire NLCD data from USGS WCS web service.
Arguments:
lctype -- string Source dataset from which NLCD tile should be extracted.
outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added.
verbose -- boolean Produce verbose output. Default: False.
overwrite -- boolean Overwrite existing output. Default: False.
"""
lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
outfile = kwargs.get('outfile', None)
verbose = kwargs.get('verbose', False)
overwrite = kwargs.get('overwrite', False)
if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
msg = "Land cover type {lctype} is not in the list of supported types {types}"
raise CommandException(
msg.format(lctype=lctype,
types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))
self.checkMetadata()
demFilename = self.manifest['dem']
demFilepath = os.path.join(self.context.projectDir, demFilename)
demFilepath = os.path.abspath(demFilepath)
bbox = getRasterExtentAsBbox(demFilepath)
if not outfile:
outfile = 'NLCD'
try:
(resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(
self.context.config,
self.context.projectDir,
bbox,
coverage=LC_TYPE_TO_COVERAGE[lctype],
filename=outfile,
srs=self.studyArea['dem_srs'],
resx=self.studyArea['dem_res_x'],
resy=self.studyArea['dem_res_y'],
overwrite=overwrite,
verbose=verbose,
outfp=self.outfp)
except Exception as e:
traceback.print_exc(file=self.outfp)
raise RunException(e)
if not resp:
raise RunException(
"Failed to download NLCD data from URL {0}".format(urlFetched))
# Write metadata entries
cmdline = GenericMetadata.getCommandLine()
GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type",
lctype)
# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'landcover'
asset.dcIdentifier = fname
asset.dcSource = urlFetched
asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(self.context)
# Write processing history
GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
