def run(self, *args, **kwargs): """ Run the command: Acquire Australian soils data. Arguments: verbose -- boolean Produce verbose output. Default: False. overwrite -- boolean Overwrite existing output. Default: False. """ verbose = kwargs.get('verbose', False) overwrite = kwargs.get('overwrite', False) self.checkMetadata() bbox = bboxFromString(self.studyArea['bbox_wgs84']) try: rasters = getSoilsRasterDataForBoundingBox( self.context.config, self.context.projectDir, bbox, srs=self.studyArea['dem_srs'], resx=self.studyArea['dem_res_x'], resy=self.studyArea['dem_res_y'], overwrite=overwrite, verbose=verbose, outfp=self.outfp) except Exception as e: traceback.print_exc(file=self.outfp) raise RunException(e) # Write metadata entries cmdline = GenericMetadata.getCommandLine() for attr in list(rasters.keys()): (filepath, url) = rasters[attr] filename = os.path.basename(filepath) asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = attr asset.dcIdentifier = filename asset.dcSource = url asset.dcTitle = attr asset.dcPublisher = soilwcs.DC_PUBLISHER asset.dcDescription = cmdline asset.writeToMetadata(self.context) # Write processing history GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs): """ Run the command: Acquire Australian soils data. Arguments: verbose -- boolean Produce verbose output. Default: False. overwrite -- boolean Overwrite existing output. Default: False. """ verbose = kwargs.get('verbose', False) overwrite = kwargs.get('overwrite', False) self.checkMetadata() bbox = bboxFromString(self.studyArea['bbox_wgs84']) try: rasters = getSoilsRasterDataForBoundingBox(self.context.config, self.context.projectDir, bbox, srs=self.studyArea['dem_srs'], resx=self.studyArea['dem_res_x'], resy=self.studyArea['dem_res_y'], overwrite=overwrite, verbose=verbose, outfp=self.outfp) except Exception as e: traceback.print_exc(file=self.outfp) raise RunException(e) # Write metadata entries cmdline = GenericMetadata.getCommandLine() for attr in rasters.keys(): (filepath, url) = rasters[attr] filename = os.path.basename(filepath) asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = attr asset.dcIdentifier = filename asset.dcSource = url asset.dcTitle = attr asset.dcPublisher = soilwcs.DC_PUBLISHER asset.dcDescription = cmdline asset.writeToMetadata(self.context) # Write processing history GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def test_provenance_overwrite(self): """ Test case writing provenance metadata, with overwrite """ asset = AssetProvenance() asset.section = GenericMetadata.MANIFEST_SECTION asset.name = "dem" asset.dcIdentifier = "dem.tif" asset.dcSource = "http://www.demexplorer.com/..." asset.dcTitle = "Study area DEM" asset.dcDate = datetime.strptime("201303", "%Y%m") asset.dcPublisher = "USGS" asset.dcDescription = "RegisterDEM.py ..." asset.writeToMetadata(self.context) assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0] self.assertTrue(asset.section == assetProvenance.section) self.assertTrue(asset.name == assetProvenance.name) self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier) self.assertTrue(asset.dcSource == assetProvenance.dcSource) self.assertTrue(asset.dcTitle == assetProvenance.dcTitle) self.assertTrue(asset.dcDate == assetProvenance.dcDate) self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher) self.assertTrue(asset.dcDescription == assetProvenance.dcDescription) asset.dcIdentifier = 'foo.img' asset.dcSource = "http://a.different.url/..." asset.dcTitle = "A different study area DEM" asset.dcDate = datetime.strptime("201304", "%Y%m") asset.dcPublisher = "NASA" asset.dcDescription = "GetDEMExplorerDEM.py ..." asset.writeToMetadata(self.context) assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0] self.assertTrue(asset.section == assetProvenance.section) self.assertTrue(asset.name == assetProvenance.name) self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier) self.assertTrue(asset.dcSource == assetProvenance.dcSource) self.assertTrue(asset.dcTitle == assetProvenance.dcTitle) self.assertTrue(asset.dcDate == assetProvenance.dcDate) self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher) self.assertTrue(asset.dcDescription == assetProvenance.dcDescription)
if result: gage_lat = result[1] gage_lon = result[0] else: gage_lat = gage_lon = "Gage not found" # Write gage coordinates to a shapefile in the project directory shpFilename = writeCoordinatePairsToPointShapefile(context.projectDir, "gage", "gage_id", [args.gageid], [(gage_lon, gage_lat)]) # Write study area metadata GenericMetadata.writeStudyAreaEntry(context, 'gage_id_attr', 'gage_id') GenericMetadata.writeStudyAreaEntry(context, 'gage_id', args.gageid) GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_reachcode', reachcode) GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_measure_pct', measure) GenericMetadata.writeStudyAreaEntry(context, 'gage_lat_wgs84', gage_lat) GenericMetadata.writeStudyAreaEntry(context, 'gage_lon_wgs84', gage_lon) # Write provenance asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = 'gage' asset.dcIdentifier = shpFilename asset.dcSource = 'http://www.horizon-systems.com/NHDPlus/NHDPlusV2_home.php' asset.dcTitle = 'Streamflow gage' asset.dcPublisher = 'USGS' asset.dcDescription = cmdline asset.writeToMetadata(context) # Write processing history GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs): """ Run the command: Acquire USGS DEM data. Arguments: coverage -- string Source dataset from which DEM tile should be extracted. outfile -- string The name of the DEM file to be written. File extension ".tif" will be added. demResolution list<float>[2] Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters. srs -- string Target spatial reference system of output, in EPSG:num format. verbose -- boolean Produce verbose output. Default: False. overwrite -- boolean Overwrite existing output. Default: False. """ coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE) outfile = kwargs.get('outfile', None) demResolution = kwargs.get('demResolution', None) srs = kwargs.get('srs', None) verbose = kwargs.get('verbose', False) overwrite = kwargs.get('overwrite', False) self.checkMetadata() bbox = bboxFromString(self.studyArea['bbox_wgs84']) if not outfile: outfile = 'DEM' demFilename = "%s.tif" % (outfile) demResolutionX = demResolutionY = None if demResolution: demResolutionX = demResolution[0] demResolutionY = demResolution[1] if srs: if not isValidSrs(srs): msg = "ERROR: '%s' is not a valid spatial reference. Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'. For more information, see: http://www.spatialreference.org/" % (srs,) raise RunException(msg) else: # Default for UTM (centerLon, centerLat) = calculateBoundingBoxCenter(bbox) (utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat) srs = getEPSGStringForUTMZone(utmZone, isNorth) try: (dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config, self.context.projectDir, demFilename, bbox, srs, coverage=coverage, resx=demResolutionX, resy=demResolutionY, scale=0.01, overwrite=overwrite, verbose=verbose, outfp=self.outfp) except Exception as e: traceback.print_exc(file=self.outfp) raise RunException(e) if not dataFetched: raise RunException("Failed to download DEM data from URL {0}".format(urlFetched)) # Write metadata entries cmdline = GenericMetadata.getCommandLine() # Write metadata demFilepath = os.path.join(self.context.projectDir, demFilename) demSrs = getSpatialReferenceForRaster(demFilepath) GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0]) GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1]) GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs) # Get rows and columns for DEM (columns, rows) = getDimensionsForRaster(demFilepath) GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns) GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows) # Write provenance asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = 'dem' asset.dcIdentifier = demFilename asset.dcSource = urlFetched asset.dcTitle = "Digital Elevation Model ({0})".format(coverage) asset.dcPublisher = 'U.S. Geological Survey' asset.dcDescription = cmdline asset.processingNotes = "Elevation values rescaled from centimeters to meters. " asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}." asset.processingNotes = asset.processingNotes.format(srs=srs, xres=demSrs[0], yres=demSrs[1]) asset.writeToMetadata(self.context) # Write processing history GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
if not context.config.has_option('HYDRO1k', 'PATH_OF_HYDRO1K_BAS'): sys.exit("Config file %s does not define option %s in section %s" % \ (args.configfile, 'HYDRO1k', 'PATH_OF_HYDRO1K_BAS')) if not context.config.has_option('HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'): sys.exit("Config file %s does not define option %s in section %s" % \ (args.configfile, 'HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME')) if args.outfile: outfile = args.outfile else: outfile = "catchment" shapeFilename = "%s.shp" % (outfile) shapeFilepath = os.path.join(context.projectDir, shapeFilename) if not os.path.exists(shapeFilepath): getCatchmentShapefileForHYDRO1kBasins(context.config, context.projectDir, shapeFilename, args.basins) # Write provenance asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = 'study_area_shapefile' asset.dcIdentifier = shapeFilename asset.dcSource = 'http://eros.usgs.gov/#/Find_Data/Products_and_Data_Available/gtopo30/hydro/namerica' asset.dcTitle = 'Study area shapefile derived from HYDRO1k Basins' asset.dcPublisher = 'USGS' asset.dcDescription = cmdline asset.writeToMetadata(context) # Write processing history GenericMetadata.appendProcessingHistoryItem(context, cmdline)
def run(self, *args, **kwargs): """ Run the command: Acquire NLCD data from USGS WCS web service. Arguments: lctype -- string Source dataset from which NLCD tile should be extracted. outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added. verbose -- boolean Produce verbose output. Default: False. overwrite -- boolean Overwrite existing output. Default: False. """ lctype = kwargs.get('lctype', DEFAULT_LC_TYPE) outfile = kwargs.get('outfile', None) verbose = kwargs.get('verbose', False) overwrite = kwargs.get('overwrite', False) if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE: msg = "Land cover type {lctype} is not in the list of supported types {types}" raise CommandException(msg.format(lctype=lctype, types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE)) self.checkMetadata() demFilename = self.manifest['dem'] demFilepath = os.path.join(self.context.projectDir, demFilename) demFilepath = os.path.abspath(demFilepath) bbox = getRasterExtentAsBbox(demFilepath) if not outfile: outfile = 'NLCD' try: (resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(self.context.config, self.context.projectDir, bbox, coverage=LC_TYPE_TO_COVERAGE[lctype], filename=outfile, srs=self.studyArea['dem_srs'], resx=self.studyArea['dem_res_x'], resy=self.studyArea['dem_res_y'], overwrite=overwrite, verbose=verbose, outfp=self.outfp) except Exception as e: traceback.print_exc(file=self.outfp) raise RunException(e) if not resp: raise RunException("Failed to download NLCD data from URL {0}".format(urlFetched)) # Write metadata entries cmdline = GenericMetadata.getCommandLine() GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type", lctype) # Write provenance asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = 'landcover' asset.dcIdentifier = fname asset.dcSource = urlFetched asset.dcTitle = "The National Landcover Database: {0}".format(lctype) asset.dcPublisher = 'USGS' asset.dcDescription = cmdline asset.writeToMetadata(self.context) # Write processing history GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
def run(self, *args, **kwargs): """ Run the command: Acquire NLCD data from USGS WCS web service. Arguments: lctype -- string Source dataset from which NLCD tile should be extracted. outfile -- string The name of the NLCD file to be written. File extension ".tif" will be added. verbose -- boolean Produce verbose output. Default: False. overwrite -- boolean Overwrite existing output. Default: False. """ lctype = kwargs.get('lctype', DEFAULT_LC_TYPE) outfile = kwargs.get('outfile', None) verbose = kwargs.get('verbose', False) overwrite = kwargs.get('overwrite', False) if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE: msg = "Land cover type {lctype} is not in the list of supported types {types}" raise CommandException( msg.format(lctype=lctype, types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE)) self.checkMetadata() demFilename = self.manifest['dem'] demFilepath = os.path.join(self.context.projectDir, demFilename) demFilepath = os.path.abspath(demFilepath) bbox = getRasterExtentAsBbox(demFilepath) if not outfile: outfile = 'NLCD' try: (resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox( self.context.config, self.context.projectDir, bbox, coverage=LC_TYPE_TO_COVERAGE[lctype], filename=outfile, srs=self.studyArea['dem_srs'], resx=self.studyArea['dem_res_x'], resy=self.studyArea['dem_res_y'], overwrite=overwrite, verbose=verbose, outfp=self.outfp) except Exception as e: traceback.print_exc(file=self.outfp) raise RunException(e) if not resp: raise RunException( "Failed to download NLCD data from URL {0}".format(urlFetched)) # Write metadata entries cmdline = GenericMetadata.getCommandLine() GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type", lctype) # Write provenance asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION) asset.name = 'landcover' asset.dcIdentifier = fname asset.dcSource = urlFetched asset.dcTitle = "The National Landcover Database: {0}".format(lctype) asset.dcPublisher = 'USGS' asset.dcDescription = cmdline asset.writeToMetadata(self.context) # Write processing history GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)