Ejemplo n.º 1
0
    def run(self, *args, **kwargs):
        """ Run the command: Acquire Australian soils data. 
        
        Arguments:
        verbose -- boolean    Produce verbose output. Default: False.
        overwrite -- boolean    Overwrite existing output.  Default: False.
        """
        verbose = kwargs.get('verbose', False)
        overwrite = kwargs.get('overwrite', False)

        self.checkMetadata()

        bbox = bboxFromString(self.studyArea['bbox_wgs84'])

        try:
            rasters = getSoilsRasterDataForBoundingBox(
                self.context.config,
                self.context.projectDir,
                bbox,
                srs=self.studyArea['dem_srs'],
                resx=self.studyArea['dem_res_x'],
                resy=self.studyArea['dem_res_y'],
                overwrite=overwrite,
                verbose=verbose,
                outfp=self.outfp)
        except Exception as e:
            traceback.print_exc(file=self.outfp)
            raise RunException(e)

        # Write metadata entries
        cmdline = GenericMetadata.getCommandLine()
        for attr in list(rasters.keys()):
            (filepath, url) = rasters[attr]
            filename = os.path.basename(filepath)
            asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
            asset.name = attr
            asset.dcIdentifier = filename
            asset.dcSource = url
            asset.dcTitle = attr
            asset.dcPublisher = soilwcs.DC_PUBLISHER
            asset.dcDescription = cmdline
            asset.writeToMetadata(self.context)

        # Write processing history
        GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
Ejemplo n.º 2
0
 def run(self, *args, **kwargs):
     """ Run the command: Acquire Australian soils data. 
     
     Arguments:
     verbose -- boolean    Produce verbose output. Default: False.
     overwrite -- boolean    Overwrite existing output.  Default: False.
     """
     verbose = kwargs.get('verbose', False)
     overwrite = kwargs.get('overwrite', False)
     
     self.checkMetadata()
     
     bbox = bboxFromString(self.studyArea['bbox_wgs84'])
     
     try: 
         rasters = getSoilsRasterDataForBoundingBox(self.context.config, 
                                                    self.context.projectDir,
                                                    bbox,
                                                    srs=self.studyArea['dem_srs'],
                                                    resx=self.studyArea['dem_res_x'],
                                                    resy=self.studyArea['dem_res_y'],
                                                    overwrite=overwrite,
                                                    verbose=verbose,
                                                    outfp=self.outfp)
     except Exception as e:
         traceback.print_exc(file=self.outfp)
         raise RunException(e)
     
     # Write metadata entries
     cmdline = GenericMetadata.getCommandLine()
     for attr in rasters.keys():
         (filepath, url) = rasters[attr]
         filename = os.path.basename(filepath)
         asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
         asset.name = attr
         asset.dcIdentifier = filename
         asset.dcSource = url
         asset.dcTitle = attr
         asset.dcPublisher = soilwcs.DC_PUBLISHER
         asset.dcDescription = cmdline
         asset.writeToMetadata(self.context)
         
     # Write processing history
     GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
Ejemplo n.º 3
0
 def test_provenance_overwrite(self):
     """ Test case writing provenance metadata, with overwrite """
     asset = AssetProvenance()
     asset.section = GenericMetadata.MANIFEST_SECTION
     asset.name = "dem"
     asset.dcIdentifier = "dem.tif"
     asset.dcSource = "http://www.demexplorer.com/..."
     asset.dcTitle = "Study area DEM"
     asset.dcDate = datetime.strptime("201303", "%Y%m")
     asset.dcPublisher = "USGS"
     asset.dcDescription = "RegisterDEM.py ..."
     asset.writeToMetadata(self.context)
     
     assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0]
     self.assertTrue(asset.section == assetProvenance.section)
     self.assertTrue(asset.name == assetProvenance.name)
     self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier)
     self.assertTrue(asset.dcSource == assetProvenance.dcSource)
     self.assertTrue(asset.dcTitle == assetProvenance.dcTitle)
     self.assertTrue(asset.dcDate == assetProvenance.dcDate)
     self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher)
     self.assertTrue(asset.dcDescription == assetProvenance.dcDescription)
     
     asset.dcIdentifier = 'foo.img'
     asset.dcSource = "http://a.different.url/..."
     asset.dcTitle = "A different study area DEM"
     asset.dcDate = datetime.strptime("201304", "%Y%m")
     asset.dcPublisher = "NASA"
     asset.dcDescription = "GetDEMExplorerDEM.py ..."
     asset.writeToMetadata(self.context)
     
     assetProvenance = GenericMetadata.readAssetProvenanceObjects(self.context)[0]
     self.assertTrue(asset.section == assetProvenance.section)
     self.assertTrue(asset.name == assetProvenance.name)
     self.assertTrue(asset.dcIdentifier == assetProvenance.dcIdentifier)
     self.assertTrue(asset.dcSource == assetProvenance.dcSource)
     self.assertTrue(asset.dcTitle == assetProvenance.dcTitle)
     self.assertTrue(asset.dcDate == assetProvenance.dcDate)
     self.assertTrue(asset.dcPublisher == assetProvenance.dcPublisher)
     self.assertTrue(asset.dcDescription == assetProvenance.dcDescription)
if result:
    gage_lat = result[1]
    gage_lon = result[0]
else:
    gage_lat = gage_lon = "Gage not found"

# Write gage coordinates to a shapefile in the project directory
shpFilename = writeCoordinatePairsToPointShapefile(context.projectDir, "gage", 
                                                   "gage_id", [args.gageid], [(gage_lon, gage_lat)])

# Write study area metadata
GenericMetadata.writeStudyAreaEntry(context, 'gage_id_attr', 'gage_id')
GenericMetadata.writeStudyAreaEntry(context, 'gage_id', args.gageid)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_reachcode', reachcode)
GenericMetadata.writeStudyAreaEntry(context, 'nhd_gage_measure_pct', measure)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lat_wgs84', gage_lat)
GenericMetadata.writeStudyAreaEntry(context, 'gage_lon_wgs84', gage_lon)

# Write provenance
asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
asset.name = 'gage'
asset.dcIdentifier = shpFilename
asset.dcSource = 'http://www.horizon-systems.com/NHDPlus/NHDPlusV2_home.php'
asset.dcTitle = 'Streamflow gage'
asset.dcPublisher = 'USGS'
asset.dcDescription = cmdline
asset.writeToMetadata(context)

# Write processing history
GenericMetadata.appendProcessingHistoryItem(context, cmdline)
Ejemplo n.º 5
0
    def run(self, *args, **kwargs):
        """ Run the command: Acquire USGS DEM data.
        
        Arguments:
        coverage -- string    Source dataset from which DEM tile should be extracted.
        outfile -- string    The name of the DEM file to be written.  File extension ".tif" will be added.
        demResolution list<float>[2]    Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
        srs -- string    Target spatial reference system of output, in EPSG:num format.
        verbose -- boolean    Produce verbose output. Default: False.
        overwrite -- boolean    Overwrite existing output.  Default: False.
        """
        coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
        outfile = kwargs.get('outfile', None)
        demResolution = kwargs.get('demResolution', None)
        srs = kwargs.get('srs', None)
        verbose = kwargs.get('verbose', False)
        overwrite = kwargs.get('overwrite', False)
        
        self.checkMetadata()
        
        bbox = bboxFromString(self.studyArea['bbox_wgs84'])
 
        if not outfile:
            outfile = 'DEM'
        demFilename = "%s.tif" % (outfile)    
        
        demResolutionX = demResolutionY = None
        if demResolution:
            demResolutionX = demResolution[0]
            demResolutionY = demResolution[1]
        
        if srs:
            if not isValidSrs(srs):
                msg = "ERROR: '%s' is not a valid spatial reference.  Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'.  For more information, see: http://www.spatialreference.org/" % (srs,)
                raise RunException(msg)
        else:
            # Default for UTM
            (centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
            (utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
            srs = getEPSGStringForUTMZone(utmZone, isNorth)
        
        try: 
            (dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config, 
                                           self.context.projectDir,
                                           demFilename,
                                           bbox,
                                           srs,
                                           coverage=coverage,
                                           resx=demResolutionX,
                                           resy=demResolutionY,
                                           scale=0.01,
                                           overwrite=overwrite,
                                           verbose=verbose,
                                           outfp=self.outfp)
        except Exception as e:
            traceback.print_exc(file=self.outfp)
            raise RunException(e)
        
        if not dataFetched:
            raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
        
        # Write metadata entries
        cmdline = GenericMetadata.getCommandLine()
        
        # Write metadata
        demFilepath = os.path.join(self.context.projectDir, demFilename)  
        demSrs = getSpatialReferenceForRaster(demFilepath)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
        
        # Get rows and columns for DEM  
        (columns, rows) = getDimensionsForRaster(demFilepath)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
        
        # Write provenance
        asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
        asset.name = 'dem'
        asset.dcIdentifier = demFilename
        asset.dcSource = urlFetched
        asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
        asset.dcPublisher = 'U.S. Geological Survey'
        asset.dcDescription = cmdline
        asset.processingNotes = "Elevation values rescaled from centimeters to meters. "   
        asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
        asset.processingNotes = asset.processingNotes.format(srs=srs,
                                                             xres=demSrs[0],
                                                             yres=demSrs[1])
        asset.writeToMetadata(self.context)
            
        # Write processing history
        GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
if not context.config.has_option('HYDRO1k', 'PATH_OF_HYDRO1K_BAS'):
    sys.exit("Config file %s does not define option %s in section %s" % \
          (args.configfile, 'HYDRO1k', 'PATH_OF_HYDRO1K_BAS'))
if not context.config.has_option('HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'):
    sys.exit("Config file %s does not define option %s in section %s" % \
          (args.configfile, 'HYDRO1k', 'HYDRO1k_BAS_LAYER_NAME'))

if args.outfile:
    outfile = args.outfile
else:
    outfile = "catchment"

shapeFilename = "%s.shp" % (outfile)
shapeFilepath = os.path.join(context.projectDir, shapeFilename)
if not os.path.exists(shapeFilepath):
    getCatchmentShapefileForHYDRO1kBasins(context.config, context.projectDir,
                                          shapeFilename, args.basins)

    # Write provenance
    asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
    asset.name = 'study_area_shapefile'
    asset.dcIdentifier = shapeFilename
    asset.dcSource = 'http://eros.usgs.gov/#/Find_Data/Products_and_Data_Available/gtopo30/hydro/namerica'
    asset.dcTitle = 'Study area shapefile derived from HYDRO1k Basins'
    asset.dcPublisher = 'USGS'
    asset.dcDescription = cmdline
    asset.writeToMetadata(context)

    # Write processing history
    GenericMetadata.appendProcessingHistoryItem(context, cmdline)
Ejemplo n.º 7
0
    def run(self, *args, **kwargs):
        """ Run the command: Acquire USGS DEM data.
        
        Arguments:
        coverage -- string    Source dataset from which DEM tile should be extracted.
        outfile -- string    The name of the DEM file to be written.  File extension ".tif" will be added.
        demResolution list<float>[2]    Two floating point numbers representing the desired X and Y output resolution of soil property raster maps; unit: meters.
        srs -- string    Target spatial reference system of output, in EPSG:num format.
        verbose -- boolean    Produce verbose output. Default: False.
        overwrite -- boolean    Overwrite existing output.  Default: False.
        """
        coverage = kwargs.get('coverage', self.DEFAULT_COVERAGE)
        outfile = kwargs.get('outfile', None)
        demResolution = kwargs.get('demResolution', None)
        srs = kwargs.get('srs', None)
        verbose = kwargs.get('verbose', False)
        overwrite = kwargs.get('overwrite', False)
        
        self.checkMetadata()
        
        bbox = bboxFromString(self.studyArea['bbox_wgs84'])
 
        if not outfile:
            outfile = 'DEM'
        demFilename = "%s.tif" % (outfile)    
        
        demResolutionX = demResolutionY = None
        if demResolution:
            demResolutionX = demResolution[0]
            demResolutionY = demResolution[1]
        
        if srs:
            if not isValidSrs(srs):
                msg = "ERROR: '%s' is not a valid spatial reference.  Spatial reference must be of the form 'EPSG:XXXX', e.g. 'EPSG:32617'.  For more information, see: http://www.spatialreference.org/" % (srs,)
                raise RunException(msg)
        else:
            # Default for UTM
            (centerLon, centerLat) = calculateBoundingBoxCenter(bbox)
            (utmZone, isNorth) = getUTMZoneFromCoordinates(centerLon, centerLat)
            srs = getEPSGStringForUTMZone(utmZone, isNorth)
        
        try: 
            (dataFetched, urlFetched) = ecohydrolib.usgs.demwcs.getDEMForBoundingBox(self.context.config, 
                                           self.context.projectDir,
                                           demFilename,
                                           bbox,
                                           srs,
                                           coverage=coverage,
                                           resx=demResolutionX,
                                           resy=demResolutionY,
                                           scale=0.01,
                                           overwrite=overwrite,
                                           verbose=verbose,
                                           outfp=self.outfp)
        except Exception as e:
            traceback.print_exc(file=self.outfp)
            raise RunException(e)
        
        if not dataFetched:
            raise RunException("Failed to download DEM data from URL {0}".format(urlFetched))
        
        # Write metadata entries
        cmdline = GenericMetadata.getCommandLine()
        
        # Write metadata
        demFilepath = os.path.join(self.context.projectDir, demFilename)  
        demSrs = getSpatialReferenceForRaster(demFilepath)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_x', demSrs[0])
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_res_y', demSrs[1])
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_srs', srs)
        
        # Get rows and columns for DEM  
        (columns, rows) = getDimensionsForRaster(demFilepath)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_columns', columns)
        GenericMetadata.writeStudyAreaEntry(self.context, 'dem_rows', rows)
        
        # Write provenance
        asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
        asset.name = 'dem'
        asset.dcIdentifier = demFilename
        asset.dcSource = urlFetched
        asset.dcTitle = "Digital Elevation Model ({0})".format(coverage)
        asset.dcPublisher = 'U.S. Geological Survey'
        asset.dcDescription = cmdline
        asset.processingNotes = "Elevation values rescaled from centimeters to meters. "   
        asset.processingNotes += "Spatial grid resampled to {srs} with X resolution {xres} and Y resolution {yres}."
        asset.processingNotes = asset.processingNotes.format(srs=srs,
                                                             xres=demSrs[0],
                                                             yres=demSrs[1])
        asset.writeToMetadata(self.context)
            
        # Write processing history
        GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
Ejemplo n.º 8
0
    def run(self, *args, **kwargs):
        """ Run the command: Acquire NLCD data from USGS WCS web service.
        
        Arguments:
        lctype -- string    Source dataset from which NLCD tile should be extracted.
        outfile -- string    The name of the NLCD file to be written.  File extension ".tif" will be added.
        verbose -- boolean    Produce verbose output. Default: False.
        overwrite -- boolean    Overwrite existing output.  Default: False.
        """
        lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
        outfile = kwargs.get('outfile', None)
        verbose = kwargs.get('verbose', False)
        overwrite = kwargs.get('overwrite', False)
        
        if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
            msg = "Land cover type {lctype} is not in the list of supported types {types}"
            raise CommandException(msg.format(lctype=lctype, 
                                              types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))
        
        self.checkMetadata()
        
        demFilename = self.manifest['dem']
        demFilepath = os.path.join(self.context.projectDir, demFilename)
        demFilepath = os.path.abspath(demFilepath)
        bbox = getRasterExtentAsBbox(demFilepath)
 
        if not outfile:
            outfile = 'NLCD'
        try:
            (resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(self.context.config, 
                                                                        self.context.projectDir,
                                                                        bbox,
                                                                        coverage=LC_TYPE_TO_COVERAGE[lctype],
                                                                        filename=outfile,
                                                                        srs=self.studyArea['dem_srs'],
                                                                        resx=self.studyArea['dem_res_x'],
                                                                        resy=self.studyArea['dem_res_y'],
                                                                        overwrite=overwrite,
                                                                        verbose=verbose,
                                                                        outfp=self.outfp)
            
        except Exception as e:
            traceback.print_exc(file=self.outfp)
            raise RunException(e)
        
        if not resp:
            raise RunException("Failed to download NLCD data from URL {0}".format(urlFetched))
        
        # Write metadata entries
        cmdline = GenericMetadata.getCommandLine()
        GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type", lctype)
        
        # Write provenance
        asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
        asset.name = 'landcover'
        asset.dcIdentifier = fname
        asset.dcSource = urlFetched
        asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
        asset.dcPublisher = 'USGS'
        asset.dcDescription = cmdline
        asset.writeToMetadata(self.context)
        
        # Write processing history
        GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)
Ejemplo n.º 9
0
    def run(self, *args, **kwargs):
        """ Run the command: Acquire NLCD data from USGS WCS web service.
        
        Arguments:
        lctype -- string    Source dataset from which NLCD tile should be extracted.
        outfile -- string    The name of the NLCD file to be written.  File extension ".tif" will be added.
        verbose -- boolean    Produce verbose output. Default: False.
        overwrite -- boolean    Overwrite existing output.  Default: False.
        """
        lctype = kwargs.get('lctype', DEFAULT_LC_TYPE)
        outfile = kwargs.get('outfile', None)
        verbose = kwargs.get('verbose', False)
        overwrite = kwargs.get('overwrite', False)

        if lctype not in ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE:
            msg = "Land cover type {lctype} is not in the list of supported types {types}"
            raise CommandException(
                msg.format(lctype=lctype,
                           types=ecohydrolib.usgs.nlcdwcs.LC_TYPE_TO_COVERAGE))

        self.checkMetadata()

        demFilename = self.manifest['dem']
        demFilepath = os.path.join(self.context.projectDir, demFilename)
        demFilepath = os.path.abspath(demFilepath)
        bbox = getRasterExtentAsBbox(demFilepath)

        if not outfile:
            outfile = 'NLCD'
        try:
            (resp, urlFetched, fname) = getNLCDRasterDataForBoundingBox(
                self.context.config,
                self.context.projectDir,
                bbox,
                coverage=LC_TYPE_TO_COVERAGE[lctype],
                filename=outfile,
                srs=self.studyArea['dem_srs'],
                resx=self.studyArea['dem_res_x'],
                resy=self.studyArea['dem_res_y'],
                overwrite=overwrite,
                verbose=verbose,
                outfp=self.outfp)

        except Exception as e:
            traceback.print_exc(file=self.outfp)
            raise RunException(e)

        if not resp:
            raise RunException(
                "Failed to download NLCD data from URL {0}".format(urlFetched))

        # Write metadata entries
        cmdline = GenericMetadata.getCommandLine()
        GenericMetadata.writeStudyAreaEntry(self.context, "landcover_type",
                                            lctype)

        # Write provenance
        asset = AssetProvenance(GenericMetadata.MANIFEST_SECTION)
        asset.name = 'landcover'
        asset.dcIdentifier = fname
        asset.dcSource = urlFetched
        asset.dcTitle = "The National Landcover Database: {0}".format(lctype)
        asset.dcPublisher = 'USGS'
        asset.dcDescription = cmdline
        asset.writeToMetadata(self.context)

        # Write processing history
        GenericMetadata.appendProcessingHistoryItem(self.context, cmdline)