Exemplo n.º 1
0
    def generate_data(self):
        '''
        Description:
            Provides the main processing algorithm for building the Surface 
            Temperature product.  It produces the final ST product.
        '''

        try:
            self.retrieve_metadata_information()
        except Exception:
            self.logger.exception('Failed reading input XML metadata file')
            raise

        # Register all the gdal drivers and choose the ENVI for our output
        gdal.AllRegister()
        envi_driver = gdal.GetDriverByName('ENVI')

        # Read the bands into memory

        # Landsat Radiance at sensor for thermal band
        self.logger.info('Loading intermediate thermal band data [{0}]'.format(
            self.thermal_name))
        dataset = gdal.Open(self.thermal_name)
        x_dim = dataset.RasterXSize  # They are all the same size
        y_dim = dataset.RasterYSize

        thermal_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Atmospheric transmittance
        self.logger.info(
            'Loading intermediate transmittance band data [{0}]'.format(
                self.transmittance_name))
        dataset = gdal.Open(self.transmittance_name)
        trans_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Atmospheric path radiance - upwelled radiance
        self.logger.info(
            'Loading intermediate upwelled band data [{0}]'.format(
                self.upwelled_name))
        dataset = gdal.Open(self.upwelled_name)
        upwelled_data = dataset.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)

        self.logger.info('Calculating surface radiance')
        # Surface radiance
        with np.errstate(invalid='ignore'):
            surface_radiance = (thermal_data - upwelled_data) / trans_data

        # Fix the no data locations
        no_data_locations = np.where(thermal_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        no_data_locations = np.where(trans_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        no_data_locations = np.where(upwelled_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        # Memory cleanup
        del thermal_data
        del trans_data
        del upwelled_data
        del no_data_locations

        # Downwelling sky irradiance
        self.logger.info(
            'Loading intermediate downwelled band data [{0}]'.format(
                self.downwelled_name))
        dataset = gdal.Open(self.downwelled_name)
        downwelled_data = dataset.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)

        # Landsat emissivity estimated from ASTER GED data
        self.logger.info(
            'Loading intermediate emissivity band data [{0}]'.format(
                self.emissivity_name))
        dataset = gdal.Open(self.emissivity_name)
        emissivity_data = dataset.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)

        # Save for the output product
        ds_srs = osr.SpatialReference()
        ds_srs.ImportFromWkt(dataset.GetProjection())
        ds_transform = dataset.GetGeoTransform()

        # Memory cleanup
        del dataset

        # Estimate Earth-emitted radiance by subtracting off the reflected
        # downwelling component
        radiance = (surface_radiance -
                    (1.0 - emissivity_data) * downwelled_data)

        # Account for surface emissivity to get Plank emitted radiance
        self.logger.info('Calculating Plank emitted radiance')
        with np.errstate(invalid='ignore'):
            radiance_emitted = radiance / emissivity_data

        # Fix the no data locations
        no_data_locations = np.where(surface_radiance == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        no_data_locations = np.where(downwelled_data == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        no_data_locations = np.where(emissivity_data == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        # Memory cleanup
        del downwelled_data
        del emissivity_data
        del surface_radiance
        del radiance
        del no_data_locations

        # Use Brightness Temperature LUT to get skin temperature
        # Read the correct one for what we are processing
        if self.satellite == 'LANDSAT_8':
            self.logger.info('Using Landsat 8 Brightness Temperature LUT')
            bt_name = 'L8_Brightness_Temperature_LUT.txt'

        elif self.satellite == 'LANDSAT_7':
            self.logger.info('Using Landsat 7 Brightness Temperature LUT')
            bt_name = 'L7_Brightness_Temperature_LUT.txt'

        elif self.satellite == 'LANDSAT_5':
            self.logger.info('Using Landsat 5 Brightness Temperature LUT')
            bt_name = 'L5_Brightness_Temperature_LUT.txt'

        elif self.satellite == 'LANDSAT_4':
            self.logger.info('Using Landsat 4 Brightness Temperature LUT')
            bt_name = 'L4_Brightness_Temperature_LUT.txt'

        bt_data = np.loadtxt(os.path.join(self.st_data_dir, bt_name),
                             dtype=float,
                             delimiter=' ')
        bt_radiance_lut = bt_data[:, 1]
        bt_temp_lut = bt_data[:, 0]

        self.logger.info('Generating ST results')
        st_data = np.interp(radiance_emitted, bt_radiance_lut, bt_temp_lut)

        # Scale the result
        st_data = st_data * MULT_FACTOR

        # Add the fill and scan gaps back into the results, since they may
        # have been lost
        self.logger.info('Adding fill and data gaps back into the Surface'
                         ' Temperature results')

        # Fix the no data locations
        no_data_locations = np.where(radiance_emitted == self.no_data_value)
        st_data[no_data_locations] = self.no_data_value

        # Memory cleanup
        del radiance_emitted
        del no_data_locations

        product_id = self.xml_filename.split('.xml')[0]
        st_img_filename = ''.join([product_id, '_st', '.img'])
        st_hdr_filename = ''.join([product_id, '_st', '.hdr'])
        st_aux_filename = ''.join([st_img_filename, '.aux', '.xml'])

        self.logger.info('Creating {0}'.format(st_img_filename))
        util.Geo.generate_raster_file(envi_driver, st_img_filename, st_data,
                                      x_dim, y_dim, ds_transform,
                                      ds_srs.ExportToWkt(), self.no_data_value,
                                      gdal.GDT_Int16)

        self.logger.info('Updating {0}'.format(st_hdr_filename))
        util.Geo.update_envi_header(st_hdr_filename, self.no_data_value)

        # Memory cleanup
        del ds_srs
        del ds_transform

        # Remove the *.aux.xml file generated by GDAL
        if os.path.exists(st_aux_filename):
            os.unlink(st_aux_filename)

        self.logger.info('Adding {0} to {1}'.format(st_img_filename,
                                                    self.xml_filename))
        # Add the estimated Surface Temperature product to the metadata
        espa_xml = metadata_api.parse(self.xml_filename, silence=True)
        bands = espa_xml.get_bands()
        sensor_code = product_id[0:4]

        # Find the TOA Band 1 to use for the specific band details
        base_band = None
        for band in bands.band:
            if band.product == 'toa_refl' and band.name == 'toa_band1':
                base_band = band

        if base_band is None:
            raise Exception('Failed to find the TOA band 1'
                            ' in the input data')

        st_band = metadata_api.band(product='st',
                                    source='toa_refl',
                                    name='surface_temperature',
                                    category='image',
                                    data_type='INT16',
                                    scale_factor=SCALE_FACTOR,
                                    add_offset=0,
                                    nlines=base_band.get_nlines(),
                                    nsamps=base_band.get_nsamps(),
                                    fill_value=str(self.no_data_value))

        st_band.set_short_name('{0}ST'.format(sensor_code))
        st_band.set_long_name('Surface Temperature')
        st_band.set_file_name(st_img_filename)
        st_band.set_data_units('temperature (kelvin)')

        pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
                                             base_band.pixel_size.x,
                                             base_band.pixel_size.units)
        st_band.set_pixel_size(pixel_size)

        st_band.set_resample_method('none')

        valid_range = metadata_api.valid_range(min=1500, max=3730)
        st_band.set_valid_range(valid_range)

        # Set the date, but first clean the microseconds off of it
        production_date = (datetime.datetime.strptime(
            datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
            '%Y-%m-%dT%H:%M:%S'))

        st_band.set_production_date(production_date)

        st_band.set_app_version(util.Version.app_version())

        bands.add_band(st_band)

        # Write the XML metadata file out
        with open(self.xml_filename, 'w') as metadata_fd:
            metadata_api.export(metadata_fd, espa_xml)

        # Memory cleanup
        del st_band
        del bands
        del espa_xml
        del st_data
    def generate_product(self):
        '''
        Description:
            Provides the main processing algorithm for generating the
            estimated Landsat emissivity product.  It produces the final
            emissivity product.
        '''

        self.logger = logging.getLogger(__name__)

        self.logger.info('Start - Estimate Landsat Emissivity')

        try:
            self.retrieve_metadata_information()
        except Exception:
            self.logger.exception('Failed reading input XML metadata file')
            raise

        try:
            self.determine_sensor_specific_coefficients()
        except Exception:
            self.logger.exception('Failed determining sensor coefficients')
            raise

        # Register all the gdal drivers and choose the GeoTiff for our temp
        # output
        gdal.AllRegister()
        geotiff_driver = gdal.GetDriverByName('GTiff')
        envi_driver = gdal.GetDriverByName('ENVI')

        # ====================================================================
        # Build NDVI in memory
        self.logger.info('Building TOA based NDVI band for Landsat data')

        # NIR ----------------------------------------------------------------
        data_set = gdal.Open(self.toa_nir_name)
        x_dim = data_set.RasterXSize  # They are all the same size
        y_dim = data_set.RasterYSize
        ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                            x_dim, y_dim)
        nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
        ls_nir_data = ls_nir_data * self.toa_nir_scale_factor

        # RED ----------------------------------------------------------------
        data_set = gdal.Open(self.toa_red_name)
        ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                            x_dim, y_dim)
        red_no_data_locations = np.where(ls_red_data == self.no_data_value)
        ls_red_data = ls_red_data * self.toa_red_scale_factor

        # NDVI ---------------------------------------------------------------
        ls_ndvi_data = ((ls_nir_data - ls_red_data) /
                        (ls_nir_data + ls_red_data))

        # Cleanup no data locations
        ls_ndvi_data[nir_no_data_locations] = self.no_data_value
        ls_ndvi_data[red_no_data_locations] = self.no_data_value

        if self.keep_intermediate_data:
            geo_transform = data_set.GetGeoTransform()
            ds_srs = osr.SpatialReference()
            ds_srs.ImportFromWkt(data_set.GetProjection())

        # Memory cleanup
        del ls_red_data
        del ls_nir_data
        del nir_no_data_locations
        del red_no_data_locations

        # ====================================================================
        # Build NDSI in memory
        self.logger.info('Building TOA based NDSI band for Landsat data')

        # GREEN --------------------------------------------------------------
        data_set = gdal.Open(self.toa_green_name)
        ls_green_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                              x_dim, y_dim)
        green_no_data_locations = (
            np.where(ls_green_data == self.no_data_value))
        ls_green_data = ls_green_data * self.toa_green_scale_factor

        # SWIR1 --------------------------------------------------------------
        data_set = gdal.Open(self.toa_swir1_name)
        ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                              x_dim, y_dim)
        swir1_no_data_locations = (
            np.where(ls_swir1_data == self.no_data_value))
        ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor

        # Build the Landsat TOA NDSI data
        self.logger.info('Building TOA based NDSI for Landsat data')
        ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
                        (ls_green_data + ls_swir1_data))

        # Cleanup no data locations
        ls_ndsi_data[green_no_data_locations] = self.no_data_value
        # Cleanup no data locations
        ls_ndsi_data[swir1_no_data_locations] = self.no_data_value

        # Memory cleanup
        del ls_green_data
        del ls_swir1_data
        del green_no_data_locations
        del swir1_no_data_locations

        # Save for the output products
        ds_tmp_srs = osr.SpatialReference()
        ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
        ds_tmp_transform = data_set.GetGeoTransform()

        # Memory cleanup
        del data_set

        # Save the locations for the specfied snow pixels
        self.logger.info('Determine snow pixel locations')
        selected_snow_locations = np.where(ls_ndsi_data > 0.4)

        # Save ndvi and ndsi no data locations
        ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
        ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)

        # Memory cleanup
        del ls_ndsi_data

        # Turn all negative values to zero
        # Use a realy small value so that we don't have negative zero (-0.0)
        ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0

        if self.keep_intermediate_data:
            self.logger.info('Writing Landsat NDVI raster')
            util.Geo.generate_raster_file(geotiff_driver,
                                          'internal_landsat_ndvi.tif',
                                          ls_ndvi_data,
                                          x_dim, y_dim,
                                          geo_transform,
                                          ds_srs.ExportToWkt(),
                                          self.no_data_value,
                                          gdal.GDT_Float32)

        # Build the estimated Landsat EMIS data from the ASTER GED data and
        # warp it to the Landsat scenes projection and image extents
        # For convenience the ASTER NDVI is also extracted and warped to the
        # Landsat scenes projection and image extents
        self.logger.info('Build thermal emissivity band and'
                         ' retrieve ASTER NDVI')
        (ls_emis_warped_name,
         aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)

        # Load the warped estimated Landsat EMIS into memory
        data_set = gdal.Open(ls_emis_warped_name)
        ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                             x_dim, y_dim)
        ls_emis_gap_locations = np.where(ls_emis_data == 0)
        ls_emis_no_data_locations = (
            np.where(ls_emis_data == self.no_data_value))

        # Load the warped ASTER NDVI into memory
        data_set = gdal.Open(aster_ndvi_warped_name)
        aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
                                                                x_dim, y_dim)
        aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
        aster_ndvi_no_data_locations = (
            np.where(aster_ndvi_data == self.no_data_value))

        # Turn all negative values to zero
        # Use a realy small value so that we don't have negative zero (-0.0)
        aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0

        # Memory cleanup
        del data_set

        if not self.keep_intermediate_data:
            # Cleanup the temp files since we have them in memory
            if os.path.exists(ls_emis_warped_name):
                os.unlink(ls_emis_warped_name)
            if os.path.exists(aster_ndvi_warped_name):
                os.unlink(aster_ndvi_warped_name)

        self.logger.info('Normalizing Landsat and ASTER NDVI')
        # Normalize Landsat NDVI by max value
        max_ls_ndvi = ls_ndvi_data.max()
        self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
        ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)

        if self.keep_intermediate_data:
            self.logger.info('Writing Landsat NDVI NORM MAX raster')
            util.Geo.generate_raster_file(geotiff_driver,
                                          'internal_landsat_ndvi_norm_max.tif',
                                          ls_ndvi_data,
                                          x_dim, y_dim,
                                          geo_transform,
                                          ds_srs.ExportToWkt(),
                                          self.no_data_value,
                                          gdal.GDT_Float32)

        # Normalize ASTER NDVI by max value
        max_aster_ndvi = aster_ndvi_data.max()
        self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
        aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)

        if self.keep_intermediate_data:
            self.logger.info('Writing Aster NDVI NORM MAX raster')
            util.Geo.generate_raster_file(geotiff_driver,
                                          'internal_aster_ndvi_norm_max.tif',
                                          aster_ndvi_data,
                                          x_dim, y_dim,
                                          geo_transform,
                                          ds_srs.ExportToWkt(),
                                          self.no_data_value,
                                          gdal.GDT_Float32)


        # Soil - From prototype code variable name
        ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
                         (1.0 - aster_ndvi_data))

        # Memory cleanup
        del aster_ndvi_data
        del ls_emis_data

        # Adjust estimated Landsat EMIS for vegetation and snow, to generate
        # the final Landsat EMIS data
        self.logger.info('Adjusting estimated EMIS for vegetation')
        ls_emis_final = (self.vegetation_coeff * ls_ndvi_data +
                         ls_emis_final * (1.0 - ls_ndvi_data))

        # Medium snow
        self.logger.info('Adjusting estimated EMIS for snow')
        ls_emis_final[selected_snow_locations] = self.snow_emis_value

        # Memory cleanup
        del ls_ndvi_data
        del selected_snow_locations

        # Add the fill and scan gaps and ASTER gaps back into the results,
        # since they may have been lost
        self.logger.info('Adding fill and data gaps back into the estimated'
                         ' Landsat emissivity results')
        ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
        ls_emis_final[ls_emis_gap_locations] = self.no_data_value
        ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
        ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
        ls_emis_final[ndvi_no_data_locations] = self.no_data_value
        ls_emis_final[ndsi_no_data_locations] = self.no_data_value

        # Memory cleanup
        del ls_emis_no_data_locations
        del ls_emis_gap_locations
        del aster_ndvi_no_data_locations
        del aster_ndvi_gap_locations

        product_id = self.xml_filename.split('.xml')[0]
        ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
        ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
        ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])

        self.logger.info('Creating {0}'.format(ls_emis_img_filename))
        util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
                                      ls_emis_final, x_dim, y_dim,
                                      ds_tmp_transform,
                                      ds_tmp_srs.ExportToWkt(),
                                      self.no_data_value, gdal.GDT_Float32)

        self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
        util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)

        # Remove the *.aux.xml file generated by GDAL
        if os.path.exists(ls_emis_aux_filename):
            os.unlink(ls_emis_aux_filename)

        self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
                                                    self.xml_filename))
        # Add the estimated Landsat emissivity to the metadata XML
        espa_xml = metadata_api.parse(self.xml_filename, silence=True)
        bands = espa_xml.get_bands()
        sensor_code = product_id[0:3]
        source_product = 'toa_refl'

        # Find the TOA Band 1 to use for the specific band details
        base_band = None
        for band in bands.band:
            if band.product == source_product and band.name == 'toa_band1':
                base_band = band

        if base_band is None:
            raise Exception('Failed to find the TOA BLUE band'
                            ' in the input data')

        emis_band = metadata_api.band(product='lst_temp',
                                      source=source_product,
                                      name='landsat_emis',
                                      category='image',
                                      data_type='FLOAT32',
                                      nlines=base_band.get_nlines(),
                                      nsamps=base_band.get_nsamps(),
                                      fill_value=str(self.no_data_value))

        emis_band.set_short_name('{0}EMIS'.format(sensor_code))
        emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
                                ' data')
        emis_band.set_file_name(ls_emis_img_filename)
        emis_band.set_data_units('Emissivity Coefficient')

        pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
                                             base_band.pixel_size.x,
                                             base_band.pixel_size.units)
        emis_band.set_pixel_size(pixel_size)

        valid_range = metadata_api.valid_range(min=0.0, max=1.0)
        emis_band.set_valid_range(valid_range)

        # Set the date, but first clean the microseconds off of it
        production_date = (
            datetime.datetime.strptime(datetime.datetime.now().
                                       strftime('%Y-%m-%dT%H:%M:%S'),
                                       '%Y-%m-%dT%H:%M:%S'))

        emis_band.set_production_date(production_date)

        emis_band.set_app_version(util.Version.app_version())

        bands.add_band(emis_band)

        # Write the XML metadata file out
        with open(self.xml_filename, 'w') as output_fd:
            metadata_api.export(output_fd, espa_xml)

        # Memory cleanup
        del ls_emis_final

        self.logger.info('Completed - Estimate Landsat Emissivity')
    def generate_product(self):
        '''
        Description:
            Provides the main processing algorithm for generating the
            estimated Landsat emissivity product.  It produces the final
            emissivity product.
        '''

        self.logger = logging.getLogger(__name__)

        self.logger.info('Start - Estimate Landsat Emissivity')

        try:
            self.retrieve_metadata_information()
        except Exception:
            self.logger.exception('Failed reading input XML metadata file')
            raise

        try:
            self.determine_sensor_specific_coefficients()
        except Exception:
            self.logger.exception('Failed determining sensor coefficients')
            raise

        # Register all the gdal drivers and choose the GeoTiff for our temp
        # output
        gdal.AllRegister()
        geotiff_driver = gdal.GetDriverByName('GTiff')
        envi_driver = gdal.GetDriverByName('ENVI')

        # ====================================================================
        # Build NDVI in memory
        self.logger.info('Building TOA based NDVI band for Landsat data')

        # NIR ----------------------------------------------------------------
        data_set = gdal.Open(self.toa_nir_name)
        x_dim = data_set.RasterXSize  # They are all the same size
        y_dim = data_set.RasterYSize
        ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
        nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
        ls_nir_data = ls_nir_data * self.toa_nir_scale_factor

        # RED ----------------------------------------------------------------
        data_set = gdal.Open(self.toa_red_name)
        ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
        red_no_data_locations = np.where(ls_red_data == self.no_data_value)
        ls_red_data = ls_red_data * self.toa_red_scale_factor

        # NDVI ---------------------------------------------------------------
        ls_ndvi_data = ((ls_nir_data - ls_red_data) /
                        (ls_nir_data + ls_red_data))

        # Cleanup no data locations
        ls_ndvi_data[nir_no_data_locations] = self.no_data_value
        ls_ndvi_data[red_no_data_locations] = self.no_data_value

        if self.keep_intermediate_data:
            geo_transform = data_set.GetGeoTransform()
            ds_srs = osr.SpatialReference()
            ds_srs.ImportFromWkt(data_set.GetProjection())

        # Memory cleanup
        del ls_red_data
        del ls_nir_data
        del nir_no_data_locations
        del red_no_data_locations

        # ====================================================================
        # Build NDSI in memory
        self.logger.info('Building TOA based NDSI band for Landsat data')

        # GREEN --------------------------------------------------------------
        data_set = gdal.Open(self.toa_green_name)
        ls_green_data = data_set.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)
        green_no_data_locations = (np.where(
            ls_green_data == self.no_data_value))
        ls_green_data = ls_green_data * self.toa_green_scale_factor

        # SWIR1 --------------------------------------------------------------
        data_set = gdal.Open(self.toa_swir1_name)
        ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)
        swir1_no_data_locations = (np.where(
            ls_swir1_data == self.no_data_value))
        ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor

        # Build the Landsat TOA NDSI data
        self.logger.info('Building TOA based NDSI for Landsat data')
        ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
                        (ls_green_data + ls_swir1_data))

        # Cleanup no data locations
        ls_ndsi_data[green_no_data_locations] = self.no_data_value
        # Cleanup no data locations
        ls_ndsi_data[swir1_no_data_locations] = self.no_data_value

        # Memory cleanup
        del ls_green_data
        del ls_swir1_data
        del green_no_data_locations
        del swir1_no_data_locations

        # Save for the output products
        ds_tmp_srs = osr.SpatialReference()
        ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
        ds_tmp_transform = data_set.GetGeoTransform()

        # Memory cleanup
        del data_set

        # Save the locations for the specfied snow pixels
        self.logger.info('Determine snow pixel locations')
        selected_snow_locations = np.where(ls_ndsi_data > 0.4)

        # Save ndvi and ndsi no data locations
        ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
        ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)

        # Memory cleanup
        del ls_ndsi_data

        # Turn all negative values to zero
        # Use a realy small value so that we don't have negative zero (-0.0)
        ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0

        if self.keep_intermediate_data:
            self.logger.info('Writing Landsat NDVI raster')
            util.Geo.generate_raster_file(geotiff_driver,
                                          'internal_landsat_ndvi.tif',
                                          ls_ndvi_data, x_dim, y_dim,
                                          geo_transform, ds_srs.ExportToWkt(),
                                          self.no_data_value, gdal.GDT_Float32)

        # Build the estimated Landsat EMIS data from the ASTER GED data and
        # warp it to the Landsat scenes projection and image extents
        # For convenience the ASTER NDVI is also extracted and warped to the
        # Landsat scenes projection and image extents
        self.logger.info('Build thermal emissivity band and'
                         ' retrieve ASTER NDVI')
        (ls_emis_warped_name,
         aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)

        # Load the warped estimated Landsat EMIS into memory
        data_set = gdal.Open(ls_emis_warped_name)
        ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)
        ls_emis_gap_locations = np.where(ls_emis_data == 0)
        ls_emis_no_data_locations = (np.where(
            ls_emis_data == self.no_data_value))

        # Load the warped ASTER NDVI into memory
        data_set = gdal.Open(aster_ndvi_warped_name)
        aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(
            0, 0, x_dim, y_dim)
        aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
        aster_ndvi_no_data_locations = (np.where(
            aster_ndvi_data == self.no_data_value))

        # Turn all negative values to zero
        # Use a realy small value so that we don't have negative zero (-0.0)
        aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0

        # Memory cleanup
        del data_set

        if not self.keep_intermediate_data:
            # Cleanup the temp files since we have them in memory
            if os.path.exists(ls_emis_warped_name):
                os.unlink(ls_emis_warped_name)
            if os.path.exists(aster_ndvi_warped_name):
                os.unlink(aster_ndvi_warped_name)

        self.logger.info('Normalizing Landsat and ASTER NDVI')
        # Normalize Landsat NDVI by max value
        max_ls_ndvi = ls_ndvi_data.max()
        self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
        ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)

        if self.keep_intermediate_data:
            self.logger.info('Writing Landsat NDVI NORM MAX raster')
            util.Geo.generate_raster_file(
                geotiff_driver, 'internal_landsat_ndvi_norm_max.tif',
                ls_ndvi_data, x_dim, y_dim, geo_transform,
                ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32)

        # Normalize ASTER NDVI by max value
        max_aster_ndvi = aster_ndvi_data.max()
        self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
        aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)

        if self.keep_intermediate_data:
            self.logger.info('Writing Aster NDVI NORM MAX raster')
            util.Geo.generate_raster_file(geotiff_driver,
                                          'internal_aster_ndvi_norm_max.tif',
                                          aster_ndvi_data, x_dim, y_dim,
                                          geo_transform, ds_srs.ExportToWkt(),
                                          self.no_data_value, gdal.GDT_Float32)

        # Soil - From prototype code variable name
        self.logger.info('Calculating EMIS Final')
        with np.errstate(divide='ignore'):
            ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
                             (1.0 - aster_ndvi_data))

        # Memory cleanup
        del aster_ndvi_data
        del ls_emis_data

        # Adjust estimated Landsat EMIS for vegetation and snow, to generate
        # the final Landsat EMIS data
        self.logger.info('Adjusting estimated EMIS for vegetation')
        ls_emis_final = (self.vegetation_coeff * ls_ndvi_data + ls_emis_final *
                         (1.0 - ls_ndvi_data))

        # Medium snow
        self.logger.info('Adjusting estimated EMIS for snow')
        ls_emis_final[selected_snow_locations] = self.snow_emis_value

        # Memory cleanup
        del ls_ndvi_data
        del selected_snow_locations

        # Add the fill and scan gaps and ASTER gaps back into the results,
        # since they may have been lost
        self.logger.info('Adding fill and data gaps back into the estimated'
                         ' Landsat emissivity results')
        ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
        ls_emis_final[ls_emis_gap_locations] = self.no_data_value
        ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
        ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
        ls_emis_final[ndvi_no_data_locations] = self.no_data_value
        ls_emis_final[ndsi_no_data_locations] = self.no_data_value

        # Memory cleanup
        del ls_emis_no_data_locations
        del ls_emis_gap_locations
        del aster_ndvi_no_data_locations
        del aster_ndvi_gap_locations

        product_id = self.xml_filename.split('.xml')[0]
        ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
        ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
        ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])

        self.logger.info('Creating {0}'.format(ls_emis_img_filename))
        util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
                                      ls_emis_final, x_dim,
                                      y_dim, ds_tmp_transform,
                                      ds_tmp_srs.ExportToWkt(),
                                      self.no_data_value, gdal.GDT_Float32)

        self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
        util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)

        # Remove the *.aux.xml file generated by GDAL
        if os.path.exists(ls_emis_aux_filename):
            os.unlink(ls_emis_aux_filename)

        self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
                                                    self.xml_filename))
        # Add the estimated Landsat emissivity to the metadata XML
        espa_xml = metadata_api.parse(self.xml_filename, silence=True)
        bands = espa_xml.get_bands()
        sensor_code = product_id[0:3]
        source_product = 'toa_refl'

        # Find the TOA Band 1 to use for the specific band details
        base_band = None
        for band in bands.band:
            if band.product == source_product and band.name == 'toa_band1':
                base_band = band

        if base_band is None:
            raise Exception('Failed to find the TOA BLUE band'
                            ' in the input data')

        emis_band = metadata_api.band(product='lst_temp',
                                      source=source_product,
                                      name='landsat_emis',
                                      category='image',
                                      data_type='FLOAT32',
                                      nlines=base_band.get_nlines(),
                                      nsamps=base_band.get_nsamps(),
                                      fill_value=str(self.no_data_value))

        emis_band.set_short_name('{0}EMIS'.format(sensor_code))
        emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
                                ' data')
        emis_band.set_file_name(ls_emis_img_filename)
        emis_band.set_data_units('Emissivity Coefficient')

        pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
                                             base_band.pixel_size.x,
                                             base_band.pixel_size.units)
        emis_band.set_pixel_size(pixel_size)

        emis_band.set_resample_method('none')

        valid_range = metadata_api.valid_range(min=0.0, max=1.0)
        emis_band.set_valid_range(valid_range)

        # Set the date, but first clean the microseconds off of it
        production_date = (datetime.datetime.strptime(
            datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
            '%Y-%m-%dT%H:%M:%S'))

        emis_band.set_production_date(production_date)

        emis_band.set_app_version(util.Version.app_version())

        bands.add_band(emis_band)

        # Write the XML metadata file out
        with open(self.xml_filename, 'w') as output_fd:
            metadata_api.export(output_fd, espa_xml)

        # Memory cleanup
        del ls_emis_final

        self.logger.info('Completed - Estimate Landsat Emissivity')
Exemplo n.º 4
0
bands = metadata_api.bandsType()

# Create a band
band = metadata_api.band(product="RDD",
                         name="band1",
                         category="image",
                         data_type="UINT8",
                         nlines="7321",
                         nsamps="7951",
                         fill_value="0")

band.set_short_name("LT5DN")
band.set_long_name("band 1 digital numbers")
band.set_file_name("LT50460282002042EDC01_B1.img")

pixel_size = metadata_api.pixel_size("30.000000", 30, "meters")
band.set_pixel_size(pixel_size)

band.set_data_units("digital numbers")

valid_range = metadata_api.valid_range(min="1", max=255)
band.set_valid_range(valid_range)

toa_reflectance = metadata_api.toa_reflectance(gain=1.448, bias="-4.28819")
band.set_toa_reflectance(toa_reflectance)

band.set_app_version("LPGS_12.3.1")

production_date = \
    datetime.datetime.strptime ('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S')
band.set_production_date(production_date)
    def generate_data(self):
        '''
        Description:
            Provides the main processing algorithm for building the Land
            Surface Temperature product.  It produces the final LST product.
        '''

        try:
            self.retrieve_metadata_information()
        except Exception:
            self.logger.exception('Failed reading input XML metadata file')
            raise

        # Register all the gdal drivers and choose the ENVI for our output
        gdal.AllRegister()
        envi_driver = gdal.GetDriverByName('ENVI')

        # Read the bands into memory

        # Landsat Radiance at sensor for thermal band
        self.logger.info('Loading intermediate thermal band data')
        ds = gdal.Open(self.thermal_name)
        x_dim = ds.RasterXSize  # They are all the same size
        y_dim = ds.RasterYSize
        thermal_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Atmospheric transmittance
        self.logger.info('Loading intermediate transmittance band data')
        ds = gdal.Open(self.transmittance_name)
        trans_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Atmospheric path radiance - upwelled radiance
        self.logger.info('Loading intermediate upwelled band data')
        ds = gdal.Open(self.upwelled_name)
        upwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        self.logger.info('Calculating surface radiance')
        # Surface radiance
        surface_radiance = (thermal_data - upwelled_data) / trans_data

        # Fix the no data locations
        no_data_locations = np.where(thermal_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        no_data_locations = np.where(trans_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        no_data_locations = np.where(upwelled_data == self.no_data_value)
        surface_radiance[no_data_locations] = self.no_data_value

        # Memory cleanup
        del (thermal_data)
        del (trans_data)
        del (upwelled_data)
        del (no_data_locations)

        # Downwelling sky irradiance
        self.logger.info('Loading intermediate downwelled band data')
        ds = gdal.Open(self.downwelled_name)
        downwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Landsat emissivity estimated from ASTER GED data
        self.logger.info('Loading intermediate emissivity band data')
        ds = gdal.Open(self.emissivity_name)
        emissivity_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)

        # Save for the output product
        ds_srs = osr.SpatialReference()
        ds_srs.ImportFromWkt(ds.GetProjection())
        ds_transform = ds.GetGeoTransform()

        # Memory cleanup
        del (ds)

        # Estimate Earth-emitted radiance by subtracting off the reflected
        # downwelling component
        radiance = (surface_radiance -
                    (1.0 - emissivity_data) * downwelled_data)

        # Account for surface emissivity to get Plank emitted radiance
        self.logger.info('Calculating Plank emitted radiance')
        radiance_emitted = radiance / emissivity_data

        # Fix the no data locations
        no_data_locations = np.where(surface_radiance == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        no_data_locations = np.where(downwelled_data == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        no_data_locations = np.where(emissivity_data == self.no_data_value)
        radiance_emitted[no_data_locations] = self.no_data_value

        # Memory cleanup
        del (downwelled_data)
        del (emissivity_data)
        del (surface_radiance)
        del (radiance)
        del (no_data_locations)

        # Use Brightness Temperature LUT to get skin temperature
        # Read the correct one for what we are processing
        if self.satellite == 'LANDSAT_7':
            self.logger.info('Using Landsat 7 Brightness Temperature LUT')
            bt_name = 'L7_Brightness_Temperature_LUT.txt'

        elif self.satellite == 'LANDSAT_5':
            self.logger.info('Using Landsat 5 Brightness Temperature LUT')
            bt_name = 'L5_Brightness_Temperature_LUT.txt'

        bt_data = np.loadtxt(os.path.join(self.lst_data_dir, bt_name),
                             dtype=float, delimiter=' ')
        bt_radiance_LUT = bt_data[:, 1]
        bt_temp_LUT = bt_data[:, 0]

        self.logger.info('Generating LST results')
        lst_data = np.interp(radiance_emitted, bt_radiance_LUT, bt_temp_LUT)

        # Scale the result and convert it to an int16
        lst_data = lst_data * MULT_FACTOR
        lst_fata = lst_data.astype(np.int16)

        # Add the fill and scan gaps back into the results, since they may
        # have been lost
        self.logger.info('Adding fill and data gaps back into the Land'
                         ' Surface Temperature results')

        # Fix the no data locations
        no_data_locations = np.where(radiance_emitted == self.no_data_value)
        lst_data[no_data_locations] = self.no_data_value

        # Memory cleanup
        del (radiance_emitted)
        del (no_data_locations)

        product_id = self.xml_filename.split('.xml')[0]
        lst_img_filename = ''.join([product_id, '_lst', '.img'])
        lst_hdr_filename = ''.join([product_id, '_lst', '.hdr'])
        lst_aux_filename = ''.join([lst_img_filename, '.aux', '.xml'])

        self.logger.info('Creating {0}'.format(lst_img_filename))
        util.Geo.generate_raster_file(envi_driver, lst_img_filename,
                                      lst_data, x_dim, y_dim, ds_transform,
                                      ds_srs.ExportToWkt(), self.no_data_value,
                                      gdal.GDT_Int16)

        self.logger.info('Updating {0}'.format(lst_hdr_filename))
        util.Geo.update_envi_header(lst_hdr_filename, self.no_data_value)

        # Memory cleanup
        del (ds_srs)
        del (ds_transform)

        # Remove the *.aux.xml file generated by GDAL
        if os.path.exists(lst_aux_filename):
            os.unlink(lst_aux_filename)

        self.logger.info('Adding {0} to {1}'.format(lst_img_filename,
                                                    self.xml_filename))
        # Add the estimated Land Surface Temperature product to the metadata
        espa_xml = metadata_api.parse(self.xml_filename, silence=True)
        bands = espa_xml.get_bands()
        sensor_code = product_id[0:3]

        # Find the TOA Band 1 to use for the specific band details
        base_band = None
        for band in bands.band:
            if band.product == 'toa_refl' and band.name == 'toa_band1':
                base_band = band

        if base_band is None:
            raise Exception('Failed to find the TOA BLUE band'
                            ' in the input data')

        lst_band = metadata_api.band(product='lst',
                                     source='toa_refl',
                                     name='land_surface_temperature',
                                     category='image',
                                     data_type='INT16',
                                     scale_factor=SCALE_FACTOR,
                                     add_offset=0,
                                     nlines=base_band.get_nlines(),
                                     nsamps=base_band.get_nsamps(),
                                     fill_value=str(self.no_data_value))

        lst_band.set_short_name('{0}LST'.format(sensor_code))
        lst_band.set_long_name('Land Surface Temperature')
        lst_band.set_file_name(lst_img_filename)
        lst_band.set_data_units('temperature (kelvin)')

        pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
                                             base_band.pixel_size.x,
                                             base_band.pixel_size.units)
        lst_band.set_pixel_size(pixel_size)

        valid_range = metadata_api.valid_range(min=1500, max=3730)
        lst_band.set_valid_range(valid_range)

        # Set the date, but first clean the microseconds off of it
        production_date = (
            datetime.datetime.strptime(datetime.datetime.now().
                                       strftime('%Y-%m-%dT%H:%M:%S'),
                                       '%Y-%m-%dT%H:%M:%S'))

        lst_band.set_production_date(production_date)

        lst_band.set_app_version(util.Version.app_version())

        bands.add_band(lst_band)

        # Write the XML metadata file out
        with open(self.xml_filename, 'w') as fd:
            metadata_api.export(fd, espa_xml)

        # Memory cleanup
        del (lst_band)
        del (bands)
        del (espa_xml)
        del (lst_data)
Exemplo n.º 6
0
xml = metadata_api.parse('LT50460282002042EDC01.xml', silence=True)

bands = xml.get_bands()

# Remove the L1T bands by creating a new list of all the others
bands.band[:] = [band for band in bands.band if band.product != 'L1T']

band = metadata_api.band(product="RDD", name="band1", category="image",
    data_type="UINT8", nlines="7321", nsamps="7951", fill_value="0")

band.set_short_name ("LT5DN")
band.set_long_name ("band 1 digital numbers")
band.set_file_name ("LT50460282002042EDC01_B1.img")

pixel_size = metadata_api.pixel_size ("30.000000", 30, "meters")
band.set_pixel_size (pixel_size)

band.set_data_units ("digital numbers")

valid_range = metadata_api.valid_range (min="1", max=255)
band.set_valid_range (valid_range)

toa_reflectance = metadata_api.toa_reflectance (gain=1.448, bias="-4.28819")
band.set_toa_reflectance (toa_reflectance)

band.set_app_version ("LPGS_12.3.1")

production_date = \
    datetime.datetime.strptime('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S')
band.set_production_date (production_date)