def generate_data(self): ''' Description: Provides the main processing algorithm for building the Surface Temperature product. It produces the final ST product. ''' try: self.retrieve_metadata_information() except Exception: self.logger.exception('Failed reading input XML metadata file') raise # Register all the gdal drivers and choose the ENVI for our output gdal.AllRegister() envi_driver = gdal.GetDriverByName('ENVI') # Read the bands into memory # Landsat Radiance at sensor for thermal band self.logger.info('Loading intermediate thermal band data [{0}]'.format( self.thermal_name)) dataset = gdal.Open(self.thermal_name) x_dim = dataset.RasterXSize # They are all the same size y_dim = dataset.RasterYSize thermal_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Atmospheric transmittance self.logger.info( 'Loading intermediate transmittance band data [{0}]'.format( self.transmittance_name)) dataset = gdal.Open(self.transmittance_name) trans_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Atmospheric path radiance - upwelled radiance self.logger.info( 'Loading intermediate upwelled band data [{0}]'.format( self.upwelled_name)) dataset = gdal.Open(self.upwelled_name) upwelled_data = dataset.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) self.logger.info('Calculating surface radiance') # Surface radiance with np.errstate(invalid='ignore'): surface_radiance = (thermal_data - upwelled_data) / trans_data # Fix the no data locations no_data_locations = np.where(thermal_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value no_data_locations = np.where(trans_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value no_data_locations = np.where(upwelled_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value # Memory cleanup del thermal_data del trans_data del upwelled_data del no_data_locations # Downwelling sky irradiance self.logger.info( 'Loading intermediate downwelled band data [{0}]'.format( self.downwelled_name)) dataset = gdal.Open(self.downwelled_name) downwelled_data = dataset.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) # Landsat emissivity estimated from ASTER GED data self.logger.info( 'Loading intermediate emissivity band data [{0}]'.format( self.emissivity_name)) dataset = gdal.Open(self.emissivity_name) emissivity_data = dataset.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) # Save for the output product ds_srs = osr.SpatialReference() ds_srs.ImportFromWkt(dataset.GetProjection()) ds_transform = dataset.GetGeoTransform() # Memory cleanup del dataset # Estimate Earth-emitted radiance by subtracting off the reflected # downwelling component radiance = (surface_radiance - (1.0 - emissivity_data) * downwelled_data) # Account for surface emissivity to get Plank emitted radiance self.logger.info('Calculating Plank emitted radiance') with np.errstate(invalid='ignore'): radiance_emitted = radiance / emissivity_data # Fix the no data locations no_data_locations = np.where(surface_radiance == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value no_data_locations = np.where(downwelled_data == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value no_data_locations = np.where(emissivity_data == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value # Memory cleanup del downwelled_data del emissivity_data del surface_radiance del radiance del no_data_locations # Use Brightness Temperature LUT to get skin temperature # Read the correct one for what we are processing if self.satellite == 'LANDSAT_8': self.logger.info('Using Landsat 8 Brightness Temperature LUT') bt_name = 'L8_Brightness_Temperature_LUT.txt' elif self.satellite == 'LANDSAT_7': self.logger.info('Using Landsat 7 Brightness Temperature LUT') bt_name = 'L7_Brightness_Temperature_LUT.txt' elif self.satellite == 'LANDSAT_5': self.logger.info('Using Landsat 5 Brightness Temperature LUT') bt_name = 'L5_Brightness_Temperature_LUT.txt' elif self.satellite == 'LANDSAT_4': self.logger.info('Using Landsat 4 Brightness Temperature LUT') bt_name = 'L4_Brightness_Temperature_LUT.txt' bt_data = np.loadtxt(os.path.join(self.st_data_dir, bt_name), dtype=float, delimiter=' ') bt_radiance_lut = bt_data[:, 1] bt_temp_lut = bt_data[:, 0] self.logger.info('Generating ST results') st_data = np.interp(radiance_emitted, bt_radiance_lut, bt_temp_lut) # Scale the result st_data = st_data * MULT_FACTOR # Add the fill and scan gaps back into the results, since they may # have been lost self.logger.info('Adding fill and data gaps back into the Surface' ' Temperature results') # Fix the no data locations no_data_locations = np.where(radiance_emitted == self.no_data_value) st_data[no_data_locations] = self.no_data_value # Memory cleanup del radiance_emitted del no_data_locations product_id = self.xml_filename.split('.xml')[0] st_img_filename = ''.join([product_id, '_st', '.img']) st_hdr_filename = ''.join([product_id, '_st', '.hdr']) st_aux_filename = ''.join([st_img_filename, '.aux', '.xml']) self.logger.info('Creating {0}'.format(st_img_filename)) util.Geo.generate_raster_file(envi_driver, st_img_filename, st_data, x_dim, y_dim, ds_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Int16) self.logger.info('Updating {0}'.format(st_hdr_filename)) util.Geo.update_envi_header(st_hdr_filename, self.no_data_value) # Memory cleanup del ds_srs del ds_transform # Remove the *.aux.xml file generated by GDAL if os.path.exists(st_aux_filename): os.unlink(st_aux_filename) self.logger.info('Adding {0} to {1}'.format(st_img_filename, self.xml_filename)) # Add the estimated Surface Temperature product to the metadata espa_xml = metadata_api.parse(self.xml_filename, silence=True) bands = espa_xml.get_bands() sensor_code = product_id[0:4] # Find the TOA Band 1 to use for the specific band details base_band = None for band in bands.band: if band.product == 'toa_refl' and band.name == 'toa_band1': base_band = band if base_band is None: raise Exception('Failed to find the TOA band 1' ' in the input data') st_band = metadata_api.band(product='st', source='toa_refl', name='surface_temperature', category='image', data_type='INT16', scale_factor=SCALE_FACTOR, add_offset=0, nlines=base_band.get_nlines(), nsamps=base_band.get_nsamps(), fill_value=str(self.no_data_value)) st_band.set_short_name('{0}ST'.format(sensor_code)) st_band.set_long_name('Surface Temperature') st_band.set_file_name(st_img_filename) st_band.set_data_units('temperature (kelvin)') pixel_size = metadata_api.pixel_size(base_band.pixel_size.x, base_band.pixel_size.x, base_band.pixel_size.units) st_band.set_pixel_size(pixel_size) st_band.set_resample_method('none') valid_range = metadata_api.valid_range(min=1500, max=3730) st_band.set_valid_range(valid_range) # Set the date, but first clean the microseconds off of it production_date = (datetime.datetime.strptime( datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), '%Y-%m-%dT%H:%M:%S')) st_band.set_production_date(production_date) st_band.set_app_version(util.Version.app_version()) bands.add_band(st_band) # Write the XML metadata file out with open(self.xml_filename, 'w') as metadata_fd: metadata_api.export(metadata_fd, espa_xml) # Memory cleanup del st_band del bands del espa_xml del st_data
def generate_product(self): ''' Description: Provides the main processing algorithm for generating the estimated Landsat emissivity product. It produces the final emissivity product. ''' self.logger = logging.getLogger(__name__) self.logger.info('Start - Estimate Landsat Emissivity') try: self.retrieve_metadata_information() except Exception: self.logger.exception('Failed reading input XML metadata file') raise try: self.determine_sensor_specific_coefficients() except Exception: self.logger.exception('Failed determining sensor coefficients') raise # Register all the gdal drivers and choose the GeoTiff for our temp # output gdal.AllRegister() geotiff_driver = gdal.GetDriverByName('GTiff') envi_driver = gdal.GetDriverByName('ENVI') # ==================================================================== # Build NDVI in memory self.logger.info('Building TOA based NDVI band for Landsat data') # NIR ---------------------------------------------------------------- data_set = gdal.Open(self.toa_nir_name) x_dim = data_set.RasterXSize # They are all the same size y_dim = data_set.RasterYSize ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) nir_no_data_locations = np.where(ls_nir_data == self.no_data_value) ls_nir_data = ls_nir_data * self.toa_nir_scale_factor # RED ---------------------------------------------------------------- data_set = gdal.Open(self.toa_red_name) ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) red_no_data_locations = np.where(ls_red_data == self.no_data_value) ls_red_data = ls_red_data * self.toa_red_scale_factor # NDVI --------------------------------------------------------------- ls_ndvi_data = ((ls_nir_data - ls_red_data) / (ls_nir_data + ls_red_data)) # Cleanup no data locations ls_ndvi_data[nir_no_data_locations] = self.no_data_value ls_ndvi_data[red_no_data_locations] = self.no_data_value if self.keep_intermediate_data: geo_transform = data_set.GetGeoTransform() ds_srs = osr.SpatialReference() ds_srs.ImportFromWkt(data_set.GetProjection()) # Memory cleanup del ls_red_data del ls_nir_data del nir_no_data_locations del red_no_data_locations # ==================================================================== # Build NDSI in memory self.logger.info('Building TOA based NDSI band for Landsat data') # GREEN -------------------------------------------------------------- data_set = gdal.Open(self.toa_green_name) ls_green_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) green_no_data_locations = ( np.where(ls_green_data == self.no_data_value)) ls_green_data = ls_green_data * self.toa_green_scale_factor # SWIR1 -------------------------------------------------------------- data_set = gdal.Open(self.toa_swir1_name) ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) swir1_no_data_locations = ( np.where(ls_swir1_data == self.no_data_value)) ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor # Build the Landsat TOA NDSI data self.logger.info('Building TOA based NDSI for Landsat data') ls_ndsi_data = ((ls_green_data - ls_swir1_data) / (ls_green_data + ls_swir1_data)) # Cleanup no data locations ls_ndsi_data[green_no_data_locations] = self.no_data_value # Cleanup no data locations ls_ndsi_data[swir1_no_data_locations] = self.no_data_value # Memory cleanup del ls_green_data del ls_swir1_data del green_no_data_locations del swir1_no_data_locations # Save for the output products ds_tmp_srs = osr.SpatialReference() ds_tmp_srs.ImportFromWkt(data_set.GetProjection()) ds_tmp_transform = data_set.GetGeoTransform() # Memory cleanup del data_set # Save the locations for the specfied snow pixels self.logger.info('Determine snow pixel locations') selected_snow_locations = np.where(ls_ndsi_data > 0.4) # Save ndvi and ndsi no data locations ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value) ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value) # Memory cleanup del ls_ndsi_data # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0 if self.keep_intermediate_data: self.logger.info('Writing Landsat NDVI raster') util.Geo.generate_raster_file(geotiff_driver, 'internal_landsat_ndvi.tif', ls_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Build the estimated Landsat EMIS data from the ASTER GED data and # warp it to the Landsat scenes projection and image extents # For convenience the ASTER NDVI is also extracted and warped to the # Landsat scenes projection and image extents self.logger.info('Build thermal emissivity band and' ' retrieve ASTER NDVI') (ls_emis_warped_name, aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver) # Load the warped estimated Landsat EMIS into memory data_set = gdal.Open(ls_emis_warped_name) ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) ls_emis_gap_locations = np.where(ls_emis_data == 0) ls_emis_no_data_locations = ( np.where(ls_emis_data == self.no_data_value)) # Load the warped ASTER NDVI into memory data_set = gdal.Open(aster_ndvi_warped_name) aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0) aster_ndvi_no_data_locations = ( np.where(aster_ndvi_data == self.no_data_value)) # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0 # Memory cleanup del data_set if not self.keep_intermediate_data: # Cleanup the temp files since we have them in memory if os.path.exists(ls_emis_warped_name): os.unlink(ls_emis_warped_name) if os.path.exists(aster_ndvi_warped_name): os.unlink(aster_ndvi_warped_name) self.logger.info('Normalizing Landsat and ASTER NDVI') # Normalize Landsat NDVI by max value max_ls_ndvi = ls_ndvi_data.max() self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi)) ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi) if self.keep_intermediate_data: self.logger.info('Writing Landsat NDVI NORM MAX raster') util.Geo.generate_raster_file(geotiff_driver, 'internal_landsat_ndvi_norm_max.tif', ls_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Normalize ASTER NDVI by max value max_aster_ndvi = aster_ndvi_data.max() self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi)) aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi) if self.keep_intermediate_data: self.logger.info('Writing Aster NDVI NORM MAX raster') util.Geo.generate_raster_file(geotiff_driver, 'internal_aster_ndvi_norm_max.tif', aster_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Soil - From prototype code variable name ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) / (1.0 - aster_ndvi_data)) # Memory cleanup del aster_ndvi_data del ls_emis_data # Adjust estimated Landsat EMIS for vegetation and snow, to generate # the final Landsat EMIS data self.logger.info('Adjusting estimated EMIS for vegetation') ls_emis_final = (self.vegetation_coeff * ls_ndvi_data + ls_emis_final * (1.0 - ls_ndvi_data)) # Medium snow self.logger.info('Adjusting estimated EMIS for snow') ls_emis_final[selected_snow_locations] = self.snow_emis_value # Memory cleanup del ls_ndvi_data del selected_snow_locations # Add the fill and scan gaps and ASTER gaps back into the results, # since they may have been lost self.logger.info('Adding fill and data gaps back into the estimated' ' Landsat emissivity results') ls_emis_final[ls_emis_no_data_locations] = self.no_data_value ls_emis_final[ls_emis_gap_locations] = self.no_data_value ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value ls_emis_final[ndvi_no_data_locations] = self.no_data_value ls_emis_final[ndsi_no_data_locations] = self.no_data_value # Memory cleanup del ls_emis_no_data_locations del ls_emis_gap_locations del aster_ndvi_no_data_locations del aster_ndvi_gap_locations product_id = self.xml_filename.split('.xml')[0] ls_emis_img_filename = ''.join([product_id, '_emis', '.img']) ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr']) ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml']) self.logger.info('Creating {0}'.format(ls_emis_img_filename)) util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename, ls_emis_final, x_dim, y_dim, ds_tmp_transform, ds_tmp_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) self.logger.info('Updating {0}'.format(ls_emis_hdr_filename)) util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value) # Remove the *.aux.xml file generated by GDAL if os.path.exists(ls_emis_aux_filename): os.unlink(ls_emis_aux_filename) self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename, self.xml_filename)) # Add the estimated Landsat emissivity to the metadata XML espa_xml = metadata_api.parse(self.xml_filename, silence=True) bands = espa_xml.get_bands() sensor_code = product_id[0:3] source_product = 'toa_refl' # Find the TOA Band 1 to use for the specific band details base_band = None for band in bands.band: if band.product == source_product and band.name == 'toa_band1': base_band = band if base_band is None: raise Exception('Failed to find the TOA BLUE band' ' in the input data') emis_band = metadata_api.band(product='lst_temp', source=source_product, name='landsat_emis', category='image', data_type='FLOAT32', nlines=base_band.get_nlines(), nsamps=base_band.get_nsamps(), fill_value=str(self.no_data_value)) emis_band.set_short_name('{0}EMIS'.format(sensor_code)) emis_band.set_long_name('Landsat emissivity estimated from ASTER GED' ' data') emis_band.set_file_name(ls_emis_img_filename) emis_band.set_data_units('Emissivity Coefficient') pixel_size = metadata_api.pixel_size(base_band.pixel_size.x, base_band.pixel_size.x, base_band.pixel_size.units) emis_band.set_pixel_size(pixel_size) valid_range = metadata_api.valid_range(min=0.0, max=1.0) emis_band.set_valid_range(valid_range) # Set the date, but first clean the microseconds off of it production_date = ( datetime.datetime.strptime(datetime.datetime.now(). strftime('%Y-%m-%dT%H:%M:%S'), '%Y-%m-%dT%H:%M:%S')) emis_band.set_production_date(production_date) emis_band.set_app_version(util.Version.app_version()) bands.add_band(emis_band) # Write the XML metadata file out with open(self.xml_filename, 'w') as output_fd: metadata_api.export(output_fd, espa_xml) # Memory cleanup del ls_emis_final self.logger.info('Completed - Estimate Landsat Emissivity')
def generate_product(self): ''' Description: Provides the main processing algorithm for generating the estimated Landsat emissivity product. It produces the final emissivity product. ''' self.logger = logging.getLogger(__name__) self.logger.info('Start - Estimate Landsat Emissivity') try: self.retrieve_metadata_information() except Exception: self.logger.exception('Failed reading input XML metadata file') raise try: self.determine_sensor_specific_coefficients() except Exception: self.logger.exception('Failed determining sensor coefficients') raise # Register all the gdal drivers and choose the GeoTiff for our temp # output gdal.AllRegister() geotiff_driver = gdal.GetDriverByName('GTiff') envi_driver = gdal.GetDriverByName('ENVI') # ==================================================================== # Build NDVI in memory self.logger.info('Building TOA based NDVI band for Landsat data') # NIR ---------------------------------------------------------------- data_set = gdal.Open(self.toa_nir_name) x_dim = data_set.RasterXSize # They are all the same size y_dim = data_set.RasterYSize ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) nir_no_data_locations = np.where(ls_nir_data == self.no_data_value) ls_nir_data = ls_nir_data * self.toa_nir_scale_factor # RED ---------------------------------------------------------------- data_set = gdal.Open(self.toa_red_name) ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) red_no_data_locations = np.where(ls_red_data == self.no_data_value) ls_red_data = ls_red_data * self.toa_red_scale_factor # NDVI --------------------------------------------------------------- ls_ndvi_data = ((ls_nir_data - ls_red_data) / (ls_nir_data + ls_red_data)) # Cleanup no data locations ls_ndvi_data[nir_no_data_locations] = self.no_data_value ls_ndvi_data[red_no_data_locations] = self.no_data_value if self.keep_intermediate_data: geo_transform = data_set.GetGeoTransform() ds_srs = osr.SpatialReference() ds_srs.ImportFromWkt(data_set.GetProjection()) # Memory cleanup del ls_red_data del ls_nir_data del nir_no_data_locations del red_no_data_locations # ==================================================================== # Build NDSI in memory self.logger.info('Building TOA based NDSI band for Landsat data') # GREEN -------------------------------------------------------------- data_set = gdal.Open(self.toa_green_name) ls_green_data = data_set.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) green_no_data_locations = (np.where( ls_green_data == self.no_data_value)) ls_green_data = ls_green_data * self.toa_green_scale_factor # SWIR1 -------------------------------------------------------------- data_set = gdal.Open(self.toa_swir1_name) ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) swir1_no_data_locations = (np.where( ls_swir1_data == self.no_data_value)) ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor # Build the Landsat TOA NDSI data self.logger.info('Building TOA based NDSI for Landsat data') ls_ndsi_data = ((ls_green_data - ls_swir1_data) / (ls_green_data + ls_swir1_data)) # Cleanup no data locations ls_ndsi_data[green_no_data_locations] = self.no_data_value # Cleanup no data locations ls_ndsi_data[swir1_no_data_locations] = self.no_data_value # Memory cleanup del ls_green_data del ls_swir1_data del green_no_data_locations del swir1_no_data_locations # Save for the output products ds_tmp_srs = osr.SpatialReference() ds_tmp_srs.ImportFromWkt(data_set.GetProjection()) ds_tmp_transform = data_set.GetGeoTransform() # Memory cleanup del data_set # Save the locations for the specfied snow pixels self.logger.info('Determine snow pixel locations') selected_snow_locations = np.where(ls_ndsi_data > 0.4) # Save ndvi and ndsi no data locations ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value) ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value) # Memory cleanup del ls_ndsi_data # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0 if self.keep_intermediate_data: self.logger.info('Writing Landsat NDVI raster') util.Geo.generate_raster_file(geotiff_driver, 'internal_landsat_ndvi.tif', ls_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Build the estimated Landsat EMIS data from the ASTER GED data and # warp it to the Landsat scenes projection and image extents # For convenience the ASTER NDVI is also extracted and warped to the # Landsat scenes projection and image extents self.logger.info('Build thermal emissivity band and' ' retrieve ASTER NDVI') (ls_emis_warped_name, aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver) # Load the warped estimated Landsat EMIS into memory data_set = gdal.Open(ls_emis_warped_name) ls_emis_data = data_set.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) ls_emis_gap_locations = np.where(ls_emis_data == 0) ls_emis_no_data_locations = (np.where( ls_emis_data == self.no_data_value)) # Load the warped ASTER NDVI into memory data_set = gdal.Open(aster_ndvi_warped_name) aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray( 0, 0, x_dim, y_dim) aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0) aster_ndvi_no_data_locations = (np.where( aster_ndvi_data == self.no_data_value)) # Turn all negative values to zero # Use a realy small value so that we don't have negative zero (-0.0) aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0 # Memory cleanup del data_set if not self.keep_intermediate_data: # Cleanup the temp files since we have them in memory if os.path.exists(ls_emis_warped_name): os.unlink(ls_emis_warped_name) if os.path.exists(aster_ndvi_warped_name): os.unlink(aster_ndvi_warped_name) self.logger.info('Normalizing Landsat and ASTER NDVI') # Normalize Landsat NDVI by max value max_ls_ndvi = ls_ndvi_data.max() self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi)) ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi) if self.keep_intermediate_data: self.logger.info('Writing Landsat NDVI NORM MAX raster') util.Geo.generate_raster_file( geotiff_driver, 'internal_landsat_ndvi_norm_max.tif', ls_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Normalize ASTER NDVI by max value max_aster_ndvi = aster_ndvi_data.max() self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi)) aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi) if self.keep_intermediate_data: self.logger.info('Writing Aster NDVI NORM MAX raster') util.Geo.generate_raster_file(geotiff_driver, 'internal_aster_ndvi_norm_max.tif', aster_ndvi_data, x_dim, y_dim, geo_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) # Soil - From prototype code variable name self.logger.info('Calculating EMIS Final') with np.errstate(divide='ignore'): ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) / (1.0 - aster_ndvi_data)) # Memory cleanup del aster_ndvi_data del ls_emis_data # Adjust estimated Landsat EMIS for vegetation and snow, to generate # the final Landsat EMIS data self.logger.info('Adjusting estimated EMIS for vegetation') ls_emis_final = (self.vegetation_coeff * ls_ndvi_data + ls_emis_final * (1.0 - ls_ndvi_data)) # Medium snow self.logger.info('Adjusting estimated EMIS for snow') ls_emis_final[selected_snow_locations] = self.snow_emis_value # Memory cleanup del ls_ndvi_data del selected_snow_locations # Add the fill and scan gaps and ASTER gaps back into the results, # since they may have been lost self.logger.info('Adding fill and data gaps back into the estimated' ' Landsat emissivity results') ls_emis_final[ls_emis_no_data_locations] = self.no_data_value ls_emis_final[ls_emis_gap_locations] = self.no_data_value ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value ls_emis_final[ndvi_no_data_locations] = self.no_data_value ls_emis_final[ndsi_no_data_locations] = self.no_data_value # Memory cleanup del ls_emis_no_data_locations del ls_emis_gap_locations del aster_ndvi_no_data_locations del aster_ndvi_gap_locations product_id = self.xml_filename.split('.xml')[0] ls_emis_img_filename = ''.join([product_id, '_emis', '.img']) ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr']) ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml']) self.logger.info('Creating {0}'.format(ls_emis_img_filename)) util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename, ls_emis_final, x_dim, y_dim, ds_tmp_transform, ds_tmp_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32) self.logger.info('Updating {0}'.format(ls_emis_hdr_filename)) util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value) # Remove the *.aux.xml file generated by GDAL if os.path.exists(ls_emis_aux_filename): os.unlink(ls_emis_aux_filename) self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename, self.xml_filename)) # Add the estimated Landsat emissivity to the metadata XML espa_xml = metadata_api.parse(self.xml_filename, silence=True) bands = espa_xml.get_bands() sensor_code = product_id[0:3] source_product = 'toa_refl' # Find the TOA Band 1 to use for the specific band details base_band = None for band in bands.band: if band.product == source_product and band.name == 'toa_band1': base_band = band if base_band is None: raise Exception('Failed to find the TOA BLUE band' ' in the input data') emis_band = metadata_api.band(product='lst_temp', source=source_product, name='landsat_emis', category='image', data_type='FLOAT32', nlines=base_band.get_nlines(), nsamps=base_band.get_nsamps(), fill_value=str(self.no_data_value)) emis_band.set_short_name('{0}EMIS'.format(sensor_code)) emis_band.set_long_name('Landsat emissivity estimated from ASTER GED' ' data') emis_band.set_file_name(ls_emis_img_filename) emis_band.set_data_units('Emissivity Coefficient') pixel_size = metadata_api.pixel_size(base_band.pixel_size.x, base_band.pixel_size.x, base_band.pixel_size.units) emis_band.set_pixel_size(pixel_size) emis_band.set_resample_method('none') valid_range = metadata_api.valid_range(min=0.0, max=1.0) emis_band.set_valid_range(valid_range) # Set the date, but first clean the microseconds off of it production_date = (datetime.datetime.strptime( datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), '%Y-%m-%dT%H:%M:%S')) emis_band.set_production_date(production_date) emis_band.set_app_version(util.Version.app_version()) bands.add_band(emis_band) # Write the XML metadata file out with open(self.xml_filename, 'w') as output_fd: metadata_api.export(output_fd, espa_xml) # Memory cleanup del ls_emis_final self.logger.info('Completed - Estimate Landsat Emissivity')
bands = metadata_api.bandsType() # Create a band band = metadata_api.band(product="RDD", name="band1", category="image", data_type="UINT8", nlines="7321", nsamps="7951", fill_value="0") band.set_short_name("LT5DN") band.set_long_name("band 1 digital numbers") band.set_file_name("LT50460282002042EDC01_B1.img") pixel_size = metadata_api.pixel_size("30.000000", 30, "meters") band.set_pixel_size(pixel_size) band.set_data_units("digital numbers") valid_range = metadata_api.valid_range(min="1", max=255) band.set_valid_range(valid_range) toa_reflectance = metadata_api.toa_reflectance(gain=1.448, bias="-4.28819") band.set_toa_reflectance(toa_reflectance) band.set_app_version("LPGS_12.3.1") production_date = \ datetime.datetime.strptime ('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S') band.set_production_date(production_date)
def generate_data(self): ''' Description: Provides the main processing algorithm for building the Land Surface Temperature product. It produces the final LST product. ''' try: self.retrieve_metadata_information() except Exception: self.logger.exception('Failed reading input XML metadata file') raise # Register all the gdal drivers and choose the ENVI for our output gdal.AllRegister() envi_driver = gdal.GetDriverByName('ENVI') # Read the bands into memory # Landsat Radiance at sensor for thermal band self.logger.info('Loading intermediate thermal band data') ds = gdal.Open(self.thermal_name) x_dim = ds.RasterXSize # They are all the same size y_dim = ds.RasterYSize thermal_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Atmospheric transmittance self.logger.info('Loading intermediate transmittance band data') ds = gdal.Open(self.transmittance_name) trans_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Atmospheric path radiance - upwelled radiance self.logger.info('Loading intermediate upwelled band data') ds = gdal.Open(self.upwelled_name) upwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) self.logger.info('Calculating surface radiance') # Surface radiance surface_radiance = (thermal_data - upwelled_data) / trans_data # Fix the no data locations no_data_locations = np.where(thermal_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value no_data_locations = np.where(trans_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value no_data_locations = np.where(upwelled_data == self.no_data_value) surface_radiance[no_data_locations] = self.no_data_value # Memory cleanup del (thermal_data) del (trans_data) del (upwelled_data) del (no_data_locations) # Downwelling sky irradiance self.logger.info('Loading intermediate downwelled band data') ds = gdal.Open(self.downwelled_name) downwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Landsat emissivity estimated from ASTER GED data self.logger.info('Loading intermediate emissivity band data') ds = gdal.Open(self.emissivity_name) emissivity_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim) # Save for the output product ds_srs = osr.SpatialReference() ds_srs.ImportFromWkt(ds.GetProjection()) ds_transform = ds.GetGeoTransform() # Memory cleanup del (ds) # Estimate Earth-emitted radiance by subtracting off the reflected # downwelling component radiance = (surface_radiance - (1.0 - emissivity_data) * downwelled_data) # Account for surface emissivity to get Plank emitted radiance self.logger.info('Calculating Plank emitted radiance') radiance_emitted = radiance / emissivity_data # Fix the no data locations no_data_locations = np.where(surface_radiance == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value no_data_locations = np.where(downwelled_data == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value no_data_locations = np.where(emissivity_data == self.no_data_value) radiance_emitted[no_data_locations] = self.no_data_value # Memory cleanup del (downwelled_data) del (emissivity_data) del (surface_radiance) del (radiance) del (no_data_locations) # Use Brightness Temperature LUT to get skin temperature # Read the correct one for what we are processing if self.satellite == 'LANDSAT_7': self.logger.info('Using Landsat 7 Brightness Temperature LUT') bt_name = 'L7_Brightness_Temperature_LUT.txt' elif self.satellite == 'LANDSAT_5': self.logger.info('Using Landsat 5 Brightness Temperature LUT') bt_name = 'L5_Brightness_Temperature_LUT.txt' bt_data = np.loadtxt(os.path.join(self.lst_data_dir, bt_name), dtype=float, delimiter=' ') bt_radiance_LUT = bt_data[:, 1] bt_temp_LUT = bt_data[:, 0] self.logger.info('Generating LST results') lst_data = np.interp(radiance_emitted, bt_radiance_LUT, bt_temp_LUT) # Scale the result and convert it to an int16 lst_data = lst_data * MULT_FACTOR lst_fata = lst_data.astype(np.int16) # Add the fill and scan gaps back into the results, since they may # have been lost self.logger.info('Adding fill and data gaps back into the Land' ' Surface Temperature results') # Fix the no data locations no_data_locations = np.where(radiance_emitted == self.no_data_value) lst_data[no_data_locations] = self.no_data_value # Memory cleanup del (radiance_emitted) del (no_data_locations) product_id = self.xml_filename.split('.xml')[0] lst_img_filename = ''.join([product_id, '_lst', '.img']) lst_hdr_filename = ''.join([product_id, '_lst', '.hdr']) lst_aux_filename = ''.join([lst_img_filename, '.aux', '.xml']) self.logger.info('Creating {0}'.format(lst_img_filename)) util.Geo.generate_raster_file(envi_driver, lst_img_filename, lst_data, x_dim, y_dim, ds_transform, ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Int16) self.logger.info('Updating {0}'.format(lst_hdr_filename)) util.Geo.update_envi_header(lst_hdr_filename, self.no_data_value) # Memory cleanup del (ds_srs) del (ds_transform) # Remove the *.aux.xml file generated by GDAL if os.path.exists(lst_aux_filename): os.unlink(lst_aux_filename) self.logger.info('Adding {0} to {1}'.format(lst_img_filename, self.xml_filename)) # Add the estimated Land Surface Temperature product to the metadata espa_xml = metadata_api.parse(self.xml_filename, silence=True) bands = espa_xml.get_bands() sensor_code = product_id[0:3] # Find the TOA Band 1 to use for the specific band details base_band = None for band in bands.band: if band.product == 'toa_refl' and band.name == 'toa_band1': base_band = band if base_band is None: raise Exception('Failed to find the TOA BLUE band' ' in the input data') lst_band = metadata_api.band(product='lst', source='toa_refl', name='land_surface_temperature', category='image', data_type='INT16', scale_factor=SCALE_FACTOR, add_offset=0, nlines=base_band.get_nlines(), nsamps=base_band.get_nsamps(), fill_value=str(self.no_data_value)) lst_band.set_short_name('{0}LST'.format(sensor_code)) lst_band.set_long_name('Land Surface Temperature') lst_band.set_file_name(lst_img_filename) lst_band.set_data_units('temperature (kelvin)') pixel_size = metadata_api.pixel_size(base_band.pixel_size.x, base_band.pixel_size.x, base_band.pixel_size.units) lst_band.set_pixel_size(pixel_size) valid_range = metadata_api.valid_range(min=1500, max=3730) lst_band.set_valid_range(valid_range) # Set the date, but first clean the microseconds off of it production_date = ( datetime.datetime.strptime(datetime.datetime.now(). strftime('%Y-%m-%dT%H:%M:%S'), '%Y-%m-%dT%H:%M:%S')) lst_band.set_production_date(production_date) lst_band.set_app_version(util.Version.app_version()) bands.add_band(lst_band) # Write the XML metadata file out with open(self.xml_filename, 'w') as fd: metadata_api.export(fd, espa_xml) # Memory cleanup del (lst_band) del (bands) del (espa_xml) del (lst_data)
xml = metadata_api.parse('LT50460282002042EDC01.xml', silence=True) bands = xml.get_bands() # Remove the L1T bands by creating a new list of all the others bands.band[:] = [band for band in bands.band if band.product != 'L1T'] band = metadata_api.band(product="RDD", name="band1", category="image", data_type="UINT8", nlines="7321", nsamps="7951", fill_value="0") band.set_short_name ("LT5DN") band.set_long_name ("band 1 digital numbers") band.set_file_name ("LT50460282002042EDC01_B1.img") pixel_size = metadata_api.pixel_size ("30.000000", 30, "meters") band.set_pixel_size (pixel_size) band.set_data_units ("digital numbers") valid_range = metadata_api.valid_range (min="1", max=255) band.set_valid_range (valid_range) toa_reflectance = metadata_api.toa_reflectance (gain=1.448, bias="-4.28819") band.set_toa_reflectance (toa_reflectance) band.set_app_version ("LPGS_12.3.1") production_date = \ datetime.datetime.strptime('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S') band.set_production_date (production_date)