def generate_data(self):
'''
Description:
Provides the main processing algorithm for building the Surface
Temperature product. It produces the final ST product.
'''
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
# Register all the gdal drivers and choose the ENVI for our output
gdal.AllRegister()
envi_driver = gdal.GetDriverByName('ENVI')
# Read the bands into memory
# Landsat Radiance at sensor for thermal band
self.logger.info('Loading intermediate thermal band data [{0}]'.format(
self.thermal_name))
dataset = gdal.Open(self.thermal_name)
x_dim = dataset.RasterXSize # They are all the same size
y_dim = dataset.RasterYSize
thermal_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric transmittance
self.logger.info(
'Loading intermediate transmittance band data [{0}]'.format(
self.transmittance_name))
dataset = gdal.Open(self.transmittance_name)
trans_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric path radiance - upwelled radiance
self.logger.info(
'Loading intermediate upwelled band data [{0}]'.format(
self.upwelled_name))
dataset = gdal.Open(self.upwelled_name)
upwelled_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
self.logger.info('Calculating surface radiance')
# Surface radiance
with np.errstate(invalid='ignore'):
surface_radiance = (thermal_data - upwelled_data) / trans_data
# Fix the no data locations
no_data_locations = np.where(thermal_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(trans_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(upwelled_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
# Memory cleanup
del thermal_data
del trans_data
del upwelled_data
del no_data_locations
# Downwelling sky irradiance
self.logger.info(
'Loading intermediate downwelled band data [{0}]'.format(
self.downwelled_name))
dataset = gdal.Open(self.downwelled_name)
downwelled_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
# Landsat emissivity estimated from ASTER GED data
self.logger.info(
'Loading intermediate emissivity band data [{0}]'.format(
self.emissivity_name))
dataset = gdal.Open(self.emissivity_name)
emissivity_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
# Save for the output product
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(dataset.GetProjection())
ds_transform = dataset.GetGeoTransform()
# Memory cleanup
del dataset
# Estimate Earth-emitted radiance by subtracting off the reflected
# downwelling component
radiance = (surface_radiance -
(1.0 - emissivity_data) * downwelled_data)
# Account for surface emissivity to get Plank emitted radiance
self.logger.info('Calculating Plank emitted radiance')
with np.errstate(invalid='ignore'):
radiance_emitted = radiance / emissivity_data
# Fix the no data locations
no_data_locations = np.where(surface_radiance == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(downwelled_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(emissivity_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
# Memory cleanup
del downwelled_data
del emissivity_data
del surface_radiance
del radiance
del no_data_locations
# Use Brightness Temperature LUT to get skin temperature
# Read the correct one for what we are processing
if self.satellite == 'LANDSAT_8':
self.logger.info('Using Landsat 8 Brightness Temperature LUT')
bt_name = 'L8_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_7':
self.logger.info('Using Landsat 7 Brightness Temperature LUT')
bt_name = 'L7_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_5':
self.logger.info('Using Landsat 5 Brightness Temperature LUT')
bt_name = 'L5_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_4':
self.logger.info('Using Landsat 4 Brightness Temperature LUT')
bt_name = 'L4_Brightness_Temperature_LUT.txt'
bt_data = np.loadtxt(os.path.join(self.st_data_dir, bt_name),
dtype=float,
delimiter=' ')
bt_radiance_lut = bt_data[:, 1]
bt_temp_lut = bt_data[:, 0]
self.logger.info('Generating ST results')
st_data = np.interp(radiance_emitted, bt_radiance_lut, bt_temp_lut)
# Scale the result
st_data = st_data * MULT_FACTOR
# Add the fill and scan gaps back into the results, since they may
# have been lost
self.logger.info('Adding fill and data gaps back into the Surface'
' Temperature results')
# Fix the no data locations
no_data_locations = np.where(radiance_emitted == self.no_data_value)
st_data[no_data_locations] = self.no_data_value
# Memory cleanup
del radiance_emitted
del no_data_locations
product_id = self.xml_filename.split('.xml')[0]
st_img_filename = ''.join([product_id, '_st', '.img'])
st_hdr_filename = ''.join([product_id, '_st', '.hdr'])
st_aux_filename = ''.join([st_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(st_img_filename))
util.Geo.generate_raster_file(envi_driver, st_img_filename, st_data,
x_dim, y_dim, ds_transform,
ds_srs.ExportToWkt(), self.no_data_value,
gdal.GDT_Int16)
self.logger.info('Updating {0}'.format(st_hdr_filename))
util.Geo.update_envi_header(st_hdr_filename, self.no_data_value)
# Memory cleanup
del ds_srs
del ds_transform
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(st_aux_filename):
os.unlink(st_aux_filename)
self.logger.info('Adding {0} to {1}'.format(st_img_filename,
self.xml_filename))
# Add the estimated Surface Temperature product to the metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:4]
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA band 1'
' in the input data')
st_band = metadata_api.band(product='st',
source='toa_refl',
name='surface_temperature',
category='image',
data_type='INT16',
scale_factor=SCALE_FACTOR,
add_offset=0,
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
st_band.set_short_name('{0}ST'.format(sensor_code))
st_band.set_long_name('Surface Temperature')
st_band.set_file_name(st_img_filename)
st_band.set_data_units('temperature (kelvin)')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
st_band.set_pixel_size(pixel_size)
st_band.set_resample_method('none')
valid_range = metadata_api.valid_range(min=1500, max=3730)
st_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (datetime.datetime.strptime(
datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
st_band.set_production_date(production_date)
st_band.set_app_version(util.Version.app_version())
bands.add_band(st_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as metadata_fd:
metadata_api.export(metadata_fd, espa_xml)
# Memory cleanup
del st_band
del bands
del espa_xml
del st_data
def generate_product(self):
'''
Description:
Provides the main processing algorithm for generating the
estimated Landsat emissivity product. It produces the final
emissivity product.
'''
self.logger = logging.getLogger(__name__)
self.logger.info('Start - Estimate Landsat Emissivity')
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
try:
self.determine_sensor_specific_coefficients()
except Exception:
self.logger.exception('Failed determining sensor coefficients')
raise
# Register all the gdal drivers and choose the GeoTiff for our temp
# output
gdal.AllRegister()
geotiff_driver = gdal.GetDriverByName('GTiff')
envi_driver = gdal.GetDriverByName('ENVI')
# ====================================================================
# Build NDVI in memory
self.logger.info('Building TOA based NDVI band for Landsat data')
# NIR ----------------------------------------------------------------
data_set = gdal.Open(self.toa_nir_name)
x_dim = data_set.RasterXSize # They are all the same size
y_dim = data_set.RasterYSize
ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
ls_nir_data = ls_nir_data * self.toa_nir_scale_factor
# RED ----------------------------------------------------------------
data_set = gdal.Open(self.toa_red_name)
ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
red_no_data_locations = np.where(ls_red_data == self.no_data_value)
ls_red_data = ls_red_data * self.toa_red_scale_factor
# NDVI ---------------------------------------------------------------
ls_ndvi_data = ((ls_nir_data - ls_red_data) /
(ls_nir_data + ls_red_data))
# Cleanup no data locations
ls_ndvi_data[nir_no_data_locations] = self.no_data_value
ls_ndvi_data[red_no_data_locations] = self.no_data_value
if self.keep_intermediate_data:
geo_transform = data_set.GetGeoTransform()
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(data_set.GetProjection())
# Memory cleanup
del ls_red_data
del ls_nir_data
del nir_no_data_locations
del red_no_data_locations
# ====================================================================
# Build NDSI in memory
self.logger.info('Building TOA based NDSI band for Landsat data')
# GREEN --------------------------------------------------------------
data_set = gdal.Open(self.toa_green_name)
ls_green_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
green_no_data_locations = (
np.where(ls_green_data == self.no_data_value))
ls_green_data = ls_green_data * self.toa_green_scale_factor
# SWIR1 --------------------------------------------------------------
data_set = gdal.Open(self.toa_swir1_name)
ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
swir1_no_data_locations = (
np.where(ls_swir1_data == self.no_data_value))
ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor
# Build the Landsat TOA NDSI data
self.logger.info('Building TOA based NDSI for Landsat data')
ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
(ls_green_data + ls_swir1_data))
# Cleanup no data locations
ls_ndsi_data[green_no_data_locations] = self.no_data_value
# Cleanup no data locations
ls_ndsi_data[swir1_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_green_data
del ls_swir1_data
del green_no_data_locations
del swir1_no_data_locations
# Save for the output products
ds_tmp_srs = osr.SpatialReference()
ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
ds_tmp_transform = data_set.GetGeoTransform()
# Memory cleanup
del data_set
# Save the locations for the specfied snow pixels
self.logger.info('Determine snow pixel locations')
selected_snow_locations = np.where(ls_ndsi_data > 0.4)
# Save ndvi and ndsi no data locations
ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)
# Memory cleanup
del ls_ndsi_data
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi.tif',
ls_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Build the estimated Landsat EMIS data from the ASTER GED data and
# warp it to the Landsat scenes projection and image extents
# For convenience the ASTER NDVI is also extracted and warped to the
# Landsat scenes projection and image extents
self.logger.info('Build thermal emissivity band and'
' retrieve ASTER NDVI')
(ls_emis_warped_name,
aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)
# Load the warped estimated Landsat EMIS into memory
data_set = gdal.Open(ls_emis_warped_name)
ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
ls_emis_gap_locations = np.where(ls_emis_data == 0)
ls_emis_no_data_locations = (
np.where(ls_emis_data == self.no_data_value))
# Load the warped ASTER NDVI into memory
data_set = gdal.Open(aster_ndvi_warped_name)
aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
aster_ndvi_no_data_locations = (
np.where(aster_ndvi_data == self.no_data_value))
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0
# Memory cleanup
del data_set
if not self.keep_intermediate_data:
# Cleanup the temp files since we have them in memory
if os.path.exists(ls_emis_warped_name):
os.unlink(ls_emis_warped_name)
if os.path.exists(aster_ndvi_warped_name):
os.unlink(aster_ndvi_warped_name)
self.logger.info('Normalizing Landsat and ASTER NDVI')
# Normalize Landsat NDVI by max value
max_ls_ndvi = ls_ndvi_data.max()
self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi_norm_max.tif',
ls_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Normalize ASTER NDVI by max value
max_aster_ndvi = aster_ndvi_data.max()
self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Aster NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_aster_ndvi_norm_max.tif',
aster_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Soil - From prototype code variable name
ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
(1.0 - aster_ndvi_data))
# Memory cleanup
del aster_ndvi_data
del ls_emis_data
# Adjust estimated Landsat EMIS for vegetation and snow, to generate
# the final Landsat EMIS data
self.logger.info('Adjusting estimated EMIS for vegetation')
ls_emis_final = (self.vegetation_coeff * ls_ndvi_data +
ls_emis_final * (1.0 - ls_ndvi_data))
# Medium snow
self.logger.info('Adjusting estimated EMIS for snow')
ls_emis_final[selected_snow_locations] = self.snow_emis_value
# Memory cleanup
del ls_ndvi_data
del selected_snow_locations
# Add the fill and scan gaps and ASTER gaps back into the results,
# since they may have been lost
self.logger.info('Adding fill and data gaps back into the estimated'
' Landsat emissivity results')
ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
ls_emis_final[ls_emis_gap_locations] = self.no_data_value
ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
ls_emis_final[ndvi_no_data_locations] = self.no_data_value
ls_emis_final[ndsi_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_emis_no_data_locations
del ls_emis_gap_locations
del aster_ndvi_no_data_locations
del aster_ndvi_gap_locations
product_id = self.xml_filename.split('.xml')[0]
ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(ls_emis_img_filename))
util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
ls_emis_final, x_dim, y_dim,
ds_tmp_transform,
ds_tmp_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(ls_emis_aux_filename):
os.unlink(ls_emis_aux_filename)
self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
self.xml_filename))
# Add the estimated Landsat emissivity to the metadata XML
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
source_product = 'toa_refl'
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == source_product and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
emis_band = metadata_api.band(product='lst_temp',
source=source_product,
name='landsat_emis',
category='image',
data_type='FLOAT32',
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
emis_band.set_short_name('{0}EMIS'.format(sensor_code))
emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
' data')
emis_band.set_file_name(ls_emis_img_filename)
emis_band.set_data_units('Emissivity Coefficient')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
emis_band.set_pixel_size(pixel_size)
valid_range = metadata_api.valid_range(min=0.0, max=1.0)
emis_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (
datetime.datetime.strptime(datetime.datetime.now().
strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
emis_band.set_production_date(production_date)
emis_band.set_app_version(util.Version.app_version())
bands.add_band(emis_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as output_fd:
metadata_api.export(output_fd, espa_xml)
# Memory cleanup
del ls_emis_final
self.logger.info('Completed - Estimate Landsat Emissivity')
def generate_product(self):
'''
Description:
Provides the main processing algorithm for generating the
estimated Landsat emissivity product. It produces the final
emissivity product.
'''
self.logger = logging.getLogger(__name__)
self.logger.info('Start - Estimate Landsat Emissivity')
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
try:
self.determine_sensor_specific_coefficients()
except Exception:
self.logger.exception('Failed determining sensor coefficients')
raise
# Register all the gdal drivers and choose the GeoTiff for our temp
# output
gdal.AllRegister()
geotiff_driver = gdal.GetDriverByName('GTiff')
envi_driver = gdal.GetDriverByName('ENVI')
# ====================================================================
# Build NDVI in memory
self.logger.info('Building TOA based NDVI band for Landsat data')
# NIR ----------------------------------------------------------------
data_set = gdal.Open(self.toa_nir_name)
x_dim = data_set.RasterXSize # They are all the same size
y_dim = data_set.RasterYSize
ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
ls_nir_data = ls_nir_data * self.toa_nir_scale_factor
# RED ----------------------------------------------------------------
data_set = gdal.Open(self.toa_red_name)
ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
red_no_data_locations = np.where(ls_red_data == self.no_data_value)
ls_red_data = ls_red_data * self.toa_red_scale_factor
# NDVI ---------------------------------------------------------------
ls_ndvi_data = ((ls_nir_data - ls_red_data) /
(ls_nir_data + ls_red_data))
# Cleanup no data locations
ls_ndvi_data[nir_no_data_locations] = self.no_data_value
ls_ndvi_data[red_no_data_locations] = self.no_data_value
if self.keep_intermediate_data:
geo_transform = data_set.GetGeoTransform()
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(data_set.GetProjection())
# Memory cleanup
del ls_red_data
del ls_nir_data
del nir_no_data_locations
del red_no_data_locations
# ====================================================================
# Build NDSI in memory
self.logger.info('Building TOA based NDSI band for Landsat data')
# GREEN --------------------------------------------------------------
data_set = gdal.Open(self.toa_green_name)
ls_green_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
green_no_data_locations = (np.where(
ls_green_data == self.no_data_value))
ls_green_data = ls_green_data * self.toa_green_scale_factor
# SWIR1 --------------------------------------------------------------
data_set = gdal.Open(self.toa_swir1_name)
ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
swir1_no_data_locations = (np.where(
ls_swir1_data == self.no_data_value))
ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor
# Build the Landsat TOA NDSI data
self.logger.info('Building TOA based NDSI for Landsat data')
ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
(ls_green_data + ls_swir1_data))
# Cleanup no data locations
ls_ndsi_data[green_no_data_locations] = self.no_data_value
# Cleanup no data locations
ls_ndsi_data[swir1_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_green_data
del ls_swir1_data
del green_no_data_locations
del swir1_no_data_locations
# Save for the output products
ds_tmp_srs = osr.SpatialReference()
ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
ds_tmp_transform = data_set.GetGeoTransform()
# Memory cleanup
del data_set
# Save the locations for the specfied snow pixels
self.logger.info('Determine snow pixel locations')
selected_snow_locations = np.where(ls_ndsi_data > 0.4)
# Save ndvi and ndsi no data locations
ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)
# Memory cleanup
del ls_ndsi_data
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi.tif',
ls_ndvi_data, x_dim, y_dim,
geo_transform, ds_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
# Build the estimated Landsat EMIS data from the ASTER GED data and
# warp it to the Landsat scenes projection and image extents
# For convenience the ASTER NDVI is also extracted and warped to the
# Landsat scenes projection and image extents
self.logger.info('Build thermal emissivity band and'
' retrieve ASTER NDVI')
(ls_emis_warped_name,
aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)
# Load the warped estimated Landsat EMIS into memory
data_set = gdal.Open(ls_emis_warped_name)
ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
ls_emis_gap_locations = np.where(ls_emis_data == 0)
ls_emis_no_data_locations = (np.where(
ls_emis_data == self.no_data_value))
# Load the warped ASTER NDVI into memory
data_set = gdal.Open(aster_ndvi_warped_name)
aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
aster_ndvi_no_data_locations = (np.where(
aster_ndvi_data == self.no_data_value))
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0
# Memory cleanup
del data_set
if not self.keep_intermediate_data:
# Cleanup the temp files since we have them in memory
if os.path.exists(ls_emis_warped_name):
os.unlink(ls_emis_warped_name)
if os.path.exists(aster_ndvi_warped_name):
os.unlink(aster_ndvi_warped_name)
self.logger.info('Normalizing Landsat and ASTER NDVI')
# Normalize Landsat NDVI by max value
max_ls_ndvi = ls_ndvi_data.max()
self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI NORM MAX raster')
util.Geo.generate_raster_file(
geotiff_driver, 'internal_landsat_ndvi_norm_max.tif',
ls_ndvi_data, x_dim, y_dim, geo_transform,
ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32)
# Normalize ASTER NDVI by max value
max_aster_ndvi = aster_ndvi_data.max()
self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Aster NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_aster_ndvi_norm_max.tif',
aster_ndvi_data, x_dim, y_dim,
geo_transform, ds_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
# Soil - From prototype code variable name
self.logger.info('Calculating EMIS Final')
with np.errstate(divide='ignore'):
ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
(1.0 - aster_ndvi_data))
# Memory cleanup
del aster_ndvi_data
del ls_emis_data
# Adjust estimated Landsat EMIS for vegetation and snow, to generate
# the final Landsat EMIS data
self.logger.info('Adjusting estimated EMIS for vegetation')
ls_emis_final = (self.vegetation_coeff * ls_ndvi_data + ls_emis_final *
(1.0 - ls_ndvi_data))
# Medium snow
self.logger.info('Adjusting estimated EMIS for snow')
ls_emis_final[selected_snow_locations] = self.snow_emis_value
# Memory cleanup
del ls_ndvi_data
del selected_snow_locations
# Add the fill and scan gaps and ASTER gaps back into the results,
# since they may have been lost
self.logger.info('Adding fill and data gaps back into the estimated'
' Landsat emissivity results')
ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
ls_emis_final[ls_emis_gap_locations] = self.no_data_value
ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
ls_emis_final[ndvi_no_data_locations] = self.no_data_value
ls_emis_final[ndsi_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_emis_no_data_locations
del ls_emis_gap_locations
del aster_ndvi_no_data_locations
del aster_ndvi_gap_locations
product_id = self.xml_filename.split('.xml')[0]
ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(ls_emis_img_filename))
util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
ls_emis_final, x_dim,
y_dim, ds_tmp_transform,
ds_tmp_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(ls_emis_aux_filename):
os.unlink(ls_emis_aux_filename)
self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
self.xml_filename))
# Add the estimated Landsat emissivity to the metadata XML
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
source_product = 'toa_refl'
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == source_product and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
emis_band = metadata_api.band(product='lst_temp',
source=source_product,
name='landsat_emis',
category='image',
data_type='FLOAT32',
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
emis_band.set_short_name('{0}EMIS'.format(sensor_code))
emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
' data')
emis_band.set_file_name(ls_emis_img_filename)
emis_band.set_data_units('Emissivity Coefficient')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
emis_band.set_pixel_size(pixel_size)
emis_band.set_resample_method('none')
valid_range = metadata_api.valid_range(min=0.0, max=1.0)
emis_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (datetime.datetime.strptime(
datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
emis_band.set_production_date(production_date)
emis_band.set_app_version(util.Version.app_version())
bands.add_band(emis_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as output_fd:
metadata_api.export(output_fd, espa_xml)
# Memory cleanup
del ls_emis_final
self.logger.info('Completed - Estimate Landsat Emissivity')
bands = metadata_api.bandsType()
# Create a band
band = metadata_api.band(product="RDD",
name="band1",
category="image",
data_type="UINT8",
nlines="7321",
nsamps="7951",
fill_value="0")
band.set_short_name("LT5DN")
band.set_long_name("band 1 digital numbers")
band.set_file_name("LT50460282002042EDC01_B1.img")
pixel_size = metadata_api.pixel_size("30.000000", 30, "meters")
band.set_pixel_size(pixel_size)
band.set_data_units("digital numbers")
valid_range = metadata_api.valid_range(min="1", max=255)
band.set_valid_range(valid_range)
toa_reflectance = metadata_api.toa_reflectance(gain=1.448, bias="-4.28819")
band.set_toa_reflectance(toa_reflectance)
band.set_app_version("LPGS_12.3.1")
production_date = \
datetime.datetime.strptime ('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S')
band.set_production_date(production_date)
def generate_data(self):
'''
Description:
Provides the main processing algorithm for building the Land
Surface Temperature product. It produces the final LST product.
'''
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
# Register all the gdal drivers and choose the ENVI for our output
gdal.AllRegister()
envi_driver = gdal.GetDriverByName('ENVI')
# Read the bands into memory
# Landsat Radiance at sensor for thermal band
self.logger.info('Loading intermediate thermal band data')
ds = gdal.Open(self.thermal_name)
x_dim = ds.RasterXSize # They are all the same size
y_dim = ds.RasterYSize
thermal_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric transmittance
self.logger.info('Loading intermediate transmittance band data')
ds = gdal.Open(self.transmittance_name)
trans_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric path radiance - upwelled radiance
self.logger.info('Loading intermediate upwelled band data')
ds = gdal.Open(self.upwelled_name)
upwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
self.logger.info('Calculating surface radiance')
# Surface radiance
surface_radiance = (thermal_data - upwelled_data) / trans_data
# Fix the no data locations
no_data_locations = np.where(thermal_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(trans_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(upwelled_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
# Memory cleanup
del (thermal_data)
del (trans_data)
del (upwelled_data)
del (no_data_locations)
# Downwelling sky irradiance
self.logger.info('Loading intermediate downwelled band data')
ds = gdal.Open(self.downwelled_name)
downwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Landsat emissivity estimated from ASTER GED data
self.logger.info('Loading intermediate emissivity band data')
ds = gdal.Open(self.emissivity_name)
emissivity_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Save for the output product
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(ds.GetProjection())
ds_transform = ds.GetGeoTransform()
# Memory cleanup
del (ds)
# Estimate Earth-emitted radiance by subtracting off the reflected
# downwelling component
radiance = (surface_radiance -
(1.0 - emissivity_data) * downwelled_data)
# Account for surface emissivity to get Plank emitted radiance
self.logger.info('Calculating Plank emitted radiance')
radiance_emitted = radiance / emissivity_data
# Fix the no data locations
no_data_locations = np.where(surface_radiance == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(downwelled_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(emissivity_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
# Memory cleanup
del (downwelled_data)
del (emissivity_data)
del (surface_radiance)
del (radiance)
del (no_data_locations)
# Use Brightness Temperature LUT to get skin temperature
# Read the correct one for what we are processing
if self.satellite == 'LANDSAT_7':
self.logger.info('Using Landsat 7 Brightness Temperature LUT')
bt_name = 'L7_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_5':
self.logger.info('Using Landsat 5 Brightness Temperature LUT')
bt_name = 'L5_Brightness_Temperature_LUT.txt'
bt_data = np.loadtxt(os.path.join(self.lst_data_dir, bt_name),
dtype=float, delimiter=' ')
bt_radiance_LUT = bt_data[:, 1]
bt_temp_LUT = bt_data[:, 0]
self.logger.info('Generating LST results')
lst_data = np.interp(radiance_emitted, bt_radiance_LUT, bt_temp_LUT)
# Scale the result and convert it to an int16
lst_data = lst_data * MULT_FACTOR
lst_fata = lst_data.astype(np.int16)
# Add the fill and scan gaps back into the results, since they may
# have been lost
self.logger.info('Adding fill and data gaps back into the Land'
' Surface Temperature results')
# Fix the no data locations
no_data_locations = np.where(radiance_emitted == self.no_data_value)
lst_data[no_data_locations] = self.no_data_value
# Memory cleanup
del (radiance_emitted)
del (no_data_locations)
product_id = self.xml_filename.split('.xml')[0]
lst_img_filename = ''.join([product_id, '_lst', '.img'])
lst_hdr_filename = ''.join([product_id, '_lst', '.hdr'])
lst_aux_filename = ''.join([lst_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(lst_img_filename))
util.Geo.generate_raster_file(envi_driver, lst_img_filename,
lst_data, x_dim, y_dim, ds_transform,
ds_srs.ExportToWkt(), self.no_data_value,
gdal.GDT_Int16)
self.logger.info('Updating {0}'.format(lst_hdr_filename))
util.Geo.update_envi_header(lst_hdr_filename, self.no_data_value)
# Memory cleanup
del (ds_srs)
del (ds_transform)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(lst_aux_filename):
os.unlink(lst_aux_filename)
self.logger.info('Adding {0} to {1}'.format(lst_img_filename,
self.xml_filename))
# Add the estimated Land Surface Temperature product to the metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
lst_band = metadata_api.band(product='lst',
source='toa_refl',
name='land_surface_temperature',
category='image',
data_type='INT16',
scale_factor=SCALE_FACTOR,
add_offset=0,
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
lst_band.set_short_name('{0}LST'.format(sensor_code))
lst_band.set_long_name('Land Surface Temperature')
lst_band.set_file_name(lst_img_filename)
lst_band.set_data_units('temperature (kelvin)')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
lst_band.set_pixel_size(pixel_size)
valid_range = metadata_api.valid_range(min=1500, max=3730)
lst_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (
datetime.datetime.strptime(datetime.datetime.now().
strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
lst_band.set_production_date(production_date)
lst_band.set_app_version(util.Version.app_version())
bands.add_band(lst_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as fd:
metadata_api.export(fd, espa_xml)
# Memory cleanup
del (lst_band)
del (bands)
del (espa_xml)
del (lst_data)
xml = metadata_api.parse('LT50460282002042EDC01.xml', silence=True)
bands = xml.get_bands()
# Remove the L1T bands by creating a new list of all the others
bands.band[:] = [band for band in bands.band if band.product != 'L1T']
band = metadata_api.band(product="RDD", name="band1", category="image",
data_type="UINT8", nlines="7321", nsamps="7951", fill_value="0")
band.set_short_name ("LT5DN")
band.set_long_name ("band 1 digital numbers")
band.set_file_name ("LT50460282002042EDC01_B1.img")
pixel_size = metadata_api.pixel_size ("30.000000", 30, "meters")
band.set_pixel_size (pixel_size)
band.set_data_units ("digital numbers")
valid_range = metadata_api.valid_range (min="1", max=255)
band.set_valid_range (valid_range)
toa_reflectance = metadata_api.toa_reflectance (gain=1.448, bias="-4.28819")
band.set_toa_reflectance (toa_reflectance)
band.set_app_version ("LPGS_12.3.1")
production_date = \
datetime.datetime.strptime('2014-01-13T06:49:56', '%Y-%m-%dT%H:%M:%S')
band.set_production_date (production_date)