def remove_products(xml_filename, product_list):
'''
Description:
Removes the specified products from the file system, as well as
from the XML file.
'''
if not product_list:
# We don't error, just nothing to do.
return
espa_xml = metadata_api.parse(xml_filename, silence=True)
bands = espa_xml.get_bands()
# Gather all the filenames to be removed
filenames = list()
for band in bands.band:
if band.product in product_list:
# Add the .img file
filenames.append(band.file_name)
# Add the .hdr file
hdr_filename = band.file_name.replace('.img', '.hdr')
filenames.append(hdr_filename)
# If we found some then remove them
if len(filenames) > 0:
# First remove from disk
for filename in filenames:
if os.path.exists(filename):
os.unlink(filename)
# Second remove from metadata XML
# Remove them from the XML by creating a new list of all the
# others
bands.band[:] = [
band for band in bands.band if band.product not in product_list
]
try:
# Export to the file with validation
with open(xml_filename, 'w') as xml_fd:
metadata_api.export(xml_fd, espa_xml)
except Exception:
raise
del bands
del espa_xml
def remove_products(xml_filename, product_list):
'''
Description:
Removes the specified products from the file system, as well as
from the XML file.
'''
if not product_list:
# We don't error, just nothing to do.
return
espa_xml = metadata_api.parse(xml_filename, silence=True)
bands = espa_xml.get_bands()
# Gather all the filenames to be removed
filenames = list()
for band in bands.band:
if band.product in product_list:
# Add the .img file
filenames.append(band.file_name)
# Add the .hdr file
hdr_filename = band.file_name.replace('.img', '.hdr')
filenames.append(hdr_filename)
# If we found some then remove them
if len(filenames) > 0:
# First remove from disk
for filename in filenames:
if os.path.exists(filename):
os.unlink(filename)
# Second remove from metadata XML
# Remove them from the XML by creating a new list of all the
# others
bands.band[:] = [band for band in bands.band
if band.product not in product_list]
try:
# Export to the file with validation
with open(xml_filename, 'w') as xml_fd:
metadata_api.export(xml_fd, espa_xml)
except Exception:
raise
del bands
del espa_xml
def warp_espa_data(parms, scene, xml_filename=None):
'''
Description:
Warp each espa science product to the parameters specified in the parms
'''
logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)
# Validate the parameters
validate_parameters(parms, scene)
logger.debug(parms)
# ------------------------------------------------------------------------
# De-register the DOQ drivers since they may cause a problem with some of
# our generated imagery. And we are only processing envi format today
# inside the processing code.
doq1 = gdal.GetDriverByName('DOQ1')
doq2 = gdal.GetDriverByName('DOQ2')
doq1.Deregister()
doq2.Deregister()
# ------------------------------------------------------------------------
# Verify something was provided for the XML filename
if xml_filename is None or xml_filename == '':
raise ee.ESPAException(ee.ErrorCodes.warping, "Missing XML Filename")
# Change to the working directory
current_directory = os.getcwd()
os.chdir(parms['work_directory'])
try:
xml = metadata_api.parse(xml_filename, silence=True)
bands = xml.get_bands()
global_metadata = xml.get_global_metadata()
satellite = global_metadata.get_satellite()
# Might need this for the base warp command image extents
original_proj4 = get_original_projection(bands.band[0].get_file_name())
# Build the base warp command to use
base_warp_command = \
build_base_warp_command(parms, original_proj4=str(original_proj4))
# Determine the user specified resample method
user_resample_method = 'near' # default
if parms['resample_method'] is not None:
user_resample_method = parms['resample_method']
# Process through the bands in the XML file
for band in bands.band:
img_filename = band.get_file_name()
hdr_filename = img_filename.replace('.img', '.hdr')
logger.info("Processing %s" % img_filename)
# Reset the resample method to the user specified value
resample_method = user_resample_method
# Always use near for qa bands
category = band.get_category()
if category == 'qa':
resample_method = 'near' # over-ride with 'near'
# Update the XML metadata object for the resampling method used
# Later update_espa_xml is used to update the XML file
if resample_method == 'near':
band.set_resample_method('nearest neighbor')
if resample_method == 'bilinear':
band.set_resample_method('bilinear')
if resample_method == 'cubic':
band.set_resample_method('cubic convolution')
# Figure out the pixel size to use
pixel_size = parms['pixel_size']
# EXECUTIVE DECISION(Calli) - ESPA Issue 185
# - If the band is (Landsat 7 or 8) and Band 8 do not resize
# the pixels.
if ((satellite == 'LANDSAT_7' or satellite == 'LANDSAT_8') and
band.get_name() == 'band8'):
if parms['target_projection'] == 'lonlat':
pixel_size = settings.DEG_FOR_15_METERS
else:
pixel_size = float(band.pixel_size.x)
# Open the image to read the no data value out since the internal
# ENVI driver for GDAL does not output it, even if it is known
ds = gdal.Open(img_filename)
if ds is None:
raise RuntimeError("GDAL failed to open (%s)" % img_filename)
ds_band = None
try:
ds_band = ds.GetRasterBand(1)
except Exception as excep:
raise ee.ESPAException(ee.ErrorCodes.warping,
str(excep)), None, sys.exc_info()[2]
# Save the no data value since gdalwarp does not write it out when
# using the ENVI format
no_data_value = ds_band.GetNoDataValue()
if no_data_value is not None:
# TODO - We don't process any floating point data types. Yet
# Convert to an integer then string
no_data_value = str(int(no_data_value))
# Force a freeing of the memory
del ds_band
del ds
tmp_img_filename = 'tmp-%s' % img_filename
tmp_hdr_filename = 'tmp-%s' % hdr_filename
warp_image(img_filename, tmp_img_filename,
base_warp_command=base_warp_command,
resample_method=resample_method,
pixel_size=pixel_size,
no_data_value=no_data_value)
##################################################################
##################################################################
# Get new everything for the re-projected band
##################################################################
##################################################################
# Update the tmp ENVI header with our own values for some fields
sb = StringIO()
with open(tmp_hdr_filename, 'r') as tmp_fd:
while True:
line = tmp_fd.readline()
if not line:
break
if (line.startswith('data ignore value') or
line.startswith('description')):
pass
else:
sb.write(line)
if line.startswith('description'):
# This may be on multiple lines so read lines until
# we find the closing brace
if not line.strip().endswith('}'):
while 1:
next_line = tmp_fd.readline()
if (not next_line or
next_line.strip().endswith('}')):
break
sb.write('description = {ESPA-generated file}\n')
elif (line.startswith('data type') and
(no_data_value is not None)):
sb.write('data ignore value = %s\n' % no_data_value)
# END - with tmp_fd
# Do the actual replace here
with open(tmp_hdr_filename, 'w') as tmp_fd:
tmp_fd.write(sb.getvalue())
# Remove the original files, they are replaced in following code
if os.path.exists(img_filename):
os.unlink(img_filename)
if os.path.exists(hdr_filename):
os.unlink(hdr_filename)
# Rename the temps file back to the original name
os.rename(tmp_img_filename, img_filename)
os.rename(tmp_hdr_filename, hdr_filename)
# END for each band in the XML file
# Update the XML to reflect the new warped output
update_espa_xml(parms, xml, xml_filename)
del xml
except Exception as excep:
raise ee.ESPAException(ee.ErrorCodes.warping,
str(excep)), None, sys.exc_info()[2]
finally:
# Change back to the previous directory
os.chdir(current_directory)
def createXML(self, scene_xml_file=None, output_xml_file=None,
start_year=None, end_year=None, fill_value=None, imgfile=None,
log_handler=None):
"""Creates an XML file for the products produced by
runAnnualBurnSummaries.
Description: routine to create the XML file for the burned area summary
bands. The sample scene-based XML file will be used as the basis
for the projection information for the output XML file. The image
size, extents, etc. will need to be updated, as will the band
information.
History:
Created on May 12, 2014 by Gail Schmidt, USGS/EROS LSRD Project
Args:
scene_xml_file - scene-based XML file to be used as the base XML
information for the projection metadata.
output_xml_file - name of the XML file to be written
start_year - starting year of the scenes to process
end_year - ending year of the scenes to process
fill_value - fill or nodata value for this dataset
imgfile - name of burned area image file with associated ENVI header
which can be used to obtain the extents and geographic
information for these products
log_handler - handler for the logging information
Returns:
ERROR - error creating the XML file
SUCCESS - successful creation of the XML file
"""
# parse the scene-based XML file, just as a basis for the output XML
# file. the global attributes will be similar, but the extents and
# size of the image will be different. the bands will be based on the
# bands that are output from this routine.
xml = metadata_api.parse (scene_xml_file, silence=True)
meta_bands = xml.get_bands()
meta_global = xml.get_global_metadata()
# update the global information
meta_global.set_data_provider("USGS/EROS")
meta_global.set_satellite("LANDSAT")
meta_global.set_instrument("combination")
del (meta_global.acquisition_date)
meta_global.set_acquisition_date(None)
# open the image file to obtain the geospatial and spatial reference
# information
ds = gdal.Open (imgfile)
if ds is None:
msg = "GDAL failed to open %s" % imgfile
logIt (msg, log_handler)
return ERROR
ds_band = ds.GetRasterBand (1)
if ds_band is None:
msg = "GDAL failed to get the first band in %s" % imgfile
logIt (msg, log_handler)
return ERROR
nlines = float(ds_band.YSize)
nsamps = float(ds_band.XSize)
nlines_int = ds_band.YSize
nsamps_int = ds_band.XSize
del (ds_band)
ds_transform = ds.GetGeoTransform()
if ds_transform is None:
msg = "GDAL failed to get the geographic transform information " \
"from %s" % imgfile
logIt (msg, log_handler)
return ERROR
ds_srs = osr.SpatialReference()
if ds_srs is None:
msg = "GDAL failed to get the spatial reference information " \
"from %s" % imgfile
logIt (msg, log_handler)
return ERROR
ds_srs.ImportFromWkt (ds.GetProjection())
del (ds)
# get the UL and LR center of pixel map coordinates
(map_ul_x, map_ul_y) = convert_imageXY_to_mapXY (0.5, 0.5,
ds_transform)
(map_lr_x, map_lr_y) = convert_imageXY_to_mapXY (
nsamps - 0.5, nlines - 0.5, ds_transform)
# update the UL and LR projection corners along with the origin of the
# corners, for the center of the pixel (global projection information)
for mycorner in meta_global.projection_information.corner_point:
if mycorner.location == 'UL':
mycorner.set_x (map_ul_x)
mycorner.set_y (map_ul_y)
if mycorner.location == 'LR':
mycorner.set_x (map_lr_x)
mycorner.set_y (map_lr_y)
meta_global.projection_information.set_grid_origin("CENTER")
# update the UL and LR latitude and longitude coordinates, using the
# center of the pixel
srs_lat_lon = ds_srs.CloneGeogCS()
coord_tf = osr.CoordinateTransformation (ds_srs, srs_lat_lon)
for mycorner in meta_global.corner:
if mycorner.location == 'UL':
(lon, lat, height) = \
coord_tf.TransformPoint (map_ul_x, map_ul_y)
mycorner.set_longitude (lon)
mycorner.set_latitude (lat)
if mycorner.location == 'LR':
(lon, lat, height) = \
coord_tf.TransformPoint (map_lr_x, map_lr_y)
mycorner.set_longitude (lon)
mycorner.set_latitude (lat)
# determine the bounding coordinates; initialize using the UL and LR
# then work around the scene edges
# UL
(map_x, map_y) = convert_imageXY_to_mapXY (0.0, 0.0, ds_transform)
(ul_lon, ul_lat, height) = coord_tf.TransformPoint (map_x, map_y)
# LR
(map_x, map_y) = convert_imageXY_to_mapXY (nsamps, nlines, ds_transform)
(lr_lon, lr_lat, height) = coord_tf.TransformPoint (map_x, map_y)
# find the min and max values accordingly, for initialization
west_lon = min (ul_lon, lr_lon)
east_lon = max (ul_lon, lr_lon)
north_lat = max (ul_lat, lr_lat)
south_lat = min (ul_lat, lr_lat)
# traverse the boundaries of the image to determine the bounding
# coords; traverse one extra line and sample to get the the outer
# extents of the image vs. just the UL of the outer edge.
# top and bottom edges
for samp in range (0, nsamps_int+1):
# top edge
(map_x, map_y) = convert_imageXY_to_mapXY (samp, 0.0, ds_transform)
(top_lon, top_lat, height) = coord_tf.TransformPoint (map_x, map_y)
# lower edge
(map_x, map_y) = convert_imageXY_to_mapXY (samp, nlines,
ds_transform)
(low_lon, low_lat, height) = coord_tf.TransformPoint (map_x, map_y)
# update the min and max values
west_lon = min (top_lon, low_lon, west_lon)
east_lon = max (top_lon, low_lon, east_lon)
north_lat = max (top_lat, low_lat, north_lat)
south_lat = min (top_lat, low_lat, south_lat)
# left and right edges
for line in range (0, nlines_int+1):
# left edge
(map_x, map_y) = convert_imageXY_to_mapXY (0.0, line, ds_transform)
(left_lon, left_lat, height) = coord_tf.TransformPoint (map_x,
map_y)
# right edge
(map_x, map_y) = convert_imageXY_to_mapXY (nsamps, line,
ds_transform)
(right_lon, right_lat, height) = coord_tf.TransformPoint (map_x,
map_y)
# update the min and max values
west_lon = min (left_lon, right_lon, west_lon)
east_lon = max (left_lon, right_lon, east_lon)
north_lat = max (left_lat, right_lat, north_lat)
south_lat = min (left_lat, right_lat, south_lat)
# update the XML
bounding_coords = meta_global.get_bounding_coordinates()
bounding_coords.set_west (west_lon)
bounding_coords.set_east (east_lon)
bounding_coords.set_north (north_lat)
bounding_coords.set_south (south_lat)
del (ds_transform)
del (ds_srs)
# clear some of the global information that doesn't apply for these
# products
del (meta_global.scene_center_time)
meta_global.set_scene_center_time(None)
del (meta_global.lpgs_metadata_file)
meta_global.set_lpgs_metadata_file(None)
del (meta_global.orientation_angle)
meta_global.set_orientation_angle(None)
del (meta_global.level1_production_date)
meta_global.set_level1_production_date(None)
# clear the solar angles
del (meta_global.solar_angles)
meta_global.set_solar_angles(None)
# save the first band and then wipe the bands out so that new bands
# can be added for the burned area bands
myband_save = meta_bands.band[0]
del (meta_bands.band)
meta_bands.band = []
# create the band information; there are 4 output products per year
# for the burned area dataset; add enough bands to cover the products
# and years
# 1. first date a burned area was observed (burned_area)
# 2. number of times burn was observed (burn_count)
# 3. number of good looks (good_looks_count)
# 4. maximum probability for burned area (max_burn_prob)
nproducts = 4
nyears = end_year - start_year + 1
nbands = nproducts * nyears
for i in range (0, nbands):
# add the new band
myband = metadata_api.band()
meta_bands.band.append(myband)
# how many bands are there in the new XML file
num_scene_bands = len (meta_bands.band)
print "New XML file has %d bands" % num_scene_bands
# loop through the products and years to create the band metadata
band_count = 0
for product in range (1, nproducts+1):
for year in range (start_year, end_year+1):
myband = meta_bands.band[band_count]
myband.set_product("burned_area")
myband.set_short_name("LNDBA")
myband.set_data_type("INT16")
myband.set_pixel_size(myband_save.get_pixel_size())
myband.set_fill_value(fill_value)
myband.set_nlines(nlines)
myband.set_nsamps(nsamps)
myband.set_app_version(self.burned_area_version)
production_date = time.strftime("%Y-%m-%dT%H:%M:%S",
time.gmtime())
myband.set_production_date ( \
datetime_.datetime.strptime(production_date,
'%Y-%m-%dT%H:%M:%S'))
# clear some of the band-specific fields that don't apply for
# this product
del (myband.source)
myband.set_source(None)
del (myband.saturate_value)
myband.set_saturate_value(None)
del (myband.scale_factor)
myband.set_scale_factor(None)
del (myband.add_offset)
myband.set_add_offset(None)
del (myband.toa_reflectance)
myband.set_toa_reflectance(None)
del (myband.bitmap_description)
myband.set_bitmap_description(None)
del (myband.class_values)
myband.set_class_values(None)
del (myband.qa_description)
myband.set_qa_description(None)
del (myband.calibrated_nt)
myband.set_calibrated_nt(None)
# handle the band-specific differences
valid_range = metadata_api.valid_range()
if product == 1:
name = "burned_area_%d" % year
long_name = "first DOY a burn was observed"
file_name = "burned_area_%d.img" % year
category = "image"
data_units = "day of year"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no burn observed"
elif product == 2:
name = "burn_count_%d" % year
long_name = "number of times a burn was observed"
file_name = "burn_count_%d.img" % year
category = "image"
data_units = "count"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no burn observed"
elif product == 3:
name = "good_looks_count_%d" % year
long_name = "number of good looks (pixels with good QA)"
file_name = "good_looks_count_%d.img" % year
category = "qa"
data_units = "count"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no valid pixels (water, cloud, " \
"snow, etc.)"
elif product == 4:
name = "max_burn_prob_%d" % year
long_name = "maximum probability for burned area"
file_name = "max_burn_prob_%d.img" % year
category = "image"
data_units = "probability"
valid_range.min = 0
valid_range.max = 100
qa_description = "-9998: bad QA (water, cloud, snow, etc.)"
myband.set_name(name)
myband.set_long_name(long_name)
myband.set_file_name(file_name)
myband.set_category(category)
myband.set_data_units(data_units)
myband.set_valid_range(valid_range)
myband.set_qa_description(qa_description)
# increment the band counter
band_count += 1
# end for year
# end for nproducts
# write out a the XML file after validation
# call the export with validation
fd = open (output_xml_file, 'w')
if fd == None:
msg = "Unable to open the output XML file (%s) for writing." % \
output_xml_file
logIt (msg, log_handler)
return ERROR
metadata_api.export (fd, xml)
fd.flush()
fd.close()
return SUCCESS
def generate_product(self):
'''
Description:
Provides the main processing algorithm for generating the
estimated Landsat emissivity product. It produces the final
emissivity product.
'''
self.logger = logging.getLogger(__name__)
self.logger.info('Start - Estimate Landsat Emissivity')
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
try:
self.determine_sensor_specific_coefficients()
except Exception:
self.logger.exception('Failed determining sensor coefficients')
raise
# Register all the gdal drivers and choose the GeoTiff for our temp
# output
gdal.AllRegister()
geotiff_driver = gdal.GetDriverByName('GTiff')
envi_driver = gdal.GetDriverByName('ENVI')
# ====================================================================
# Build NDVI in memory
self.logger.info('Building TOA based NDVI band for Landsat data')
# NIR ----------------------------------------------------------------
data_set = gdal.Open(self.toa_nir_name)
x_dim = data_set.RasterXSize # They are all the same size
y_dim = data_set.RasterYSize
ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
ls_nir_data = ls_nir_data * self.toa_nir_scale_factor
# RED ----------------------------------------------------------------
data_set = gdal.Open(self.toa_red_name)
ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
red_no_data_locations = np.where(ls_red_data == self.no_data_value)
ls_red_data = ls_red_data * self.toa_red_scale_factor
# NDVI ---------------------------------------------------------------
ls_ndvi_data = ((ls_nir_data - ls_red_data) /
(ls_nir_data + ls_red_data))
# Cleanup no data locations
ls_ndvi_data[nir_no_data_locations] = self.no_data_value
ls_ndvi_data[red_no_data_locations] = self.no_data_value
if self.keep_intermediate_data:
geo_transform = data_set.GetGeoTransform()
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(data_set.GetProjection())
# Memory cleanup
del ls_red_data
del ls_nir_data
del nir_no_data_locations
del red_no_data_locations
# ====================================================================
# Build NDSI in memory
self.logger.info('Building TOA based NDSI band for Landsat data')
# GREEN --------------------------------------------------------------
data_set = gdal.Open(self.toa_green_name)
ls_green_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
green_no_data_locations = (
np.where(ls_green_data == self.no_data_value))
ls_green_data = ls_green_data * self.toa_green_scale_factor
# SWIR1 --------------------------------------------------------------
data_set = gdal.Open(self.toa_swir1_name)
ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
swir1_no_data_locations = (
np.where(ls_swir1_data == self.no_data_value))
ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor
# Build the Landsat TOA NDSI data
self.logger.info('Building TOA based NDSI for Landsat data')
ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
(ls_green_data + ls_swir1_data))
# Cleanup no data locations
ls_ndsi_data[green_no_data_locations] = self.no_data_value
# Cleanup no data locations
ls_ndsi_data[swir1_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_green_data
del ls_swir1_data
del green_no_data_locations
del swir1_no_data_locations
# Save for the output products
ds_tmp_srs = osr.SpatialReference()
ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
ds_tmp_transform = data_set.GetGeoTransform()
# Memory cleanup
del data_set
# Save the locations for the specfied snow pixels
self.logger.info('Determine snow pixel locations')
selected_snow_locations = np.where(ls_ndsi_data > 0.4)
# Save ndvi and ndsi no data locations
ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)
# Memory cleanup
del ls_ndsi_data
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi.tif',
ls_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Build the estimated Landsat EMIS data from the ASTER GED data and
# warp it to the Landsat scenes projection and image extents
# For convenience the ASTER NDVI is also extracted and warped to the
# Landsat scenes projection and image extents
self.logger.info('Build thermal emissivity band and'
' retrieve ASTER NDVI')
(ls_emis_warped_name,
aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)
# Load the warped estimated Landsat EMIS into memory
data_set = gdal.Open(ls_emis_warped_name)
ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
ls_emis_gap_locations = np.where(ls_emis_data == 0)
ls_emis_no_data_locations = (
np.where(ls_emis_data == self.no_data_value))
# Load the warped ASTER NDVI into memory
data_set = gdal.Open(aster_ndvi_warped_name)
aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(0, 0,
x_dim, y_dim)
aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
aster_ndvi_no_data_locations = (
np.where(aster_ndvi_data == self.no_data_value))
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0
# Memory cleanup
del data_set
if not self.keep_intermediate_data:
# Cleanup the temp files since we have them in memory
if os.path.exists(ls_emis_warped_name):
os.unlink(ls_emis_warped_name)
if os.path.exists(aster_ndvi_warped_name):
os.unlink(aster_ndvi_warped_name)
self.logger.info('Normalizing Landsat and ASTER NDVI')
# Normalize Landsat NDVI by max value
max_ls_ndvi = ls_ndvi_data.max()
self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi_norm_max.tif',
ls_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Normalize ASTER NDVI by max value
max_aster_ndvi = aster_ndvi_data.max()
self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Aster NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_aster_ndvi_norm_max.tif',
aster_ndvi_data,
x_dim, y_dim,
geo_transform,
ds_srs.ExportToWkt(),
self.no_data_value,
gdal.GDT_Float32)
# Soil - From prototype code variable name
ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
(1.0 - aster_ndvi_data))
# Memory cleanup
del aster_ndvi_data
del ls_emis_data
# Adjust estimated Landsat EMIS for vegetation and snow, to generate
# the final Landsat EMIS data
self.logger.info('Adjusting estimated EMIS for vegetation')
ls_emis_final = (self.vegetation_coeff * ls_ndvi_data +
ls_emis_final * (1.0 - ls_ndvi_data))
# Medium snow
self.logger.info('Adjusting estimated EMIS for snow')
ls_emis_final[selected_snow_locations] = self.snow_emis_value
# Memory cleanup
del ls_ndvi_data
del selected_snow_locations
# Add the fill and scan gaps and ASTER gaps back into the results,
# since they may have been lost
self.logger.info('Adding fill and data gaps back into the estimated'
' Landsat emissivity results')
ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
ls_emis_final[ls_emis_gap_locations] = self.no_data_value
ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
ls_emis_final[ndvi_no_data_locations] = self.no_data_value
ls_emis_final[ndsi_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_emis_no_data_locations
del ls_emis_gap_locations
del aster_ndvi_no_data_locations
del aster_ndvi_gap_locations
product_id = self.xml_filename.split('.xml')[0]
ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(ls_emis_img_filename))
util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
ls_emis_final, x_dim, y_dim,
ds_tmp_transform,
ds_tmp_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(ls_emis_aux_filename):
os.unlink(ls_emis_aux_filename)
self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
self.xml_filename))
# Add the estimated Landsat emissivity to the metadata XML
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
source_product = 'toa_refl'
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == source_product and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
emis_band = metadata_api.band(product='lst_temp',
source=source_product,
name='landsat_emis',
category='image',
data_type='FLOAT32',
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
emis_band.set_short_name('{0}EMIS'.format(sensor_code))
emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
' data')
emis_band.set_file_name(ls_emis_img_filename)
emis_band.set_data_units('Emissivity Coefficient')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
emis_band.set_pixel_size(pixel_size)
valid_range = metadata_api.valid_range(min=0.0, max=1.0)
emis_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (
datetime.datetime.strptime(datetime.datetime.now().
strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
emis_band.set_production_date(production_date)
emis_band.set_app_version(util.Version.app_version())
bands.add_band(emis_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as output_fd:
metadata_api.export(output_fd, espa_xml)
# Memory cleanup
del ls_emis_final
self.logger.info('Completed - Estimate Landsat Emissivity')
def retrieve_metadata_information(self):
'''
Description:
Loads and reads required information from the metadata XML file.
'''
# Read the XML metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
# Grab the global metadata object
gm = espa_xml.get_global_metadata()
# Grab the bands metadata object
bands = espa_xml.get_bands()
toa_bt_name = '' # Only one that is local
# Find the TOA bands to extract information from
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band2':
self.toa_green_name = band.get_file_name()
self.toa_green_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band3':
self.toa_red_name = band.get_file_name()
self.toa_red_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band4':
self.toa_nir_name = band.get_file_name()
self.toa_nir_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band5':
self.toa_swir1_name = band.get_file_name()
self.toa_swir1_scale_factor = float(band.scale_factor)
if band.product == 'toa_bt' and band.category == 'image':
# Get the output pixel size
self.ls_info.x_pixel_size = band.pixel_size.x
self.ls_info.y_pixel_size = band.pixel_size.y
toa_bt_name = band.get_file_name()
# Get the output proj4 string
self.ls_info.dest_proj4 = (
util.Geo.get_proj4_projection_string(toa_bt_name))
# Error if we didn't find the required TOA bands in the data
if len(self.toa_green_name) <= 0:
raise Exception('Failed to find the TOA GREEN band'
' in the input data')
if len(self.toa_red_name) <= 0:
raise Exception('Failed to find the TOA RED band'
' in the input data')
if len(self.toa_nir_name) <= 0:
raise Exception('Failed to find the TOA NIR band'
' in the input data')
if len(self.toa_swir1_name) <= 0:
raise Exception('Failed to find the TOA SWIR1 band'
' in the input data')
if len(toa_bt_name) <= 0:
raise Exception('Failed to find the TOA BT band'
' in the input data')
# Determine the bounding geographic coordinates for the ASTER tiles we
# will need
self.ls_info.north = math.ceil(gm.bounding_coordinates.north)
self.ls_info.south = math.floor(gm.bounding_coordinates.south)
self.ls_info.east = math.ceil(gm.bounding_coordinates.east)
self.ls_info.west = math.floor(gm.bounding_coordinates.west)
# Determine the UTM projection corner points
for cp in gm.projection_information.corner_point:
if cp.location == 'UL':
self.ls_info.min_x_extent = cp.x
self.ls_info.max_y_extent = cp.y
if cp.location == 'LR':
self.ls_info.max_x_extent = cp.x
self.ls_info.min_y_extent = cp.y
# Adjust the UTM coordinates for image extents becuse they are in
# center of pixel, and we need to supply the warping with actual
# extents
self.ls_info.min_x_extent = (self.ls_info.min_x_extent -
self.ls_info.x_pixel_size * 0.5)
self.ls_info.max_x_extent = (self.ls_info.max_x_extent +
self.ls_info.x_pixel_size * 0.5)
self.ls_info.min_y_extent = (self.ls_info.min_y_extent -
self.ls_info.y_pixel_size * 0.5)
self.ls_info.max_y_extent = (self.ls_info.max_y_extent +
self.ls_info.y_pixel_size * 0.5)
# Save for later
self.satellite = gm.satellite
del bands
del gm
del espa_xml
def createXML(self,
scene_xml_file=None,
output_xml_file=None,
start_year=None,
end_year=None,
fill_value=None,
imgfile=None,
log_handler=None):
"""Creates an XML file for the products produced by
runAnnualBurnSummaries.
Description: routine to create the XML file for the burned area summary
bands. The sample scene-based XML file will be used as the basis
for the projection information for the output XML file. The image
size, extents, etc. will need to be updated, as will the band
information.
History:
Created on May 12, 2014 by Gail Schmidt, USGS/EROS LSRD Project
Args:
scene_xml_file - scene-based XML file to be used as the base XML
information for the projection metadata.
output_xml_file - name of the XML file to be written
start_year - starting year of the scenes to process
end_year - ending year of the scenes to process
fill_value - fill or nodata value for this dataset
imgfile - name of burned area image file with associated ENVI header
which can be used to obtain the extents and geographic
information for these products
log_handler - handler for the logging information
Returns:
ERROR - error creating the XML file
SUCCESS - successful creation of the XML file
"""
# parse the scene-based XML file, just as a basis for the output XML
# file. the global attributes will be similar, but the extents and
# size of the image will be different. the bands will be based on the
# bands that are output from this routine.
xml = metadata_api.parse(scene_xml_file, silence=True)
meta_bands = xml.get_bands()
meta_global = xml.get_global_metadata()
# update the global information
meta_global.set_data_provider("USGS/EROS")
meta_global.set_satellite("LANDSAT")
meta_global.set_instrument("combination")
del (meta_global.acquisition_date)
meta_global.set_acquisition_date(None)
# open the image file to obtain the geospatial and spatial reference
# information
ds = gdal.Open(imgfile)
if ds is None:
msg = "GDAL failed to open %s" % imgfile
logIt(msg, log_handler)
return ERROR
ds_band = ds.GetRasterBand(1)
if ds_band is None:
msg = "GDAL failed to get the first band in %s" % imgfile
logIt(msg, log_handler)
return ERROR
nlines = float(ds_band.YSize)
nsamps = float(ds_band.XSize)
nlines_int = ds_band.YSize
nsamps_int = ds_band.XSize
del (ds_band)
ds_transform = ds.GetGeoTransform()
if ds_transform is None:
msg = "GDAL failed to get the geographic transform information " \
"from %s" % imgfile
logIt(msg, log_handler)
return ERROR
ds_srs = osr.SpatialReference()
if ds_srs is None:
msg = "GDAL failed to get the spatial reference information " \
"from %s" % imgfile
logIt(msg, log_handler)
return ERROR
ds_srs.ImportFromWkt(ds.GetProjection())
del (ds)
# get the UL and LR center of pixel map coordinates
(map_ul_x, map_ul_y) = convert_imageXY_to_mapXY(0.5, 0.5, ds_transform)
(map_lr_x,
map_lr_y) = convert_imageXY_to_mapXY(nsamps - 0.5, nlines - 0.5,
ds_transform)
# update the UL and LR projection corners along with the origin of the
# corners, for the center of the pixel (global projection information)
for mycorner in meta_global.projection_information.corner_point:
if mycorner.location == 'UL':
mycorner.set_x(map_ul_x)
mycorner.set_y(map_ul_y)
if mycorner.location == 'LR':
mycorner.set_x(map_lr_x)
mycorner.set_y(map_lr_y)
meta_global.projection_information.set_grid_origin("CENTER")
# update the UL and LR latitude and longitude coordinates, using the
# center of the pixel
srs_lat_lon = ds_srs.CloneGeogCS()
coord_tf = osr.CoordinateTransformation(ds_srs, srs_lat_lon)
for mycorner in meta_global.corner:
if mycorner.location == 'UL':
(lon, lat, height) = \
coord_tf.TransformPoint (map_ul_x, map_ul_y)
mycorner.set_longitude(lon)
mycorner.set_latitude(lat)
if mycorner.location == 'LR':
(lon, lat, height) = \
coord_tf.TransformPoint (map_lr_x, map_lr_y)
mycorner.set_longitude(lon)
mycorner.set_latitude(lat)
# determine the bounding coordinates; initialize using the UL and LR
# then work around the scene edges
# UL
(map_x, map_y) = convert_imageXY_to_mapXY(0.0, 0.0, ds_transform)
(ul_lon, ul_lat, height) = coord_tf.TransformPoint(map_x, map_y)
# LR
(map_x, map_y) = convert_imageXY_to_mapXY(nsamps, nlines, ds_transform)
(lr_lon, lr_lat, height) = coord_tf.TransformPoint(map_x, map_y)
# find the min and max values accordingly, for initialization
west_lon = min(ul_lon, lr_lon)
east_lon = max(ul_lon, lr_lon)
north_lat = max(ul_lat, lr_lat)
south_lat = min(ul_lat, lr_lat)
# traverse the boundaries of the image to determine the bounding
# coords; traverse one extra line and sample to get the the outer
# extents of the image vs. just the UL of the outer edge.
# top and bottom edges
for samp in range(0, nsamps_int + 1):
# top edge
(map_x, map_y) = convert_imageXY_to_mapXY(samp, 0.0, ds_transform)
(top_lon, top_lat, height) = coord_tf.TransformPoint(map_x, map_y)
# lower edge
(map_x, map_y) = convert_imageXY_to_mapXY(samp, nlines,
ds_transform)
(low_lon, low_lat, height) = coord_tf.TransformPoint(map_x, map_y)
# update the min and max values
west_lon = min(top_lon, low_lon, west_lon)
east_lon = max(top_lon, low_lon, east_lon)
north_lat = max(top_lat, low_lat, north_lat)
south_lat = min(top_lat, low_lat, south_lat)
# left and right edges
for line in range(0, nlines_int + 1):
# left edge
(map_x, map_y) = convert_imageXY_to_mapXY(0.0, line, ds_transform)
(left_lon, left_lat,
height) = coord_tf.TransformPoint(map_x, map_y)
# right edge
(map_x, map_y) = convert_imageXY_to_mapXY(nsamps, line,
ds_transform)
(right_lon, right_lat,
height) = coord_tf.TransformPoint(map_x, map_y)
# update the min and max values
west_lon = min(left_lon, right_lon, west_lon)
east_lon = max(left_lon, right_lon, east_lon)
north_lat = max(left_lat, right_lat, north_lat)
south_lat = min(left_lat, right_lat, south_lat)
# update the XML
bounding_coords = meta_global.get_bounding_coordinates()
bounding_coords.set_west(west_lon)
bounding_coords.set_east(east_lon)
bounding_coords.set_north(north_lat)
bounding_coords.set_south(south_lat)
del (ds_transform)
del (ds_srs)
# clear some of the global information that doesn't apply for these
# products
del (meta_global.scene_center_time)
meta_global.set_scene_center_time(None)
del (meta_global.lpgs_metadata_file)
meta_global.set_lpgs_metadata_file(None)
del (meta_global.orientation_angle)
meta_global.set_orientation_angle(None)
del (meta_global.level1_production_date)
meta_global.set_level1_production_date(None)
# clear the solar angles
del (meta_global.solar_angles)
meta_global.set_solar_angles(None)
# save the first band and then wipe the bands out so that new bands
# can be added for the burned area bands
myband_save = meta_bands.band[0]
del (meta_bands.band)
meta_bands.band = []
# create the band information; there are 4 output products per year
# for the burned area dataset; add enough bands to cover the products
# and years
# 1. first date a burned area was observed (burned_area)
# 2. number of times burn was observed (burn_count)
# 3. number of good looks (good_looks_count)
# 4. maximum probability for burned area (max_burn_prob)
nproducts = 4
nyears = end_year - start_year + 1
nbands = nproducts * nyears
for i in range(0, nbands):
# add the new band
myband = metadata_api.band()
meta_bands.band.append(myband)
# how many bands are there in the new XML file
num_scene_bands = len(meta_bands.band)
print "New XML file has %d bands" % num_scene_bands
# loop through the products and years to create the band metadata
band_count = 0
for product in range(1, nproducts + 1):
for year in range(start_year, end_year + 1):
myband = meta_bands.band[band_count]
myband.set_product("burned_area")
myband.set_short_name("LNDBA")
myband.set_data_type("INT16")
myband.set_pixel_size(myband_save.get_pixel_size())
myband.set_fill_value(fill_value)
myband.set_nlines(nlines)
myband.set_nsamps(nsamps)
myband.set_app_version(self.burned_area_version)
production_date = time.strftime("%Y-%m-%dT%H:%M:%S",
time.gmtime())
myband.set_production_date ( \
datetime_.datetime.strptime(production_date,
'%Y-%m-%dT%H:%M:%S'))
# clear some of the band-specific fields that don't apply for
# this product (see metadata_api.py for the full list of
# band-related values under class band)
del (myband.source)
myband.set_source(None)
del (myband.saturate_value)
myband.set_saturate_value(None)
del (myband.scale_factor)
myband.set_scale_factor(None)
del (myband.add_offset)
myband.set_add_offset(None)
del (myband.radiance)
myband.set_radiance(None)
del (myband.reflectance)
myband.set_reflectance(None)
del (myband.thermal_const)
myband.set_thermal_const(None)
del (myband.bitmap_description)
myband.set_bitmap_description(None)
del (myband.class_values)
myband.set_class_values(None)
del (myband.qa_description)
myband.set_qa_description(None)
del (myband.calibrated_nt)
myband.set_calibrated_nt(None)
# handle the band-specific differences
valid_range = metadata_api.valid_range()
if product == 1:
name = "burned_area_%d" % year
long_name = "first DOY a burn was observed"
file_name = "burned_area_%d.img" % year
category = "image"
data_units = "day of year"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no burn observed"
elif product == 2:
name = "burn_count_%d" % year
long_name = "number of times a burn was observed"
file_name = "burn_count_%d.img" % year
category = "image"
data_units = "count"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no burn observed"
elif product == 3:
name = "good_looks_count_%d" % year
long_name = "number of good looks (pixels with good QA)"
file_name = "good_looks_count_%d.img" % year
category = "qa"
data_units = "count"
valid_range.min = 0
valid_range.max = 366
qa_description = "0: no valid pixels (water, cloud, " \
"snow, etc.)"
elif product == 4:
name = "max_burn_prob_%d" % year
long_name = "maximum probability for burned area"
file_name = "max_burn_prob_%d.img" % year
category = "image"
data_units = "probability"
valid_range.min = 0
valid_range.max = 100
qa_description = "-9998: bad QA (water, cloud, snow, etc.)"
myband.set_name(name)
myband.set_long_name(long_name)
myband.set_file_name(file_name)
myband.set_category(category)
myband.set_data_units(data_units)
myband.set_valid_range(valid_range)
myband.set_qa_description(qa_description)
# increment the band counter
band_count += 1
# end for year
# end for nproducts
# write out a the XML file after validation
# call the export with validation
fd = open(output_xml_file, 'w')
if fd == None:
msg = "Unable to open the output XML file (%s) for writing." % \
output_xml_file
logIt(msg, log_handler)
return ERROR
metadata_api.export(fd, xml)
fd.flush()
fd.close()
return SUCCESS
def generate_data(self):
'''
Description:
Provides the main processing algorithm for building the Surface
Temperature product. It produces the final ST product.
'''
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
# Register all the gdal drivers and choose the ENVI for our output
gdal.AllRegister()
envi_driver = gdal.GetDriverByName('ENVI')
# Read the bands into memory
# Landsat Radiance at sensor for thermal band
self.logger.info('Loading intermediate thermal band data [{0}]'.format(
self.thermal_name))
dataset = gdal.Open(self.thermal_name)
x_dim = dataset.RasterXSize # They are all the same size
y_dim = dataset.RasterYSize
thermal_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric transmittance
self.logger.info(
'Loading intermediate transmittance band data [{0}]'.format(
self.transmittance_name))
dataset = gdal.Open(self.transmittance_name)
trans_data = dataset.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric path radiance - upwelled radiance
self.logger.info(
'Loading intermediate upwelled band data [{0}]'.format(
self.upwelled_name))
dataset = gdal.Open(self.upwelled_name)
upwelled_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
self.logger.info('Calculating surface radiance')
# Surface radiance
with np.errstate(invalid='ignore'):
surface_radiance = (thermal_data - upwelled_data) / trans_data
# Fix the no data locations
no_data_locations = np.where(thermal_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(trans_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(upwelled_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
# Memory cleanup
del thermal_data
del trans_data
del upwelled_data
del no_data_locations
# Downwelling sky irradiance
self.logger.info(
'Loading intermediate downwelled band data [{0}]'.format(
self.downwelled_name))
dataset = gdal.Open(self.downwelled_name)
downwelled_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
# Landsat emissivity estimated from ASTER GED data
self.logger.info(
'Loading intermediate emissivity band data [{0}]'.format(
self.emissivity_name))
dataset = gdal.Open(self.emissivity_name)
emissivity_data = dataset.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
# Save for the output product
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(dataset.GetProjection())
ds_transform = dataset.GetGeoTransform()
# Memory cleanup
del dataset
# Estimate Earth-emitted radiance by subtracting off the reflected
# downwelling component
radiance = (surface_radiance -
(1.0 - emissivity_data) * downwelled_data)
# Account for surface emissivity to get Plank emitted radiance
self.logger.info('Calculating Plank emitted radiance')
with np.errstate(invalid='ignore'):
radiance_emitted = radiance / emissivity_data
# Fix the no data locations
no_data_locations = np.where(surface_radiance == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(downwelled_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(emissivity_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
# Memory cleanup
del downwelled_data
del emissivity_data
del surface_radiance
del radiance
del no_data_locations
# Use Brightness Temperature LUT to get skin temperature
# Read the correct one for what we are processing
if self.satellite == 'LANDSAT_8':
self.logger.info('Using Landsat 8 Brightness Temperature LUT')
bt_name = 'L8_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_7':
self.logger.info('Using Landsat 7 Brightness Temperature LUT')
bt_name = 'L7_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_5':
self.logger.info('Using Landsat 5 Brightness Temperature LUT')
bt_name = 'L5_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_4':
self.logger.info('Using Landsat 4 Brightness Temperature LUT')
bt_name = 'L4_Brightness_Temperature_LUT.txt'
bt_data = np.loadtxt(os.path.join(self.st_data_dir, bt_name),
dtype=float,
delimiter=' ')
bt_radiance_lut = bt_data[:, 1]
bt_temp_lut = bt_data[:, 0]
self.logger.info('Generating ST results')
st_data = np.interp(radiance_emitted, bt_radiance_lut, bt_temp_lut)
# Scale the result
st_data = st_data * MULT_FACTOR
# Add the fill and scan gaps back into the results, since they may
# have been lost
self.logger.info('Adding fill and data gaps back into the Surface'
' Temperature results')
# Fix the no data locations
no_data_locations = np.where(radiance_emitted == self.no_data_value)
st_data[no_data_locations] = self.no_data_value
# Memory cleanup
del radiance_emitted
del no_data_locations
product_id = self.xml_filename.split('.xml')[0]
st_img_filename = ''.join([product_id, '_st', '.img'])
st_hdr_filename = ''.join([product_id, '_st', '.hdr'])
st_aux_filename = ''.join([st_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(st_img_filename))
util.Geo.generate_raster_file(envi_driver, st_img_filename, st_data,
x_dim, y_dim, ds_transform,
ds_srs.ExportToWkt(), self.no_data_value,
gdal.GDT_Int16)
self.logger.info('Updating {0}'.format(st_hdr_filename))
util.Geo.update_envi_header(st_hdr_filename, self.no_data_value)
# Memory cleanup
del ds_srs
del ds_transform
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(st_aux_filename):
os.unlink(st_aux_filename)
self.logger.info('Adding {0} to {1}'.format(st_img_filename,
self.xml_filename))
# Add the estimated Surface Temperature product to the metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:4]
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA band 1'
' in the input data')
st_band = metadata_api.band(product='st',
source='toa_refl',
name='surface_temperature',
category='image',
data_type='INT16',
scale_factor=SCALE_FACTOR,
add_offset=0,
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
st_band.set_short_name('{0}ST'.format(sensor_code))
st_band.set_long_name('Surface Temperature')
st_band.set_file_name(st_img_filename)
st_band.set_data_units('temperature (kelvin)')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
st_band.set_pixel_size(pixel_size)
st_band.set_resample_method('none')
valid_range = metadata_api.valid_range(min=1500, max=3730)
st_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (datetime.datetime.strptime(
datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
st_band.set_production_date(production_date)
st_band.set_app_version(util.Version.app_version())
bands.add_band(st_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as metadata_fd:
metadata_api.export(metadata_fd, espa_xml)
# Memory cleanup
del st_band
del bands
del espa_xml
del st_data
#! /usr/bin/env python
import datetime
from lxml import etree
import metadata_api
bands_element_path = '{http://espa.cr.usgs.gov/v1.0}bands'
xml = metadata_api.parse('LT50460282002042EDC01.xml', silence=True)
bands = xml.get_bands()
# Remove the L1T bands by creating a new list of all the others
bands.band[:] = [band for band in bands.band if band.product != 'L1T']
band = metadata_api.band(product="RDD",
name="band1",
category="image",
data_type="UINT8",
nlines="7321",
nsamps="7951",
fill_value="0")
band.set_short_name("LT5DN")
band.set_long_name("band 1 digital numbers")
band.set_file_name("LT50460282002042EDC01_B1.img")
pixel_size = metadata_api.pixel_size("30.000000", 30, "meters")
band.set_pixel_size(pixel_size)
def retrieve_metadata_information(self):
'''
Description:
Loads and reads required information from the metadata XML file.
'''
# Read the XML metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
# Grab the global metadata object
gm = espa_xml.get_global_metadata()
# Grab the bands metadata object
bands = espa_xml.get_bands()
self.thermal_name = ''
self.transmittance_name = ''
self.upwelled_name = ''
self.downwelled_name = ''
self.emissivity_name = ''
# Find the TOA bands to extract information from
for band in bands.band:
if (band.product == 'lst_temp' and
band.name == 'lst_thermal_radiance'):
self.thermal_name = band.get_file_name()
if (band.product == 'lst_temp' and
band.name == 'lst_atmospheric_transmittance'):
self.transmittance_name = band.get_file_name()
if (band.product == 'lst_temp' and
band.name == 'lst_upwelled_radiance'):
self.upwelled_name = band.get_file_name()
if (band.product == 'lst_temp' and
band.name == 'lst_downwelled_radiance'):
self.downwelled_name = band.get_file_name()
if (band.product == 'lst_temp' and
band.name == 'landsat_emis'):
self.emissivity_name = band.get_file_name()
# Error if we didn't find the required TOA bands in the data
if len(self.thermal_name) <= 0:
raise Exception('Failed to find the lst_thermal_radiance band'
' in the input data')
if len(self.transmittance_name) <= 0:
raise Exception('Failed to find the lst_atmospheric_transmittance'
' in the input data')
if len(self.upwelled_name) <= 0:
raise Exception('Failed to find the lst_upwelled_radiance'
' in the input data')
if len(self.downwelled_name) <= 0:
raise Exception('Failed to find the lst_downwelled_radiance'
' in the input data')
if len(self.emissivity_name) <= 0:
raise Exception('Failed to find the landsat_emis'
' in the input data')
# Save for later
self.satellite = gm.satellite
del (bands)
del (gm)
del (espa_xml)
def extract_aux_data(self):
'''
Description:
Builds the strings required to locate the auxillary data in the
archive then extracts the parameters into parameter named
directories.
'''
xml = metadata_api.parse(self.xml_filename, silence=True)
global_metadata = xml.get_global_metadata()
acq_date = str(global_metadata.get_acquisition_date())
scene_center_time = str(global_metadata.get_scene_center_time())
# Extract the individual parts from the date
year = int(acq_date[:4])
month = int(acq_date[5:7])
day = int(acq_date[8:])
# Extract the hour parts from the time and convert to an int
hour = int(scene_center_time[:2])
self.logger.debug('Using Acq. Date = {0} {1} {2}'
.format(year, month, day))
self.logger.debug('Using Scene Center Hour = {0:0>2}'.format(hour))
del global_metadata
del xml
# Determine the 3hr increments to use from the auxillary data
# We want the one before and after the scene acquisition time
# and convert back to formatted strings
hour_1 = hour - (hour % 3)
t_delta = timedelta(hours=3) # allows easy advance to the next day
date_1 = datetime(year, month, day, hour_1)
date_2 = date_1 + t_delta
self.logger.debug('Date 1 = {0}'.format(str(date_1)))
self.logger.debug('Date 2 = {0}'.format(str(date_2)))
for parm in self.parms_to_extract:
# Build the source filenames for date 1
filename = self.aux_name_template.format(parm,
date_1.year,
date_1.month,
date_1.day,
date_1.hour * 100,
'hdr')
aux_path = (self.aux_path_template.format(date_1.year,
date_1.month,
date_1.day))
hdr_1_path = self.dir_template.format(aux_path, filename)
grb_1_path = hdr_1_path.replace('.hdr', '.grb')
self.logger.info('Using {0}'.format(hdr_1_path))
self.logger.info('Using {0}'.format(grb_1_path))
# Build the source filenames for date 2
filename = self.aux_name_template.format(parm,
date_2.year,
date_2.month,
date_2.day,
date_2.hour * 100,
'hdr')
aux_path = (self.aux_path_template.format(date_2.year,
date_2.month,
date_2.day))
hdr_2_path = self.dir_template.format(aux_path, filename)
grb_2_path = hdr_2_path.replace('.hdr', '.grb')
self.logger.info('Using {0}'.format(hdr_2_path))
self.logger.info('Using {0}'.format(grb_2_path))
# Verify that the files we need exist
if (not os.path.exists(hdr_1_path) or
not os.path.exists(hdr_2_path) or
not os.path.exists(grb_1_path) or
not os.path.exists(grb_2_path)):
raise Exception('Required LST AUX files are missing')
# Date 1
output_dir = '{0}_1'.format(parm)
self.extract_grib_data(hdr_1_path, grb_1_path, output_dir)
# Date 2
output_dir = '{0}_2'.format(parm)
self.extract_grib_data(hdr_2_path, grb_2_path, output_dir)
def generate_data(self):
'''
Description:
Provides the main processing algorithm for building the Land
Surface Temperature product. It produces the final LST product.
'''
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
# Register all the gdal drivers and choose the ENVI for our output
gdal.AllRegister()
envi_driver = gdal.GetDriverByName('ENVI')
# Read the bands into memory
# Landsat Radiance at sensor for thermal band
self.logger.info('Loading intermediate thermal band data')
ds = gdal.Open(self.thermal_name)
x_dim = ds.RasterXSize # They are all the same size
y_dim = ds.RasterYSize
thermal_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric transmittance
self.logger.info('Loading intermediate transmittance band data')
ds = gdal.Open(self.transmittance_name)
trans_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Atmospheric path radiance - upwelled radiance
self.logger.info('Loading intermediate upwelled band data')
ds = gdal.Open(self.upwelled_name)
upwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
self.logger.info('Calculating surface radiance')
# Surface radiance
surface_radiance = (thermal_data - upwelled_data) / trans_data
# Fix the no data locations
no_data_locations = np.where(thermal_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(trans_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
no_data_locations = np.where(upwelled_data == self.no_data_value)
surface_radiance[no_data_locations] = self.no_data_value
# Memory cleanup
del (thermal_data)
del (trans_data)
del (upwelled_data)
del (no_data_locations)
# Downwelling sky irradiance
self.logger.info('Loading intermediate downwelled band data')
ds = gdal.Open(self.downwelled_name)
downwelled_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Landsat emissivity estimated from ASTER GED data
self.logger.info('Loading intermediate emissivity band data')
ds = gdal.Open(self.emissivity_name)
emissivity_data = ds.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
# Save for the output product
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(ds.GetProjection())
ds_transform = ds.GetGeoTransform()
# Memory cleanup
del (ds)
# Estimate Earth-emitted radiance by subtracting off the reflected
# downwelling component
radiance = (surface_radiance -
(1.0 - emissivity_data) * downwelled_data)
# Account for surface emissivity to get Plank emitted radiance
self.logger.info('Calculating Plank emitted radiance')
radiance_emitted = radiance / emissivity_data
# Fix the no data locations
no_data_locations = np.where(surface_radiance == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(downwelled_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
no_data_locations = np.where(emissivity_data == self.no_data_value)
radiance_emitted[no_data_locations] = self.no_data_value
# Memory cleanup
del (downwelled_data)
del (emissivity_data)
del (surface_radiance)
del (radiance)
del (no_data_locations)
# Use Brightness Temperature LUT to get skin temperature
# Read the correct one for what we are processing
if self.satellite == 'LANDSAT_7':
self.logger.info('Using Landsat 7 Brightness Temperature LUT')
bt_name = 'L7_Brightness_Temperature_LUT.txt'
elif self.satellite == 'LANDSAT_5':
self.logger.info('Using Landsat 5 Brightness Temperature LUT')
bt_name = 'L5_Brightness_Temperature_LUT.txt'
bt_data = np.loadtxt(os.path.join(self.lst_data_dir, bt_name),
dtype=float, delimiter=' ')
bt_radiance_LUT = bt_data[:, 1]
bt_temp_LUT = bt_data[:, 0]
self.logger.info('Generating LST results')
lst_data = np.interp(radiance_emitted, bt_radiance_LUT, bt_temp_LUT)
# Scale the result and convert it to an int16
lst_data = lst_data * MULT_FACTOR
lst_fata = lst_data.astype(np.int16)
# Add the fill and scan gaps back into the results, since they may
# have been lost
self.logger.info('Adding fill and data gaps back into the Land'
' Surface Temperature results')
# Fix the no data locations
no_data_locations = np.where(radiance_emitted == self.no_data_value)
lst_data[no_data_locations] = self.no_data_value
# Memory cleanup
del (radiance_emitted)
del (no_data_locations)
product_id = self.xml_filename.split('.xml')[0]
lst_img_filename = ''.join([product_id, '_lst', '.img'])
lst_hdr_filename = ''.join([product_id, '_lst', '.hdr'])
lst_aux_filename = ''.join([lst_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(lst_img_filename))
util.Geo.generate_raster_file(envi_driver, lst_img_filename,
lst_data, x_dim, y_dim, ds_transform,
ds_srs.ExportToWkt(), self.no_data_value,
gdal.GDT_Int16)
self.logger.info('Updating {0}'.format(lst_hdr_filename))
util.Geo.update_envi_header(lst_hdr_filename, self.no_data_value)
# Memory cleanup
del (ds_srs)
del (ds_transform)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(lst_aux_filename):
os.unlink(lst_aux_filename)
self.logger.info('Adding {0} to {1}'.format(lst_img_filename,
self.xml_filename))
# Add the estimated Land Surface Temperature product to the metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
lst_band = metadata_api.band(product='lst',
source='toa_refl',
name='land_surface_temperature',
category='image',
data_type='INT16',
scale_factor=SCALE_FACTOR,
add_offset=0,
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
lst_band.set_short_name('{0}LST'.format(sensor_code))
lst_band.set_long_name('Land Surface Temperature')
lst_band.set_file_name(lst_img_filename)
lst_band.set_data_units('temperature (kelvin)')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
lst_band.set_pixel_size(pixel_size)
valid_range = metadata_api.valid_range(min=1500, max=3730)
lst_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (
datetime.datetime.strptime(datetime.datetime.now().
strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
lst_band.set_production_date(production_date)
lst_band.set_app_version(util.Version.app_version())
bands.add_band(lst_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as fd:
metadata_api.export(fd, espa_xml)
# Memory cleanup
del (lst_band)
del (bands)
del (espa_xml)
del (lst_data)
import metadata_api
if __name__ == '__main__':
# Create a command line argument parser
description = "Validate an ESPA XML using two methods"
parser = ArgumentParser (description=description)
parser.add_argument ('--xml-file',
action='store', dest='xml_file', required=True,
help="ESPA XML file to validate")
# Parse the command line arguments
args = parser.parse_args()
xml = metadata_api.parse (args.xml_file, silence=True)
# Export with validation
f = open ('val_01-' + args.xml_file, 'w')
# Create the file and specify the namespace/schema
metadata_api.export (f, xml)
f.close()
# LXML - Validation Example
try:
f = open ('../../../htdocs/schema/espa_internal_metadata_v1_0.xsd')
schema_root = etree.parse (f)
f.close()
schema = etree.XMLSchema (schema_root)
#! /usr/bin/env python
import datetime
from lxml import etree
import metadata_api
bands_element_path = '{http://espa.cr.usgs.gov/v1.0}bands'
xml = metadata_api.parse('LT50460282002042EDC01.xml', silence=True)
bands = xml.get_bands()
# Remove the L1T bands by creating a new list of all the others
bands.band[:] = [band for band in bands.band if band.product != 'L1T']
band = metadata_api.band(product="RDD", name="band1", category="image",
data_type="UINT8", nlines="7321", nsamps="7951", fill_value="0")
band.set_short_name ("LT5DN")
band.set_long_name ("band 1 digital numbers")
band.set_file_name ("LT50460282002042EDC01_B1.img")
pixel_size = metadata_api.pixel_size ("30.000000", 30, "meters")
band.set_pixel_size (pixel_size)
band.set_data_units ("digital numbers")
valid_range = metadata_api.valid_range (min="1", max=255)
band.set_valid_range (valid_range)
def extract_aux_data(args, base_aux_dir):
'''
Description:
Builds the strings required to locate the auxillary data in the
archive then extracts the parameters into paremeter named directories.
'''
logger = logging.getLogger(__name__)
xml = metadata_api.parse(args.xml_filename, silence=True)
global_metadata = xml.get_global_metadata()
acq_date = str(global_metadata.get_acquisition_date())
scene_center_time = str(global_metadata.get_scene_center_time())
# Extract the individual parts from the date
year = int(acq_date[:4])
month = int(acq_date[5:7])
day = int(acq_date[8:])
# Extract the hour parts from the time and convert to an int
hour = int(scene_center_time[:2])
logger.debug("Using Acq. Date = {0} {1} {2}".format(year, month, day))
logger.debug("Using Scene Center Hour = {0:0>2}".format(hour))
del (global_metadata)
del (xml)
# Determine the 3hr increments to use from the auxillary data
# We want the one before and after the scene acquisition time
# and convert back to formatted strings
hour_1 = hour - (hour % 3)
td = timedelta(hours=3) # allows us to easily advance to the next day
date_1 = datetime(year, month, day, hour_1)
date_2 = date_1 + td
logger.debug("Date 1 = {0}".format(str(date_1)))
logger.debug("Date 2 = {0}".format(str(date_2)))
parms_to_extract = ['HGT', 'SPFH', 'TMP']
AUX_PATH_TEMPLATE = '{0:0>4}/{1:0>2}/{2:0>2}'
AUX_NAME_TEMPLATE = 'narr-a_221_{0}_{1:0>2}00_000_{2}.{3}'
for parm in parms_to_extract:
# Build the source filenames for date 1
yyyymmdd = '{0:0>4}{1:0>2}{2:0>2}'.format(date_1.year,
date_1.month,
date_1.day)
logger.debug("Date 1 yyyymmdd = {0}".format(yyyymmdd))
hdr_1_name = AUX_NAME_TEMPLATE.format(yyyymmdd, date_1.hour,
parm, 'hdr')
grb_1_name = AUX_NAME_TEMPLATE.format(yyyymmdd, date_1.hour,
parm, 'grb')
logger.debug("hdr 1 = {0}".format(hdr_1_name))
logger.debug("grb 1 = {0}".format(grb_1_name))
tmp = AUX_PATH_TEMPLATE.format(date_1.year, date_1.month, date_1.day)
hdr_1_path = '{0}/{1}/{2}'.format(base_aux_dir, tmp, hdr_1_name)
grb_1_path = '{0}/{1}/{2}'.format(base_aux_dir, tmp, grb_1_name)
logger.info("Using {0}".format(hdr_1_path))
logger.info("Using {0}".format(grb_1_path))
# Build the source filenames for date 2
yyyymmdd = '{0:0>4}{1:0>2}{2:0>2}'.format(date_2.year,
date_2.month,
date_2.day)
logger.debug("Date 2 yyyymmdd = {0}".format(yyyymmdd))
hdr_2_name = AUX_NAME_TEMPLATE.format(yyyymmdd, date_2.hour,
parm, 'hdr')
grb_2_name = AUX_NAME_TEMPLATE.format(yyyymmdd, date_2.hour,
parm, 'grb')
logger.debug("hdr 2 = {0}".format(hdr_2_name))
logger.debug("grb 2 = {0}".format(grb_2_name))
tmp = AUX_PATH_TEMPLATE.format(date_2.year, date_2.month, date_2.day)
hdr_2_path = '{0}/{1}/{2}'.format(base_aux_dir, tmp, hdr_2_name)
grb_2_path = '{0}/{1}/{2}'.format(base_aux_dir, tmp, grb_2_name)
logger.info("Using {0}".format(hdr_2_path))
logger.info("Using {0}".format(grb_2_path))
# Verify that the files we need exist
if (not os.path.exists(hdr_1_path)
or not os.path.exists(hdr_2_path)
or not os.path.exists(grb_1_path)
or not os.path.exists(grb_2_path)):
raise Exception("Required LST AUX files are missing")
output_dir = '{0}_1'.format(parm)
extract_grib_data(hdr_1_path, grb_1_path, output_dir)
output_dir = '{0}_2'.format(parm)
extract_grib_data(hdr_2_path, grb_2_path, output_dir)
def retrieve_metadata_information(self):
'''
Description:
Loads and reads required information from the metadata XML file.
'''
# Read the XML metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
# Grab the global metadata object
gm = espa_xml.get_global_metadata()
# Grab the bands metadata object
bands = espa_xml.get_bands()
toa_bt_name = '' # Only one that is local
# Find the TOA bands to extract information from
for band in bands.band:
if band.product == 'toa_refl' and band.name == 'toa_band2':
self.toa_green_name = band.get_file_name()
self.toa_green_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band3':
self.toa_red_name = band.get_file_name()
self.toa_red_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band4':
self.toa_nir_name = band.get_file_name()
self.toa_nir_scale_factor = float(band.scale_factor)
if band.product == 'toa_refl' and band.name == 'toa_band5':
self.toa_swir1_name = band.get_file_name()
self.toa_swir1_scale_factor = float(band.scale_factor)
if band.product == 'toa_bt' and band.category == 'image':
# Get the output pixel size
self.ls_info.x_pixel_size = band.pixel_size.x
self.ls_info.y_pixel_size = band.pixel_size.y
toa_bt_name = band.get_file_name()
# Get the output proj4 string
self.ls_info.dest_proj4 = (
util.Geo.get_proj4_projection_string(toa_bt_name))
# Error if we didn't find the required TOA bands in the data
if len(self.toa_green_name) <= 0:
raise Exception('Failed to find the TOA GREEN band'
' in the input data')
if len(self.toa_red_name) <= 0:
raise Exception('Failed to find the TOA RED band'
' in the input data')
if len(self.toa_nir_name) <= 0:
raise Exception('Failed to find the TOA NIR band'
' in the input data')
if len(self.toa_swir1_name) <= 0:
raise Exception('Failed to find the TOA SWIR1 band'
' in the input data')
if len(toa_bt_name) <= 0:
raise Exception('Failed to find the TOA BT band'
' in the input data')
# Determine the bounding geographic coordinates for the ASTER tiles we
# will need
self.ls_info.north = math.ceil(gm.bounding_coordinates.north)
self.ls_info.south = math.floor(gm.bounding_coordinates.south)
self.ls_info.east = math.ceil(gm.bounding_coordinates.east)
self.ls_info.west = math.floor(gm.bounding_coordinates.west)
# Determine the UTM projection corner points
for cp in gm.projection_information.corner_point:
if cp.location == 'UL':
self.ls_info.min_x_extent = cp.x
self.ls_info.max_y_extent = cp.y
if cp.location == 'LR':
self.ls_info.max_x_extent = cp.x
self.ls_info.min_y_extent = cp.y
# Adjust the UTM coordinates for image extents becuse they are in
# center of pixel, and we need to supply the warping with actual
# extents
self.ls_info.min_x_extent = (self.ls_info.min_x_extent -
self.ls_info.x_pixel_size * 0.5)
self.ls_info.max_x_extent = (self.ls_info.max_x_extent +
self.ls_info.x_pixel_size * 0.5)
self.ls_info.min_y_extent = (self.ls_info.min_y_extent -
self.ls_info.y_pixel_size * 0.5)
self.ls_info.max_y_extent = (self.ls_info.max_y_extent +
self.ls_info.y_pixel_size * 0.5)
# Save for later
self.satellite = gm.satellite
del bands
del gm
del espa_xml
def extract_aux_data(self):
'''
Description:
Builds the strings required to locate the auxillary data in the
archive then extracts the parameters into parameter named
directories.
'''
xml = metadata_api.parse(self.xml_filename, silence=True)
global_metadata = xml.get_global_metadata()
acq_date = str(global_metadata.get_acquisition_date())
scene_center_time = str(global_metadata.get_scene_center_time())
# Extract the individual parts from the date
year = int(acq_date[:4])
month = int(acq_date[5:7])
day = int(acq_date[8:])
# Extract the hour parts from the time and convert to an int
hour = int(scene_center_time[:2])
self.logger.debug('Using Acq. Date = {0} {1} {2}'
.format(year, month, day))
self.logger.debug('Using Scene Center Hour = {0:0>2}'.format(hour))
del global_metadata
del xml
# Determine the 3hr increments to use from the auxillary data
# We want the one before and after the scene acquisition time
# and convert back to formatted strings
hour_1 = hour - (hour % 3)
t_delta = timedelta(hours=3) # allows easy advance to the next day
date_1 = datetime(year, month, day, hour_1)
date_2 = date_1 + t_delta
self.logger.debug('Date 1 = {0}'.format(str(date_1)))
self.logger.debug('Date 2 = {0}'.format(str(date_2)))
for parm in self.parms_to_extract:
# Build the source filenames for date 1
filename = self.aux_name_template.format(parm,
date_1.year,
date_1.month,
date_1.day,
date_1.hour * 100,
'hdr')
aux_path = (self.aux_path_template.format(date_1.year,
date_1.month,
date_1.day))
hdr_1_path = self.dir_template.format(aux_path, filename)
grb_1_path = hdr_1_path.replace('.hdr', '.grb')
self.logger.info('Using {0}'.format(hdr_1_path))
self.logger.info('Using {0}'.format(grb_1_path))
# Build the source filenames for date 2
filename = self.aux_name_template.format(parm,
date_2.year,
date_2.month,
date_2.day,
date_2.hour * 100,
'hdr')
aux_path = (self.aux_path_template.format(date_2.year,
date_2.month,
date_2.day))
hdr_2_path = self.dir_template.format(aux_path, filename)
grb_2_path = hdr_2_path.replace('.hdr', '.grb')
self.logger.info('Using {0}'.format(hdr_2_path))
self.logger.info('Using {0}'.format(grb_2_path))
# Verify that the files we need exist
if (not os.path.exists(hdr_1_path) or
not os.path.exists(hdr_2_path) or
not os.path.exists(grb_1_path) or
not os.path.exists(grb_2_path)):
raise Exception('Required LST AUX files are missing')
# Date 1
output_dir = '{0}_1'.format(parm)
self.extract_grib_data(hdr_1_path, grb_1_path, output_dir)
# Date 2
output_dir = '{0}_2'.format(parm)
self.extract_grib_data(hdr_2_path, grb_2_path, output_dir)
def generate_product(self):
'''
Description:
Provides the main processing algorithm for generating the
estimated Landsat emissivity product. It produces the final
emissivity product.
'''
self.logger = logging.getLogger(__name__)
self.logger.info('Start - Estimate Landsat Emissivity')
try:
self.retrieve_metadata_information()
except Exception:
self.logger.exception('Failed reading input XML metadata file')
raise
try:
self.determine_sensor_specific_coefficients()
except Exception:
self.logger.exception('Failed determining sensor coefficients')
raise
# Register all the gdal drivers and choose the GeoTiff for our temp
# output
gdal.AllRegister()
geotiff_driver = gdal.GetDriverByName('GTiff')
envi_driver = gdal.GetDriverByName('ENVI')
# ====================================================================
# Build NDVI in memory
self.logger.info('Building TOA based NDVI band for Landsat data')
# NIR ----------------------------------------------------------------
data_set = gdal.Open(self.toa_nir_name)
x_dim = data_set.RasterXSize # They are all the same size
y_dim = data_set.RasterYSize
ls_nir_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
nir_no_data_locations = np.where(ls_nir_data == self.no_data_value)
ls_nir_data = ls_nir_data * self.toa_nir_scale_factor
# RED ----------------------------------------------------------------
data_set = gdal.Open(self.toa_red_name)
ls_red_data = data_set.GetRasterBand(1).ReadAsArray(0, 0, x_dim, y_dim)
red_no_data_locations = np.where(ls_red_data == self.no_data_value)
ls_red_data = ls_red_data * self.toa_red_scale_factor
# NDVI ---------------------------------------------------------------
ls_ndvi_data = ((ls_nir_data - ls_red_data) /
(ls_nir_data + ls_red_data))
# Cleanup no data locations
ls_ndvi_data[nir_no_data_locations] = self.no_data_value
ls_ndvi_data[red_no_data_locations] = self.no_data_value
if self.keep_intermediate_data:
geo_transform = data_set.GetGeoTransform()
ds_srs = osr.SpatialReference()
ds_srs.ImportFromWkt(data_set.GetProjection())
# Memory cleanup
del ls_red_data
del ls_nir_data
del nir_no_data_locations
del red_no_data_locations
# ====================================================================
# Build NDSI in memory
self.logger.info('Building TOA based NDSI band for Landsat data')
# GREEN --------------------------------------------------------------
data_set = gdal.Open(self.toa_green_name)
ls_green_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
green_no_data_locations = (np.where(
ls_green_data == self.no_data_value))
ls_green_data = ls_green_data * self.toa_green_scale_factor
# SWIR1 --------------------------------------------------------------
data_set = gdal.Open(self.toa_swir1_name)
ls_swir1_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
swir1_no_data_locations = (np.where(
ls_swir1_data == self.no_data_value))
ls_swir1_data = ls_swir1_data * self.toa_swir1_scale_factor
# Build the Landsat TOA NDSI data
self.logger.info('Building TOA based NDSI for Landsat data')
ls_ndsi_data = ((ls_green_data - ls_swir1_data) /
(ls_green_data + ls_swir1_data))
# Cleanup no data locations
ls_ndsi_data[green_no_data_locations] = self.no_data_value
# Cleanup no data locations
ls_ndsi_data[swir1_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_green_data
del ls_swir1_data
del green_no_data_locations
del swir1_no_data_locations
# Save for the output products
ds_tmp_srs = osr.SpatialReference()
ds_tmp_srs.ImportFromWkt(data_set.GetProjection())
ds_tmp_transform = data_set.GetGeoTransform()
# Memory cleanup
del data_set
# Save the locations for the specfied snow pixels
self.logger.info('Determine snow pixel locations')
selected_snow_locations = np.where(ls_ndsi_data > 0.4)
# Save ndvi and ndsi no data locations
ndvi_no_data_locations = np.where(ls_ndvi_data == self.no_data_value)
ndsi_no_data_locations = np.where(ls_ndsi_data == self.no_data_value)
# Memory cleanup
del ls_ndsi_data
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
ls_ndvi_data[ls_ndvi_data < 0.0000001] = 0
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_landsat_ndvi.tif',
ls_ndvi_data, x_dim, y_dim,
geo_transform, ds_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
# Build the estimated Landsat EMIS data from the ASTER GED data and
# warp it to the Landsat scenes projection and image extents
# For convenience the ASTER NDVI is also extracted and warped to the
# Landsat scenes projection and image extents
self.logger.info('Build thermal emissivity band and'
' retrieve ASTER NDVI')
(ls_emis_warped_name,
aster_ndvi_warped_name) = self.build_ls_emis_data(geotiff_driver)
# Load the warped estimated Landsat EMIS into memory
data_set = gdal.Open(ls_emis_warped_name)
ls_emis_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
ls_emis_gap_locations = np.where(ls_emis_data == 0)
ls_emis_no_data_locations = (np.where(
ls_emis_data == self.no_data_value))
# Load the warped ASTER NDVI into memory
data_set = gdal.Open(aster_ndvi_warped_name)
aster_ndvi_data = data_set.GetRasterBand(1).ReadAsArray(
0, 0, x_dim, y_dim)
aster_ndvi_gap_locations = np.where(aster_ndvi_data == 0)
aster_ndvi_no_data_locations = (np.where(
aster_ndvi_data == self.no_data_value))
# Turn all negative values to zero
# Use a realy small value so that we don't have negative zero (-0.0)
aster_ndvi_data[aster_ndvi_data < 0.0000001] = 0
# Memory cleanup
del data_set
if not self.keep_intermediate_data:
# Cleanup the temp files since we have them in memory
if os.path.exists(ls_emis_warped_name):
os.unlink(ls_emis_warped_name)
if os.path.exists(aster_ndvi_warped_name):
os.unlink(aster_ndvi_warped_name)
self.logger.info('Normalizing Landsat and ASTER NDVI')
# Normalize Landsat NDVI by max value
max_ls_ndvi = ls_ndvi_data.max()
self.logger.info('Max LS NDVI {0}'.format(max_ls_ndvi))
ls_ndvi_data = ls_ndvi_data / float(max_ls_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Landsat NDVI NORM MAX raster')
util.Geo.generate_raster_file(
geotiff_driver, 'internal_landsat_ndvi_norm_max.tif',
ls_ndvi_data, x_dim, y_dim, geo_transform,
ds_srs.ExportToWkt(), self.no_data_value, gdal.GDT_Float32)
# Normalize ASTER NDVI by max value
max_aster_ndvi = aster_ndvi_data.max()
self.logger.info('Max ASTER NDVI {0}'.format(max_aster_ndvi))
aster_ndvi_data = aster_ndvi_data / float(max_aster_ndvi)
if self.keep_intermediate_data:
self.logger.info('Writing Aster NDVI NORM MAX raster')
util.Geo.generate_raster_file(geotiff_driver,
'internal_aster_ndvi_norm_max.tif',
aster_ndvi_data, x_dim, y_dim,
geo_transform, ds_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
# Soil - From prototype code variable name
self.logger.info('Calculating EMIS Final')
with np.errstate(divide='ignore'):
ls_emis_final = ((ls_emis_data - 0.975 * aster_ndvi_data) /
(1.0 - aster_ndvi_data))
# Memory cleanup
del aster_ndvi_data
del ls_emis_data
# Adjust estimated Landsat EMIS for vegetation and snow, to generate
# the final Landsat EMIS data
self.logger.info('Adjusting estimated EMIS for vegetation')
ls_emis_final = (self.vegetation_coeff * ls_ndvi_data + ls_emis_final *
(1.0 - ls_ndvi_data))
# Medium snow
self.logger.info('Adjusting estimated EMIS for snow')
ls_emis_final[selected_snow_locations] = self.snow_emis_value
# Memory cleanup
del ls_ndvi_data
del selected_snow_locations
# Add the fill and scan gaps and ASTER gaps back into the results,
# since they may have been lost
self.logger.info('Adding fill and data gaps back into the estimated'
' Landsat emissivity results')
ls_emis_final[ls_emis_no_data_locations] = self.no_data_value
ls_emis_final[ls_emis_gap_locations] = self.no_data_value
ls_emis_final[aster_ndvi_no_data_locations] = self.no_data_value
ls_emis_final[aster_ndvi_gap_locations] = self.no_data_value
ls_emis_final[ndvi_no_data_locations] = self.no_data_value
ls_emis_final[ndsi_no_data_locations] = self.no_data_value
# Memory cleanup
del ls_emis_no_data_locations
del ls_emis_gap_locations
del aster_ndvi_no_data_locations
del aster_ndvi_gap_locations
product_id = self.xml_filename.split('.xml')[0]
ls_emis_img_filename = ''.join([product_id, '_emis', '.img'])
ls_emis_hdr_filename = ''.join([product_id, '_emis', '.hdr'])
ls_emis_aux_filename = ''.join([ls_emis_img_filename, '.aux', '.xml'])
self.logger.info('Creating {0}'.format(ls_emis_img_filename))
util.Geo.generate_raster_file(envi_driver, ls_emis_img_filename,
ls_emis_final, x_dim,
y_dim, ds_tmp_transform,
ds_tmp_srs.ExportToWkt(),
self.no_data_value, gdal.GDT_Float32)
self.logger.info('Updating {0}'.format(ls_emis_hdr_filename))
util.Geo.update_envi_header(ls_emis_hdr_filename, self.no_data_value)
# Remove the *.aux.xml file generated by GDAL
if os.path.exists(ls_emis_aux_filename):
os.unlink(ls_emis_aux_filename)
self.logger.info('Adding {0} to {1}'.format(ls_emis_img_filename,
self.xml_filename))
# Add the estimated Landsat emissivity to the metadata XML
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
bands = espa_xml.get_bands()
sensor_code = product_id[0:3]
source_product = 'toa_refl'
# Find the TOA Band 1 to use for the specific band details
base_band = None
for band in bands.band:
if band.product == source_product and band.name == 'toa_band1':
base_band = band
if base_band is None:
raise Exception('Failed to find the TOA BLUE band'
' in the input data')
emis_band = metadata_api.band(product='lst_temp',
source=source_product,
name='landsat_emis',
category='image',
data_type='FLOAT32',
nlines=base_band.get_nlines(),
nsamps=base_band.get_nsamps(),
fill_value=str(self.no_data_value))
emis_band.set_short_name('{0}EMIS'.format(sensor_code))
emis_band.set_long_name('Landsat emissivity estimated from ASTER GED'
' data')
emis_band.set_file_name(ls_emis_img_filename)
emis_band.set_data_units('Emissivity Coefficient')
pixel_size = metadata_api.pixel_size(base_band.pixel_size.x,
base_band.pixel_size.x,
base_band.pixel_size.units)
emis_band.set_pixel_size(pixel_size)
emis_band.set_resample_method('none')
valid_range = metadata_api.valid_range(min=0.0, max=1.0)
emis_band.set_valid_range(valid_range)
# Set the date, but first clean the microseconds off of it
production_date = (datetime.datetime.strptime(
datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
'%Y-%m-%dT%H:%M:%S'))
emis_band.set_production_date(production_date)
emis_band.set_app_version(util.Version.app_version())
bands.add_band(emis_band)
# Write the XML metadata file out
with open(self.xml_filename, 'w') as output_fd:
metadata_api.export(output_fd, espa_xml)
# Memory cleanup
del ls_emis_final
self.logger.info('Completed - Estimate Landsat Emissivity')
def retrieve_metadata_information(self):
'''
Description:
Loads and reads required information from the metadata XML file.
'''
# Read the XML metadata
espa_xml = metadata_api.parse(self.xml_filename, silence=True)
# Grab the global metadata object
global_metadata = espa_xml.get_global_metadata()
# Grab the bands metadata object
bands = espa_xml.get_bands()
self.thermal_name = ''
self.transmittance_name = ''
self.upwelled_name = ''
self.downwelled_name = ''
self.emissivity_name = ''
# Find the TOA bands to extract information from
for band in bands.band:
if (band.product == 'st_intermediate'
and band.name == 'st_thermal_radiance'):
self.thermal_name = band.get_file_name()
if (band.product == 'st_intermediate'
and band.name == 'st_atmospheric_transmittance'):
self.transmittance_name = band.get_file_name()
if (band.product == 'st_intermediate'
and band.name == 'st_upwelled_radiance'):
self.upwelled_name = band.get_file_name()
if (band.product == 'st_intermediate'
and band.name == 'st_downwelled_radiance'):
self.downwelled_name = band.get_file_name()
if (band.product == 'st_intermediate' and band.name == 'emis'):
self.emissivity_name = band.get_file_name()
# Error if we didn't find the required TOA bands in the data
if len(self.thermal_name) <= 0:
raise Exception('Failed to find the st_thermal_radiance band'
' in the input data')
if len(self.transmittance_name) <= 0:
raise Exception('Failed to find the st_atmospheric_transmittance'
' in the input data')
if len(self.upwelled_name) <= 0:
raise Exception('Failed to find the st_upwelled_radiance'
' in the input data')
if len(self.downwelled_name) <= 0:
raise Exception('Failed to find the st_downwelled_radiance'
' in the input data')
if len(self.emissivity_name) <= 0:
raise Exception('Failed to find the emis' ' in the input data')
# Save for later
self.satellite = global_metadata.satellite
del bands
del global_metadata
del espa_xml
def read_info_from_metadata(xml_filename):
# Read the XML metadata
espa_xml = metadata_api.parse(xml_filename, silence=True)
# Grab the global metadata object
gm = espa_xml.get_global_metadata()
# Grab the bands metadata object
bands = espa_xml.get_bands()
ls_info = LandsatInfo()
toa_bt_name = ""
toa_green_name = ""
toa_red_name = ""
toa_nir_name = ""
toa_swir1_name = ""
toa_green_scale_factor = 1.0
toa_red_scale_factor = 1.0
toa_nir_scale_factor = 1.0
toa_swir1_scale_factor = 1.0
# Find the TOA bands to extract information from
for band in bands.band:
if band.product == "toa_refl" and band.name == "toa_band2":
toa_green_name = band.get_file_name()
toa_green_scale_factor = float(band.scale_factor)
if band.product == "toa_refl" and band.name == "toa_band3":
toa_red_name = band.get_file_name()
toa_red_scale_factor = float(band.scale_factor)
if band.product == "toa_refl" and band.name == "toa_band4":
toa_nir_name = band.get_file_name()
toa_nir_scale_factor = float(band.scale_factor)
if band.product == "toa_refl" and band.name == "toa_band5":
toa_swir1_name = band.get_file_name()
toa_swir1_scale_factor = float(band.scale_factor)
if band.product == "toa_bt" and band.category == "image":
# Get the output pixel size
ls_info.x_pixel_size = band.pixel_size.x
ls_info.y_pixel_size = band.pixel_size.y
toa_bt_name = band.get_file_name()
# Get the output proj4 string
ls_info.dest_proj4 = get_proj4_projection_string(toa_bt_name)
# Error if we didn't find the required TOA bands in the data
if len(toa_green_name) <= 0:
raise Exception("Failed to find the TOA GREEN band in the input data")
if len(toa_red_name) <= 0:
raise Exception("Failed to find the TOA RED band in the input data")
if len(toa_nir_name) <= 0:
raise Exception("Failed to find the TOA NIR band in the input data")
if len(toa_swir1_name) <= 0:
raise Exception("Failed to find the TOA SWIR1 band in the input data")
if len(toa_bt_name) <= 0:
raise Exception("Failed to find the TOA BT band in the input data")
# Determine the bounding geographic coordinates for the ASTER tiles we
# will need
ls_info.north = math.ceil(gm.bounding_coordinates.north)
ls_info.south = math.floor(gm.bounding_coordinates.south)
ls_info.east = math.ceil(gm.bounding_coordinates.east)
ls_info.west = math.floor(gm.bounding_coordinates.west)
# Determine the UTM projection corner points
for cp in gm.projection_information.corner_point:
if cp.location == "UL":
ls_info.min_x_extent = cp.x
ls_info.max_y_extent = cp.y
if cp.location == "LR":
ls_info.max_x_extent = cp.x
ls_info.min_y_extent = cp.y
# Adjust the UTM coordinates for image extents becuse they are in center
# of pixel, and we need to supply the warping with actual extents
ls_info.min_x_extent = ls_info.min_x_extent - ls_info.x_pixel_size * 0.5
ls_info.max_x_extent = ls_info.max_x_extent + ls_info.x_pixel_size * 0.5
ls_info.min_y_extent = ls_info.min_y_extent - ls_info.y_pixel_size * 0.5
ls_info.max_y_extent = ls_info.max_y_extent + ls_info.y_pixel_size * 0.5
# Save for later
satellite = gm.satellite
del (bands)
del (gm)
del (espa_xml)
return (
ls_info,
toa_bt_name,
toa_green_name,
toa_red_name,
toa_nir_name,
toa_swir1_name,
toa_green_scale_factor,
toa_red_scale_factor,
toa_nir_scale_factor,
toa_swir1_scale_factor,
satellite,
)
