def _read(self, filename):
""" reads the contents of an MTL file """
# if the "filename" input is actually already a metadata class object, return it back.
if inspect.isclass(filename):
return filename
fields = []
values = []
metafile = open(filename, 'r')
metadata = metafile.readlines()
for line in metadata:
# skips lines that contain "bad flags" denoting useless data AND lines
# greater than 1000 characters. 1000 character limit works around an odd LC5
# issue where the metadata has 40,000+ characters of whitespace
bad_flags = ["END", "GROUP"]
if not any(x in line for x in bad_flags) and len(line) <= 1000:
try:
line = line.replace(" ", "")
line = line.replace("\n", "")
field_name, field_value = line.split(' = ')
fields.append(field_name)
values.append(field_value)
except:
pass
for i in range(len(fields)):
# format fields without quotes,dates, or times in them as floats
if not any(['"' in values[i], 'DATE' in fields[i], 'TIME' in fields[i]]):
setattr(self, fields[i], float(values[i]))
else:
values[i] = values[i].replace('"', '')
setattr(self, fields[i], values[i])
# create datetime_obj attribute (drop decimal seconds)
dto_string = self.DATE_ACQUIRED + self.SCENE_CENTER_TIME
self.DATETIME_OBJ = datetime.strptime(dto_string.split(".")[0], "%Y-%m-%d%H:%M:%S")
# only landsat 8 includes sun-earth-distance in MTL file, so calculate it
# for the Landsats 4,5,7 using solar module.
if not self.SPACECRAFT_ID == "LANDSAT_8":
# use 0s for lat and lon, sun_earth_distance is not a function of any one location on earth.
s = solar(0, 0, self.DATETIME_OBJ, 0)
self.EARTH_SUN_DISTANCE = s.get_rad_vector()
print("Scene {0} center time is {1}".format(self.LANDSAT_SCENE_ID, self.DATETIME_OBJ))
def surface_reflectance(meta_path, toa_folder, dem_path, dew_point, outdir = False, kt = 1.0):
"""
This function will estimate surface reflectance for Landsat 4 TM, 5 TM, 7 ETM+, or 8 OLI data.
Note this function will calculate surface reflectance for Landsat 4, 5, and 7
bands 1, 2, 3, 4, 5, and 7 or Landsat 8 bands 2, 3, 4, 5, 6, and 7. All 6 bands should be in the
toa_folder and have "TOA_Ref" in their filenames as per the landsat.toa_reflectance convention.
The Landsat 8 Coastal Aerosol band (1) and Cirrus band (9) will not be calculated as they do not
have a corresponding band in TM or ETM+. To be performed on Top-of-Atmosphere Reflectance data
processed with the landsat.toa_reflectance_457 or the landsat.toa_reflectance_8 function.
Resources
- DEM
- NASA Shuttle Radar Topography Mission (SRTM) up to 30m resolution.
[http://www2.jpl.nasa.gov/srtm/cbanddataproducts.html]
- NASA Advanced Spaceborne Thermal Emission and Reflection Radiometer (ASTER)
Global Digital Elevation Model (GDEM) [http://asterweb.jpl.nasa.gov/gdem.asp]
- USGS National Elevation Dataset (NED) [http://nationalmap.gov/elevation.html]
downloadable in 30 meter resolution through The National Map Viewer
- Dew Point
- NOAA Daily Observational Data: Global Summary of the Day (GSOD) - GIS Data Locator
[http://gis.ncdc.noaa.gov/map/viewer/#app=clim&cfg=cdo&theme=daily&layers=0001&node=gis].
Select a weather station within the Landsat scene's extent, follow the link,
and enter the scene's acquisition date. This will output a text file, with
dew point under DEWP. More NOAA climatic data available at
[https://www.climate.gov/data/maps-and-data]
:param meta_path: The full filepath to the metadata file (ending in MTL.txt) for the dataset
:param toa_folder: The filepath to the folder containing the TOA_Ref tiffs to be processed
:param dem_path: The full filepath to a DEM tif that covers the desired Landsat scene
Note that the DEM should be resampled to the Landsat bands' resolution (30m)
and pixel alignment. Also ensure there are no gaps in the dataset.
:param dew_point: The number (e.g. 57.7) for the dew point at the time and place of scene acquisition
:param outdir: Output directory to save converted files. If left False it will save
output files in the toa_folder directory.
:param kt: Unitless turbidity coefficient. Default set at 1.0 for clean air.
Set at 0.5 for extremely turbid, dusty, or polluted air.
:return output_filelist: A list of all files created by this function
"""
meta_path = os.path.abspath(meta_path)
toa_folder = os.path.abspath(toa_folder)
dem_path = os.path.abspath(dem_path)
output_filelist = []
#define the list of constants for effective narrowband transmissivity for incoming solar radiation
constants_enbt1 = [[0.987, -0.00071, 0.000036, 0.0880, 0.0789],
[2.319, -0.00016, 0.000105, 0.0437, -1.2697],
[0.951, -0.00033, 0.000280, 0.0875, 0.1014],
[0.375, -0.00048, 0.005018, 0.1355, 0.6621],
[0.234, -0.00101, 0.004336, 0.0560, 0.7757],
[0.365, -0.00097, 0.004296, 0.0155, 0.6390]]
#define the list of constants for effective narrowband transmissivity for shortwave radiation
#reflected from the surface
constants_enbt2 = [[0.987, -0.00071, 0.000036, 0.0880, 0.0789],
[2.319, -0.00016, 0.000105, 0.0437, -1.2697],
[0.951, -0.00033, 0.000280, 0.0875, 0.1014],
[0.375, -0.00048, 0.005018, 0.1355, 0.6621],
[0.234, -0.00101, 0.004336, 0.0560, 0.7757],
[0.365, -0.00097, 0.004296, 0.0155, 0.6390]]
#enforce the list of band numbers, grab metadata from the MTL file, and define the band numbers needed from each sensor
meta = landsat_metadata(meta_path)
OLI_bands = ['2','3','4','5','6','7']
TM_ETM_bands = ['1','2','3','4','5','7']
#define the tile name for the landsat scene based on the metadata file's name
#Open the metadata text file and read to set the scene's tilename
f = open(meta_path)
MText = f.read()
if "PRODUCT_CREATION_TIME" in MText:
tilename = getattr(meta, "BAND1_FILE_NAME")
else:
tilename = getattr(meta, "LANDSAT_SCENE_ID")
#construct the list of TOA reflectance band tiffs and populate it based on the above definitions
toa_list = []
out_list = []
n = 0
for file in os.listdir(toa_folder):
if ("TOA_Ref" in file) and (file[-4:] == ".tif" or file[-4:] == ".TIF"):
if "LC8" in meta_path:
tile = "{0}_B{1}".format(tilename, OLI_bands[n])
if tile in file:
path = "{0}\\{1}".format(toa_folder, file)
out_file = file.replace("TOA", "Surf")
toa_list.append(path)
out_list.append(out_file)
n = n + 1
if n > 5:
break
elif ("LE7" in file) or ("LT5" in file) or ("LT4" in file):
tile = "{0}_B{1}".format(tilename, TM_ETM_bands[n])
if tile in file:
path = "{0}\\{1}".format(toa_folder, file)
out_file = file.replace("TOA", "Surf")
toa_list.append(path)
out_list.append(out_file)
n = n + 1
if n > 5:
break
#grab the corner lat/lon coordinates to calculate the approximate scene center lat/lon
ul_lat = getattr(meta, "CORNER_UL_LAT_PRODUCT")
ul_lon = getattr(meta, "CORNER_UL_LON_PRODUCT")
ur_lat = getattr(meta, "CORNER_UR_LAT_PRODUCT")
ur_lon = getattr(meta, "CORNER_UR_LON_PRODUCT")
ll_lat = getattr(meta, "CORNER_LL_LAT_PRODUCT")
ll_lon = getattr(meta, "CORNER_LL_LON_PRODUCT")
lr_lat = getattr(meta, "CORNER_LR_LAT_PRODUCT")
lr_lon = getattr(meta, "CORNER_LR_LON_PRODUCT")
u_lon_avg = np.mean([ul_lon, ur_lon])
l_lon_avg = np.mean([ll_lon, lr_lon])
l_lat_avg = np.mean([ul_lat, ll_lat])
r_lat_avg = np.mean([ur_lat, lr_lat])
center_lat = np.mean([l_lat_avg, r_lat_avg])
center_lon = np.mean([u_lon_avg, l_lon_avg])
#construct the datetime object from the date acquired and scene center time attributes
date = getattr(meta, "DATE_ACQUIRED")
dl = date.split("-")
time = getattr(meta, "SCENE_CENTER_TIME")
tl = time.split(":")
dt = datetime.datetime(int(dl[0]), int(dl[1]), int(dl[2]), int(tl[0]), int(tl[1]), int(tl[2][0:2]))
#use the dnppy.solar module to calculate the solar characteristics at the scene center at the time of acquisition
sc = solar.solar(center_lat, center_lon, dt, 0)
sc.compute_all()
#Cosine of Solar Zenith over horizontal surface
declination = math.degrees(sc.get_declination())
hour_angle = math.degrees(sc.get_hour_angle())
lat = math.degrees(center_lat)
cth = (math.sin(declination) * math.sin(lat)) + (math.cos(declination) * math.cos(center_lat) * math.cos(hour_angle))
#Saturation Vapor Pressure
svp = 0.6108 * math.exp((17.27 * dew_point) / (dew_point + 237.3))
#Atmospheric Pressure
DEM = arcpy.sa.Raster(dem_path)
ap = 101.3 * ((( 293 - (0.0065 * DEM))/ 293) ** 5.26)
#Water in Atmosphere
wia = (0.14 * svp * ap) + 2.1
#Effective Narrowband Transmittance for incoming solar radiation
entisr_bands = []
for i in xrange(6):
c1 = constants_enbt1[i][0]
c2 = constants_enbt1[i][1]
c3 = constants_enbt1[i][2]
c4 = constants_enbt1[i][3]
c5 = constants_enbt1[i][4]
enbt1 = c1 * ((arcpy.sa.Exp((c2 * ap)/(kt * cth))) - (((c3 * wia) + c4)/cth)) + c5
entisr_bands.append(enbt1)
#Effective Narrowband Transmittance for shortwave radiation reflected from surface
entsrrs_bands = []
#cos_n always 1 for sensor pointing straight nadir
cos_n = 1
for i in xrange(6):
c1 = constants_enbt2[i][0]
c2 = constants_enbt2[i][1]
c3 = constants_enbt2[i][2]
c4 = constants_enbt2[i][3]
c5 = constants_enbt2[i][4]
enbt2 = c1 * ((arcpy.sa.Exp((c2 * ap)/(kt * cos_n))) - (((c3 * wia) + c4))) + c5
entsrrs_bands.append(enbt2)
#Per Band Path Reflectance
pr_bands = []
pr_constants = [0.254, 0.149, 0.147, 0.311, 0.103, 0.036]
for j in xrange(6):
pr = pr_constants[j] * (1 - entisr_bands[j])
pr_bands.append(pr)
#Calculate and save At-Surface Reflectance band tiffs
for k in xrange(6):
if outdir:
outdir = os.path.abspath(outdir)
asr_path = "{0}\\{1}".format(outdir, out_list[k])
else:
asr_path = "{0}\\{1}".format(toa_folder, out_list[k])
refl_surf = (toa_list[k] - pr_bands[k])/(entisr_bands[k] * entsrrs_bands[k])
refl_surf.save(asr_path)
output_filelist.append(asr_path)
return output_filelist
def surface_reflectance(meta_path, toa_folder, dem_path, dew_point, outdir = False, kt = 1.0):
"""
This function will calculate surface reflectance for Landsat 4 TM, 5 TM, 7 ETM+, or 8 OLI data.
*Note this function will calculate surface reflectance for Landsat 4, 5, and 7
bands 1, 2, 3, 4, 5, and 7 or Landsat 8 bands 2, 3, 4, 5, 6, and 7. All 6 bands should be in the
toa_folder and have "TOA_Ref" in their filenames as per the landsat.toa_reflectance convention.
*The Landsat 8 Coastal Aerosol band (1) and Cirrus band (9) will not be calculated as they do not
have a corresponding band in TM or ETM+.
To be performed on Top-of-Atmosphere Reflectance data processed with the landsat.toa_reflectance_457
or the landsat.toa_reflectance_8 function.
Resources:
DEM:
-NASA Shuttle Radar Topography Mission (SRTM):
http://www2.jpl.nasa.gov/srtm/cbanddataproducts.html
*SRTM version 2 has a 30m resolution, but contains some gaps
-NASA Advanced Spaceborne Thermal Emission and Reflection Radiometer (ASTER)
Global Digital Elevation Model (GDEM):
http://asterweb.jpl.nasa.gov/gdem.asp
-USGS National Elevation Dataset (NED):
http://nationalmap.gov/elevation.html
*downloadable in 30 meter resolution through The National Map Viewer
Dew Point:
-NOAA Daily Observational Data: Global Summary of the Day (GSOD) - GIS Data Locator
http://gis.ncdc.noaa.gov/map/viewer/#app=clim&cfg=cdo&theme=daily&layers=0001&node=gis
*Select a weather station within the Landsat scene's extent, follow the link,
and enter the scene's acquisition date. This will output a text file, with
dew point under DEWP.
*More NOAA climatic data available at https://www.climate.gov/data/maps-and-data
Inputs:
meta_path The full filepath to the metadata file (ending in MTL.txt) for the dataset
toa_folder The filepath to the folder containing the TOA_Ref tiffs to be processed
dem_path The full filepath to a DEM tif that covers the desired Landsat scene
*Note that the DEM should be resampled to the Landsat bands' resolution (30m)
and pixel alignment. Also ensure there are no gaps in the dataset.
dew_point The number (e.g. 57.7) for the dew point at the time and place of scene acquisition
outdir Output directory to save converted files. If left False it will save
output files in the toa_folder directory.
kt Unitless turbidity coefficient. Default set at 1.0 for clean air.
*Set at 0.5 for extremeley turbid, dusty, or polluted air
"""
meta_path = os.path.abspath(meta_path)
toa_folder = os.path.abspath(toa_folder)
dem_path = os.path.abspath(dem_path)
outlist = []
#define the list of constants for effective narrowband transmissivity for incoming solar radiation
constants_enbt1 = [[0.987, -0.00071, 0.000036, 0.0880, 0.0789],
[2.319, -0.00016, 0.000105, 0.0437, -1.2697],
[0.951, -0.00033, 0.000280, 0.0875, 0.1014],
[0.375, -0.00048, 0.005018, 0.1355, 0.6621],
[0.234, -0.00101, 0.004336, 0.0560, 0.7757],
[0.365, -0.00097, 0.004296, 0.0155, 0.6390]]
#define the list of constants for effective narrowband transmissivity for shortwave radiation
#reflected from the surface
constants_enbt2 = [[0.987, -0.00071, 0.000036, 0.0880, 0.0789],
[2.319, -0.00016, 0.000105, 0.0437, -1.2697],
[0.951, -0.00033, 0.000280, 0.0875, 0.1014],
[0.375, -0.00048, 0.005018, 0.1355, 0.6621],
[0.234, -0.00101, 0.004336, 0.0560, 0.7757],
[0.365, -0.00097, 0.004296, 0.0155, 0.6390]]
#enforce the list of band numbers, grab metadata from the MTL file, and define the band numbers needed from each sensor
meta = grab_meta(meta_path)
OLI_bands = ['2','3','4','5','6','7']
TM_ETM_bands = ['1','2','3','4','5','7']
#define the tile name for the landsat scene based on the metadata file's name
#Open the metadata text file and read to set the scene's tilename
f = open(meta_path)
MText = f.read()
if "PRODUCT_CREATION_TIME" in MText:
tilename = getattr(meta, "BAND1_FILE_NAME")
else:
tilename = getattr(meta, "LANDSAT_SCENE_ID")
#construct the list of TOA reflectance band tiffs and populate it based on the above definitions
toa_list = []
out_list = []
n = 0
for file in os.listdir(toa_folder):
if ("TOA_Ref" in file) and (file[-4:] == ".tif" or file[-4:] == ".TIF"):
if "LC8" in meta_path:
tile = "{0}_B{1}".format(tilename, OLI_bands[n])
if tile in file:
path = "{0}\\{1}".format(toa_folder, file)
out_file = file.replace("TOA", "Surf")
toa_list.append(path)
out_list.append(out_file)
n = n + 1
if n > 5:
break
elif ("LE7" in file) or ("LT5" in file) or ("LT4" in file):
tile = "{0}_B{1}".format(tilename, TM_ETM_bands[n])
if tile in file:
path = "{0}\\{1}".format(toa_folder, file)
out_file = file.replace("TOA", "Surf")
toa_list.append(path)
out_list.append(out_file)
n = n + 1
if n > 5:
break
#grab the corner lat/lon coordinates to calculate the approximate scene center lat/lon
ul_lat = getattr(meta, "CORNER_UL_LAT_PRODUCT")
ul_lon = getattr(meta, "CORNER_UL_LON_PRODUCT")
ur_lat = getattr(meta, "CORNER_UR_LAT_PRODUCT")
ur_lon = getattr(meta, "CORNER_UR_LON_PRODUCT")
ll_lat = getattr(meta, "CORNER_LL_LAT_PRODUCT")
ll_lon = getattr(meta, "CORNER_LL_LON_PRODUCT")
lr_lat = getattr(meta, "CORNER_LR_LAT_PRODUCT")
lr_lon = getattr(meta, "CORNER_LR_LON_PRODUCT")
u_lon_avg = np.mean([ul_lon, ur_lon])
l_lon_avg = np.mean([ll_lon, lr_lon])
l_lat_avg = np.mean([ul_lat, ll_lat])
r_lat_avg = np.mean([ur_lat, lr_lat])
center_lat = np.mean([l_lat_avg, r_lat_avg])
center_lon = np.mean([u_lon_avg, l_lon_avg])
#construct the datetime object from the date acquired and scene center time attributes
date = getattr(meta, "DATE_ACQUIRED")
dl = date.split("-")
time = getattr(meta, "SCENE_CENTER_TIME")
tl = time.split(":")
dt = datetime.datetime(int(dl[0]), int(dl[1]), int(dl[2]), int(tl[0]), int(tl[1]), int(tl[2][0:2]))
#use the dnppy.solar module to calculate the solar characteristics at the scene center at the time of acquisition
sc = solar.solar(center_lat, center_lon, dt, 0)
sc.compute_all()
#Cosine of Solar Zenith over horizontal surface
declination = math.degrees(sc.get_declination())
hour_angle = math.degrees(sc.get_hour_angle())
lat = math.degrees(center_lat)
cth = (math.sin(declination) * math.sin(lat)) + (math.cos(declination) * math.cos(center_lat) * math.cos(hour_angle))
#Saturation Vapor Pressure
svp = 0.6108 * math.exp((17.27 * dew_point) / (dew_point + 237.3))
#Atmospheric Pressure
DEM = arcpy.sa.Raster(dem_path)
ap = 101.3 * ((( 293 - (0.0065 * DEM))/ 293) ** 5.26)
#Water in Atmosphere
wia = (0.14 * svp * ap) + 2.1
#Effective Narrowband Transmittance for incoming solar radiation
entisr_bands = []
for i in xrange(6):
c1 = constants_enbt1[i][0]
c2 = constants_enbt1[i][1]
c3 = constants_enbt1[i][2]
c4 = constants_enbt1[i][3]
c5 = constants_enbt1[i][4]
enbt1 = c1 * ((arcpy.sa.Exp((c2 * ap)/(kt * cth))) - (((c3 * wia) + c4)/cth)) + c5
entisr_bands.append(enbt1)
#Effective Narrowband Transmittance for shortwave radiation reflected from surface
entsrrs_bands = []
#cos_n always 1 for sensor pointing straight nadir
cos_n = 1
for i in xrange(6):
c1 = constants_enbt2[i][0]
c2 = constants_enbt2[i][1]
c3 = constants_enbt2[i][2]
c4 = constants_enbt2[i][3]
c5 = constants_enbt2[i][4]
enbt2 = c1 * ((arcpy.sa.Exp((c2 * ap)/(kt * cos_n))) - (((c3 * wia) + c4))) + c5
entsrrs_bands.append(enbt2)
#Per Band Path Reflectance
pr_bands = []
pr_constants = [0.254, 0.149, 0.147, 0.311, 0.103, 0.036]
for j in xrange(6):
pr = pr_constants[j] * (1 - entisr_bands[j])
pr_bands.append(pr)
#Calculate and save At-Surface Reflectance band tiffs
for k in xrange(6):
if outdir:
outdir = os.path.abspath(outdir)
asr_path = "{0}\\{1}".format(outdir, out_list[k])
else:
asr_path = "{0}\\{1}".format(toa_folder, out_list[k])
refl_surf = (toa_list[k] - pr_bands[k])/(entisr_bands[k] * entsrrs_bands[k])
refl_surf.save(asr_path)
outlist.append(asr_path)
return outlist