def test_load_txt_mtl_2():
txtmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.txt')
jsonmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.json')
for k in jsonmtl['L1_METADATA_FILE'].keys():
assert k in txtmtl['L1_METADATA_FILE']
assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_load_txt_mtl_1():
txtmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.txt')
jsonmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.json')
for k in jsonmtl['L1_METADATA_FILE'].keys():
assert k in txtmtl['L1_METADATA_FILE']
assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_load_txt_mtl_1():
txtmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.txt')
jsonmtl = _load_mtl('tests/data/LC81060712016134LGN00_MTL.json')
for k in jsonmtl['L1_METADATA_FILE'].keys():
assert k in txtmtl['L1_METADATA_FILE']
assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_load_txt_mtl_2():
txtmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.txt')
jsonmtl = _load_mtl('tests/data/LC80100202015018LGN00_MTL.json')
for k in jsonmtl['L1_METADATA_FILE'].keys():
assert k in txtmtl['L1_METADATA_FILE']
assert jsonmtl['L1_METADATA_FILE'][k] == txtmtl['L1_METADATA_FILE'][k]
def test_data():
mtl1 = toa_utils._load_mtl('tests/data/LC81060712016134LGN00_MTL.json')
mtl2 = toa_utils._load_mtl('tests/data/LC80430302016140LGN00_MTL.json')
mtl3 = toa_utils._load_mtl('tests/data/LC82290902015304LGN00_MTL.json')
mtl4 = toa_utils._load_mtl('tests/data/LC80100202015018LGN00_MTL.json')
return mtl1, mtl2, mtl3, mtl4
def processBandSet(i, row, dfTemp):
mtl = toa_utils._load_mtl(image_path + dfTemp['image_id'][0] + '_MTL.txt')
metadata = mtl['L1_METADATA_FILE']
for b in range(1, 10, 1):
dfTemp['REFLECTANCE_MULT_BAND_{}'.format(b)] = metadata[
'RADIOMETRIC_RESCALING']['REFLECTANCE_MULT_BAND_{}'.format(b)]
dfTemp['REFLECTANCE_ADD_BAND_{}'.format(b)] = metadata[
'RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_{}'.format(b)]
for b in range(10, 12, 1):
dfTemp['K1_CONSTANT_BAND_{}'.format(b)] = metadata[
'TIRS_THERMAL_CONSTANTS']['K1_CONSTANT_BAND_{}'.format(b)]
dfTemp['K2_CONSTANT_BAND_{}'.format(b)] = metadata[
'TIRS_THERMAL_CONSTANTS']['K2_CONSTANT_BAND_{}'.format(b)]
dfTemp['RADIANCE_MULT_BAND_{}'.format(b)] = metadata[
'RADIOMETRIC_RESCALING']['RADIANCE_MULT_BAND_{}'.format(b)]
dfTemp['RADIANCE_ADD_BAND_{}'.format(b)] = metadata[
'RADIOMETRIC_RESCALING']['RADIANCE_ADD_BAND_{}'.format(b)]
dfTemp['SUN_ELEVATION'] = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
dfTemp['DATE_ACQUIRED'] = date_collected = metadata['PRODUCT_METADATA'][
'DATE_ACQUIRED']
dfTemp['SCENE_CENTER_TIME'] = metadata['PRODUCT_METADATA'][
'SCENE_CENTER_TIME']
for elem in pollutant:
dfTemp[elem] = row[elem]
for bandNo in bands:
dfTemp = processBand(bandNo, dfTemp)
return dfTemp
def calculate_landsat_reflectance(src_path, src_mtl, dst_path,
creation_options, band, dst_dtype,
processes):
"""
Parameters
------------
src_path: string
src_mtl: string
Returns
---------
out: None
Output is written to dst_path
"""
mtl = toa_utils._load_mtl(src_mtl)
M = toa_utils._load_mtl_key(mtl, [
'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'
], band)
A = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
band)
E = toa_utils._load_mtl_key(
mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION'])
dst_dtype = np.__dict__[dst_dtype]
with rio.open(src_path) as src:
dst_profile = src.profile.copy()
src_nodata = src.nodata
for co in creation_options:
dst_profile[co] = creation_options[co]
dst_profile['dtype'] = dst_dtype
global_args = {
'A': A,
'M': M,
'E': E,
'src_nodata': 0,
'dst_dtype': dst_dtype
}
with riomucho.RioMucho([src_path],
dst_path,
_reflectance_worker,
options=dst_profile,
global_args=global_args) as rm:
rm.run(processes)
def calculate_landsat_reflectance(src_path, src_mtl, dst_path, creation_options, band, dst_dtype, processes):
"""
Parameters
------------
src_path: string
src_mtl: string
Returns
---------
out: None
Output is written to dst_path
"""
mtl = toa_utils._load_mtl(src_mtl)
M = toa_utils._load_mtl_key(mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'],
band)
A = toa_utils._load_mtl_key(mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
band)
E = toa_utils._load_mtl_key(mtl,
['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES','SUN_ELEVATION'])
dst_dtype = np.__dict__[dst_dtype]
with rio.open(src_path) as src:
dst_profile = src.profile.copy()
src_nodata = src.nodata
for co in creation_options:
dst_profile[co] = creation_options[co]
dst_profile['dtype'] = dst_dtype
global_args = {
'A': A,
'M': M,
'E': E,
'src_nodata': 0,
'dst_dtype': dst_dtype
}
with riomucho.RioMucho([src_path], dst_path, _reflectance_worker,
options=dst_profile,
global_args=global_args) as rm:
rm.run(processes)
def dn2toa(self, platform, mtl_file=None, wavelengths=None):
"""This method converts digital numbers to top of atmosphere reflectance, like described here:
https://www.usgs.gov/land-resources/nli/landsat/using-usgs-landsat-level-1-data-product
:param platform: image platform, possible Platform.Landsat[5, 7, 8] or Platform.Sentinel2 (<enum 'Platform'>).
:param mtl_file: path to Landsat MTL file that holds the band specific rescale factors (str).
:param wavelengths: like ["Blue", "Green", "Red", "NIR", "SWIR1", "TIRS", "SWIR2"] for Landsat-5 (list of str).
"""
if platform in [
Platform.Landsat5,
Platform.Landsat7,
Platform.Landsat8,
]:
if mtl_file is None:
raise AttributeError(
f"'mtl_file' has to be set if platform is {platform}.")
else:
# get rescale factors from mtl file
mtl = toa_utils._load_mtl(
str(mtl_file)) # no obvious reason not to call this
metadata = mtl["L1_METADATA_FILE"]
sun_elevation = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
toa = []
for idx, b in enumerate(
self._lookup_bands(platform, wavelengths)):
if (platform == Platform.Landsat8 and b
in ["10", "11"]) or (platform != Platform.Landsat8
and b.startswith("6")):
if platform == Platform.Landsat8:
thermal_conversion_constant1 = metadata[
"TIRS_THERMAL_CONSTANTS"][
f"K1_CONSTANT_BAND_{b}"]
thermal_conversion_constant2 = metadata[
"TIRS_THERMAL_CONSTANTS"][
f"K2_CONSTANT_BAND_{b}"]
else:
thermal_conversion_constant1 = metadata[
"THERMAL_CONSTANTS"][f"K1_CONSTANT_BAND_{b}"]
thermal_conversion_constant2 = metadata[
"THERMAL_CONSTANTS"][f"K2_CONSTANT_BAND_{b}"]
multiplicative_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"RADIANCE_MULT_BAND_{b}"]
additive_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"RADIANCE_ADD_BAND_{b}"]
# rescale thermal bands
toa.append(
brightness_temp.brightness_temp(
self.__arr[idx, :, :],
ML=multiplicative_rescaling_factors,
AL=additive_rescaling_factors,
K1=thermal_conversion_constant1,
K2=thermal_conversion_constant2,
))
continue
# rescale reflectance bands
multiplicative_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"REFLECTANCE_MULT_BAND_{b}"]
additive_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{b}"]
toa.append(
reflectance.reflectance(
self.__arr[idx, :, :],
MR=multiplicative_rescaling_factors,
AR=additive_rescaling_factors,
E=sun_elevation,
))
self.__arr = np.array(np.stack(toa, axis=0))
elif platform == Platform.Sentinel2:
self.__arr = self.__arr.astype(np.float32) / 10000.0
else:
raise AttributeError(
f"Cannot convert dn2toa. Platform {platform} not supported [Landsat-5, Landsat-7, Landsat-8, "
f"Sentinel-2]. ")
self.__update_dataset(self.dataset.crs,
self.dataset.transform,
nodata=self.dataset.nodata)
def test_load_mtl():
src_mtl = 'tests/data/LC80100202015018LGN00_MTL.json'
mtl = _load_mtl(src_mtl)
assert isinstance(mtl, dict)
cval.append(-1000000)
cval.append(1000000)
cval.append(-1000000.0)
cval.append(1000000.0)
dfTemp = pd.DataFrame([cval, cval], columns=ccol)
pd.reset_option('mode.chained_assignment')
totC = []
step = 50
bin_range = np.arange(0, 65535 + step, step)
for i, row in df.iterrows():
image_id = row['displayId']
if os.path.isfile(image_path + image_id + '_MTL.txt'):
mtl = toa_utils._load_mtl(image_path + image_id + '_MTL.txt')
metadata = mtl['L1_METADATA_FILE']
for fe in fileExt:
srcFile = image_path + image_id + fe
band = int(fe[2:-4])
if os.path.isfile(srcFile):
with rasterio.open(srcFile) as bandImg:
band_in = bandImg.read(1)
band_in.resize((band_in.shape[0] * band_in.shape[1]),
refcheck=False)
for scope in range(0, 2, 1):
if scope == 1:
band_in = band_in[np.where(band_in > 0)]
band_in = band_in[np.where(band_in < 0xFFFF)]
max = band_in.max()
classifier.summary()
# -------------------------------------------------------------------------------------------------------
dst_data = np.empty(rescaler[0] * rescaler[1])
print(time.strftime("%d.%m.%Y %H:%M:%S"), 'Starting data preparation for',
os.path.split(model)[-1][:-3])
logFile.write("{} {}\n".format(time.strftime("%d.%m.%Y %H:%M:%S"),
'Starting data preparation'))
# -------------------------------------------------------------------------------------------------------
# prepare vector for learning
Xinput = np.empty([rescaler[0] * rescaler[1], 17])
for band in range(0, 12, 1):
print(time.strftime("%d.%m.%Y %H:%M:%S"),
'Building band: B{}'.format(band))
# metadata
mtl = toa_utils._load_mtl(file_path + displayId + '_MTL.txt')
metadata = mtl['L1_METADATA_FILE']
src_data[band].resize(
(src_data[band].shape[0] * src_data[band].shape[1]),
refcheck=False)
if (band < 9):
MR = metadata['RADIOMETRIC_RESCALING'][
'REFLECTANCE_MULT_BAND_{}'.format(band + 1)]
AR = metadata['RADIOMETRIC_RESCALING'][
'REFLECTANCE_ADD_BAND_{}'.format(band + 1)]
SE = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
# REFLECTANCE TOA
Xinput[:, band + 6] = src_data[band]
Xinput[:, band + 6] = (MR * Xinput[:, band + 6] + AR) / np.sin(
np.deg2rad(SE))
elif band < 11:
def calculate_landsat_brightness_temperature(src_path, src_mtl, dst_path,
temp_scale, creation_options,
band, dst_dtype, processes):
"""Parameters
------------
src_path: list
list of src_paths(strings)
src_mtl: string
mtl file path
dst_path: string
destination file path
rescale_factor: float [default] float(55000.0/2**16)
rescale post-TOA tifs to 55,000 or to full 16-bit
creation_options: dictionary
rio.options.creation_options
band: list
list of integers
dst_dtype: strings [default] uint16
destination data dtype
Returns
---------
out: None
Output is written to dst_path
"""
mtl = toa_utils._load_mtl(src_mtl)
M = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_MULT_BAND_'],
band)
A = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'RADIANCE_ADD_BAND_'],
band)
K1 = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K1_CONSTANT_BAND_'],
band)
K2 = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'TIRS_THERMAL_CONSTANTS', 'K2_CONSTANT_BAND_'],
band)
dst_dtype = np.__dict__[dst_dtype]
with rio.open(src_path) as src:
dst_profile = src.profile.copy()
src_nodata = src.nodata
for co in creation_options:
dst_profile[co] = creation_options[co]
dst_profile['dtype'] = dst_dtype
global_args = {
'M': M,
'A': A,
'K1': K1,
'K2': K2,
'src_nodata': 0,
'temp_scale': temp_scale,
'dst_dtype': dst_dtype
}
with riomucho.RioMucho([src_path],
dst_path,
_brightness_temp_worker,
options=dst_profile,
global_args=global_args) as rm:
rm.run(processes)
def calculate_landsat_radiance(src_path, src_mtl, dst_path, rescale_factor,
creation_options, band, dst_dtype, processes,
clip=True):
"""
Parameters
------------
src_path: strings
src_mtl: string
dst_path: string
rescale_factor: float
creation_options: dict
bands: list
dst_dtype: string
processes: integer
pixel_sunangle: boolean
clip: boolean
Returns
---------
None
Output is written to dst_path
"""
mtl = toa_utils._load_mtl(src_mtl)
M = toa_utils._load_mtl_key(mtl,
['L1_METADATA_FILE',
'RADIOMETRIC_RESCALING',
'RADIANCE_MULT_BAND_'],
band)
A = toa_utils._load_mtl_key(mtl,
['L1_METADATA_FILE',
'RADIOMETRIC_RESCALING',
'RADIANCE_ADD_BAND_'],
band)
rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype)
dst_dtype = np.__dict__[dst_dtype]
with rasterio.open(src_path) as src:
dst_profile = src.profile.copy()
src_nodata = src.nodata
for co in creation_options:
dst_profile[co] = creation_options[co]
dst_profile['dtype'] = dst_dtype
global_args = {
'A': A,
'M': M,
'src_nodata': src_nodata,
'rescale_factor': rescale_factor,
'clip': clip,
'dst_dtype': dst_dtype
}
with riomucho.RioMucho([src_path],
dst_path,
_radiance_worker,
options=dst_profile,
global_args=global_args) as rm:
rm.run(processes)
def calculate_landsat_reflectance(src_paths, src_mtl, dst_path, rescale_factor,
creation_options, bands, dst_dtype,
processes, pixel_sunangle, clip=True):
"""
Parameters
------------
src_paths: list of strings
src_mtl: string
dst_path: string
rescale_factor: float
creation_options: dict
bands: list
dst_dtype: string
processes: integer
pixel_sunangle: boolean
clip: boolean
Returns
---------
None
Output is written to dst_path
"""
mtl = toa_utils._load_mtl(src_mtl)
metadata = mtl['L1_METADATA_FILE']
M = [metadata['RADIOMETRIC_RESCALING']
['REFLECTANCE_MULT_BAND_{}'.format(b)]
for b in bands]
A = [metadata['RADIOMETRIC_RESCALING']
['REFLECTANCE_ADD_BAND_{}'.format(b)]
for b in bands]
E = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
date_collected = metadata['PRODUCT_METADATA']['DATE_ACQUIRED']
time_collected_utc = metadata['PRODUCT_METADATA']['SCENE_CENTER_TIME']
rescale_factor = toa_utils.normalize_scale(rescale_factor, dst_dtype)
dst_dtype = np.__dict__[dst_dtype]
for src_path in src_paths:
with rasterio.open(src_path) as src:
dst_profile = src.profile.copy()
src_nodata = src.nodata
for co in creation_options:
dst_profile[co] = creation_options[co]
dst_profile['dtype'] = dst_dtype
global_args = {
'A': A,
'M': M,
'E': E,
'src_nodata': src_nodata,
'src_crs': dst_profile['crs'],
'dst_dtype': dst_dtype,
'rescale_factor': rescale_factor,
'clip': clip,
'pixel_sunangle': pixel_sunangle,
'date_collected': date_collected,
'time_collected_utc': time_collected_utc,
'bands': len(bands)
}
dst_profile.update(count=len(bands))
if len(bands) == 3:
dst_profile.update(photometric='rgb')
else:
dst_profile.update(photometric='minisblack')
with riomucho.RioMucho(list(src_paths),
dst_path,
_reflectance_worker,
options=dst_profile,
global_args=global_args,
mode='manual_read') as rm:
rm.run(processes)
def test_load_mtl(): src_mtl = 'tests/data/LC80100202015018LGN00_MTL.json' mtl = _load_mtl(src_mtl) assert isinstance(mtl, dict)
# load image
raster = gdal.Open(my_file, gdal.GA_ReadOnly)
array = raster.GetRasterBand(1).ReadAsArray()
msk_array = np.ma.masked_where(((array == 0) | (array == 65535)), array)
#msk_array = np.ma.masked_equal(array, value = 65535)
print('Raster Projection:\n', raster.GetProjection())
print('Raster GeoTransform:\n', raster.GetGeoTransform())
mlt = None
# find file & load lon lat cooridanes from image
for fdir in os.walk(GetImagesPath()):
my_txt_file = os.path.join(
fdir[0], fname[0] + '_' + fname[1] + '_' + fname[2] + '_' + fname[3] +
'_' + fname[4] + '_' + fname[5] + '_' + fname[6] + '_MTL.txt')
if os.path.isfile(my_txt_file):
mtl = toa_utils._load_mtl(my_txt_file)
break
metadata = mtl['L1_METADATA_FILE']
CORNER_UL_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UL_LON_PRODUCT']
CORNER_LL_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_LL_LAT_PRODUCT']
CORNER_UR_LON_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LON_PRODUCT']
CORNER_UR_LAT_PRODUCT = metadata['PRODUCT_METADATA']['CORNER_UR_LAT_PRODUCT']
# map coordinates
ul_lon = math.floor(CORNER_UL_LON_PRODUCT) - map_margin
ll_lat = math.floor(CORNER_LL_LAT_PRODUCT) - map_margin
ur_lon = math.ceil(CORNER_UR_LON_PRODUCT) + map_margin
ur_lat = math.ceil(CORNER_UR_LAT_PRODUCT) + map_margin
print('Map coordinates : {},{},{},{}'.format(ul_lon, ll_lat, ur_lon, ur_lat))
#ul_lon = 0.0
#ll_lat = 40.0
