def _load_e_tm_clouds(self, qa_arr: XDS_TYPE,
band_list: Union[list, BandNames]) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read Landsat-(E)TM clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2 TM)[https://www.usgs.gov/media/files/landsat-4-5-tm-collection-2-level-1-data-format-control-book]
- (COL 2 ETM)[https://www.usgs.gov/media/files/landsat-7-etm-collection-2-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata = None
cld = None
shd = None
if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list):
if self._collection == LandsatCollection.COL_1:
# Bit id
nodata_id = 0
cloud_id = 4 # Clouds with high confidence
shd_conf_1_id = 7
shd_conf_2_id = 8
nodata, cld, shd_conf_1, shd_conf_2 = rasters.read_bit_array(
qa_arr,
[nodata_id, cloud_id, shd_conf_1_id, shd_conf_2_id])
shd = shd_conf_1 & shd_conf_2
else:
# Bit ids
nodata_id = 0
cloud_id = 3 # Clouds with high confidence
shd_id = 4 # Shadows with high confidence
nodata, cld, shd = rasters.read_bit_array(
qa_arr, [nodata_id, cloud_id, shd_id])
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = self._create_mask(qa_arr, cld | shd, nodata)
elif band == SHADOWS:
bands[band] = self._create_mask(qa_arr, shd, nodata)
elif band == CLOUDS:
bands[band] = self._create_mask(qa_arr, cld, nodata)
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-(E)TM sensor: {band}"
)
return bands
def _create_mask(
self,
bit_array: xr.DataArray,
bit_ids: Union[int, list],
nodata: np.ndarray,
) -> xr.DataArray:
"""
Create a mask masked array (uint8) from a bit array, bit IDs and a nodata mask.
Args:
bit_array (xr.DataArray): Conditional array
bit_ids (Union[int, list]): Bit IDs
nodata (np.ndarray): Nodata mask
Returns:
xr.DataArray: Mask masked array
"""
if not isinstance(bit_ids, list):
bit_ids = [bit_ids]
conds = rasters.read_bit_array(bit_array, bit_ids)
cond = reduce(lambda x, y: x | y,
conds) # Use every conditions (bitwise or)
cond_arr = np.where(cond, self._mask_true,
self._mask_false).astype(np.uint8)
cond_arr = np.squeeze(cond_arr)
cond_arr = features.sieve(cond_arr, size=10, connectivity=4)
cond_arr = np.expand_dims(cond_arr, axis=0)
return super()._create_mask(bit_array, cond_arr, nodata)
def _load_mss_clouds(self, qa_arr: XDS_TYPE, band_list: list) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read Landsat-MSS clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2)[https://www.usgs.gov/media/files/landsat-1-5-mss-collection-2-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata_id = 0
cloud_id = (4 if self._collection == LandsatCollection.COL_1 else 3
) # Clouds with high confidence
clouds = None
if ALL_CLOUDS in band_list or CLOUDS in band_list:
nodata, cld = rasters.read_bit_array(qa_arr, [nodata_id, cloud_id])
clouds = self._create_mask(qa_arr, cld, nodata)
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = clouds
elif band == CLOUDS:
bands[band] = clouds
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-MSS sensor: {band}")
return bands
def _create_mask(
self, bit_array: XDS_TYPE, bit_ids: Union[int, list], nodata: np.ndarray
) -> xr.DataArray:
"""
Create a mask masked array (uint8) from a bit array, bit IDs and a nodata mask.
Args:
bit_array (XDS_TYPE): Conditional array
bit_ids (Union[int, list]): Bit IDs
nodata (np.ndarray): Nodata mask
Returns:
xr.DataArray: Mask masked array
"""
if not isinstance(bit_ids, list):
bit_ids = [bit_ids]
conds = rasters.read_bit_array(bit_array.astype(np.uint8), bit_ids)
cond = reduce(lambda x, y: x | y, conds) # Use every conditions (bitwise or)
return super()._create_mask(bit_array, cond, nodata)
def _manage_invalid_pixels_olci(
self,
band_arr: XDS_TYPE,
band: obn,
resolution: float = None,
size: Union[list, tuple] = None,
) -> XDS_TYPE:
"""
Manage invalid pixels (Nodata, saturated, defective...) for OLCI data.
See there:
https://sentinel.esa.int/documents/247904/1872756/Sentinel-3-OLCI-Product-Data-Format-Specification-OLCI-Level-1
QUALITY FLAGS (From end to start of the 32 bits):
| Bit | Flag |
|----|----------------------|
| 0 | saturated21 |
| 1 | saturated20 |
| 2 | saturated19 |
| 3 | saturated18 |
| 4 | saturated17 |
| 5 | saturated16 |
| 6 | saturated15 |
| 7 | saturated14 |
| 8 | saturated13 |
| 9 | saturated12 |
| 10 | saturated11 |
| 11 | saturated10 |
| 11 | saturated09 |
| 12 | saturated08 |
| 13 | saturated07 |
| 14 | saturated06 |
| 15 | saturated05 |
| 16 | saturated04 |
| 17 | saturated03 |
| 18 | saturated02 |
| 19 | saturated01 |
| 20 | dubious |
| 21 | sun-glint_risk |
| 22 | duplicated |
| 23 | cosmetic |
| 24 | invalid |
| 25 | straylight_risk |
| 26 | bright |
| 27 | tidal_region |
| 28 | fresh_inland_water |
| 19 | coastline |
| 30 | land |
Args:
band_arr (XDS_TYPE): Band array
band (obn): Band name as an OpticalBandNames
resolution (float): Band resolution in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
XDS_TYPE: Cleaned band array
"""
nodata_true = 1
nodata_false = 0
# Bit ids
band_bit_id = {
obn.CA: 18, # Band 2
obn.BLUE: 17, # Band 3
obn.GREEN: 14, # Band 6
obn.RED: 12, # Band 8
obn.VRE_1: 10, # Band 11
obn.VRE_2: 9, # Band 12
obn.VRE_3: 5, # Band 16
obn.NIR: 4, # Band 17
obn.NARROW_NIR: 4, # Band 17
obn.WV: 1, # Band 20
obn.FAR_NIR: 0, # Band 21
}
invalid_id = 24
sat_band_id = band_bit_id[band]
# Open quality flags
qual_flags_path = os.path.join(self._get_band_folder(),
"quality_flags.tif")
if not os.path.isfile(qual_flags_path):
LOGGER.warning(
"Impossible to open quality flags %s. Taking the band as is.",
qual_flags_path,
)
return band_arr
# Open flag file
qual_arr, _ = rasters_rio.read(
qual_flags_path,
resolution=resolution,
size=size,
resampling=Resampling.nearest, # Nearest to keep the flags
masked=False,
)
invalid, sat = rasters.read_bit_array(qual_arr.astype(np.uint32),
[invalid_id, sat_band_id])
# Get nodata mask
no_data = np.where(np.isnan(band_arr), nodata_true, nodata_false)
# Combine masks
mask = no_data | invalid | sat
# DO not set 0 to epsilons as they are a part of the
return self._set_nodata_mask(band_arr, mask)
def _manage_invalid_pixels(
self,
band_arr: XDS_TYPE,
band: obn,
resolution: float = None,
size: Union[list, tuple] = None,
) -> XDS_TYPE:
"""
Manage invalid pixels (Nodata, saturated, defective...)
Args:
band_arr (XDS_TYPE): Band array
band (obn): Band name as an OpticalBandNames
resolution (float): Band resolution in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
XDS_TYPE: Cleaned band array
"""
# Open QA band
landsat_qa_path = self._get_path(self._quality_id)
qa_arr = self._read_band(landsat_qa_path,
resolution=resolution,
size=size).data # To np array
if self._collection == LandsatCollection.COL_1:
# https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-1-level-1-quality-assessment-band
# Bit ids
nodata_id = 0 # Fill value
dropped_id = 1 # Dropped pixel or terrain occlusion
# Set nodata to every saturated pixel, even if only 1-2 bands are touched by it
# -> 01 or 10 or 11
# -> bit 2 or bit 3
sat_id_1 = 2
sat_id_2 = 3
nodata, dropped, sat_1, sat_2 = rasters.read_bit_array(
qa_arr, [nodata_id, dropped_id, sat_id_1, sat_id_2])
mask = nodata | dropped | sat_1 | sat_2
else:
# https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2-quality-assessment-bands
# SATURATED & OTHER PIXELS
band_nb = int(self.band_names[band])
# Bit ids
sat_id = band_nb - 1 # Saturated pixel
if self.product_type != LandsatProductType.L1_OLCI:
other_id = 11 # Terrain occlusion
else:
other_id = 9 # Dropped pixels
sat, other = rasters.read_bit_array(qa_arr, [sat_id, other_id])
# If collection 2, nodata has to be found in pixel QA file
landsat_stat_path = self._get_path(self._nodata_band_id)
pixel_arr = self._read_band(landsat_stat_path,
resolution=resolution,
size=size).data
nodata = np.where(pixel_arr == 1, 1, 0)
mask = sat | other | nodata
return self._set_nodata_mask(band_arr, mask)
def _load_olci_clouds(self, qa_arr: XDS_TYPE,
band_list: Union[list, BandNames]) -> dict:
"""
Load cloud files as xarrays.
Read Landsat-OLCI clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2)[https://www.usgs.gov/media/files/landsat-8-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata = None
cld = None
shd = None
cir = None
if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list):
if self._collection == LandsatCollection.COL_1:
# Bit ids
nodata_id = 0
cloud_id = 4 # Clouds with high confidence
shd_conf_1_id = 7
shd_conf_2_id = 8
cir_conf_1_id = 11
cir_conf_2_id = 12
# Read binary mask
(
nodata,
cld,
shd_conf_1,
shd_conf_2,
cir_conf_1,
cir_conf_2,
) = rasters.read_bit_array(
qa_arr,
[
nodata_id,
cloud_id,
shd_conf_1_id,
shd_conf_2_id,
cir_conf_1_id,
cir_conf_2_id,
],
)
shd = shd_conf_1 & shd_conf_2
cir = cir_conf_1 & cir_conf_2
else:
# Bit ids
nodata_id = 0
cloud_id = 3 # Clouds with high confidence
shd_id = 4 # Shadows with high confidence
cir_id = 2 # Cirrus with high confidence
nodata, cld, shd, cir = rasters.read_bit_array(
qa_arr, [nodata_id, cloud_id, shd_id, cir_id])
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = self._create_mask(qa_arr, cld | shd | cir,
nodata)
elif band == SHADOWS:
bands[band] = self._create_mask(qa_arr, shd, nodata)
elif band == CLOUDS:
bands[band] = self._create_mask(qa_arr, cld, nodata)
elif band == CIRRUS:
bands[band] = self._create_mask(qa_arr, cir, nodata)
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-OLCI sensor: {band}")
return bands