def _set_product_type(self) -> None:
"""Set products type"""
# Product type
if self.name[7] != "1":
raise InvalidTypeError(
"Only L1 products are used for Sentinel-3 data.")
if "OL" in self.name:
# Instrument
self._instrument_name = S3Instrument.OLCI
# Data type
if S3DataTypes.EFR.value in self.name:
self._data_type = S3DataTypes.EFR
self.product_type = S3ProductType.OLCI_EFR
else:
raise InvalidTypeError(
"Only EFR data type is used for Sentinel-3 OLCI data.")
# Bands
self.band_names.map_bands({
obn.CA: "02",
obn.BLUE: "03",
obn.GREEN: "06",
obn.RED: "08",
obn.VRE_1: "11",
obn.VRE_2: "12",
obn.VRE_3: "16",
obn.NIR: "17",
obn.NARROW_NIR: "17",
obn.WV: "20",
obn.FAR_NIR: "21",
})
elif "SL" in self.name:
# Instrument
self._instrument_name = S3Instrument.SLSTR
# Data type
if S3DataTypes.RBT.value in self.name:
self._data_type = S3DataTypes.RBT
self.product_type = S3ProductType.SLSTR_RBT
else:
raise InvalidTypeError(
"Only RBT data type is used for Sentinel-3 SLSTR data.")
# Bands
self.band_names.map_bands({
obn.GREEN: "1", # radiance, 500m
obn.RED: "2", # radiance, 500m
obn.NIR: "3", # radiance, 500m
obn.NARROW_NIR: "3", # radiance, 500m
obn.SWIR_CIRRUS: "4", # radiance, 500m
obn.SWIR_1: "5", # radiance, 500m
obn.SWIR_2: "6", # radiance, 500m
obn.MIR: "7", # brilliance temperature, 1km
obn.TIR_1: "8", # brilliance temperature, 1km
obn.TIR_2: "9", # brilliance temperature, 1km
})
else:
raise InvalidProductError(f"Invalid Sentinel-3 name: {self.name}")
def _set_sensor_mode(self) -> None:
"""
Get products type from RADARSAT-2 products name (could check the metadata too)
"""
# Get metadata
root, namespace = self.read_mtd()
# Get sensor mode
sensor_mode_xml = None
for element in root:
if element.tag == namespace + "sourceAttributes":
radar_param = element.find(namespace + "radarParameters")
# WARNING: this word may differ from the Enum !!! (no docs available)
# Get the closest match
sensor_mode_xml = radar_param.findtext(namespace +
"acquisitionType")
break
if sensor_mode_xml:
sensor_mode = difflib.get_close_matches(
sensor_mode_xml, Rs2SensorMode.list_values())[0]
try:
self.sensor_mode = Rs2SensorMode.from_value(sensor_mode)
except ValueError as ex:
raise InvalidTypeError(
f"Invalid sensor mode for {self.name}") from ex
else:
raise InvalidTypeError(f"Invalid sensor mode for {self.name}")
def _get_band_filename(self, band: Union[obn, str]) -> str:
"""
Get band filename from its band type
Args:
band ( Union[obn, str]): Band as an OpticalBandNames or directly the snap_name
Returns:
str: Band name
"""
if isinstance(band, obn):
snap_name = self._get_snap_band_name(band)
elif isinstance(band, str):
snap_name = band
else:
raise InvalidTypeError(
"The given band should be an OpticalBandNames or directly the snap_name"
)
# Remove _an/_in for SLSTR products
if self._data_type == S3DataTypes.RBT:
if "cloud" not in snap_name:
snap_name = snap_name[:-3]
elif "an" in snap_name:
snap_name = snap_name[:-3] + "_RAD"
else:
# in
snap_name = snap_name[:-3] + "_BT"
return snap_name
def _get_slstr_quality_flags_name(self, band: obn) -> str:
"""
Get SNAP band name.
Args:
band (obn): Band as an OpticalBandNames
Returns:
str: Quality flag name with SNAP format
"""
# Get band number
band_nb = self.band_names[band]
if band_nb is None:
raise InvalidProductError(
f"Non existing band ({band.name}) for S3-{self._data_type.name} products"
)
# Get quality flag name
if self._data_type == S3DataTypes.RBT:
snap_bn = f"S{band_nb}_exception_{'i' if band in BT_BANDS else 'a'}n"
else:
raise InvalidTypeError(
f"This function only works for Sentinel-3 SLSTR data: {self._data_type}"
)
return snap_bn
def _get_snap_band_name(self, band: obn) -> str:
"""
Get SNAP band name.
Args:
band (obn): Band as an OpticalBandNames
Returns:
str: Band name with SNAP format
"""
# Get band number
band_nb = self.band_names[band]
if band_nb is None:
raise InvalidProductError(
f"Non existing band ({band.name}) for S3-{self._data_type.name} products"
)
# Get band name
if self._data_type == S3DataTypes.EFR:
snap_bn = f"Oa{band_nb}_reflectance" # Converted into reflectance previously in the graph
elif self._data_type == S3DataTypes.RBT:
if band in BT_BANDS:
snap_bn = f"S{band_nb}_BT_in"
else:
snap_bn = f"S{band_nb}_reflectance_an" # Conv into reflectance previously in the graph
else:
raise InvalidTypeError(
f"Unknown data type for Sentinel-3 data: {self._data_type}")
return snap_bn
def to_value_list(cls, name_list: list = None) -> list:
"""
Get a list from the values of the bands
.. code-block:: python
>>> SarBandNames.to_name_list([SarBandNames.HV_DSPK, SarBandNames.VV])
['HV_DSPK', 'VV']
>>> SarBandNames.to_name_list()
['VV', 'VV_DSPK', 'HH', 'HH_DSPK', 'VH', 'VH_DSPK', 'HV', 'HV_DSPK']
Args:
name_list (list): List of band names
Returns:
list: List of band values
"""
if name_list:
out_list = []
for key in name_list:
if isinstance(key, str):
out_list.append(getattr(cls, key).value)
elif isinstance(key, cls):
out_list.append(key.value)
else:
raise InvalidTypeError(
"The list should either contain strings or SarBandNames"
)
else:
out_list = cls.list_values()
return out_list
def from_list(cls, name_list: Union[list, str]) -> list:
"""
Get the band enums from list of band names
.. code-block:: python
>>> SarBandNames.from_list("VV")
[<SarBandNames.VV: 'VV'>]
Args:
name_list (Union[list, str]): List of names
Returns:
list: List of enums
"""
if not isinstance(name_list, list):
name_list = [name_list]
try:
band_names = [cls(name) for name in name_list]
except ValueError as ex:
raise InvalidTypeError(
f"Band names ({name_list}) should be chosen among: {cls.list_names()}"
) from ex
return band_names
def map_bands(self, band_map: dict) -> None:
"""
Mapping band names to specific satellite band numbers, as strings.
.. code-block:: python
>>> # Example for Sentinel-2 L1C data
>>> ob = OpticalBands()
>>> ob.map_bands({
CA: '01',
BLUE: '02',
GREEN: '03',
RED: '04',
VRE_1: '05',
VRE_2: '06',
VRE_3: '07',
NIR: '08',
NNIR: '8A',
WV: '09',
SWIR_1: '11',
SWIR_2: '12'
})
Args:
band_map (dict): Band mapping as {OpticalBandNames: Band number for loading band}
"""
for band_name, band_nb in band_map.items():
if band_name not in self._band_map or not isinstance(
band_name, OpticalBandNames):
raise InvalidTypeError(
f"{band_name} should be an OpticalBandNames object")
# Set number
self._band_map[band_name] = band_nb
def _set_resolution(self) -> float:
"""
Set product default resolution (in meters)
"""
def_res = None
# Read metadata for default resolution
try:
root, _ = self.read_mtd()
def_res = float(root.findtext(".//GroundRangeGeometricResolution"))
except (InvalidProductError, TypeError):
pass
# If we cannot read it in MTD, initiate survival mode
if not def_res:
if self.sensor_mode == CskSensorMode.S2:
def_res = 1.0
elif self.sensor_mode == CskSensorMode.HI:
def_res = 5.0
elif self.sensor_mode == CskSensorMode.PP:
def_res = 20.0
elif self.sensor_mode == CskSensorMode.WR:
def_res = 30.0
elif self.sensor_mode == CskSensorMode.HR:
def_res = 100.0
else:
raise InvalidTypeError(
f"Unknown sensor mode {self.sensor_mode}")
return def_res
def get_default_band(self) -> BandNames:
"""
Get default band:
The first existing one between `VV` and `HH` for SAR data.
.. code-block:: python
>>> from eoreader.reader import Reader
>>> path = r"S1A_IW_GRDH_1SDV_20191215T060906_20191215T060931_030355_0378F7_3696.zip"
>>> prod = Reader().open(path)
>>> prod.get_default_band()
<SarBandNames.VV: 'VV'>
Returns:
str: Default band
"""
existing_bands = self._get_raw_bands()
if not existing_bands:
raise InvalidProductError(
f"No band exists for products: {self.name}")
# The order matters, as we almost always prefer VV and HH
if sbn.VV in existing_bands:
default_band = sbn.VV
elif sbn.HH in existing_bands:
default_band = sbn.HH
elif sbn.VH in existing_bands:
default_band = sbn.VH
elif sbn.HV in existing_bands:
default_band = sbn.HV
else:
raise InvalidTypeError(
f"Invalid bands for products: {existing_bands}")
return default_band
def _set_resolution(self) -> float:
"""
Set product default resolution (in meters)
.. WARNING:: We assume being in High Resolution (except for WV where we must be in medium resolution)
"""
def_res = None
# Read metadata
try:
root, _ = self.read_mtd()
def_res = float(root.findtext(".//rangePixelSpacing"))
except (InvalidProductError, TypeError):
pass
# If we cannot read it in MTD, initiate survival mode
if not def_res:
if self.sensor_mode in [S1SensorMode.SM, S1SensorMode.IW]:
def_res = 10.0
elif self.sensor_mode in [S1SensorMode.EW, S1SensorMode.WV]:
def_res = 25.0
else:
raise InvalidTypeError(
f"Unknown sensor mode {self.sensor_mode}")
LOGGER.debug(
f"Default resolution is set to {def_res}. "
f"The product is considered being in "
f"{'Medium' if self.sensor_mode == S1SensorMode.WV else 'High'}-Resolution"
)
return def_res
def _set_resolution(self) -> float:
"""
Set product default resolution (in meters)
"""
def_res = None
# Read metadata
try:
root, nsmap = self.read_mtd()
namespace = nsmap[None]
def_res = float(root.findtext(f"{namespace}sampledPixelSpacing"))
except (InvalidProductError, TypeError):
pass
# If we cannot read it in MTD, initiate survival mode
if not def_res:
if self.sensor_mode == Rs2SensorMode.SLA:
def_res = 1.0 if self.product_type == Rs2ProductType.SGX else 0.5
elif self.sensor_mode in [Rs2SensorMode.U, Rs2SensorMode.WU]:
def_res = 1.0 if self.product_type == Rs2ProductType.SGX else 1.56
elif self.sensor_mode in [
Rs2SensorMode.MF,
Rs2SensorMode.WMF,
Rs2SensorMode.F,
Rs2SensorMode.WF,
]:
def_res = 3.13 if self.product_type == Rs2ProductType.SGX else 6.25
elif self.sensor_mode == Rs2SensorMode.XF:
def_res = 2.0 if self.product_type == Rs2ProductType.SGX else 3.13
if self.product_type in [Rs2ProductType.SGF, Rs2ProductType.SGX]:
LOGGER.debug(
"This product is considered to have one look (not checked in metadata)"
) # TODO
elif self.sensor_mode in [Rs2SensorMode.S, Rs2SensorMode.EH]:
def_res = 8.0 if self.product_type == Rs2ProductType.SGX else 12.5
elif self.sensor_mode in [Rs2SensorMode.W, Rs2SensorMode.EL]:
def_res = 10.0 if self.product_type == Rs2ProductType.SGX else 12.5
elif self.sensor_mode in [Rs2SensorMode.FQ, Rs2SensorMode.WQ]:
def_res = 3.13
elif self.sensor_mode in [Rs2SensorMode.SQ, Rs2SensorMode.WSQ]:
raise NotImplementedError(
"Not squared pixels management are not implemented in EOReader."
)
elif self.sensor_mode == Rs2SensorMode.SCN:
def_res = 25.0
elif self.sensor_mode == Rs2SensorMode.SCW:
def_res = 50.0
elif self.sensor_mode == Rs2SensorMode.DVWF:
def_res = 40.0 if self.product_type == Rs2ProductType.SCF else 20.0
elif self.sensor_mode == Rs2SensorMode.SCW:
if self.product_type == Rs2ProductType.SCF:
def_res = 50.0
else:
raise NotImplementedError(
"Not squared pixels management are not implemented in EOReader."
)
else:
raise InvalidTypeError(f"Unknown sensor mode {self.sensor_mode}")
return def_res
def _set_resolution(self) -> float:
"""
Set product default resolution (in meters)
"""
def_res = None
# Read metadata
try:
root, _ = self.read_mtd()
for element in root:
if element.tag == "ProductCharacteristics":
def_res = float(
element.findtext("GroundRangeGeometricResolution"))
break
except (InvalidProductError, AttributeError):
pass
# If we cannot read it in MTD, initiate survival mode
if not def_res:
if self.sensor_mode == CskSensorMode.S2:
def_res = 1.0
elif self.sensor_mode == CskSensorMode.HI:
def_res = 5.0
elif self.sensor_mode == CskSensorMode.PP:
def_res = 20.0
elif self.sensor_mode == CskSensorMode.WR:
def_res = 30.0
elif self.sensor_mode == CskSensorMode.HR:
def_res = 100.0
else:
raise InvalidTypeError(
f"Unknown sensor mode {self.sensor_mode}")
return def_res
def _load_e_tm_clouds(self, qa_arr: XDS_TYPE,
band_list: Union[list, BandNames]) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read Landsat-(E)TM clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2 TM)[https://www.usgs.gov/media/files/landsat-4-5-tm-collection-2-level-1-data-format-control-book]
- (COL 2 ETM)[https://www.usgs.gov/media/files/landsat-7-etm-collection-2-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata = None
cld = None
shd = None
if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list):
if self._collection == LandsatCollection.COL_1:
# Bit id
nodata_id = 0
cloud_id = 4 # Clouds with high confidence
shd_conf_1_id = 7
shd_conf_2_id = 8
nodata, cld, shd_conf_1, shd_conf_2 = rasters.read_bit_array(
qa_arr,
[nodata_id, cloud_id, shd_conf_1_id, shd_conf_2_id])
shd = shd_conf_1 & shd_conf_2
else:
# Bit ids
nodata_id = 0
cloud_id = 3 # Clouds with high confidence
shd_id = 4 # Shadows with high confidence
nodata, cld, shd = rasters.read_bit_array(
qa_arr, [nodata_id, cloud_id, shd_id])
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = self._create_mask(qa_arr, cld | shd, nodata)
elif band == SHADOWS:
bands[band] = self._create_mask(qa_arr, shd, nodata)
elif band == CLOUDS:
bands[band] = self._create_mask(qa_arr, cld, nodata)
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-(E)TM sensor: {band}"
)
return bands
def _set_sensor_mode(self) -> None:
"""
Get products type from TerraSAR-X products name (could check the metadata too)
"""
# Get sensor mode
try:
self.sensor_mode = TsxSensorMode.from_value(self.split_name[4])
except ValueError as ex:
raise InvalidTypeError(f"Invalid sensor mode for {self.name}") from ex
def _load_clouds(self,
bands: list,
resolution: float = None,
size: Union[list, tuple] = None) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read S2 cloud mask .GML files (both valid for L2A and L1C products).
https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi/level-1c/cloud-masks
Args:
bands (list): List of the wanted bands
resolution (int): Band resolution in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
dict: Dictionary {band_name, band_xarray}
"""
band_dict = {}
if bands:
def_band = self.get_default_band()
cloud_vec = self.open_mask("CLOUDS", "00")
# Open a bands to mask it
def_band = self.load(def_band, resolution=resolution)[def_band]
nodata = np.where(np.isnan(def_band), 1, 0)
for band in bands:
if band == ALL_CLOUDS:
band_dict[band] = self._rasterize(def_band, cloud_vec,
nodata)
elif band == CIRRUS:
try:
cirrus = cloud_vec[cloud_vec.maskType == "CIRRUS"]
except AttributeError:
# No masktype -> empty
cirrus = gpd.GeoDataFrame(geometry=[],
crs=cloud_vec.crs)
band_dict[band] = self._rasterize(def_band, cirrus, nodata)
elif band == CLOUDS:
try:
clouds = cloud_vec[cloud_vec.maskType == "OPAQUE"]
except AttributeError:
# No masktype -> empty
clouds = gpd.GeoDataFrame(geometry=[],
crs=cloud_vec.crs)
band_dict[band] = self._rasterize(def_band, clouds, nodata)
elif band == RAW_CLOUDS:
band_dict[band] = self._rasterize(def_band, cloud_vec,
nodata)
else:
raise InvalidTypeError(
f"Non existing cloud band for Sentinel-2: {band}")
return band_dict
def _load(self,
bands: list,
resolution: float = None,
size: Union[list, tuple] = None) -> dict:
"""
Core function loading SAR data bands
Args:
bands (list): Band list
resolution (float): Resolution of the band, in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
Dictionary {band_name, band_xarray}
"""
band_list = []
dem_list = []
for band in bands:
if is_index(band):
raise NotImplementedError(
"For now, no index is implemented for SAR data.")
elif is_optical_band(band):
raise TypeError(
f"You should ask for SAR bands as {self.name} is a SAR product."
)
elif is_sar_band(band):
if not self.has_band(band):
raise InvalidBandError(
f"{band} cannot be retrieved from {self.condensed_name}"
)
else:
band_list.append(band)
elif is_dem(band):
dem_list.append(band)
elif is_clouds(band):
raise NotImplementedError(
f"Clouds cannot be retrieved from SAR data ({self.condensed_name})."
)
else:
raise InvalidTypeError(
f"{band} is neither a band nor an index !")
# Check if DEM is set and exists
if dem_list:
self._check_dem_path()
# Load bands
bands = self._load_bands(band_list, resolution=resolution, size=size)
# Add DEM
bands.update(self._load_dem(dem_list, resolution=resolution,
size=size))
return bands
def _set_sensor_mode(self) -> None:
"""
Get products type from RADARSAT-2 products name (could check the metadata too)
"""
# Get metadata
root, nsmap = self.read_mtd()
namespace = nsmap[None]
# Get sensor mode
# WARNING: this word may differ from the Enum !!! (no docs available)
# Get the closest match
sensor_mode_xml = root.findtext(f".//{namespace}acquisitionType")
if sensor_mode_xml:
sensor_mode = difflib.get_close_matches(
sensor_mode_xml, Rs2SensorMode.list_values()
)[0]
try:
self.sensor_mode = Rs2SensorMode.from_value(sensor_mode)
except ValueError as ex:
raise InvalidTypeError(f"Invalid sensor mode for {self.name}") from ex
else:
raise InvalidTypeError(f"Invalid sensor mode for {self.name}")
def get_mean_sun_angles(self) -> (float, float):
"""
Get Mean Sun angles (Azimuth and Zenith angles)
.. code-block:: python
>>> from eoreader.reader import Reader
>>> path = "S3B_SL_1_RBT____20191115T233722_20191115T234022_20191117T031722_0179_032_144_3420_LN2_O_NT_003.SEN3"
>>> prod = Reader().open(path)
>>> prod.get_mean_sun_angles()
(78.55043955912154, 31.172127033319388)
Returns:
(float, float): Mean Azimuth and Zenith angle
"""
if self._data_type == S3DataTypes.EFR:
geom_file = os.path.join(self.path, "tie_geometries.nc")
sun_az = "SAA"
sun_ze = "SZA"
elif self._data_type == S3DataTypes.RBT:
geom_file = os.path.join(self.path,
"geometry_tn.nc") # Only use nadir files
sun_az = "solar_azimuth_tn"
sun_ze = "solar_zenith_tn"
else:
raise InvalidTypeError(
f"Unknown/Unsupported data type for Sentinel-3 data: {self._data_type}"
)
# Open file
if os.path.isfile(geom_file):
# Bug pylint with netCDF4
# pylint: disable=E1101
netcdf_ds = netCDF4.Dataset(geom_file)
# Get variables
sun_az_var = netcdf_ds.variables[sun_az]
sun_ze_var = netcdf_ds.variables[sun_ze]
# Get sun angles as the mean of whole arrays
azimuth_angle = float(np.mean(sun_az_var[:]))
zenith_angle = float(np.mean(sun_ze_var[:]))
# Close dataset
netcdf_ds.close()
else:
raise InvalidProductError(f"Geometry file {geom_file} not found")
return azimuth_angle, zenith_angle
def _get_sar_product_type(
self,
prod_type_pos: int,
gdrg_types: Union[ListEnum, list],
cplx_types: Union[ListEnum, list],
) -> None:
"""
Get products type, special function for SAR satellites.
Args:
prod_type_pos (int): Position of the products type in the file name
gdrg_types (Union[ListEnum, list]): Ground Range products types
cplx_types (Union[ListEnum, list]): Complex products types
"""
# Get and check products type class
if not isinstance(gdrg_types, list):
gdrg_types = [gdrg_types]
if not isinstance(cplx_types, list):
cplx_types = [cplx_types]
all_types = gdrg_types + cplx_types
prod_type_class = all_types[0].__class__
assert all(
isinstance(prod_type, prod_type_class) for prod_type in all_types)
# Get products type
try:
# All products types can be found in the filename and are 3 characters long
self.product_type = prod_type_class.from_value(
self.split_name[prod_type_pos][:3])
except ValueError as ex:
raise InvalidTypeError(
f"Invalid products type for {self.name}") from ex
# Link to SAR generic products types
if self.product_type in gdrg_types:
self.sar_prod_type = SarProductType.GDRG
elif self.product_type in cplx_types:
self.sar_prod_type = SarProductType.CPLX
else:
self.sar_prod_type = SarProductType.OTHER
# Discard invalid products types
if self.sar_prod_type == SarProductType.OTHER:
raise NotImplementedError(
f"{self.product_type.value} product type is not available ({self.name})"
)
def _load_mss_clouds(self, qa_arr: XDS_TYPE, band_list: list) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read Landsat-MSS clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2)[https://www.usgs.gov/media/files/landsat-1-5-mss-collection-2-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata_id = 0
cloud_id = (4 if self._collection == LandsatCollection.COL_1 else 3
) # Clouds with high confidence
clouds = None
if ALL_CLOUDS in band_list or CLOUDS in band_list:
nodata, cld = rasters.read_bit_array(qa_arr, [nodata_id, cloud_id])
clouds = self._create_mask(qa_arr, cld, nodata)
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = clouds
elif band == CLOUDS:
bands[band] = clouds
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-MSS sensor: {band}")
return bands
def _get_band_from_filename(self, band_filename: str) -> obn:
"""
Get band from filename
Args:
band_filename (str): Band filename
Returns:
obn: Band name with SNAP format
"""
# Get band name
if self._data_type == S3DataTypes.EFR:
band_nb = band_filename[2:4]
elif self._data_type == S3DataTypes.RBT:
band_nb = band_filename[1]
else:
raise InvalidTypeError(
f"Invalid Sentinel-3 datatype: {self._data_type}")
# Get band
band = list(self.band_names.keys())[list(
self.band_names.values()).index(band_nb)]
return band
def _load_olci_clouds(self, qa_arr: XDS_TYPE,
band_list: Union[list, BandNames]) -> dict:
"""
Load cloud files as xarrays.
Read Landsat-OLCI clouds from QA mask.
See here for clouds_values:
- (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band]
- (COL 2)[https://www.usgs.gov/media/files/landsat-8-level-1-data-format-control-book]
Args:
qa_arr (XDS_TYPE): Quality array
band_list (list): List of the wanted bands
Returns:
dict, dict: Dictionary {band_name, band_array}
"""
bands = {}
# Get clouds and nodata
nodata = None
cld = None
shd = None
cir = None
if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list):
if self._collection == LandsatCollection.COL_1:
# Bit ids
nodata_id = 0
cloud_id = 4 # Clouds with high confidence
shd_conf_1_id = 7
shd_conf_2_id = 8
cir_conf_1_id = 11
cir_conf_2_id = 12
# Read binary mask
(
nodata,
cld,
shd_conf_1,
shd_conf_2,
cir_conf_1,
cir_conf_2,
) = rasters.read_bit_array(
qa_arr,
[
nodata_id,
cloud_id,
shd_conf_1_id,
shd_conf_2_id,
cir_conf_1_id,
cir_conf_2_id,
],
)
shd = shd_conf_1 & shd_conf_2
cir = cir_conf_1 & cir_conf_2
else:
# Bit ids
nodata_id = 0
cloud_id = 3 # Clouds with high confidence
shd_id = 4 # Shadows with high confidence
cir_id = 2 # Cirrus with high confidence
nodata, cld, shd, cir = rasters.read_bit_array(
qa_arr, [nodata_id, cloud_id, shd_id, cir_id])
for band in band_list:
if band == ALL_CLOUDS:
bands[band] = self._create_mask(qa_arr, cld | shd | cir,
nodata)
elif band == SHADOWS:
bands[band] = self._create_mask(qa_arr, shd, nodata)
elif band == CLOUDS:
bands[band] = self._create_mask(qa_arr, cld, nodata)
elif band == CIRRUS:
bands[band] = self._create_mask(qa_arr, cir, nodata)
elif band == RAW_CLOUDS:
bands[band] = qa_arr
else:
raise InvalidTypeError(
f"Non existing cloud band for Landsat-OLCI sensor: {band}")
return bands
def extent(self) -> gpd.GeoDataFrame:
"""
Get UTM extent of the tile, managing the case with not orthorectified bands.
.. code-block:: python
>>> from eoreader.reader import Reader
>>> path = "S3B_SL_1_RBT____20191115T233722_20191115T234022_20191117T031722_0179_032_144_3420_LN2_O_NT_003.SEN3"
>>> prod = Reader().open(path)
>>> prod.utm_extent()
geometry
0 POLYGON ((1488846.028 6121896.451, 1488846.028...
Returns:
gpd.GeoDataFrame: Footprint in UTM
"""
try:
extent = super().extent()
except (FileNotFoundError, TypeError) as ex:
def get_min_max(substr: str, subdatasets: list) -> (float, float):
"""
Get min/max of a subdataset array
Args:
substr: Substring to identfy the subdataset
subdatasets: List of subdatasets
Returns:
float, float: min/max of the subdataset
"""
path = [path for path in subdatasets if substr in path][0]
with rasterio.open(path, "r") as sub_ds:
# Open the 4 corners of the array
height = sub_ds.height
width = sub_ds.width
scales = sub_ds.scales
pt1 = sub_ds.read(1, window=Window(0, 0, 1, 1)) * scales
pt2 = sub_ds.read(1, window=Window(width - 1, 0, width,
1)) * scales
pt3 = (sub_ds.read(
1, window=Window(0, height - 1, 1, height)) * scales)
pt4 = (sub_ds.read(1,
window=Window(width - 1, height - 1,
width, height)) * scales)
pt_list = [pt1, pt2, pt3, pt4]
# Return min and max
return np.min(pt_list), np.max(pt_list)
if self.product_type == S3ProductType.OLCI_EFR:
# Open geodetic_an.nc
geom_file = os.path.join(
self.path, "geo_coordinates.nc") # Only use nadir files
with rasterio.open(geom_file, "r") as geom_ds:
lat_min, lat_max = get_min_max("latitude",
geom_ds.subdatasets)
lon_min, lon_max = get_min_max("longitude",
geom_ds.subdatasets)
elif self.product_type == S3ProductType.SLSTR_RBT:
# Open geodetic_an.nc
geom_file = os.path.join(
self.path, "geodetic_an.nc") # Only use nadir files
with rasterio.open(geom_file, "r") as geom_ds:
lat_min, lat_max = get_min_max("latitude_an",
geom_ds.subdatasets)
lon_min, lon_max = get_min_max("longitude_an",
geom_ds.subdatasets)
else:
raise InvalidTypeError(
f"Invalid products type {self.product_type}") from ex
# Create wgs84 extent (left, bottom, right, top)
extent_wgs84 = gpd.GeoDataFrame(
geometry=[
vectors.from_bounds_to_polygon(lon_min, lat_min, lon_max,
lat_max)
],
crs=vectors.WGS84,
)
# Get upper-left corner and deduce UTM proj from it
utm = vectors.corresponding_utm_projection(
extent_wgs84.bounds.minx, extent_wgs84.bounds.maxy)
extent = extent_wgs84.to_crs(utm)
return extent
def to_band(to_convert: list) -> list:
"""
Convert a string (or real value) to any alias, band or index.
You can pass the name or the value of the bands.
.. code-block:: python
>>> to_band(["NDVI", "GREEN", RED, "VH_DSPK", "SLOPE", DEM, "CLOUDS", CLOUDS])
[<function NDVI at 0x00000154DDB12488>,
<OpticalBandNames.GREEN: 'GREEN'>,
<OpticalBandNames.RED: 'RED'>,
<SarBandNames.VH_DSPK: 'VH_DSPK'>,
<DemBandNames.SLOPE: 'SLOPE'>,
<DemBandNames.DEM: 'DEM'>,
<ClassifBandNames.CLOUDS: 'CLOUDS'>,
<ClassifBandNames.CLOUDS: 'CLOUDS'>]
Args:
to_convert (list): Values to convert into band objects
Returns:
list: converted values
"""
if not isinstance(to_convert, list):
to_convert = [to_convert]
bands = []
for tc in to_convert:
band_or_idx = None
# Try legit types
if isinstance(tc, str):
# Try index
if hasattr(_idx, tc):
band_or_idx = getattr(_idx, tc)
else:
try:
band_or_idx = _sbn.convert_from(tc)[0]
except TypeError:
try:
band_or_idx = _obn.convert_from(tc)[0]
except TypeError:
try:
band_or_idx = _dem.convert_from(tc)[0]
except TypeError:
try:
band_or_idx = _clouds.convert_from(tc)[0]
except TypeError:
pass
elif is_index(tc) or is_band(tc) or is_dem(tc) or is_clouds(tc):
band_or_idx = tc
# Store it
if band_or_idx:
bands.append(band_or_idx)
else:
raise InvalidTypeError(f"Unknown band or index: {tc}")
return bands
def _load_clouds(
self, bands: list, resolution: float = None, size: Union[list, tuple] = None
) -> dict:
"""
Load cloud files as numpy arrays with the same resolution (and same metadata).
Read S2 Theia cloud mask:
https://labo.obs-mip.fr/multitemp/sentinel-2/theias-sentinel-2-l2a-product-format/
> A cloud mask for each resolution (CLM_R1.tif ou CLM_R2.tif):
- bit 0 (1) : all clouds except the thinnest and all shadows
- bit 1 (2) : all clouds (except the thinnest)
- bit 2 (4) : clouds detected via mono-temporal thresholds
- bit 3 (8) : clouds detected via multi-temporal thresholds
- bit 4 (16) : thinnest clouds
- bit 5 (32) : cloud shadows cast by a detected cloud
- bit 6 (64) : cloud shadows cast by a cloud outside image
- bit 7 (128) : high clouds detected by 1.38 µm
Args:
bands (list): List of the wanted bands
resolution (int): Band resolution in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
dict: Dictionary {band_name, band_xarray}
"""
band_dict = {}
if bands:
# Open 20m cloud file if resolution >= 20m
res_id = "R2" if resolution >= 20 else "R1"
cloud_path = self.get_mask_path("CLM", res_id)
clouds_mask = rasters.read(
cloud_path,
resolution=resolution,
size=size,
resampling=Resampling.nearest,
)
# Get nodata mask
nodata = self.open_mask("EDG", res_id, resolution=resolution, size=size)
# Bit ids
clouds_shadows_id = 0
clouds_id = 1
cirrus_id = 4
shadows_in_id = 5
shadows_out_id = 6
for res_id in bands:
if res_id == ALL_CLOUDS:
band_dict[res_id] = self._create_mask(
clouds_mask, [clouds_shadows_id, cirrus_id], nodata
)
elif res_id == SHADOWS:
band_dict[res_id] = self._create_mask(
clouds_mask, [shadows_in_id, shadows_out_id], nodata
)
elif res_id == CLOUDS:
band_dict[res_id] = self._create_mask(
clouds_mask, clouds_id, nodata
)
elif res_id == CIRRUS:
band_dict[res_id] = self._create_mask(
clouds_mask, cirrus_id, nodata
)
elif res_id == RAW_CLOUDS:
band_dict[res_id] = clouds_mask
else:
raise InvalidTypeError(
f"Non existing cloud band for Sentinel-2 THEIA: {res_id}"
)
return band_dict
def _load_clouds(self,
bands: list,
resolution: float = None,
size: Union[list, tuple] = None) -> dict:
"""
Load cloud files as xarrays.
Read S3 SLSTR clouds from the flags file:cloud netcdf file.
https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-3-slstr/level-1/cloud-identification
bit_id flag_masks (ushort) flag_meanings
=== === ===
0 1US visible
1 2US 1.37_threshold
2 4US 1.6_small_histogram
3 8US 1.6_large_histogram
4 16US 2.25_small_histogram
5 32US 2.25_large_histogram
6 64US 11_spatial_coherence
7 128US gross_cloud
8 256US thin_cirrus
9 512US medium_high
10 1024US fog_low_stratus
11 2048US 11_12_view_difference
12 4096US 3.7_11_view_difference
13 8192US thermal_histogram
14 16384US spare
15 32768US spare
Args:
bands (list): List of the wanted bands
resolution (int): Band resolution in meters
size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided.
Returns:
dict: Dictionary {band_name, band_xarray}
"""
band_dict = {}
if bands:
if self._instrument_name == S3Instrument.OLCI:
raise InvalidTypeError(
"Sentinel-3 OLCI sensor does not provide any cloud file.")
all_ids = list(np.arange(0, 14))
cir_id = 8
cloud_ids = [id for id in all_ids if id != cir_id]
try:
cloud_path = files.get_file_in_dir(self._get_band_folder(),
"cloud_RAD.tif")
except FileNotFoundError:
self._preprocess_s3(resolution)
cloud_path = files.get_file_in_dir(self.output,
"cloud_RAD.tif")
if not cloud_path:
raise FileNotFoundError(
f"Unable to find the cloud mask for {self.path}")
# Open cloud file
clouds_array = rasters.read(
cloud_path,
resolution=resolution,
size=size,
resampling=Resampling.nearest,
masked=False,
).astype(np.uint16)
# Get nodata mask
# nodata = np.where(np.isnan(clouds_array), 1, 0)
nodata = np.where(clouds_array == 65535, 1, 0)
for band in bands:
if band == ALL_CLOUDS:
band_dict[band] = self._create_mask(
clouds_array, all_ids, nodata)
elif band == CLOUDS:
band_dict[band] = self._create_mask(
clouds_array, cloud_ids, nodata)
elif band == CIRRUS:
band_dict[band] = self._create_mask(
clouds_array, cir_id, nodata)
elif band == RAW_CLOUDS:
band_dict[band] = clouds_array
else:
raise InvalidTypeError(
f"Non existing cloud band for Sentinel-3 SLSTR: {band}"
)
return band_dict