def _set_product_type(self) -> None: """Set products type""" # Product type if self.name[7] != "1": raise InvalidTypeError( "Only L1 products are used for Sentinel-3 data.") if "OL" in self.name: # Instrument self._instrument_name = S3Instrument.OLCI # Data type if S3DataTypes.EFR.value in self.name: self._data_type = S3DataTypes.EFR self.product_type = S3ProductType.OLCI_EFR else: raise InvalidTypeError( "Only EFR data type is used for Sentinel-3 OLCI data.") # Bands self.band_names.map_bands({ obn.CA: "02", obn.BLUE: "03", obn.GREEN: "06", obn.RED: "08", obn.VRE_1: "11", obn.VRE_2: "12", obn.VRE_3: "16", obn.NIR: "17", obn.NARROW_NIR: "17", obn.WV: "20", obn.FAR_NIR: "21", }) elif "SL" in self.name: # Instrument self._instrument_name = S3Instrument.SLSTR # Data type if S3DataTypes.RBT.value in self.name: self._data_type = S3DataTypes.RBT self.product_type = S3ProductType.SLSTR_RBT else: raise InvalidTypeError( "Only RBT data type is used for Sentinel-3 SLSTR data.") # Bands self.band_names.map_bands({ obn.GREEN: "1", # radiance, 500m obn.RED: "2", # radiance, 500m obn.NIR: "3", # radiance, 500m obn.NARROW_NIR: "3", # radiance, 500m obn.SWIR_CIRRUS: "4", # radiance, 500m obn.SWIR_1: "5", # radiance, 500m obn.SWIR_2: "6", # radiance, 500m obn.MIR: "7", # brilliance temperature, 1km obn.TIR_1: "8", # brilliance temperature, 1km obn.TIR_2: "9", # brilliance temperature, 1km }) else: raise InvalidProductError(f"Invalid Sentinel-3 name: {self.name}")
def _set_sensor_mode(self) -> None: """ Get products type from RADARSAT-2 products name (could check the metadata too) """ # Get metadata root, namespace = self.read_mtd() # Get sensor mode sensor_mode_xml = None for element in root: if element.tag == namespace + "sourceAttributes": radar_param = element.find(namespace + "radarParameters") # WARNING: this word may differ from the Enum !!! (no docs available) # Get the closest match sensor_mode_xml = radar_param.findtext(namespace + "acquisitionType") break if sensor_mode_xml: sensor_mode = difflib.get_close_matches( sensor_mode_xml, Rs2SensorMode.list_values())[0] try: self.sensor_mode = Rs2SensorMode.from_value(sensor_mode) except ValueError as ex: raise InvalidTypeError( f"Invalid sensor mode for {self.name}") from ex else: raise InvalidTypeError(f"Invalid sensor mode for {self.name}")
def _get_band_filename(self, band: Union[obn, str]) -> str: """ Get band filename from its band type Args: band ( Union[obn, str]): Band as an OpticalBandNames or directly the snap_name Returns: str: Band name """ if isinstance(band, obn): snap_name = self._get_snap_band_name(band) elif isinstance(band, str): snap_name = band else: raise InvalidTypeError( "The given band should be an OpticalBandNames or directly the snap_name" ) # Remove _an/_in for SLSTR products if self._data_type == S3DataTypes.RBT: if "cloud" not in snap_name: snap_name = snap_name[:-3] elif "an" in snap_name: snap_name = snap_name[:-3] + "_RAD" else: # in snap_name = snap_name[:-3] + "_BT" return snap_name
def _get_slstr_quality_flags_name(self, band: obn) -> str: """ Get SNAP band name. Args: band (obn): Band as an OpticalBandNames Returns: str: Quality flag name with SNAP format """ # Get band number band_nb = self.band_names[band] if band_nb is None: raise InvalidProductError( f"Non existing band ({band.name}) for S3-{self._data_type.name} products" ) # Get quality flag name if self._data_type == S3DataTypes.RBT: snap_bn = f"S{band_nb}_exception_{'i' if band in BT_BANDS else 'a'}n" else: raise InvalidTypeError( f"This function only works for Sentinel-3 SLSTR data: {self._data_type}" ) return snap_bn
def _get_snap_band_name(self, band: obn) -> str: """ Get SNAP band name. Args: band (obn): Band as an OpticalBandNames Returns: str: Band name with SNAP format """ # Get band number band_nb = self.band_names[band] if band_nb is None: raise InvalidProductError( f"Non existing band ({band.name}) for S3-{self._data_type.name} products" ) # Get band name if self._data_type == S3DataTypes.EFR: snap_bn = f"Oa{band_nb}_reflectance" # Converted into reflectance previously in the graph elif self._data_type == S3DataTypes.RBT: if band in BT_BANDS: snap_bn = f"S{band_nb}_BT_in" else: snap_bn = f"S{band_nb}_reflectance_an" # Conv into reflectance previously in the graph else: raise InvalidTypeError( f"Unknown data type for Sentinel-3 data: {self._data_type}") return snap_bn
def to_value_list(cls, name_list: list = None) -> list: """ Get a list from the values of the bands .. code-block:: python >>> SarBandNames.to_name_list([SarBandNames.HV_DSPK, SarBandNames.VV]) ['HV_DSPK', 'VV'] >>> SarBandNames.to_name_list() ['VV', 'VV_DSPK', 'HH', 'HH_DSPK', 'VH', 'VH_DSPK', 'HV', 'HV_DSPK'] Args: name_list (list): List of band names Returns: list: List of band values """ if name_list: out_list = [] for key in name_list: if isinstance(key, str): out_list.append(getattr(cls, key).value) elif isinstance(key, cls): out_list.append(key.value) else: raise InvalidTypeError( "The list should either contain strings or SarBandNames" ) else: out_list = cls.list_values() return out_list
def from_list(cls, name_list: Union[list, str]) -> list: """ Get the band enums from list of band names .. code-block:: python >>> SarBandNames.from_list("VV") [<SarBandNames.VV: 'VV'>] Args: name_list (Union[list, str]): List of names Returns: list: List of enums """ if not isinstance(name_list, list): name_list = [name_list] try: band_names = [cls(name) for name in name_list] except ValueError as ex: raise InvalidTypeError( f"Band names ({name_list}) should be chosen among: {cls.list_names()}" ) from ex return band_names
def map_bands(self, band_map: dict) -> None: """ Mapping band names to specific satellite band numbers, as strings. .. code-block:: python >>> # Example for Sentinel-2 L1C data >>> ob = OpticalBands() >>> ob.map_bands({ CA: '01', BLUE: '02', GREEN: '03', RED: '04', VRE_1: '05', VRE_2: '06', VRE_3: '07', NIR: '08', NNIR: '8A', WV: '09', SWIR_1: '11', SWIR_2: '12' }) Args: band_map (dict): Band mapping as {OpticalBandNames: Band number for loading band} """ for band_name, band_nb in band_map.items(): if band_name not in self._band_map or not isinstance( band_name, OpticalBandNames): raise InvalidTypeError( f"{band_name} should be an OpticalBandNames object") # Set number self._band_map[band_name] = band_nb
def _set_resolution(self) -> float: """ Set product default resolution (in meters) """ def_res = None # Read metadata for default resolution try: root, _ = self.read_mtd() def_res = float(root.findtext(".//GroundRangeGeometricResolution")) except (InvalidProductError, TypeError): pass # If we cannot read it in MTD, initiate survival mode if not def_res: if self.sensor_mode == CskSensorMode.S2: def_res = 1.0 elif self.sensor_mode == CskSensorMode.HI: def_res = 5.0 elif self.sensor_mode == CskSensorMode.PP: def_res = 20.0 elif self.sensor_mode == CskSensorMode.WR: def_res = 30.0 elif self.sensor_mode == CskSensorMode.HR: def_res = 100.0 else: raise InvalidTypeError( f"Unknown sensor mode {self.sensor_mode}") return def_res
def get_default_band(self) -> BandNames: """ Get default band: The first existing one between `VV` and `HH` for SAR data. .. code-block:: python >>> from eoreader.reader import Reader >>> path = r"S1A_IW_GRDH_1SDV_20191215T060906_20191215T060931_030355_0378F7_3696.zip" >>> prod = Reader().open(path) >>> prod.get_default_band() <SarBandNames.VV: 'VV'> Returns: str: Default band """ existing_bands = self._get_raw_bands() if not existing_bands: raise InvalidProductError( f"No band exists for products: {self.name}") # The order matters, as we almost always prefer VV and HH if sbn.VV in existing_bands: default_band = sbn.VV elif sbn.HH in existing_bands: default_band = sbn.HH elif sbn.VH in existing_bands: default_band = sbn.VH elif sbn.HV in existing_bands: default_band = sbn.HV else: raise InvalidTypeError( f"Invalid bands for products: {existing_bands}") return default_band
def _set_resolution(self) -> float: """ Set product default resolution (in meters) .. WARNING:: We assume being in High Resolution (except for WV where we must be in medium resolution) """ def_res = None # Read metadata try: root, _ = self.read_mtd() def_res = float(root.findtext(".//rangePixelSpacing")) except (InvalidProductError, TypeError): pass # If we cannot read it in MTD, initiate survival mode if not def_res: if self.sensor_mode in [S1SensorMode.SM, S1SensorMode.IW]: def_res = 10.0 elif self.sensor_mode in [S1SensorMode.EW, S1SensorMode.WV]: def_res = 25.0 else: raise InvalidTypeError( f"Unknown sensor mode {self.sensor_mode}") LOGGER.debug( f"Default resolution is set to {def_res}. " f"The product is considered being in " f"{'Medium' if self.sensor_mode == S1SensorMode.WV else 'High'}-Resolution" ) return def_res
def _set_resolution(self) -> float: """ Set product default resolution (in meters) """ def_res = None # Read metadata try: root, nsmap = self.read_mtd() namespace = nsmap[None] def_res = float(root.findtext(f"{namespace}sampledPixelSpacing")) except (InvalidProductError, TypeError): pass # If we cannot read it in MTD, initiate survival mode if not def_res: if self.sensor_mode == Rs2SensorMode.SLA: def_res = 1.0 if self.product_type == Rs2ProductType.SGX else 0.5 elif self.sensor_mode in [Rs2SensorMode.U, Rs2SensorMode.WU]: def_res = 1.0 if self.product_type == Rs2ProductType.SGX else 1.56 elif self.sensor_mode in [ Rs2SensorMode.MF, Rs2SensorMode.WMF, Rs2SensorMode.F, Rs2SensorMode.WF, ]: def_res = 3.13 if self.product_type == Rs2ProductType.SGX else 6.25 elif self.sensor_mode == Rs2SensorMode.XF: def_res = 2.0 if self.product_type == Rs2ProductType.SGX else 3.13 if self.product_type in [Rs2ProductType.SGF, Rs2ProductType.SGX]: LOGGER.debug( "This product is considered to have one look (not checked in metadata)" ) # TODO elif self.sensor_mode in [Rs2SensorMode.S, Rs2SensorMode.EH]: def_res = 8.0 if self.product_type == Rs2ProductType.SGX else 12.5 elif self.sensor_mode in [Rs2SensorMode.W, Rs2SensorMode.EL]: def_res = 10.0 if self.product_type == Rs2ProductType.SGX else 12.5 elif self.sensor_mode in [Rs2SensorMode.FQ, Rs2SensorMode.WQ]: def_res = 3.13 elif self.sensor_mode in [Rs2SensorMode.SQ, Rs2SensorMode.WSQ]: raise NotImplementedError( "Not squared pixels management are not implemented in EOReader." ) elif self.sensor_mode == Rs2SensorMode.SCN: def_res = 25.0 elif self.sensor_mode == Rs2SensorMode.SCW: def_res = 50.0 elif self.sensor_mode == Rs2SensorMode.DVWF: def_res = 40.0 if self.product_type == Rs2ProductType.SCF else 20.0 elif self.sensor_mode == Rs2SensorMode.SCW: if self.product_type == Rs2ProductType.SCF: def_res = 50.0 else: raise NotImplementedError( "Not squared pixels management are not implemented in EOReader." ) else: raise InvalidTypeError(f"Unknown sensor mode {self.sensor_mode}") return def_res
def _set_resolution(self) -> float: """ Set product default resolution (in meters) """ def_res = None # Read metadata try: root, _ = self.read_mtd() for element in root: if element.tag == "ProductCharacteristics": def_res = float( element.findtext("GroundRangeGeometricResolution")) break except (InvalidProductError, AttributeError): pass # If we cannot read it in MTD, initiate survival mode if not def_res: if self.sensor_mode == CskSensorMode.S2: def_res = 1.0 elif self.sensor_mode == CskSensorMode.HI: def_res = 5.0 elif self.sensor_mode == CskSensorMode.PP: def_res = 20.0 elif self.sensor_mode == CskSensorMode.WR: def_res = 30.0 elif self.sensor_mode == CskSensorMode.HR: def_res = 100.0 else: raise InvalidTypeError( f"Unknown sensor mode {self.sensor_mode}") return def_res
def _load_e_tm_clouds(self, qa_arr: XDS_TYPE, band_list: Union[list, BandNames]) -> dict: """ Load cloud files as numpy arrays with the same resolution (and same metadata). Read Landsat-(E)TM clouds from QA mask. See here for clouds_values: - (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band] - (COL 2 TM)[https://www.usgs.gov/media/files/landsat-4-5-tm-collection-2-level-1-data-format-control-book] - (COL 2 ETM)[https://www.usgs.gov/media/files/landsat-7-etm-collection-2-level-1-data-format-control-book] Args: qa_arr (XDS_TYPE): Quality array band_list (list): List of the wanted bands Returns: dict, dict: Dictionary {band_name, band_array} """ bands = {} # Get clouds and nodata nodata = None cld = None shd = None if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list): if self._collection == LandsatCollection.COL_1: # Bit id nodata_id = 0 cloud_id = 4 # Clouds with high confidence shd_conf_1_id = 7 shd_conf_2_id = 8 nodata, cld, shd_conf_1, shd_conf_2 = rasters.read_bit_array( qa_arr, [nodata_id, cloud_id, shd_conf_1_id, shd_conf_2_id]) shd = shd_conf_1 & shd_conf_2 else: # Bit ids nodata_id = 0 cloud_id = 3 # Clouds with high confidence shd_id = 4 # Shadows with high confidence nodata, cld, shd = rasters.read_bit_array( qa_arr, [nodata_id, cloud_id, shd_id]) for band in band_list: if band == ALL_CLOUDS: bands[band] = self._create_mask(qa_arr, cld | shd, nodata) elif band == SHADOWS: bands[band] = self._create_mask(qa_arr, shd, nodata) elif band == CLOUDS: bands[band] = self._create_mask(qa_arr, cld, nodata) elif band == RAW_CLOUDS: bands[band] = qa_arr else: raise InvalidTypeError( f"Non existing cloud band for Landsat-(E)TM sensor: {band}" ) return bands
def _set_sensor_mode(self) -> None: """ Get products type from TerraSAR-X products name (could check the metadata too) """ # Get sensor mode try: self.sensor_mode = TsxSensorMode.from_value(self.split_name[4]) except ValueError as ex: raise InvalidTypeError(f"Invalid sensor mode for {self.name}") from ex
def _load_clouds(self, bands: list, resolution: float = None, size: Union[list, tuple] = None) -> dict: """ Load cloud files as numpy arrays with the same resolution (and same metadata). Read S2 cloud mask .GML files (both valid for L2A and L1C products). https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi/level-1c/cloud-masks Args: bands (list): List of the wanted bands resolution (int): Band resolution in meters size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided. Returns: dict: Dictionary {band_name, band_xarray} """ band_dict = {} if bands: def_band = self.get_default_band() cloud_vec = self.open_mask("CLOUDS", "00") # Open a bands to mask it def_band = self.load(def_band, resolution=resolution)[def_band] nodata = np.where(np.isnan(def_band), 1, 0) for band in bands: if band == ALL_CLOUDS: band_dict[band] = self._rasterize(def_band, cloud_vec, nodata) elif band == CIRRUS: try: cirrus = cloud_vec[cloud_vec.maskType == "CIRRUS"] except AttributeError: # No masktype -> empty cirrus = gpd.GeoDataFrame(geometry=[], crs=cloud_vec.crs) band_dict[band] = self._rasterize(def_band, cirrus, nodata) elif band == CLOUDS: try: clouds = cloud_vec[cloud_vec.maskType == "OPAQUE"] except AttributeError: # No masktype -> empty clouds = gpd.GeoDataFrame(geometry=[], crs=cloud_vec.crs) band_dict[band] = self._rasterize(def_band, clouds, nodata) elif band == RAW_CLOUDS: band_dict[band] = self._rasterize(def_band, cloud_vec, nodata) else: raise InvalidTypeError( f"Non existing cloud band for Sentinel-2: {band}") return band_dict
def _load(self, bands: list, resolution: float = None, size: Union[list, tuple] = None) -> dict: """ Core function loading SAR data bands Args: bands (list): Band list resolution (float): Resolution of the band, in meters size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided. Returns: Dictionary {band_name, band_xarray} """ band_list = [] dem_list = [] for band in bands: if is_index(band): raise NotImplementedError( "For now, no index is implemented for SAR data.") elif is_optical_band(band): raise TypeError( f"You should ask for SAR bands as {self.name} is a SAR product." ) elif is_sar_band(band): if not self.has_band(band): raise InvalidBandError( f"{band} cannot be retrieved from {self.condensed_name}" ) else: band_list.append(band) elif is_dem(band): dem_list.append(band) elif is_clouds(band): raise NotImplementedError( f"Clouds cannot be retrieved from SAR data ({self.condensed_name})." ) else: raise InvalidTypeError( f"{band} is neither a band nor an index !") # Check if DEM is set and exists if dem_list: self._check_dem_path() # Load bands bands = self._load_bands(band_list, resolution=resolution, size=size) # Add DEM bands.update(self._load_dem(dem_list, resolution=resolution, size=size)) return bands
def _set_sensor_mode(self) -> None: """ Get products type from RADARSAT-2 products name (could check the metadata too) """ # Get metadata root, nsmap = self.read_mtd() namespace = nsmap[None] # Get sensor mode # WARNING: this word may differ from the Enum !!! (no docs available) # Get the closest match sensor_mode_xml = root.findtext(f".//{namespace}acquisitionType") if sensor_mode_xml: sensor_mode = difflib.get_close_matches( sensor_mode_xml, Rs2SensorMode.list_values() )[0] try: self.sensor_mode = Rs2SensorMode.from_value(sensor_mode) except ValueError as ex: raise InvalidTypeError(f"Invalid sensor mode for {self.name}") from ex else: raise InvalidTypeError(f"Invalid sensor mode for {self.name}")
def get_mean_sun_angles(self) -> (float, float): """ Get Mean Sun angles (Azimuth and Zenith angles) .. code-block:: python >>> from eoreader.reader import Reader >>> path = "S3B_SL_1_RBT____20191115T233722_20191115T234022_20191117T031722_0179_032_144_3420_LN2_O_NT_003.SEN3" >>> prod = Reader().open(path) >>> prod.get_mean_sun_angles() (78.55043955912154, 31.172127033319388) Returns: (float, float): Mean Azimuth and Zenith angle """ if self._data_type == S3DataTypes.EFR: geom_file = os.path.join(self.path, "tie_geometries.nc") sun_az = "SAA" sun_ze = "SZA" elif self._data_type == S3DataTypes.RBT: geom_file = os.path.join(self.path, "geometry_tn.nc") # Only use nadir files sun_az = "solar_azimuth_tn" sun_ze = "solar_zenith_tn" else: raise InvalidTypeError( f"Unknown/Unsupported data type for Sentinel-3 data: {self._data_type}" ) # Open file if os.path.isfile(geom_file): # Bug pylint with netCDF4 # pylint: disable=E1101 netcdf_ds = netCDF4.Dataset(geom_file) # Get variables sun_az_var = netcdf_ds.variables[sun_az] sun_ze_var = netcdf_ds.variables[sun_ze] # Get sun angles as the mean of whole arrays azimuth_angle = float(np.mean(sun_az_var[:])) zenith_angle = float(np.mean(sun_ze_var[:])) # Close dataset netcdf_ds.close() else: raise InvalidProductError(f"Geometry file {geom_file} not found") return azimuth_angle, zenith_angle
def _get_sar_product_type( self, prod_type_pos: int, gdrg_types: Union[ListEnum, list], cplx_types: Union[ListEnum, list], ) -> None: """ Get products type, special function for SAR satellites. Args: prod_type_pos (int): Position of the products type in the file name gdrg_types (Union[ListEnum, list]): Ground Range products types cplx_types (Union[ListEnum, list]): Complex products types """ # Get and check products type class if not isinstance(gdrg_types, list): gdrg_types = [gdrg_types] if not isinstance(cplx_types, list): cplx_types = [cplx_types] all_types = gdrg_types + cplx_types prod_type_class = all_types[0].__class__ assert all( isinstance(prod_type, prod_type_class) for prod_type in all_types) # Get products type try: # All products types can be found in the filename and are 3 characters long self.product_type = prod_type_class.from_value( self.split_name[prod_type_pos][:3]) except ValueError as ex: raise InvalidTypeError( f"Invalid products type for {self.name}") from ex # Link to SAR generic products types if self.product_type in gdrg_types: self.sar_prod_type = SarProductType.GDRG elif self.product_type in cplx_types: self.sar_prod_type = SarProductType.CPLX else: self.sar_prod_type = SarProductType.OTHER # Discard invalid products types if self.sar_prod_type == SarProductType.OTHER: raise NotImplementedError( f"{self.product_type.value} product type is not available ({self.name})" )
def _load_mss_clouds(self, qa_arr: XDS_TYPE, band_list: list) -> dict: """ Load cloud files as numpy arrays with the same resolution (and same metadata). Read Landsat-MSS clouds from QA mask. See here for clouds_values: - (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band] - (COL 2)[https://www.usgs.gov/media/files/landsat-1-5-mss-collection-2-level-1-data-format-control-book] Args: qa_arr (XDS_TYPE): Quality array band_list (list): List of the wanted bands Returns: dict, dict: Dictionary {band_name, band_array} """ bands = {} # Get clouds and nodata nodata_id = 0 cloud_id = (4 if self._collection == LandsatCollection.COL_1 else 3 ) # Clouds with high confidence clouds = None if ALL_CLOUDS in band_list or CLOUDS in band_list: nodata, cld = rasters.read_bit_array(qa_arr, [nodata_id, cloud_id]) clouds = self._create_mask(qa_arr, cld, nodata) for band in band_list: if band == ALL_CLOUDS: bands[band] = clouds elif band == CLOUDS: bands[band] = clouds elif band == RAW_CLOUDS: bands[band] = qa_arr else: raise InvalidTypeError( f"Non existing cloud band for Landsat-MSS sensor: {band}") return bands
def _get_band_from_filename(self, band_filename: str) -> obn: """ Get band from filename Args: band_filename (str): Band filename Returns: obn: Band name with SNAP format """ # Get band name if self._data_type == S3DataTypes.EFR: band_nb = band_filename[2:4] elif self._data_type == S3DataTypes.RBT: band_nb = band_filename[1] else: raise InvalidTypeError( f"Invalid Sentinel-3 datatype: {self._data_type}") # Get band band = list(self.band_names.keys())[list( self.band_names.values()).index(band_nb)] return band
def _load_olci_clouds(self, qa_arr: XDS_TYPE, band_list: Union[list, BandNames]) -> dict: """ Load cloud files as xarrays. Read Landsat-OLCI clouds from QA mask. See here for clouds_values: - (COL 1)[https://www.usgs.gov/land-resources/nli/landsat/landsat-collection-1-level-1-quality-assessment-band] - (COL 2)[https://www.usgs.gov/media/files/landsat-8-level-1-data-format-control-book] Args: qa_arr (XDS_TYPE): Quality array band_list (list): List of the wanted bands Returns: dict, dict: Dictionary {band_name, band_array} """ bands = {} # Get clouds and nodata nodata = None cld = None shd = None cir = None if any(band in [ALL_CLOUDS, CLOUDS, SHADOWS] for band in band_list): if self._collection == LandsatCollection.COL_1: # Bit ids nodata_id = 0 cloud_id = 4 # Clouds with high confidence shd_conf_1_id = 7 shd_conf_2_id = 8 cir_conf_1_id = 11 cir_conf_2_id = 12 # Read binary mask ( nodata, cld, shd_conf_1, shd_conf_2, cir_conf_1, cir_conf_2, ) = rasters.read_bit_array( qa_arr, [ nodata_id, cloud_id, shd_conf_1_id, shd_conf_2_id, cir_conf_1_id, cir_conf_2_id, ], ) shd = shd_conf_1 & shd_conf_2 cir = cir_conf_1 & cir_conf_2 else: # Bit ids nodata_id = 0 cloud_id = 3 # Clouds with high confidence shd_id = 4 # Shadows with high confidence cir_id = 2 # Cirrus with high confidence nodata, cld, shd, cir = rasters.read_bit_array( qa_arr, [nodata_id, cloud_id, shd_id, cir_id]) for band in band_list: if band == ALL_CLOUDS: bands[band] = self._create_mask(qa_arr, cld | shd | cir, nodata) elif band == SHADOWS: bands[band] = self._create_mask(qa_arr, shd, nodata) elif band == CLOUDS: bands[band] = self._create_mask(qa_arr, cld, nodata) elif band == CIRRUS: bands[band] = self._create_mask(qa_arr, cir, nodata) elif band == RAW_CLOUDS: bands[band] = qa_arr else: raise InvalidTypeError( f"Non existing cloud band for Landsat-OLCI sensor: {band}") return bands
def extent(self) -> gpd.GeoDataFrame: """ Get UTM extent of the tile, managing the case with not orthorectified bands. .. code-block:: python >>> from eoreader.reader import Reader >>> path = "S3B_SL_1_RBT____20191115T233722_20191115T234022_20191117T031722_0179_032_144_3420_LN2_O_NT_003.SEN3" >>> prod = Reader().open(path) >>> prod.utm_extent() geometry 0 POLYGON ((1488846.028 6121896.451, 1488846.028... Returns: gpd.GeoDataFrame: Footprint in UTM """ try: extent = super().extent() except (FileNotFoundError, TypeError) as ex: def get_min_max(substr: str, subdatasets: list) -> (float, float): """ Get min/max of a subdataset array Args: substr: Substring to identfy the subdataset subdatasets: List of subdatasets Returns: float, float: min/max of the subdataset """ path = [path for path in subdatasets if substr in path][0] with rasterio.open(path, "r") as sub_ds: # Open the 4 corners of the array height = sub_ds.height width = sub_ds.width scales = sub_ds.scales pt1 = sub_ds.read(1, window=Window(0, 0, 1, 1)) * scales pt2 = sub_ds.read(1, window=Window(width - 1, 0, width, 1)) * scales pt3 = (sub_ds.read( 1, window=Window(0, height - 1, 1, height)) * scales) pt4 = (sub_ds.read(1, window=Window(width - 1, height - 1, width, height)) * scales) pt_list = [pt1, pt2, pt3, pt4] # Return min and max return np.min(pt_list), np.max(pt_list) if self.product_type == S3ProductType.OLCI_EFR: # Open geodetic_an.nc geom_file = os.path.join( self.path, "geo_coordinates.nc") # Only use nadir files with rasterio.open(geom_file, "r") as geom_ds: lat_min, lat_max = get_min_max("latitude", geom_ds.subdatasets) lon_min, lon_max = get_min_max("longitude", geom_ds.subdatasets) elif self.product_type == S3ProductType.SLSTR_RBT: # Open geodetic_an.nc geom_file = os.path.join( self.path, "geodetic_an.nc") # Only use nadir files with rasterio.open(geom_file, "r") as geom_ds: lat_min, lat_max = get_min_max("latitude_an", geom_ds.subdatasets) lon_min, lon_max = get_min_max("longitude_an", geom_ds.subdatasets) else: raise InvalidTypeError( f"Invalid products type {self.product_type}") from ex # Create wgs84 extent (left, bottom, right, top) extent_wgs84 = gpd.GeoDataFrame( geometry=[ vectors.from_bounds_to_polygon(lon_min, lat_min, lon_max, lat_max) ], crs=vectors.WGS84, ) # Get upper-left corner and deduce UTM proj from it utm = vectors.corresponding_utm_projection( extent_wgs84.bounds.minx, extent_wgs84.bounds.maxy) extent = extent_wgs84.to_crs(utm) return extent
def to_band(to_convert: list) -> list: """ Convert a string (or real value) to any alias, band or index. You can pass the name or the value of the bands. .. code-block:: python >>> to_band(["NDVI", "GREEN", RED, "VH_DSPK", "SLOPE", DEM, "CLOUDS", CLOUDS]) [<function NDVI at 0x00000154DDB12488>, <OpticalBandNames.GREEN: 'GREEN'>, <OpticalBandNames.RED: 'RED'>, <SarBandNames.VH_DSPK: 'VH_DSPK'>, <DemBandNames.SLOPE: 'SLOPE'>, <DemBandNames.DEM: 'DEM'>, <ClassifBandNames.CLOUDS: 'CLOUDS'>, <ClassifBandNames.CLOUDS: 'CLOUDS'>] Args: to_convert (list): Values to convert into band objects Returns: list: converted values """ if not isinstance(to_convert, list): to_convert = [to_convert] bands = [] for tc in to_convert: band_or_idx = None # Try legit types if isinstance(tc, str): # Try index if hasattr(_idx, tc): band_or_idx = getattr(_idx, tc) else: try: band_or_idx = _sbn.convert_from(tc)[0] except TypeError: try: band_or_idx = _obn.convert_from(tc)[0] except TypeError: try: band_or_idx = _dem.convert_from(tc)[0] except TypeError: try: band_or_idx = _clouds.convert_from(tc)[0] except TypeError: pass elif is_index(tc) or is_band(tc) or is_dem(tc) or is_clouds(tc): band_or_idx = tc # Store it if band_or_idx: bands.append(band_or_idx) else: raise InvalidTypeError(f"Unknown band or index: {tc}") return bands
def _load_clouds( self, bands: list, resolution: float = None, size: Union[list, tuple] = None ) -> dict: """ Load cloud files as numpy arrays with the same resolution (and same metadata). Read S2 Theia cloud mask: https://labo.obs-mip.fr/multitemp/sentinel-2/theias-sentinel-2-l2a-product-format/ > A cloud mask for each resolution (CLM_R1.tif ou CLM_R2.tif): - bit 0 (1) : all clouds except the thinnest and all shadows - bit 1 (2) : all clouds (except the thinnest) - bit 2 (4) : clouds detected via mono-temporal thresholds - bit 3 (8) : clouds detected via multi-temporal thresholds - bit 4 (16) : thinnest clouds - bit 5 (32) : cloud shadows cast by a detected cloud - bit 6 (64) : cloud shadows cast by a cloud outside image - bit 7 (128) : high clouds detected by 1.38 µm Args: bands (list): List of the wanted bands resolution (int): Band resolution in meters size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided. Returns: dict: Dictionary {band_name, band_xarray} """ band_dict = {} if bands: # Open 20m cloud file if resolution >= 20m res_id = "R2" if resolution >= 20 else "R1" cloud_path = self.get_mask_path("CLM", res_id) clouds_mask = rasters.read( cloud_path, resolution=resolution, size=size, resampling=Resampling.nearest, ) # Get nodata mask nodata = self.open_mask("EDG", res_id, resolution=resolution, size=size) # Bit ids clouds_shadows_id = 0 clouds_id = 1 cirrus_id = 4 shadows_in_id = 5 shadows_out_id = 6 for res_id in bands: if res_id == ALL_CLOUDS: band_dict[res_id] = self._create_mask( clouds_mask, [clouds_shadows_id, cirrus_id], nodata ) elif res_id == SHADOWS: band_dict[res_id] = self._create_mask( clouds_mask, [shadows_in_id, shadows_out_id], nodata ) elif res_id == CLOUDS: band_dict[res_id] = self._create_mask( clouds_mask, clouds_id, nodata ) elif res_id == CIRRUS: band_dict[res_id] = self._create_mask( clouds_mask, cirrus_id, nodata ) elif res_id == RAW_CLOUDS: band_dict[res_id] = clouds_mask else: raise InvalidTypeError( f"Non existing cloud band for Sentinel-2 THEIA: {res_id}" ) return band_dict
def _load_clouds(self, bands: list, resolution: float = None, size: Union[list, tuple] = None) -> dict: """ Load cloud files as xarrays. Read S3 SLSTR clouds from the flags file:cloud netcdf file. https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-3-slstr/level-1/cloud-identification bit_id flag_masks (ushort) flag_meanings === === === 0 1US visible 1 2US 1.37_threshold 2 4US 1.6_small_histogram 3 8US 1.6_large_histogram 4 16US 2.25_small_histogram 5 32US 2.25_large_histogram 6 64US 11_spatial_coherence 7 128US gross_cloud 8 256US thin_cirrus 9 512US medium_high 10 1024US fog_low_stratus 11 2048US 11_12_view_difference 12 4096US 3.7_11_view_difference 13 8192US thermal_histogram 14 16384US spare 15 32768US spare Args: bands (list): List of the wanted bands resolution (int): Band resolution in meters size (Union[tuple, list]): Size of the array (width, height). Not used if resolution is provided. Returns: dict: Dictionary {band_name, band_xarray} """ band_dict = {} if bands: if self._instrument_name == S3Instrument.OLCI: raise InvalidTypeError( "Sentinel-3 OLCI sensor does not provide any cloud file.") all_ids = list(np.arange(0, 14)) cir_id = 8 cloud_ids = [id for id in all_ids if id != cir_id] try: cloud_path = files.get_file_in_dir(self._get_band_folder(), "cloud_RAD.tif") except FileNotFoundError: self._preprocess_s3(resolution) cloud_path = files.get_file_in_dir(self.output, "cloud_RAD.tif") if not cloud_path: raise FileNotFoundError( f"Unable to find the cloud mask for {self.path}") # Open cloud file clouds_array = rasters.read( cloud_path, resolution=resolution, size=size, resampling=Resampling.nearest, masked=False, ).astype(np.uint16) # Get nodata mask # nodata = np.where(np.isnan(clouds_array), 1, 0) nodata = np.where(clouds_array == 65535, 1, 0) for band in bands: if band == ALL_CLOUDS: band_dict[band] = self._create_mask( clouds_array, all_ids, nodata) elif band == CLOUDS: band_dict[band] = self._create_mask( clouds_array, cloud_ids, nodata) elif band == CIRRUS: band_dict[band] = self._create_mask( clouds_array, cir_id, nodata) elif band == RAW_CLOUDS: band_dict[band] = clouds_array else: raise InvalidTypeError( f"Non existing cloud band for Sentinel-3 SLSTR: {band}" ) return band_dict