from typing import List, Dict
from uuid import UUID
from datacube import Datacube
from datacube.model import Dataset as ODCDataset
import numpy as np
import rasterio as rio
import xarray as xa
import cfsi
from cfsi.exceptions import ProductNotFoundException
from cfsi.utils.load_datasets import xadataset_from_odcdataset, odcdataset_from_uri
from cfsi.utils.logger import create_logger
from cfsi.utils.write_utils import array_to_geotiff, rio_params_for_xadataset, create_overviews
LOGGER = create_logger("mosaic", level=DEBUG)
# TODO: option to write latest image as reference
config = cfsi.config()
class MosaicCreator:
def __init__(self,
mask_product_name: str,
date_: str = "today",
days: int = 30):
""" Constructor method """
self.__product_name = mask_product_name
self.__use_masks = True
if mask_product_name == "scl":
from typing import List, Optional
from datacube import Datacube
from datacube.model import Dataset as ODCDataset
import numpy as np
import rasterio as rio
from rasterio.warp import reproject, Resampling
from sentinelhub import AwsTile, AwsTileRequest, DataCollection
import cfsi
from cfsi.utils import check_existing_mask_directory, get_s2_tile_ids
from cfsi.utils.logger import create_logger
from cfsi.utils.write_utils import write_l1c_dataset
config = cfsi.config()
LOGGER = create_logger("cloud_mask_generator", level=DEBUG)
class CloudMaskGenerator:
l1c_product_name = "s2_level1c_granule"
def __init__(self):
self.i = 1
self.indexed_masks: List[ODCDataset] = []
self.mask_product_name: Optional[str] = None
self.max_iterations: Optional[int] = None
self.total_iterations: Optional[int] = None
def create_masks(self):
l1c_datasets = self.get_l1c_datasets()
if len(l1c_datasets) < self.max_iterations or not self.max_iterations:
from pathlib import Path
from typing import List, Tuple, Union, Dict
import numpy as np
import xarray as xa
import rasterio as rio
from rasterio.crs import CRS
from rasterio.enums import Resampling
from rasterio.transform import Affine
from cfsi.utils import generate_s2_tif_path
from cfsi.utils.logger import create_logger
from cfsi.utils.load_datasets import xadataset_from_odcdataset
from datacube.model import Dataset as ODCDataset
LOGGER = create_logger("write_utils")
def write_l1c_dataset(dataset: ODCDataset, rgb: bool = True):
""" Writes a ODC S2 L1C dataset to a rgb .tif file """
measurements = None
product_name = "l1c"
if rgb:
measurements = ['B02', 'B03', 'B04']
product_name = "rgb"
output_dir = generate_s2_tif_path(dataset, product_name).parent
if output_dir.exists():
LOGGER.info(f"Directory {product_name} for L1C output already exists, skipping")
return
LOGGER.info(f"Writing L1C output for dataset {dataset}")
ds = xadataset_from_odcdataset(dataset, measurements=measurements)
from hashlib import md5
from pathlib import Path
from typing import Dict
from datacube.model import Dataset as ODCDataset
from cfsi.scripts.index import ODCIndexer
from cfsi.utils.logger import create_logger
LOGGER = create_logger("s2cloudless_index")
class S2CloudlessIndexer(ODCIndexer):
def __init__(self):
""" Set up indexer """
super().__init__("S2CloudlessIndexer")
def generate_eo3_dataset_doc(self, l1c_dataset: ODCDataset,
s2cloudless_masks: Dict[str, Path]):
""" Generates and returns a s2cloudless eo3 metadata document """
uri = self._generate_mask_uri(s2cloudless_masks)
measurements = {
name: {
"path": str(mask_path)
}
for name, mask_path in s2cloudless_masks.items()
}
properties, grids = self.generate_mask_properties(l1c_dataset)
eo3 = {
"id": md5(str(uri).encode("utf-8")).hexdigest(),
from logging import DEBUG
from pathlib import Path
from datacube.model import Dataset as ODCDataset
from fmask.cmdline import sentinel2Stacked
import cfsi
from cfsi.scripts.index.fmask_index import FmaskIndexer
from cfsi.scripts.masks.cloud_mask_generator import CloudMaskGenerator
from cfsi.utils.logger import create_logger
from cfsi.utils import get_s2_tile_ids, generate_s2_tif_path
config = cfsi.config()
LOGGER = create_logger("fmask", level=DEBUG)
class FmaskGenerator(CloudMaskGenerator):
def __init__(self):
super().__init__()
self.max_iterations = config.masks.fmask_masks.max_iterations
self.mask_product_name = "s2_level1c_fmask"
def _create_mask(self, l1c_dataset: ODCDataset) -> bool:
if not config.masks.fmask_masks.generate:
LOGGER.info("Skipping Fmask mask generation due to config")
return False
if not self._should_process(l1c_dataset):
return True
LOGGER.info(
f"Iteration {self.i}/{self.total_iterations} ({self.max_iterations}): {l1c_dataset.uris[0]}"
from pathlib import Path
from queue import Empty
from typing import Dict, List
from xml.etree import ElementTree
from hashlib import md5
import cfsi
from cfsi.scripts.index import ODCIndexer
from cfsi.utils.logger import create_logger
from cfsi.constants import (GUARDIAN, L2A_BUCKET, S2_MEASUREMENTS,
S2_PRODUCT_NAMES)
config = cfsi.config()
LOGGER = create_logger("s2_index")
class S2Indexer(ODCIndexer):
""" Index Sentinel 2 scenes from AWS S3 """
def __init__(self, name: str = "S2Indexer"):
super().__init__(name)
def add_to_index(self):
""" Index S2 datasets from S3 to ODC """
LOGGER.info("Indexing Sentinel 2 datasets from S3")
for bucket_name in config.index.s2_index.s3_buckets:
LOGGER.info(f"Indexing bucket {bucket_name}")
self.__index_s3_bucket(bucket_name)
LOGGER.info(f"Bucket {bucket_name} indexed")
from urllib.parse import urlparse
from xml.etree import ElementTree
from boto3 import Session
from datacube import Datacube
from datacube.index.hl import Doc2Dataset
from datacube.model import Dataset as ODCDataset
from datacube.utils import changes
from datacube.utils.changes import DocumentMismatchError
from cfsi.exceptions import ProductNotFoundException
from cfsi.utils.load_datasets import odcdataset_from_uri
from cfsi.utils.logger import create_logger
from cfsi.utils.utils import swap_s2_bucket_names
LOGGER = create_logger("ODCIndexer")
class ODCIndexer:
""" Index data to CFSI ODC - base class """
def __init__(self, name: str = "ODCIndexer"):
""" Sets up the indexer """
self.dc: Datacube = Datacube(app=name)
self.session: Session = Session(
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
region_name='eu-central-1')
def index_masks(self, l1c_dataset: ODCDataset,
mask_output: Dict[str, Path]) -> ODCDataset:
""" Indexes output cloud masks to ODC.
from hashlib import md5
from pathlib import Path
from typing import Dict
import rasterio
import xarray as xa
from cfsi.scripts.index import ODCIndexer
from cfsi.utils.logger import create_logger
LOGGER = create_logger("mosaic_index", level=10)
class MosaicIndexer(ODCIndexer):
""" Index output cloudless mosaics to ODC """
def __init__(self):
super().__init__("MosaicIndexer")
def index_mosaic(self, mosaic_ds: xa.Dataset, file_path: Path):
""" Create new cloudless mosaic ODCDataset from mosaic in Path """
eo3_doc = self.generate_eo3_dataset_doc(mosaic_ds, file_path)
dataset, exception = self.add_dataset(eo3_doc)
if not exception:
LOGGER.info(f"Indexed cloudless mosaic {dataset}")
@staticmethod
def generate_eo3_dataset_doc(mosaic_ds: xa.Dataset,
file_path: Path) -> Dict:
""" Generates and returns a cloudless mosaic eo3 metadata document """
mask_name = file_path.name.split("_")[
-2] # TODO: more robust mask name checking