Exemplo n.º 1
0
    def destination_folder(self, base: Path):
        self._check_enough_properties_to_name()
        # DEA naming conventions folder hierarchy.
        # Example: "ga_ls8c_ard_3/092/084/2016/06/28"

        parts = [self.product_name]

        # Cut the region code in subfolders
        region_code = self.dataset.region_code
        if region_code:
            parts.extend(utils.subfolderise(region_code))

        parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/"))

        # If it's not a final product, append the maturity to the folder.
        maturity: str = self.dataset.properties.get("dea:dataset_maturity")
        if maturity and maturity != "final":
            parts[-1] = f"{parts[-1]}_{maturity}"

        if self.dataset_separator_field is not None:
            val = self.dataset.properties[self.dataset_separator_field]
            # TODO: choosable formatter?
            if isinstance(val, datetime):
                val = f"{val:%Y%m%dT%H%M%S}"
            parts.append(val)
        return base.joinpath(*parts)
Exemplo n.º 2
0
 def destination_folder(self, base: Path):
     self._check_enough_properties_to_name()
     parts = [
         self.product_name,
         self.dataset.dataset_version.replace(".", "-")
     ]
     parts.extend(utils.subfolderise(self.dataset.region_code))
     parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/"))
     return base.joinpath(*parts)
def main(
    output_base: Optional[Path],
    datasets: List[Path],
    overwrite_existing: bool,
    producer: str,
    embed_location: bool,
    source_telemetry: Optional[Path],
    newer_than: datetime,
):
    logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
                        level=logging.INFO)
    with rasterio.Env():
        for ds in datasets:
            if output_base:
                output = output_base.absolute().joinpath(
                    *utils.subfolderise(_dataset_region_code(ds)))
                output.mkdir(parents=True, exist_ok=True)
            else:
                # Alongside the dataset itself.
                output = ds.absolute().parent

            ds_path = _normalise_dataset_path(Path(ds).absolute())
            (mode, ino, dev, nlink, uid, gid, size, atime, mtime,
             ctime) = os.stat(ds)
            create_date = datetime.utcfromtimestamp(ctime)
            if newer_than and (create_date <= newer_than):
                logging.info(
                    "Creation time %s older than start date %s ...SKIPPING %s",
                    newer_than - create_date,
                    newer_than,
                    ds_path.name,
                )
                continue

            logging.info("Processing %s", ds_path)
            output_yaml = output / f"{_dataset_name(ds_path)}.odc-metadata.yaml"

            if output_yaml.exists():
                if not overwrite_existing:
                    logging.info("Output exists: skipping. %s", output_yaml)
                    continue

                logging.info("Output exists: overwriting %s", output_yaml)

            output_uuid, output_path = prepare_and_write(
                ds_path,
                output_yaml,
                producer=producer,
                source_telemetry=source_telemetry,
                embed_location=embed_location,
            )
            logging.info("Wrote dataset %s to %s", output_uuid, output_path)
Exemplo n.º 4
0
    def destination_folder(self, base: Path):
        self._check_enough_properties_to_name()
        # DEA naming conventions folder hierarchy.
        # Example: "ga_ls8c_ard_3/092/084/2016/06/28"

        parts = [self.product_name]

        # Cut the region code in subfolders
        region_code = self.dataset.region_code
        if region_code:
            parts.extend(utils.subfolderise(region_code))

        parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/"))

        return base.joinpath(*parts)
Exemplo n.º 5
0
    def destination_folder(self, base: Path) -> Path:
        self._check_enough_properties_to_name()
        parts = [
            self.product_name,
            self.dataset.dataset_version.replace(".", "-")
        ]
        parts.extend(utils.subfolderise(self.dataset.region_code))
        parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/"))

        if self.dataset_separator_field is not None:
            val = self.dataset.properties[self.dataset_separator_field]
            # TODO: choosable formatter?
            if isinstance(val, datetime):
                val = f"{val:%Y%m%dT%H%M%S}"
            parts.append(val)

        return base.joinpath(*parts)
Exemplo n.º 6
0
 def __get__(self, c: "NamingConventions", owner) -> Optional[str]:
     # Cut the region code in subfolders
     region_code = c.metadata.region_code
     if region_code:
         return "/".join(utils.subfolderise(region_code))
     return None