def destination_folder(self, base: Path): self._check_enough_properties_to_name() # DEA naming conventions folder hierarchy. # Example: "ga_ls8c_ard_3/092/084/2016/06/28" parts = [self.product_name] # Cut the region code in subfolders region_code = self.dataset.region_code if region_code: parts.extend(utils.subfolderise(region_code)) parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) # If it's not a final product, append the maturity to the folder. maturity: str = self.dataset.properties.get("dea:dataset_maturity") if maturity and maturity != "final": parts[-1] = f"{parts[-1]}_{maturity}" if self.dataset_separator_field is not None: val = self.dataset.properties[self.dataset_separator_field] # TODO: choosable formatter? if isinstance(val, datetime): val = f"{val:%Y%m%dT%H%M%S}" parts.append(val) return base.joinpath(*parts)
def destination_folder(self, base: Path): self._check_enough_properties_to_name() parts = [ self.product_name, self.dataset.dataset_version.replace(".", "-") ] parts.extend(utils.subfolderise(self.dataset.region_code)) parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) return base.joinpath(*parts)
def main( output_base: Optional[Path], datasets: List[Path], overwrite_existing: bool, producer: str, embed_location: bool, source_telemetry: Optional[Path], newer_than: datetime, ): logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO) with rasterio.Env(): for ds in datasets: if output_base: output = output_base.absolute().joinpath( *utils.subfolderise(_dataset_region_code(ds))) output.mkdir(parents=True, exist_ok=True) else: # Alongside the dataset itself. output = ds.absolute().parent ds_path = _normalise_dataset_path(Path(ds).absolute()) (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(ds) create_date = datetime.utcfromtimestamp(ctime) if newer_than and (create_date <= newer_than): logging.info( "Creation time %s older than start date %s ...SKIPPING %s", newer_than - create_date, newer_than, ds_path.name, ) continue logging.info("Processing %s", ds_path) output_yaml = output / f"{_dataset_name(ds_path)}.odc-metadata.yaml" if output_yaml.exists(): if not overwrite_existing: logging.info("Output exists: skipping. %s", output_yaml) continue logging.info("Output exists: overwriting %s", output_yaml) output_uuid, output_path = prepare_and_write( ds_path, output_yaml, producer=producer, source_telemetry=source_telemetry, embed_location=embed_location, ) logging.info("Wrote dataset %s to %s", output_uuid, output_path)
def destination_folder(self, base: Path): self._check_enough_properties_to_name() # DEA naming conventions folder hierarchy. # Example: "ga_ls8c_ard_3/092/084/2016/06/28" parts = [self.product_name] # Cut the region code in subfolders region_code = self.dataset.region_code if region_code: parts.extend(utils.subfolderise(region_code)) parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) return base.joinpath(*parts)
def destination_folder(self, base: Path) -> Path: self._check_enough_properties_to_name() parts = [ self.product_name, self.dataset.dataset_version.replace(".", "-") ] parts.extend(utils.subfolderise(self.dataset.region_code)) parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) if self.dataset_separator_field is not None: val = self.dataset.properties[self.dataset_separator_field] # TODO: choosable formatter? if isinstance(val, datetime): val = f"{val:%Y%m%dT%H%M%S}" parts.append(val) return base.joinpath(*parts)
def __get__(self, c: "NamingConventions", owner) -> Optional[str]: # Cut the region code in subfolders region_code = c.metadata.region_code if region_code: return "/".join(utils.subfolderise(region_code)) return None