def run_single_item(catalog, item_id, band_keys, metadata_key, shadow_transform, bbox, macaroon_path): """ Run the shadow-enhancement workflow on a single item of the catalog, include the output files as catalog assets and upload these to the storage. :param catalog: output STAC catalog object :param item_id: ID of the item to work on :param band_keys: list of bands required for the shadow transform :param metadata_key: metadata asset key :param shadow_transform: type of shadow transform algorithm employed :param bbox: bounds of the area of interest :param macaroon_path: path to dCache token :return: STAC item object with assets included """ # configure connection to dCache dcache = stac2dcache.configure(filesystem="dcache", token_filename=macaroon_path) # get item of the catalog where output will be saved item = catalog.get_item(item_id, recursive=True) # get linked item from the scene catalog and get all input assets input_item = _get_linked_object(item, "computed_from") asset_keys = [*band_keys, metadata_key] assets = _get_assets(_get_linked_object(input_item, "parent"), asset_keys, item_id, dcache) metadata = assets.pop(metadata_key) # extract data from bands _band = assets[band_keys[0]] bbox_indices = _get_bbox_indices(_band.x, _band.y, bbox) \ if bbox is not None else None bands = [assets[k] for k in band_keys] bands = bands if bbox is None else [b.rio.slice_xy(*bbox) for b in bands] crs = bands[0].spatial_ref.crs_wkt transform = bands[0].rio.transform() blue, green, red, NIR = [b.squeeze().data for b in bands] # work in temporary directory properties = dict(shadow_transform=shadow_transform) date_created = item.datetime.strftime("%Y:%m:%d %H:%M:%S") with tempfile.TemporaryDirectory(dir="./") as tmpdir: work_path = os.path.abspath(tmpdir) assets = _generate_shadow_images(shadow_transform, metadata, work_path, crs, transform, blue, green, red, NIR, bbox_indices, date_created) for key, href in assets.items(): asset = pystac.Asset(href=href, properties=properties) item.add_asset(key, asset) copy_asset(catalog, asset_key=key, update_catalog=True, item_id=item_id, filesystem_to=dcache, max_workers=1) return catalog.get_item(item_id, recursive=True).assets
def test_copy_asset_updates_catalog(catalog_with_assets): copy_asset(catalog_with_assets, asset_key="tile", update_catalog=True) for item in catalog_with_assets.get_all_items(): item_path = pathlib.Path(item.get_self_href()) item_dir = item_path.parent for asset in item.assets.values(): asset_path = pathlib.Path(asset.get_absolute_href()) # assets should be in the dir folders assert asset_path.name in [el.name for el in item_dir.iterdir()] # assets href should have been updated assert item_dir.as_posix() in asset_path.as_posix()
def test_copy_asset_to_custom_path(catalog_with_assets): with tempfile.TemporaryDirectory() as tmpdir: copy_asset(catalog_with_assets, asset_key="tile", to_uri=tmpdir) tmpdir_path = pathlib.Path(tmpdir) for item in catalog_with_assets.get_all_items(): elements = [el.name for el in (tmpdir_path / item.id).iterdir()] for asset in item.assets.values(): asset_path = pathlib.Path(asset.get_absolute_href()) # assets should be in the tmp dir assert asset_path.name in elements # assets href should have not been updated assert tmpdir_path.as_posix() not in asset_path.as_posix()
def test_copy_asset_for_a_single_item(catalog_with_assets): item_id = "tile_1" copy_asset(catalog_with_assets, asset_key="tile", item_id=item_id) for item in catalog_with_assets.get_all_items(): item_path = pathlib.Path(item.get_self_href()) item_dir = item_path.parent for asset in item.assets.values(): asset_path = pathlib.Path(asset.get_absolute_href()) dir_elements = [el.name for el in item_dir.iterdir()] # assets should be in the dir folders if item.id == item_id: assert asset_path.name in dir_elements else: assert asset_path.name not in dir_elements
def download_assets(catalog, fs_from=None, fs_to=None, asset_keys=None, max_workers=1): """ Download the catalog's assets. :param catalog: PySTAC Catalog object :param fs_from: FS-SPEC-like file system from where to get the assets from :param fs_to: FS-SPEC-like file system where to save the assets :param asset_keys: list of assets to download (if None, copy all assets) :param max_workers: number of parallel processes """ asset_keys = asset_keys or [None] for asset_key in asset_keys: copy_asset( catalog, asset_key, update_catalog=True, # update the catalog's links to the assets filesystem_from=fs_from, filesystem_to=fs_to, max_workers=max_workers)