def run_single_item(catalog, item_id, band_keys, metadata_key,
shadow_transform, bbox, macaroon_path):
"""
Run the shadow-enhancement workflow on a single item of the catalog,
include the output files as catalog assets and upload these to the storage.
:param catalog: output STAC catalog object
:param item_id: ID of the item to work on
:param band_keys: list of bands required for the shadow transform
:param metadata_key: metadata asset key
:param shadow_transform: type of shadow transform algorithm employed
:param bbox: bounds of the area of interest
:param macaroon_path: path to dCache token
:return: STAC item object with assets included
"""
# configure connection to dCache
dcache = stac2dcache.configure(filesystem="dcache",
token_filename=macaroon_path)
# get item of the catalog where output will be saved
item = catalog.get_item(item_id, recursive=True)
# get linked item from the scene catalog and get all input assets
input_item = _get_linked_object(item, "computed_from")
asset_keys = [*band_keys, metadata_key]
assets = _get_assets(_get_linked_object(input_item, "parent"), asset_keys,
item_id, dcache)
metadata = assets.pop(metadata_key)
# extract data from bands
_band = assets[band_keys[0]]
bbox_indices = _get_bbox_indices(_band.x, _band.y, bbox) \
if bbox is not None else None
bands = [assets[k] for k in band_keys]
bands = bands if bbox is None else [b.rio.slice_xy(*bbox) for b in bands]
crs = bands[0].spatial_ref.crs_wkt
transform = bands[0].rio.transform()
blue, green, red, NIR = [b.squeeze().data for b in bands]
# work in temporary directory
properties = dict(shadow_transform=shadow_transform)
date_created = item.datetime.strftime("%Y:%m:%d %H:%M:%S")
with tempfile.TemporaryDirectory(dir="./") as tmpdir:
work_path = os.path.abspath(tmpdir)
assets = _generate_shadow_images(shadow_transform, metadata, work_path,
crs, transform, blue, green, red, NIR,
bbox_indices, date_created)
for key, href in assets.items():
asset = pystac.Asset(href=href, properties=properties)
item.add_asset(key, asset)
copy_asset(catalog, asset_key=key, update_catalog=True,
item_id=item_id, filesystem_to=dcache,
max_workers=1)
return catalog.get_item(item_id, recursive=True).assets
def test_copy_asset_updates_catalog(catalog_with_assets):
copy_asset(catalog_with_assets, asset_key="tile", update_catalog=True)
for item in catalog_with_assets.get_all_items():
item_path = pathlib.Path(item.get_self_href())
item_dir = item_path.parent
for asset in item.assets.values():
asset_path = pathlib.Path(asset.get_absolute_href())
# assets should be in the dir folders
assert asset_path.name in [el.name for el in item_dir.iterdir()]
# assets href should have been updated
assert item_dir.as_posix() in asset_path.as_posix()
def test_copy_asset_to_custom_path(catalog_with_assets):
with tempfile.TemporaryDirectory() as tmpdir:
copy_asset(catalog_with_assets, asset_key="tile", to_uri=tmpdir)
tmpdir_path = pathlib.Path(tmpdir)
for item in catalog_with_assets.get_all_items():
elements = [el.name for el in (tmpdir_path / item.id).iterdir()]
for asset in item.assets.values():
asset_path = pathlib.Path(asset.get_absolute_href())
# assets should be in the tmp dir
assert asset_path.name in elements
# assets href should have not been updated
assert tmpdir_path.as_posix() not in asset_path.as_posix()
def test_copy_asset_for_a_single_item(catalog_with_assets):
item_id = "tile_1"
copy_asset(catalog_with_assets, asset_key="tile", item_id=item_id)
for item in catalog_with_assets.get_all_items():
item_path = pathlib.Path(item.get_self_href())
item_dir = item_path.parent
for asset in item.assets.values():
asset_path = pathlib.Path(asset.get_absolute_href())
dir_elements = [el.name for el in item_dir.iterdir()]
# assets should be in the dir folders
if item.id == item_id:
assert asset_path.name in dir_elements
else:
assert asset_path.name not in dir_elements
def download_assets(catalog,
fs_from=None,
fs_to=None,
asset_keys=None,
max_workers=1):
"""
Download the catalog's assets.
:param catalog: PySTAC Catalog object
:param fs_from: FS-SPEC-like file system from where to get the assets from
:param fs_to: FS-SPEC-like file system where to save the assets
:param asset_keys: list of assets to download (if None, copy all assets)
:param max_workers: number of parallel processes
"""
asset_keys = asset_keys or [None]
for asset_key in asset_keys:
copy_asset(
catalog,
asset_key,
update_catalog=True, # update the catalog's links to the assets
filesystem_from=fs_from,
filesystem_to=fs_to,
max_workers=max_workers)