def download_metadata_command(destination, id, quiet):
"""Creates a 3DEP collection in DESTINATION."""
base_ids = id # not sure how to rename arguments in click
for product in PRODUCTS:
if base_ids:
ids = base_ids
else:
ids = utils.fetch_ids(product)
for id in ids:
path = utils.path(product,
id,
extension="xml",
base=destination)
if os.path.exists(path):
if not quiet:
print("{} exists, skipping download...".format(path))
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
source_path = utils.path(product,
id,
extension="xml",
base=USGS_FTP_BASE)
if not quiet:
print("{} -> {}".format(source_path, path))
text = STAC_IO.read_text(source_path)
with open(path, "w") as f:
f.write(text)
def from_product_and_id(cls,
product: str,
id: str,
base: str = None) -> Metadata:
"""Creates a Metadata from a product and id."""
if base is None:
base = DEFAULT_BASE
href = utils.path(product, id, extension="xml", base=base)
return cls.from_href(href)
def _asset_href_with_extension(self,
base: str,
extension: str,
id_only: bool = False) -> str:
if base is None:
base = DEFAULT_BASE
return utils.path(self.product,
self.id,
base=base,
extension=extension,
id_only=id_only)
def _fetch_ids_from_aws(product: str) -> [str]:
path = os.path.dirname(utils.path(product, ""))
prefix = os.path.join(AWS_PREFIX, path)
client = boto3.client("s3")
paginator = client.get_paginator("list_objects_v2")
page_iterator = paginator.paginate(Bucket=AWS_BUCKET, Prefix=prefix)
filtered_iterator = page_iterator.search(
"Contents[?ends_with(Key, `.xml`)].Key")
# The main tif/xml files for each id are named simply like "USGS_1_n41w106.tif".
# If there's been updates, older versions will have a datetime on the end, e.g.
# "USGS_1_n41w106_20210330.tif" or something similar. By splitting on underscores,
# we're hoping to catch only the "main" files.
return [
os.path.basename(os.path.dirname(key)) for key in filtered_iterator
if len(os.path.basename(key).split("_")) == 3
]
def test_simple(self):
path = utils.path("1", "n41w106")
self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106")
def test_base(self):
path = utils.path("1", "n41w106", base="foo")
self.assertEqual(path, "foo/1/TIFF/n41w106/USGS_1_n41w106")
def test_extension(self):
path = utils.path("1", "n41w106", extension="foo")
self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106.foo")