def download_metadata_command(destination, id, quiet): """Creates a 3DEP collection in DESTINATION.""" base_ids = id # not sure how to rename arguments in click for product in PRODUCTS: if base_ids: ids = base_ids else: ids = utils.fetch_ids(product) for id in ids: path = utils.path(product, id, extension="xml", base=destination) if os.path.exists(path): if not quiet: print("{} exists, skipping download...".format(path)) continue os.makedirs(os.path.dirname(path), exist_ok=True) source_path = utils.path(product, id, extension="xml", base=USGS_FTP_BASE) if not quiet: print("{} -> {}".format(source_path, path)) text = STAC_IO.read_text(source_path) with open(path, "w") as f: f.write(text)
def from_product_and_id(cls, product: str, id: str, base: str = None) -> Metadata: """Creates a Metadata from a product and id.""" if base is None: base = DEFAULT_BASE href = utils.path(product, id, extension="xml", base=base) return cls.from_href(href)
def _asset_href_with_extension(self, base: str, extension: str, id_only: bool = False) -> str: if base is None: base = DEFAULT_BASE return utils.path(self.product, self.id, base=base, extension=extension, id_only=id_only)
def _fetch_ids_from_aws(product: str) -> [str]: path = os.path.dirname(utils.path(product, "")) prefix = os.path.join(AWS_PREFIX, path) client = boto3.client("s3") paginator = client.get_paginator("list_objects_v2") page_iterator = paginator.paginate(Bucket=AWS_BUCKET, Prefix=prefix) filtered_iterator = page_iterator.search( "Contents[?ends_with(Key, `.xml`)].Key") # The main tif/xml files for each id are named simply like "USGS_1_n41w106.tif". # If there's been updates, older versions will have a datetime on the end, e.g. # "USGS_1_n41w106_20210330.tif" or something similar. By splitting on underscores, # we're hoping to catch only the "main" files. return [ os.path.basename(os.path.dirname(key)) for key in filtered_iterator if len(os.path.basename(key).split("_")) == 3 ]
def test_simple(self): path = utils.path("1", "n41w106") self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106")
def test_base(self): path = utils.path("1", "n41w106", base="foo") self.assertEqual(path, "foo/1/TIFF/n41w106/USGS_1_n41w106")
def test_extension(self): path = utils.path("1", "n41w106", extension="foo") self.assertEqual(path, "1/TIFF/n41w106/USGS_1_n41w106.foo")