def cli(thredds_catalogue, skips, select, workers, outfile):
""" Download Metadata from THREDDS server to tarball
Example:
\b
Download files in directory that match `*yaml` and store them as a tar
> thredds-to-tar -c "http://dapds00.nci.org.au/thredds/catalog/if87/2018-11-29/"
-t ".*ARD-METADATA.yaml" -s '.*NBAR.*' -s '.*SUPPLEMENTARY.*'
-s '.*NBART.*' -s '.*/QA/.*' -w 8 --outfile 2018-11-29.tar.gz
"""
print("Searching {thredds_catalogue} for matching files".format(
thredds_catalogue=thredds_catalogue))
urls = thredds_find_glob(thredds_catalogue, skips, [select], workers)
print("Found {0} metadata urls".format(str(len(urls))))
yamls = download_yamls(urls, workers)
# jam it all in a tar
tar_opts = dict(name=outfile,
mode='w' + tar_mode(gzip=True, xz=True, is_pipe=False))
with tarfile.open(**tar_opts) as tar:
for yaml in yamls:
add_txt_file(tar=tar, content=yaml[0], fname=yaml[1])
print("Done!")
def cli(
skip_lineage: bool,
fail_on_missing_lineage: bool,
verify_lineage: bool,
uri: str,
product: str,
):
skips = [".*NBAR.*", ".*SUPPLEMENTARY.*", ".*NBART.*", ".*/QA/.*"]
select = [".*ARD-METADATA.yaml"]
candidate_products = product.split()
print(f"Crawling {uri} on Thredds")
print(f"Matching to {candidate_products}")
yaml_urls = thredds_find_glob(uri, skips, select)
print(f"Found {len(yaml_urls)} datasets")
yaml_contents = download_yamls(yaml_urls)
# Consume generator and fetch YAML's
dc = Datacube()
added, failed = dump_list_to_odc(
yaml_contents,
dc,
candidate_products,
skip_lineage=skip_lineage,
fail_on_missing_lineage=fail_on_missing_lineage,
verify_lineage=verify_lineage,
)
print(f"Added {added} Datasets, Failed {failed} Datasets")
def test_download_yaml():
"""Test pass/fail arms of YAML download from Thredds"""
test_urls = [
"http://dapds00.nci.org.au/thredds/fileServer/if87/2018-11-29/S2A_OPER_MSI_ARD_TL_EPAE_20181129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml",
"http://dapds00.nci.org.au/thredds/fileServer/if87/2028-11-29/S2A_OPER_MSI_ARD_TL_EPAE_20281129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml",
"http://downtime00.nci.org.au/thredds/fileServer/if87/2018-11-29/S2A_OPER_MSI_ARD_TL_EPAE_20181129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml",
]
results = download_yamls(test_urls)
assert results
assert len(results) == 3
print(results)
assert results[0][0] is not None
assert results[1][0] is None
assert results[1][2] == "Yaml not found"
assert results[2][2] == "Thredds Failed"