def list_products(index):
"""
List products that are defined in the index
"""
dc = Datacube(index)
products = dc.list_products()
echo(products.to_string(columns=('name', 'description', 'product_type', 'instrument',
'format', 'platform'),
justify='left'))
def list_products(index):
"""
List products that are defined in the index
"""
dc = Datacube(index)
products = dc.list_products()
echo(
products.to_string(columns=('name', 'description', 'product_type',
'instrument', 'format', 'platform'),
justify='left'))
async def get_products(lon1: float, lat1: float, lon2: float, lat2: float, date1: str = None, date2: str = None, limit: int = 10, days: int = 7):
lon1, lon2 = min(lon1, lon2), max(lon1, lon2)
lat1, lat2 = min(lat1, lat2), max(lat1, lat2)
if not date1 or not date2:
dates = None
else:
dates = (date1, date2)
dc = Datacube(config=config.DATACUBE_CONF)
product = dc.list_products(with_pandas=False)[0]
print(product)
datasets = dc.index.datasets.search(product=product["name"])
print(list(datasets))
for dataset in datasets:
print(dataset)
def add_update_products(dc: Datacube,
csv_path: str,
update_if_exists: Optional[bool] = False) -> List[int]:
# Parse csv file
new_products = [x for x in _parse_csv(csv_path)]
logging.info(f"Found {len(new_products)} products in the CSV {csv_path}")
# List existing products
products = dc.list_products(with_pandas=False)
existing_names = [product["name"] for product in products]
logging.info(f"Found {len(existing_names)} products in the Datacube")
added, updated, failed = 0, 0, 0
for product in new_products:
if product.doc is None:
failed += 1
continue
# Add new products
try:
if product.name not in existing_names:
dc.index.products.add_document(product.doc)
added += 1
logging.info(f"Added product {product.name}")
# Update existing products, if required
elif update_if_exists:
dc.index.products.update_document(product.doc,
allow_unsafe_updates=True)
updated += 1
logging.info(f"Updated product {product.name}")
except Exception as e:
failed += 1
logging.error(
f"Failed to add/update product {product.name} with exception: {e}"
)
# Return results
return added, updated, failed
def cli(
skip_lineage,
fail_on_missing_lineage,
verify_lineage,
stac,
absolute,
update,
update_if_exists,
allow_unsafe,
skip_check,
no_sign_request,
request_payer,
uri,
product,
):
""" Iterate through files in an S3 bucket and add them to datacube"""
transform = None
if stac:
if absolute:
transform = stac_transform_absolute
else:
transform = stac_transform
candidate_products = product.split()
opts = {}
if request_payer:
opts["RequestPayer"] = "requester"
# Check datacube connection and products
dc = Datacube()
odc_products = dc.list_products().name.values
odc_products = set(odc_products)
if not set(candidate_products).issubset(odc_products):
missing_products = list(set(candidate_products) - odc_products)
print(
f"Error: Requested Product/s {', '.join(missing_products)} {'is' if len(missing_products) == 1 else 'are'} "
"not present in the ODC Database",
file=sys.stderr,
)
sys.exit(1)
# Get a generator from supplied S3 Uri for candidate documents
fetcher = S3Fetcher(aws_unsigned=no_sign_request)
document_stream = stream_urls(
s3_find_glob(uri, skip_check=skip_check, s3=fetcher, **opts))
added, failed = dump_to_odc(
fetcher(document_stream),
dc,
candidate_products,
skip_lineage=skip_lineage,
fail_on_missing_lineage=fail_on_missing_lineage,
verify_lineage=verify_lineage,
transform=transform,
update=update,
update_if_exists=update_if_exists,
allow_unsafe=allow_unsafe,
)
print(f"Added {added} datasets and failed {failed} datasets.")
if failed > 0:
sys.exit(failed)
def cli(
skip_lineage,
fail_on_missing_lineage,
verify_lineage,
stac,
update,
allow_unsafe,
skip_check,
no_sign_request,
request_payer,
uri,
product,
):
""" Iterate through files in an S3 bucket and add them to datacube"""
transform = None
if stac:
transform = stac_transform
candidate_products = product.split()
opts = {}
if request_payer:
opts["RequestPayer"] = "requester"
# Get a generator from supplied S3 Uri for metadata definitions
fetcher = S3Fetcher(aws_unsigned=no_sign_request)
# TODO: Share Fetcher
s3_obj_stream = s3_find_glob(uri,
skip_check=skip_check,
s3=fetcher,
**opts)
# Extract URLs from output of iterator before passing to Fetcher
s3_url_stream = (o.url for o in s3_obj_stream)
# TODO: Capture S3 URL's in batches and perform bulk_location_has
# Consume generator and fetch YAML's
dc = Datacube()
odc_products = dc.list_products().name.values
odc_products = set(odc_products)
if not set(candidate_products).issubset(odc_products):
missing_products = list(set(candidate_products) - odc_products)
print(
f"Error: Requested Product/s {', '.join(missing_products)} {'is' if len(missing_products) == 1 else 'are'} "
"not present in the ODC Database",
file=sys.stderr,
)
sys.exit(1)
added, failed = dump_to_odc(
fetcher(s3_url_stream),
dc,
candidate_products,
skip_lineage=skip_lineage,
fail_on_missing_lineage=fail_on_missing_lineage,
verify_lineage=verify_lineage,
transform=transform,
update=update,
allow_unsafe=allow_unsafe,
)
print(f"Added {added} Datasets, Failed {failed} Datasets")
if failed > 0:
sys.exit(failed)
