def list_products(index): """ List products that are defined in the index """ dc = Datacube(index) products = dc.list_products() echo(products.to_string(columns=('name', 'description', 'product_type', 'instrument', 'format', 'platform'), justify='left'))
def list_products(index): """ List products that are defined in the index """ dc = Datacube(index) products = dc.list_products() echo( products.to_string(columns=('name', 'description', 'product_type', 'instrument', 'format', 'platform'), justify='left'))
async def get_products(lon1: float, lat1: float, lon2: float, lat2: float, date1: str = None, date2: str = None, limit: int = 10, days: int = 7): lon1, lon2 = min(lon1, lon2), max(lon1, lon2) lat1, lat2 = min(lat1, lat2), max(lat1, lat2) if not date1 or not date2: dates = None else: dates = (date1, date2) dc = Datacube(config=config.DATACUBE_CONF) product = dc.list_products(with_pandas=False)[0] print(product) datasets = dc.index.datasets.search(product=product["name"]) print(list(datasets)) for dataset in datasets: print(dataset)
def add_update_products(dc: Datacube, csv_path: str, update_if_exists: Optional[bool] = False) -> List[int]: # Parse csv file new_products = [x for x in _parse_csv(csv_path)] logging.info(f"Found {len(new_products)} products in the CSV {csv_path}") # List existing products products = dc.list_products(with_pandas=False) existing_names = [product["name"] for product in products] logging.info(f"Found {len(existing_names)} products in the Datacube") added, updated, failed = 0, 0, 0 for product in new_products: if product.doc is None: failed += 1 continue # Add new products try: if product.name not in existing_names: dc.index.products.add_document(product.doc) added += 1 logging.info(f"Added product {product.name}") # Update existing products, if required elif update_if_exists: dc.index.products.update_document(product.doc, allow_unsafe_updates=True) updated += 1 logging.info(f"Updated product {product.name}") except Exception as e: failed += 1 logging.error( f"Failed to add/update product {product.name} with exception: {e}" ) # Return results return added, updated, failed
def cli( skip_lineage, fail_on_missing_lineage, verify_lineage, stac, absolute, update, update_if_exists, allow_unsafe, skip_check, no_sign_request, request_payer, uri, product, ): """ Iterate through files in an S3 bucket and add them to datacube""" transform = None if stac: if absolute: transform = stac_transform_absolute else: transform = stac_transform candidate_products = product.split() opts = {} if request_payer: opts["RequestPayer"] = "requester" # Check datacube connection and products dc = Datacube() odc_products = dc.list_products().name.values odc_products = set(odc_products) if not set(candidate_products).issubset(odc_products): missing_products = list(set(candidate_products) - odc_products) print( f"Error: Requested Product/s {', '.join(missing_products)} {'is' if len(missing_products) == 1 else 'are'} " "not present in the ODC Database", file=sys.stderr, ) sys.exit(1) # Get a generator from supplied S3 Uri for candidate documents fetcher = S3Fetcher(aws_unsigned=no_sign_request) document_stream = stream_urls( s3_find_glob(uri, skip_check=skip_check, s3=fetcher, **opts)) added, failed = dump_to_odc( fetcher(document_stream), dc, candidate_products, skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, transform=transform, update=update, update_if_exists=update_if_exists, allow_unsafe=allow_unsafe, ) print(f"Added {added} datasets and failed {failed} datasets.") if failed > 0: sys.exit(failed)
def cli( skip_lineage, fail_on_missing_lineage, verify_lineage, stac, update, allow_unsafe, skip_check, no_sign_request, request_payer, uri, product, ): """ Iterate through files in an S3 bucket and add them to datacube""" transform = None if stac: transform = stac_transform candidate_products = product.split() opts = {} if request_payer: opts["RequestPayer"] = "requester" # Get a generator from supplied S3 Uri for metadata definitions fetcher = S3Fetcher(aws_unsigned=no_sign_request) # TODO: Share Fetcher s3_obj_stream = s3_find_glob(uri, skip_check=skip_check, s3=fetcher, **opts) # Extract URLs from output of iterator before passing to Fetcher s3_url_stream = (o.url for o in s3_obj_stream) # TODO: Capture S3 URL's in batches and perform bulk_location_has # Consume generator and fetch YAML's dc = Datacube() odc_products = dc.list_products().name.values odc_products = set(odc_products) if not set(candidate_products).issubset(odc_products): missing_products = list(set(candidate_products) - odc_products) print( f"Error: Requested Product/s {', '.join(missing_products)} {'is' if len(missing_products) == 1 else 'are'} " "not present in the ODC Database", file=sys.stderr, ) sys.exit(1) added, failed = dump_to_odc( fetcher(s3_url_stream), dc, candidate_products, skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, transform=transform, update=update, allow_unsafe=allow_unsafe, ) print(f"Added {added} Datasets, Failed {failed} Datasets") if failed > 0: sys.exit(failed)