def handle_json_message(metadata, transform, odc_metadata_link):
odc_yaml_uri = None
uri = None
if odc_metadata_link:
if odc_metadata_link.startswith("STAC-LINKS-REL:"):
rel_val = odc_metadata_link.replace("STAC-LINKS-REL:", "")
odc_yaml_uri = get_uri(metadata, rel_val)
else:
# if odc_metadata_link is provided, it will look for value with dict path provided
odc_yaml_uri = dicttoolz.get_in(odc_metadata_link.split("/"),
metadata)
# if odc_yaml_uri exist, it will load the metadata content from that URL
if odc_yaml_uri:
try:
content = requests.get(odc_yaml_uri).content
metadata = documents.parse_yaml(content)
uri = odc_yaml_uri
except requests.RequestException as err:
raise IndexingException(
f"Failed to load metadata from the link provided - {err}")
else:
raise IndexingException("ODC EO3 metadata link not found")
else:
# if no odc_metadata_link provided, it will look for metadata dict "href" value with "rel==self"
uri = get_uri(metadata, "self")
if transform:
metadata = transform(metadata)
return metadata, uri
def parse_doc_stream(doc_stream, on_error=None, transform=None):
"""
Replace doc bytes/strings with parsed dicts.
Stream[(uri, bytes)] -> Stream[(uri, dict)]
:param doc_stream: sequence of (uri, doc: bytes|string)
:param on_error: Callback uri, doc, exception -> None
:param transform: dict -> dict if supplied also apply further transform on parsed document
On output doc is replaced with python dict parsed from yaml, or with None
if parsing/transform error occurred.
"""
for uri, doc in doc_stream:
try:
if uri.endswith(".json"):
metadata = json.loads(doc)
else:
metadata = parse_yaml(doc)
if transform is not None:
metadata = transform(metadata)
except Exception as e: # pylint: disable=broad-except
if on_error is not None:
on_error(uri, doc, e)
metadata = None
yield uri, metadata
def catalog_from_yaml(catalog_body: str, name_resolver=None) -> Catalog:
"""
Load a catalog of virtual products from a yaml document.
"""
if name_resolver is None:
name_resolver = DEFAULT_RESOLVER
return Catalog(DEFAULT_RESOLVER, parse_yaml(catalog_body))
def construct_from_yaml(recipe: str, name_resolver=None) -> VirtualProduct:
"""
Create a virtual product from a yaml recipe.
"""
if name_resolver is None:
name_resolver = DEFAULT_RESOLVER
return construct(**parse_yaml(recipe))
def mk_sample_eo(name='eo'):
eo_yaml = f"""
name: {name}
description: Sample
dataset:
id: ['id']
label: ['ga_label']
creation_time: ['creation_dt']
measurements: ['image', 'bands']
sources: ['lineage', 'source_datasets']
format: ['format', 'name']
grid_spatial: ['grid_spatial', 'projection']
search_fields:
time:
type: 'datetime-range'
min_offset: [['time']]
max_offset: [['time']]
"""
return MetadataType(parse_yaml(eo_yaml))
def sample_doc_180():
return parse_yaml(SAMPLE_DOC_180)
def sample_doc():
return parse_yaml(SAMPLE_DOC)
def test_parse_yaml():
assert parse_yaml('a: 10') == {'a': 10}
def construct_from_yaml(recipe: str) -> VirtualProduct:
"""
Create a virtual product from a yaml recipe.
"""
return construct(**parse_yaml(recipe))
