def handle_json_message(metadata, transform, odc_metadata_link): odc_yaml_uri = None uri = None if odc_metadata_link: if odc_metadata_link.startswith("STAC-LINKS-REL:"): rel_val = odc_metadata_link.replace("STAC-LINKS-REL:", "") odc_yaml_uri = get_uri(metadata, rel_val) else: # if odc_metadata_link is provided, it will look for value with dict path provided odc_yaml_uri = dicttoolz.get_in(odc_metadata_link.split("/"), metadata) # if odc_yaml_uri exist, it will load the metadata content from that URL if odc_yaml_uri: try: content = requests.get(odc_yaml_uri).content metadata = documents.parse_yaml(content) uri = odc_yaml_uri except requests.RequestException as err: raise IndexingException( f"Failed to load metadata from the link provided - {err}") else: raise IndexingException("ODC EO3 metadata link not found") else: # if no odc_metadata_link provided, it will look for metadata dict "href" value with "rel==self" uri = get_uri(metadata, "self") if transform: metadata = transform(metadata) return metadata, uri
def parse_doc_stream(doc_stream, on_error=None, transform=None): """ Replace doc bytes/strings with parsed dicts. Stream[(uri, bytes)] -> Stream[(uri, dict)] :param doc_stream: sequence of (uri, doc: bytes|string) :param on_error: Callback uri, doc, exception -> None :param transform: dict -> dict if supplied also apply further transform on parsed document On output doc is replaced with python dict parsed from yaml, or with None if parsing/transform error occurred. """ for uri, doc in doc_stream: try: if uri.endswith(".json"): metadata = json.loads(doc) else: metadata = parse_yaml(doc) if transform is not None: metadata = transform(metadata) except Exception as e: # pylint: disable=broad-except if on_error is not None: on_error(uri, doc, e) metadata = None yield uri, metadata
def catalog_from_yaml(catalog_body: str, name_resolver=None) -> Catalog: """ Load a catalog of virtual products from a yaml document. """ if name_resolver is None: name_resolver = DEFAULT_RESOLVER return Catalog(DEFAULT_RESOLVER, parse_yaml(catalog_body))
def construct_from_yaml(recipe: str, name_resolver=None) -> VirtualProduct: """ Create a virtual product from a yaml recipe. """ if name_resolver is None: name_resolver = DEFAULT_RESOLVER return construct(**parse_yaml(recipe))
def mk_sample_eo(name='eo'): eo_yaml = f""" name: {name} description: Sample dataset: id: ['id'] label: ['ga_label'] creation_time: ['creation_dt'] measurements: ['image', 'bands'] sources: ['lineage', 'source_datasets'] format: ['format', 'name'] grid_spatial: ['grid_spatial', 'projection'] search_fields: time: type: 'datetime-range' min_offset: [['time']] max_offset: [['time']] """ return MetadataType(parse_yaml(eo_yaml))
def sample_doc_180(): return parse_yaml(SAMPLE_DOC_180)
def sample_doc(): return parse_yaml(SAMPLE_DOC)
def test_parse_yaml(): assert parse_yaml('a: 10') == {'a': 10}
def construct_from_yaml(recipe: str) -> VirtualProduct: """ Create a virtual product from a yaml recipe. """ return construct(**parse_yaml(recipe))