def test_to_graph_should_return_catalog_without_datasets_as_graph() -> None:
"""It returns a catalog graph isomorphic to spec."""
catalog = Catalog()
catalog.identifier = "http://example.com/catalogs/1"
dataset1 = Dataset()
dataset1.identifier = "http://example.com/datasets/1"
dataset1.title = {"nb": "Datasett 1", "en": "Dataset 1"}
catalog.datasets.append(dataset1)
dataset2 = Dataset()
dataset2.identifier = "http://example.com/datasets/2"
dataset2.title = {"nb": "Datasett 2", "en": "Dataset 2"}
catalog.datasets.append(dataset2)
src = """
@prefix dct: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
<http://example.com/catalogs/1> a dcat:Catalog ;
dcat:dataset <http://example.com/datasets/1> ,
<http://example.com/datasets/2> ;
.
"""
g1 = Graph().parse(data=catalog.to_rdf(include_datasets=False),
format="turtle")
g2 = Graph().parse(data=src, format="turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic
def test_to_graph_should_return_dataset_skolemization(
mocker: MockFixture) -> None:
"""It returns a dataset graph isomorphic to spec."""
catalog = Catalog()
catalog.identifier = "http://example.com/catalogs/1"
dataset1 = Dataset()
dataset1.title = {"nb": "Datasett 1", "en": "Dataset 1"}
catalog.datasets.append(dataset1)
dataset2 = Dataset()
dataset2.title = {"nb": "Datasett 2", "en": "Dataset 2"}
catalog.datasets.append(dataset2)
src = """
@prefix dct: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
<http://example.com/catalogs/1> a dcat:Catalog ;
dcat:dataset
<http://wwww.digdir.no/.well-known/skolem/21043186-80ce-11eb-9829-cf7c8fc855ce> ,
<http://wwww.digdir.no/.well-known/skolem/284db4d2-80c2-11eb-82c3-83e80baa2f94> ;
.
<http://wwww.digdir.no/.well-known/skolem/284db4d2-80c2-11eb-82c3-83e80baa2f94>
a dcat:Dataset ;
dct:title "Dataset 1"@en, "Datasett 1"@nb ;
.
<http://wwww.digdir.no/.well-known/skolem/21043186-80ce-11eb-9829-cf7c8fc855ce>
a dcat:Dataset ;
dct:title "Dataset 2"@en, "Datasett 2"@nb ;
.
"""
skolemutils = SkolemUtils()
mocker.patch(
"skolemizer.Skolemizer.add_skolemization",
side_effect=skolemutils.get_skolemization,
)
g1 = Graph().parse(data=catalog.to_rdf(), format="turtle")
g2 = Graph().parse(data=src, format="turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic
def _add_mandatory_dataset_props(dataset: Dataset, es_hit: Mapping) -> None:
dataset.title = {"nb": utils.remove_new_line(es_hit["title"])}
dataset.identifier = URI(os.environ["DATASET_CONCEPT_IDENTIFIER"] +
es_hit["id"])
dataset.description = {"nb": es_hit["description"]}
dataset.publisher = URI(os.environ["PUBLISHER"])
dataset.language = utils.create_language(es_hit["language"])
dataset.access_rights = utils.create_access_rights(es_hit["accessRights"])
dataset.spatial_coverage = utils.create_location(es_hit["spatial"])
def test_to_graph_should_return_title() -> None:
"""It returns a title graph isomorphic to spec."""
resource = Dataset()
resource.identifier = "http://example.com/datasets/1"
resource.title = {"nb": "Tittel 1", "en": "Title 1"}
src = """
@prefix dct: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
<http://example.com/datasets/1> a dcat:Dataset ;
dct:title "Title 1"@en, "Tittel 1"@nb ;
.
"""
g1 = Graph().parse(data=resource.to_rdf(), format="turtle")
g2 = Graph().parse(data=src, format="turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic
from datacatalogtordf import Catalog, Dataset
# Create catalog object
catalog = Catalog()
catalog.identifier = "http://example.com/catalogs/1"
catalog.title = {"en": "A dataset catalog"}
catalog.publisher = "https://example.com/publishers/1"
# Create a dataset:
dataset = Dataset()
dataset.identifier = "http://example.com/datasets/1"
dataset.title = {"nb": "inntektsAPI", "en": "incomeAPI"}
#
# Add concept to catalog:
catalog.datasets.append(dataset)
# get rdf representation in turtle (default)
rdf = catalog.to_rdf()
print(rdf.decode())
def main(req: func.HttpRequest, msg: func.Out[str]) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
search = req.params.get('search')
if not search:
try:
req_body = req.get_json()
except ValueError:
pass
else:
search = req_body.get('search')
assetType = req.params.get('assetType')
if not assetType:
try:
req_body = req.get_json()
except ValueError:
pass
else:
search = req_body.get('assetType')
entityType = req.params.get('entityType')
if not entityType:
try:
req_body = req.get_json()
except ValueError:
pass
else:
search = req_body.get('entityType')
if search:
logging.info(search)
logging.info(assetType)
logging.info(entityType)
# search purview
value = entitySearch(search)
if int(json.loads(value)['@search.count']) > 0:
logging.info(json.loads(value)['@search.count'])
else:
logging.info("Nothing was returned from the search")
return func.HttpResponse(
"This HTTP triggered function executed successfully, but no search criteria was returned from the Purview catalog. Pass a search in the query string or in the request body for a response.",
status_code=200
)
#parse the response
objToexport = []
score = 0
#loop through all of the results and filter on the assetType and entityType
jsonObj = json.loads(value)
for obj in jsonObj['value'] :
if (obj['assetType'][0] == assetType) and (obj['entityType'] == entityType):
#if there are multiple objects returned, select the one with the highest score
if (obj['@search.score'] > score):
objToexport = obj
score = obj['@search.score']
# return function if not match has been found in the data returned from Purview
if(len(objToexport) == 0):
logging.info("No match was found in the Purview catalog for the search, assetType and entityType combination entityType: {0} {1} {2} ".format(search, assetType, entityType))
return func.HttpResponse(
f"No match was found in the Purview catalog for search : {search} assetType: {assetType} entityType: {entityType} ",
status_code=200
)
publisher = os.environ.get('publisher')
logging.info(objToexport['assetType'])
logging.info(objToexport['entityType'])
logging.info(objToexport['@search.score'])
logging.info(objToexport['qualifiedName'])
logging.info(objToexport['name'])
logging.info(publisher)
logging.info(score)
# Create catalog object
catalog = Catalog()
catalog.identifier = objToexport['qualifiedName']
catalog.title = {"en": objToexport['name']}
catalog.publisher = publisher
# Create a dataset:
dataset = Dataset()
dataset.identifier = objToexport['qualifiedName']
dataset.title = {"nb": objToexport['name'], "en": objToexport['name']}
#
# Add dataset to catalog:
catalog.datasets.append(dataset)
# get rdf representation in turtle (default)
rdf = catalog.to_rdf()
logging.info(rdf.decode())
# write the rdf to Azure storage
msg.set(rdf.decode())
# return the function
return func.HttpResponse(f"The HTTP triggered function for {search} executed successfully. The assetType provided was: {assetType} and the entityType was: {entityType }")
else:
return func.HttpResponse(
"This HTTP triggered function executed successfully, but no search criteria was supplied. Pass a search in the query string or in the request body for a response.",
status_code=200
)