def _reshape(self, resource: Resource, keep: List[str], versioned: bool) -> Resource:
# TODO Use as base an implementation of JSONPath for Python. DKE-147.
levels = [x.split(".", maxsplit=1) for x in keep]
roots = {x[0] for x in levels}
new = Resource()
for root in roots:
leaves = [x[1] for x in levels if len(x) > 1 and x[0] == root]
value = getattr(resource, root, None)
if value is not None:
if isinstance(value, List):
new_value = self._reshape_many(value, leaves, versioned)
for i,nv in enumerate(new_value):
if nv == Resource() and isinstance(value[i],str):
new_value[i] = value[i]
elif isinstance(value, Resource):
if leaves:
new_value = self._reshape_one(value, leaves, versioned)
else:
attributes = value.__dict__.items()
properties = {k: v for k, v in attributes if k not in value._RESERVED}
new_value = Resource(**properties)
else:
if root == "id" and versioned:
new_value = self.versioned_id_template.format(x=resource)
else:
new_value = value
setattr(new, root, new_value)
else:
pass
return new
def nested_registered_resource(nested_resource):
ingredients = [Resource(id=i, type='Ingredient') for i in range(3)]
resource = Resource(id="a_recipe",
type="Recipe",
ingridients=ingredients,
author=Resource(id="a_person", type="Person"))
do_recursive(add_metadata, resource)
return resource
def _add_prov_property(self, resource, prov_type, reference_property,
reference_type, keep, versioned, **kwargs):
if versioned and isinstance(resource, str):
not_supported(("versioned with resource:str", True))
if isinstance(resource, str):
reference = Resource(type=reference_type, id=resource)
elif isinstance(resource, Resource):
reference = self._forge.reshape(resource, keep, versioned)
result = Resource(type=prov_type, **kwargs)
result.__setattr__(reference_property, reference)
return result
def test_reshape(config):
forge = KnowledgeGraphForge(config)
reshaper = Reshaper(versioned_id_template="{x.id}?_version={x._store_metadata.version}")
simple = Resource(type="Experiment", url="file.gz")
r = reshaper.reshape(simple, keep=['type'],versioned=False)
expected = { "type": "Experiment"}
assert expected == forge.as_json(r)
simple = Resource(type=["Experiment"], url="file.gz")
r = reshaper.reshape(simple, keep=['type'], versioned=True)
expected = {"type": ["Experiment"]}
assert expected == forge.as_json(r)
def test_from_resource(config, person, organization, store_metadata_value):
forge = KnowledgeGraphForge(config)
data = {
'id': 'c51f4e4e-2b30-41f4-8f7c-aced85632b03',
'type': ['Person', 'Agent'],
'name': 'Jami Booth'
}
assert isinstance(person, Resource)
dataset = Dataset.from_resource(forge, person)
assert isinstance(dataset, Dataset)
assert forge.as_json(dataset) == forge.as_json(person)
assert forge.as_json(dataset) == data
assert dataset._store_metadata is None
person_with_store_metadata = Resource(**forge.as_json(person))
person_with_store_metadata._store_metadata = store_metadata_value
dataset = Dataset.from_resource(forge,
person_with_store_metadata,
store_metadata=True)
assert isinstance(dataset, Dataset)
person_with_store_metadata_json = forge.as_json(person_with_store_metadata)
assert forge.as_json(dataset) == person_with_store_metadata_json
assert forge.as_json(dataset) == data
assert dataset._store_metadata == person_with_store_metadata._store_metadata
assert forge.as_json(dataset, store_metadata=False) != forge.as_json(
person_with_store_metadata, store_metadata=True)
assert forge.as_json(dataset, store_metadata=True) == forge.as_json(
person_with_store_metadata, store_metadata=True)
assert isinstance(organization, Resource)
dataset = Dataset.from_resource(forge, [person, organization])
assert isinstance(dataset, List)
assert len(dataset) == 2
def building():
type_ = "Building"
name = "The Empire State Building"
description = "The Empire State Building is a 102-story landmark in New York City."
image = "http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg"
geo = {"latitude": "40.75"}
return Resource(type=type_, name=name, description=description, image=image, geo=geo)
def test_collect_values():
simple = Resource(type="Experiment", url="file.gz")
r = collect_values(simple, "url")
assert simple.url in r, "url should be in the list"
deep = Resource(type="Experiment",
level1=Resource(level2=Resource(url="file.gz")))
r = collect_values(deep, "level1.level2.url")
assert deep.level1.level2.url in r, "url should be in the list"
files = [Resource(type="Experiment", url=f"file{i}") for i in range(3)]
files.append(Resource(type="Experiment", contentUrl=f"file3"))
r = collect_values(files, "url")
assert ["file0", "file1",
"file2"] == r, "three elements should be in the list"
r = collect_values(files, "contentUrl")
assert ["file3"] == r, "one element should be in the list"
data_set = Resource(type="Dataset", hasPart=files)
r = collect_values(data_set, "hasPart.contentUrl")
assert ["file3"] == r, "one element should be in the list"
r = collect_values(data_set, "hasPart.url")
assert ["file0", "file1",
"file2"] == r, "three elements should be in the list"
r = collect_values(data_set, "fake.path")
assert len(r) == 0
with pytest.raises(ValueError):
collect_values(None, "hasPart.url", ValueError)
def add_derivation(self,
resource: Resource,
versioned: bool = True,
**kwargs) -> None:
"""Add information on the derivation of an entity resulting in the dataset."""
keep = ["id", "type", "name"]
entity = self._forge.reshape(resource, keep, versioned)
derivation = Resource(type="Derivation", entity=entity, **kwargs)
_set(self, "derivation", derivation)
def test_download(config):
simple = Resource(type="Experiment", url="file.gz")
with pytest.raises(DownloadingError):
forge = KnowledgeGraphForge(config)
forge._store.download(simple,
"fake.path",
"./",
overwrite=False,
cross_bucket=False)
def _elastic(self, query: str, limit: int, offset: int = None) -> List[Resource]:
try:
response = requests.post(
self.service.elastic_endpoint["endpoint"], data=query, headers=self.service.headers_elastic)
response.raise_for_status()
except Exception as e:
raise QueryingError(e)
else:
results = response.json()
return [Resource(**{k: v for k, v in hit.items()}) for hit in results["hits"]['hits']]
def _from_json(data: Union[Any, List[Any]], na: List[Any]) -> Any:
if isinstance(data, List):
return [_from_json(x, na) for x in data]
elif isinstance(data, Dict):
properties = {
k: _from_json(v, na)
for k, v in data.items() if v not in na
}
return Resource(**properties)
else:
return data
def test_execute_lazy_actions():
fun = lambda x: x
ra = Resource(pa1="pa1",
pa2=Resource(pb1="pb1"),
pa3=LazyAction(fun, "pa3 executed"),
pa4=Resource(pc1=LazyAction(fun, "pc1 executed"), pc2="pc2"),
pa5=[
LazyAction(fun, "pa5[0] executed"), 123,
Resource(pd1=LazyAction(fun, "pd1 executed")), "string"
])
la = collect_lazy_actions(ra)
execute_lazy_actions(ra, la)
assert ra.pa1 == "pa1"
assert ra.pa2.pb1 == "pb1"
assert ra.pa3 == "pa3 executed"
assert ra.pa4.pc1 == "pc1 executed"
assert ra.pa4.pc2 == "pc2"
assert ra.pa5[0] == "pa5[0] executed"
assert ra.pa5[1] == 123
assert ra.pa5[2].pd1 == "pd1 executed"
assert ra.pa5[3] == "string"
def _sparql(self, query: str, limit: int, offset: int = None) -> List[Resource]:
s_offset = "" if offset is None else f"OFFSET {offset}"
s_limit = "" if limit is None else f"LIMIT {limit}"
query = f"{query} {s_limit} {s_offset}"
try:
response = requests.post(
self.service.sparql_endpoint["endpoint"], data=query, headers=self.service.headers_sparql)
response.raise_for_status()
except Exception as e:
raise QueryingError(e)
else:
data = response.json()
# FIXME workaround to parse a CONSTRUCT query, this fix depends on
# https://github.com/BlueBrain/nexus/issues/1155
_, q_comp = Query.parseString(query)
if q_comp.name == "ConstructQuery":
subject_triples = {}
for r in data["results"]["bindings"]:
subject = r['subject']['value']
s = f"<{r['subject']['value']}>"
p = f"<{r['predicate']['value']}>"
if r["object"]["type"] == "uri":
o = f"<{r['object']['value']}>"
else:
if "datatype" in r["object"]:
o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}"
else:
o = f"\"{r['object']['value']}\""
if subject in subject_triples:
subject_triples[subject] += f"\n{s} {p} {o} . "
else:
subject_triples[subject] = f"{s} {p} {o} . "
def triples_to_resource(iri, triples):
graph = Graph().parse(data=triples, format="nt")
data_expanded = json.loads(graph.serialize(format="json-ld").decode("utf-8"))
frame = {"@id": iri}
data_framed = jsonld.frame(data_expanded, frame)
context = self.model_context or self.context
compacted = jsonld.compact(data_framed, context.document)
resource = from_jsonld(compacted)
resource.context = context.iri if context.is_http_iri() else context.document["@context"]
return resource
return [triples_to_resource(s, t) for s, t in subject_triples.items()]
else:
# SELECT QUERY
results = data["results"]["bindings"]
return [Resource(**{k: v["value"] for k, v in x.items()}) for x in results]
def test_freeze(config, store_metadata_value):
forge = KnowledgeGraphForge(config, debug=True)
derivation1 = Dataset(forge, type="Dataset", name="A derivation dataset")
derivation1.id = "http://derivation1"
derivation1._store_metadata = wrap_dict(store_metadata_value)
generation1 = Dataset(forge, type="Dataset", name="A generation dataset")
generation1.id = "http://generation1"
generation1._store_metadata = wrap_dict(store_metadata_value)
invalidation1 = Dataset(forge, type="Activity", name="An invalidation activity")
invalidation1.id = "http://invalidation1"
invalidation1._store_metadata = wrap_dict(store_metadata_value)
contribution1 = Resource(type="Person", name="A contributor")
contribution1.id = "http://contribution1"
contribution1._store_metadata = wrap_dict(store_metadata_value)
dataset = Dataset(forge, type="Dataset", name="A dataset")
dataset._store_metadata = wrap_dict(store_metadata_value)
dataset.add_derivation(derivation1, versioned=False)
dataset.add_generation(generation1, versioned=False)
dataset.add_invalidation(invalidation1, versioned=False)
dataset.add_contribution(contribution1, versioned=False)
expected_derivation = json.loads(json.dumps({"type":"Derivation", "entity":{"id": "http://derivation1",
"type":"Dataset", "name":"A derivation dataset"}}))
assert forge.as_json(dataset.derivation) == expected_derivation
expected_generation = json.loads(json.dumps({"type": "Generation",
"activity": {"id": "http://generation1", "type": "Dataset"}}))
assert forge.as_json(dataset.generation) == expected_generation
expected_contribution = json.loads(json.dumps({"type": "Contribution",
"agent": {"id": "http://contribution1", "type": "Person"}}))
assert forge.as_json(dataset.contribution) == expected_contribution
expected_invalidation = json.loads(json.dumps({"type": "Invalidation",
"activity": {"id": "http://invalidation1", "type": "Activity"}}))
assert forge.as_json(dataset.invalidation) == expected_invalidation
dataset.id = "http://dataset"
dataset._synchronized = True
forge._store.freeze(dataset)
assert dataset.id == "http://dataset?_version=1"
assert dataset.derivation.entity.id == "http://derivation1?_version=1"
assert dataset.generation.activity.id == "http://generation1?_version=1"
assert dataset.contribution.agent.id == "http://contribution1?_version=1"
assert dataset.invalidation.activity.id == "http://invalidation1?_version=1"
def resource(valid: bool, index: int = 0) -> Resource:
rid = str(uuid4())
r = Resource(type="Person", id=rid)
if valid:
r.name = f"resource {index}"
return r
def organization(registered_person_custom_context, store_metadata_value):
contribution = Resource(type=["Organization", "Agent"], name="Reichel Inc",
founder=registered_person_custom_context)
return contribution
def person(custom_context):
return Resource(context=custom_context, type=["Person","Agent"], name="Jami Booth")
def add_files(self, path: str, content_type: str = None) -> None:
# path: DirPath.
"""Add (different) files as parts of the dataset."""
action = self._forge.attach(path, content_type)
distribution = Resource(distribution=action)
_set(self, "hasPart", distribution)
def valid_activity_resource(self, activity_json):
resource = Resource(**activity_json)
resource.id = "http://testing/123"
return resource
def test_as_dataframe_expanded(self, forge, df_expanded_from_one_resource):
r = Resource(type="Person", name="John Doe")
x = forge.as_dataframe(r, expanded=True)
assert x.equals(df_expanded_from_one_resource)
def entity_resource(self):
return Resource(type="Entity")
def add_invalidation(self, **kwargs) -> None:
"""Add information on the invalidation of the dataset."""
if not kwargs:
raise TypeError("at least one argument should be given")
invalidation = Resource(type="Invalidation", **kwargs)
_set(self, "invalidation", invalidation)
def r5():
return Resource(p5="v5e", p6="v6e")
def add_contribution(self, agent_id: str, **kwargs) -> None:
# agent: IRI.
"""Add information on the contribution of an agent during the generation of the dataset."""
agent = Resource(type="Agent", id=agent_id)
contribution = Resource(type="Contribution", agent=agent, **kwargs)
_set(self, "contribution", contribution)
def nested_resource():
contributions = [Resource(title=f"contribution {i}") for i in range(3)]
return Resource(type="Agent", name="someone", contributions=contributions)
def add_generation(self, **kwargs) -> None:
"""Add information on the activity which has resulted in the generation of the dataset."""
if not kwargs:
raise TypeError("at least one argument should be given")
generation = Resource(type="Generation", **kwargs)
_set(self, "generation", generation)
def invalid_activity_resource(self, activity_json):
return Resource(**activity_json)