def _register_one(self, resource: Resource, schema_id: str) -> None:
context = self.model_context or self.context
data = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.service.resolve_context,
na=nan)
try:
schema = quote_plus(schema_id) if schema_id else "_"
url_base = f"{self.service.url_resources}/{schema}"
url = f"{url_base}/{quote_plus(data['@id'])}" if hasattr(
data, "@id") else url_base
params_register = copy.deepcopy(
self.service.params.get("register", None))
response = requests.post(url,
headers=self.service.headers,
data=json.dumps(data, ensure_ascii=True),
params=params_register)
response.raise_for_status()
except nexus.HTTPError as e:
raise RegistrationError(_error_message(e))
else:
response_json = response.json()
resource.id = response_json['@id']
# If resource had no context, update it with the one provided by the store.
if not hasattr(resource, "context"):
resource.context = data["@context"]
self.service.sync_metadata(resource, response_json)
def _reshape(self, resource: Resource, keep: List[str], versioned: bool) -> Resource:
# TODO Use as base an implementation of JSONPath for Python. DKE-147.
levels = [x.split(".", maxsplit=1) for x in keep]
roots = {x[0] for x in levels}
new = Resource()
for root in roots:
leaves = [x[1] for x in levels if len(x) > 1 and x[0] == root]
value = getattr(resource, root, None)
if value is not None:
if isinstance(value, List):
new_value = self._reshape_many(value, leaves, versioned)
for i,nv in enumerate(new_value):
if nv == Resource() and isinstance(value[i],str):
new_value[i] = value[i]
elif isinstance(value, Resource):
if leaves:
new_value = self._reshape_one(value, leaves, versioned)
else:
attributes = value.__dict__.items()
properties = {k: v for k, v in attributes if k not in value._RESERVED}
new_value = Resource(**properties)
else:
if root == "id" and versioned:
new_value = self.versioned_id_template.format(x=resource)
else:
new_value = value
setattr(new, root, new_value)
else:
pass
return new
def test_from_resource(config, person, organization, store_metadata_value):
forge = KnowledgeGraphForge(config)
data = {
'id': 'c51f4e4e-2b30-41f4-8f7c-aced85632b03',
'type': ['Person', 'Agent'],
'name': 'Jami Booth'
}
assert isinstance(person, Resource)
dataset = Dataset.from_resource(forge, person)
assert isinstance(dataset, Dataset)
assert forge.as_json(dataset) == forge.as_json(person)
assert forge.as_json(dataset) == data
assert dataset._store_metadata is None
person_with_store_metadata = Resource(**forge.as_json(person))
person_with_store_metadata._store_metadata = store_metadata_value
dataset = Dataset.from_resource(forge,
person_with_store_metadata,
store_metadata=True)
assert isinstance(dataset, Dataset)
person_with_store_metadata_json = forge.as_json(person_with_store_metadata)
assert forge.as_json(dataset) == person_with_store_metadata_json
assert forge.as_json(dataset) == data
assert dataset._store_metadata == person_with_store_metadata._store_metadata
assert forge.as_json(dataset, store_metadata=False) != forge.as_json(
person_with_store_metadata, store_metadata=True)
assert forge.as_json(dataset, store_metadata=True) == forge.as_json(
person_with_store_metadata, store_metadata=True)
assert isinstance(organization, Resource)
dataset = Dataset.from_resource(forge, [person, organization])
assert isinstance(dataset, List)
assert len(dataset) == 2
def nested_registered_resource(nested_resource):
ingredients = [Resource(id=i, type='Ingredient') for i in range(3)]
resource = Resource(id="a_recipe",
type="Recipe",
ingridients=ingredients,
author=Resource(id="a_person", type="Person"))
do_recursive(add_metadata, resource)
return resource
def _make_registered(r: Resource, metadata, base=None):
if base:
r.id = f"{urljoin('file:', pathname2url(os.getcwd()))}/{str(uuid4())}"
else:
r.id = str(uuid4())
if metadata:
metadata["id"] = r.id
r._store_metadata = wrap_dict(metadata)
return r
def synchronize_resource(self, resource: Resource, response: Union[Exception, Dict],
action_name: str, succeeded: bool, synchronized: bool) -> None:
if succeeded:
action = Action(action_name, succeeded, None)
self.sync_metadata(resource, response)
else:
action = Action(action_name, succeeded, response)
resource._last_action = action
resource._synchronized = synchronized
def _register_one(self, resource: Resource, schema_id: str) -> None:
data = as_json(resource, expanded=False, store_metadata=False, model_context=None,
metadata_context=None, context_resolver=None)
try:
record = self.service.create(data)
except StoreLibrary.RecordExists:
raise RegistrationError("resource already exists")
else:
resource.id = record["data"]["id"]
resource._store_metadata = wrap_dict(record["metadata"])
def _add_prov_property(self, resource, prov_type, reference_property,
reference_type, keep, versioned, **kwargs):
if versioned and isinstance(resource, str):
not_supported(("versioned with resource:str", True))
if isinstance(resource, str):
reference = Resource(type=reference_type, id=resource)
elif isinstance(resource, Resource):
reference = self._forge.reshape(resource, keep, versioned)
result = Resource(type=prov_type, **kwargs)
result.__setattr__(reference_property, reference)
return result
def test_reshape(config):
forge = KnowledgeGraphForge(config)
reshaper = Reshaper(versioned_id_template="{x.id}?_version={x._store_metadata.version}")
simple = Resource(type="Experiment", url="file.gz")
r = reshaper.reshape(simple, keep=['type'],versioned=False)
expected = { "type": "Experiment"}
assert expected == forge.as_json(r)
simple = Resource(type=["Experiment"], url="file.gz")
r = reshaper.reshape(simple, keep=['type'], versioned=True)
expected = {"type": ["Experiment"]}
assert expected == forge.as_json(r)
def test_freeze(config, store_metadata_value):
forge = KnowledgeGraphForge(config, debug=True)
derivation1 = Dataset(forge, type="Dataset", name="A derivation dataset")
derivation1.id = "http://derivation1"
derivation1._store_metadata = wrap_dict(store_metadata_value)
generation1 = Dataset(forge, type="Dataset", name="A generation dataset")
generation1.id = "http://generation1"
generation1._store_metadata = wrap_dict(store_metadata_value)
invalidation1 = Dataset(forge, type="Activity", name="An invalidation activity")
invalidation1.id = "http://invalidation1"
invalidation1._store_metadata = wrap_dict(store_metadata_value)
contribution1 = Resource(type="Person", name="A contributor")
contribution1.id = "http://contribution1"
contribution1._store_metadata = wrap_dict(store_metadata_value)
dataset = Dataset(forge, type="Dataset", name="A dataset")
dataset._store_metadata = wrap_dict(store_metadata_value)
dataset.add_derivation(derivation1, versioned=False)
dataset.add_generation(generation1, versioned=False)
dataset.add_invalidation(invalidation1, versioned=False)
dataset.add_contribution(contribution1, versioned=False)
expected_derivation = json.loads(json.dumps({"type":"Derivation", "entity":{"id": "http://derivation1",
"type":"Dataset", "name":"A derivation dataset"}}))
assert forge.as_json(dataset.derivation) == expected_derivation
expected_generation = json.loads(json.dumps({"type": "Generation",
"activity": {"id": "http://generation1", "type": "Dataset"}}))
assert forge.as_json(dataset.generation) == expected_generation
expected_contribution = json.loads(json.dumps({"type": "Contribution",
"agent": {"id": "http://contribution1", "type": "Person"}}))
assert forge.as_json(dataset.contribution) == expected_contribution
expected_invalidation = json.loads(json.dumps({"type": "Invalidation",
"activity": {"id": "http://invalidation1", "type": "Activity"}}))
assert forge.as_json(dataset.invalidation) == expected_invalidation
dataset.id = "http://dataset"
dataset._synchronized = True
forge._store.freeze(dataset)
assert dataset.id == "http://dataset?_version=1"
assert dataset.derivation.entity.id == "http://derivation1?_version=1"
assert dataset.generation.activity.id == "http://generation1?_version=1"
assert dataset.contribution.agent.id == "http://contribution1?_version=1"
assert dataset.invalidation.activity.id == "http://invalidation1?_version=1"
def sync_metadata(self, resource: Resource, result: Dict) -> None:
metadata = {"id": resource.id}
keys = sorted(self.metadata_context.terms.keys())
only_meta = {k: v for k, v in result.items() if k in keys}
metadata.update(
_remove_ld_keys(only_meta, self.metadata_context, False))
resource._store_metadata = wrap_dict(metadata)
def building():
type_ = "Building"
name = "The Empire State Building"
description = "The Empire State Building is a 102-story landmark in New York City."
image = "http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg"
geo = {"latitude": "40.75"}
return Resource(type=type_, name=name, description=description, image=image, geo=geo)
def add_metadata(resource: Resource):
metadata = {
"_self": resource.id,
"_constrainedBy": "https://bluebrain.github.io/nexus/schemas/unconstrained.json",
"_project": "https://nexus/org/prj",
"_rev": 1,
"_deprecated": False,
"_createdAt": "2019-03-28T13:40:38.934Z",
"_createdBy": "https://nexus/u1",
"_updatedAt": "2019-03-28T13:40:38.934Z",
"_updatedBy": "https://nexus/u1",
"_incoming": "https:/nexus/incoming",
"_outgoing": "https://nexux/outgoing"
}
resource._synchronized = True
resource._validated = True
resource._store_metadata = wrap_dict(metadata)
def test_collect_values():
simple = Resource(type="Experiment", url="file.gz")
r = collect_values(simple, "url")
assert simple.url in r, "url should be in the list"
deep = Resource(type="Experiment",
level1=Resource(level2=Resource(url="file.gz")))
r = collect_values(deep, "level1.level2.url")
assert deep.level1.level2.url in r, "url should be in the list"
files = [Resource(type="Experiment", url=f"file{i}") for i in range(3)]
files.append(Resource(type="Experiment", contentUrl=f"file3"))
r = collect_values(files, "url")
assert ["file0", "file1",
"file2"] == r, "three elements should be in the list"
r = collect_values(files, "contentUrl")
assert ["file3"] == r, "one element should be in the list"
data_set = Resource(type="Dataset", hasPart=files)
r = collect_values(data_set, "hasPart.contentUrl")
assert ["file3"] == r, "one element should be in the list"
r = collect_values(data_set, "hasPart.url")
assert ["file0", "file1",
"file2"] == r, "three elements should be in the list"
r = collect_values(data_set, "fake.path")
assert len(r) == 0
with pytest.raises(ValueError):
collect_values(None, "hasPart.url", ValueError)
def test_download(config):
simple = Resource(type="Experiment", url="file.gz")
with pytest.raises(DownloadingError):
forge = KnowledgeGraphForge(config)
forge._store.download(simple,
"fake.path",
"./",
overwrite=False,
cross_bucket=False)
def add_derivation(self,
resource: Resource,
versioned: bool = True,
**kwargs) -> None:
"""Add information on the derivation of an entity resulting in the dataset."""
keep = ["id", "type", "name"]
entity = self._forge.reshape(resource, keep, versioned)
derivation = Resource(type="Derivation", entity=entity, **kwargs)
_set(self, "derivation", derivation)
def _elastic(self, query: str, limit: int, offset: int = None) -> List[Resource]:
try:
response = requests.post(
self.service.elastic_endpoint["endpoint"], data=query, headers=self.service.headers_elastic)
response.raise_for_status()
except Exception as e:
raise QueryingError(e)
else:
results = response.json()
return [Resource(**{k: v for k, v in hit.items()}) for hit in results["hits"]['hits']]
def _deprecate_one(self, resource: Resource) -> None:
rid = resource.id
try:
record = self.service.deprecate(rid)
except StoreLibrary.RecordMissing:
raise DeprecationError("resource not found")
except StoreLibrary.RecordDeprecated:
raise DeprecationError("resource already deprecated")
else:
resource._store_metadata = wrap_dict(record["metadata"])
def _from_json(data: Union[Any, List[Any]], na: List[Any]) -> Any:
if isinstance(data, List):
return [_from_json(x, na) for x in data]
elif isinstance(data, Dict):
properties = {
k: _from_json(v, na)
for k, v in data.items() if v not in na
}
return Resource(**properties)
else:
return data
def _update_one(self, resource: Resource) -> None:
data = as_json(resource, expanded=False, store_metadata=False, model_context=None,
metadata_context=None, context_resolver=None)
try:
record = self.service.update(data)
except StoreLibrary.RecordMissing:
raise UpdatingError("resource not found")
except StoreLibrary.RecordDeprecated:
raise UpdatingError("resource is deprecated")
else:
resource._store_metadata = wrap_dict(record["metadata"])
def test_execute_lazy_actions():
fun = lambda x: x
ra = Resource(pa1="pa1",
pa2=Resource(pb1="pb1"),
pa3=LazyAction(fun, "pa3 executed"),
pa4=Resource(pc1=LazyAction(fun, "pc1 executed"), pc2="pc2"),
pa5=[
LazyAction(fun, "pa5[0] executed"), 123,
Resource(pd1=LazyAction(fun, "pd1 executed")), "string"
])
la = collect_lazy_actions(ra)
execute_lazy_actions(ra, la)
assert ra.pa1 == "pa1"
assert ra.pa2.pb1 == "pb1"
assert ra.pa3 == "pa3 executed"
assert ra.pa4.pc1 == "pc1 executed"
assert ra.pa4.pc2 == "pc2"
assert ra.pa5[0] == "pa5[0] executed"
assert ra.pa5[1] == 123
assert ra.pa5[2].pd1 == "pd1 executed"
assert ra.pa5[3] == "string"
def _freeze_one(self, resource: Resource) -> None:
# Notify of failures with exception FreezingError including a message.
# Use self.versioned_id_template.format(x=resource) to freeze IDs.
for _, v in resource.__dict__.items():
if isinstance(v, List):
for x in v:
if isinstance(x, Resource):
self._freeze_one(x)
elif isinstance(v, Resource):
self._freeze_one(v)
if hasattr(resource, "id"):
resource.id = self.versioned_id_template.format(x=resource)
def _register_one(self, resource: Resource, schema_id: str) -> None:
context = self.model_context or self.context
data = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.service.resolve_context)
try:
response = nexus.resources.create(org_label=self.organisation,
project_label=self.project,
data=data,
schema_id=schema_id)
except nexus.HTTPError as e:
raise RegistrationError(_error_message(e))
else:
resource.id = response['@id']
# If resource had no context, update it with the one provided by the store.
if not hasattr(resource, "context"):
resource.context = data["@context"]
self.service.sync_metadata(resource, response)
def _sparql(self, query: str, limit: int, offset: int = None) -> List[Resource]:
s_offset = "" if offset is None else f"OFFSET {offset}"
s_limit = "" if limit is None else f"LIMIT {limit}"
query = f"{query} {s_limit} {s_offset}"
try:
response = requests.post(
self.service.sparql_endpoint["endpoint"], data=query, headers=self.service.headers_sparql)
response.raise_for_status()
except Exception as e:
raise QueryingError(e)
else:
data = response.json()
# FIXME workaround to parse a CONSTRUCT query, this fix depends on
# https://github.com/BlueBrain/nexus/issues/1155
_, q_comp = Query.parseString(query)
if q_comp.name == "ConstructQuery":
subject_triples = {}
for r in data["results"]["bindings"]:
subject = r['subject']['value']
s = f"<{r['subject']['value']}>"
p = f"<{r['predicate']['value']}>"
if r["object"]["type"] == "uri":
o = f"<{r['object']['value']}>"
else:
if "datatype" in r["object"]:
o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}"
else:
o = f"\"{r['object']['value']}\""
if subject in subject_triples:
subject_triples[subject] += f"\n{s} {p} {o} . "
else:
subject_triples[subject] = f"{s} {p} {o} . "
def triples_to_resource(iri, triples):
graph = Graph().parse(data=triples, format="nt")
data_expanded = json.loads(graph.serialize(format="json-ld").decode("utf-8"))
frame = {"@id": iri}
data_framed = jsonld.frame(data_expanded, frame)
context = self.model_context or self.context
compacted = jsonld.compact(data_framed, context.document)
resource = from_jsonld(compacted)
resource.context = context.iri if context.is_http_iri() else context.document["@context"]
return resource
return [triples_to_resource(s, t) for s, t in subject_triples.items()]
else:
# SELECT QUERY
results = data["results"]["bindings"]
return [Resource(**{k: v["value"] for k, v in x.items()}) for x in results]
def test_from_jsonld(self, building, model_context, building_jsonld):
building.context = model_context.document["@context"]
payload = building_jsonld(building, "compacted", False, None)
resource = from_jsonld(payload)
assert resource == building
payload_with_atvalue = deepcopy(payload)
payload_with_atvalue["status"] = {
"@type": "xsd:string",
"@value": "opened"
}
resource_with_atvalue = from_jsonld(payload_with_atvalue)
assert "value" not in LD_KEYS.keys()
assert hasattr(resource_with_atvalue, "status")
assert resource_with_atvalue.status == Resource.from_json({"type":"xsd:string", "@value":"opened"})
def reserved_attribute_error():
with pytest.raises(NotImplementedError):
Resource(_validated=True)
def nresource():
return Resource(type="Entity", contribution=Resource(type="Contribution"))
def resource():
return Resource(type="Entity")
def nested_resource():
contributions = [Resource(title=f"contribution {i}") for i in range(3)]
return Resource(type="Agent", name="someone", contributions=contributions)
def resource(valid: bool, index: int = 0) -> Resource:
rid = str(uuid4())
r = Resource(type="Person", id=rid)
if valid:
r.name = f"resource {index}"
return r