def _register_one(self, resource: Resource, schema_id: str) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context, na=nan) try: schema = quote_plus(schema_id) if schema_id else "_" url_base = f"{self.service.url_resources}/{schema}" url = f"{url_base}/{quote_plus(data['@id'])}" if hasattr( data, "@id") else url_base params_register = copy.deepcopy( self.service.params.get("register", None)) response = requests.post(url, headers=self.service.headers, data=json.dumps(data, ensure_ascii=True), params=params_register) response.raise_for_status() except nexus.HTTPError as e: raise RegistrationError(_error_message(e)) else: response_json = response.json() resource.id = response_json['@id'] # If resource had no context, update it with the one provided by the store. if not hasattr(resource, "context"): resource.context = data["@context"] self.service.sync_metadata(resource, response_json)
def _reshape(self, resource: Resource, keep: List[str], versioned: bool) -> Resource: # TODO Use as base an implementation of JSONPath for Python. DKE-147. levels = [x.split(".", maxsplit=1) for x in keep] roots = {x[0] for x in levels} new = Resource() for root in roots: leaves = [x[1] for x in levels if len(x) > 1 and x[0] == root] value = getattr(resource, root, None) if value is not None: if isinstance(value, List): new_value = self._reshape_many(value, leaves, versioned) for i,nv in enumerate(new_value): if nv == Resource() and isinstance(value[i],str): new_value[i] = value[i] elif isinstance(value, Resource): if leaves: new_value = self._reshape_one(value, leaves, versioned) else: attributes = value.__dict__.items() properties = {k: v for k, v in attributes if k not in value._RESERVED} new_value = Resource(**properties) else: if root == "id" and versioned: new_value = self.versioned_id_template.format(x=resource) else: new_value = value setattr(new, root, new_value) else: pass return new
def test_from_resource(config, person, organization, store_metadata_value): forge = KnowledgeGraphForge(config) data = { 'id': 'c51f4e4e-2b30-41f4-8f7c-aced85632b03', 'type': ['Person', 'Agent'], 'name': 'Jami Booth' } assert isinstance(person, Resource) dataset = Dataset.from_resource(forge, person) assert isinstance(dataset, Dataset) assert forge.as_json(dataset) == forge.as_json(person) assert forge.as_json(dataset) == data assert dataset._store_metadata is None person_with_store_metadata = Resource(**forge.as_json(person)) person_with_store_metadata._store_metadata = store_metadata_value dataset = Dataset.from_resource(forge, person_with_store_metadata, store_metadata=True) assert isinstance(dataset, Dataset) person_with_store_metadata_json = forge.as_json(person_with_store_metadata) assert forge.as_json(dataset) == person_with_store_metadata_json assert forge.as_json(dataset) == data assert dataset._store_metadata == person_with_store_metadata._store_metadata assert forge.as_json(dataset, store_metadata=False) != forge.as_json( person_with_store_metadata, store_metadata=True) assert forge.as_json(dataset, store_metadata=True) == forge.as_json( person_with_store_metadata, store_metadata=True) assert isinstance(organization, Resource) dataset = Dataset.from_resource(forge, [person, organization]) assert isinstance(dataset, List) assert len(dataset) == 2
def nested_registered_resource(nested_resource): ingredients = [Resource(id=i, type='Ingredient') for i in range(3)] resource = Resource(id="a_recipe", type="Recipe", ingridients=ingredients, author=Resource(id="a_person", type="Person")) do_recursive(add_metadata, resource) return resource
def _make_registered(r: Resource, metadata, base=None): if base: r.id = f"{urljoin('file:', pathname2url(os.getcwd()))}/{str(uuid4())}" else: r.id = str(uuid4()) if metadata: metadata["id"] = r.id r._store_metadata = wrap_dict(metadata) return r
def synchronize_resource(self, resource: Resource, response: Union[Exception, Dict], action_name: str, succeeded: bool, synchronized: bool) -> None: if succeeded: action = Action(action_name, succeeded, None) self.sync_metadata(resource, response) else: action = Action(action_name, succeeded, response) resource._last_action = action resource._synchronized = synchronized
def _register_one(self, resource: Resource, schema_id: str) -> None: data = as_json(resource, expanded=False, store_metadata=False, model_context=None, metadata_context=None, context_resolver=None) try: record = self.service.create(data) except StoreLibrary.RecordExists: raise RegistrationError("resource already exists") else: resource.id = record["data"]["id"] resource._store_metadata = wrap_dict(record["metadata"])
def _add_prov_property(self, resource, prov_type, reference_property, reference_type, keep, versioned, **kwargs): if versioned and isinstance(resource, str): not_supported(("versioned with resource:str", True)) if isinstance(resource, str): reference = Resource(type=reference_type, id=resource) elif isinstance(resource, Resource): reference = self._forge.reshape(resource, keep, versioned) result = Resource(type=prov_type, **kwargs) result.__setattr__(reference_property, reference) return result
def test_reshape(config): forge = KnowledgeGraphForge(config) reshaper = Reshaper(versioned_id_template="{x.id}?_version={x._store_metadata.version}") simple = Resource(type="Experiment", url="file.gz") r = reshaper.reshape(simple, keep=['type'],versioned=False) expected = { "type": "Experiment"} assert expected == forge.as_json(r) simple = Resource(type=["Experiment"], url="file.gz") r = reshaper.reshape(simple, keep=['type'], versioned=True) expected = {"type": ["Experiment"]} assert expected == forge.as_json(r)
def test_freeze(config, store_metadata_value): forge = KnowledgeGraphForge(config, debug=True) derivation1 = Dataset(forge, type="Dataset", name="A derivation dataset") derivation1.id = "http://derivation1" derivation1._store_metadata = wrap_dict(store_metadata_value) generation1 = Dataset(forge, type="Dataset", name="A generation dataset") generation1.id = "http://generation1" generation1._store_metadata = wrap_dict(store_metadata_value) invalidation1 = Dataset(forge, type="Activity", name="An invalidation activity") invalidation1.id = "http://invalidation1" invalidation1._store_metadata = wrap_dict(store_metadata_value) contribution1 = Resource(type="Person", name="A contributor") contribution1.id = "http://contribution1" contribution1._store_metadata = wrap_dict(store_metadata_value) dataset = Dataset(forge, type="Dataset", name="A dataset") dataset._store_metadata = wrap_dict(store_metadata_value) dataset.add_derivation(derivation1, versioned=False) dataset.add_generation(generation1, versioned=False) dataset.add_invalidation(invalidation1, versioned=False) dataset.add_contribution(contribution1, versioned=False) expected_derivation = json.loads(json.dumps({"type":"Derivation", "entity":{"id": "http://derivation1", "type":"Dataset", "name":"A derivation dataset"}})) assert forge.as_json(dataset.derivation) == expected_derivation expected_generation = json.loads(json.dumps({"type": "Generation", "activity": {"id": "http://generation1", "type": "Dataset"}})) assert forge.as_json(dataset.generation) == expected_generation expected_contribution = json.loads(json.dumps({"type": "Contribution", "agent": {"id": "http://contribution1", "type": "Person"}})) assert forge.as_json(dataset.contribution) == expected_contribution expected_invalidation = json.loads(json.dumps({"type": "Invalidation", "activity": {"id": "http://invalidation1", "type": "Activity"}})) assert forge.as_json(dataset.invalidation) == expected_invalidation dataset.id = "http://dataset" dataset._synchronized = True forge._store.freeze(dataset) assert dataset.id == "http://dataset?_version=1" assert dataset.derivation.entity.id == "http://derivation1?_version=1" assert dataset.generation.activity.id == "http://generation1?_version=1" assert dataset.contribution.agent.id == "http://contribution1?_version=1" assert dataset.invalidation.activity.id == "http://invalidation1?_version=1"
def sync_metadata(self, resource: Resource, result: Dict) -> None: metadata = {"id": resource.id} keys = sorted(self.metadata_context.terms.keys()) only_meta = {k: v for k, v in result.items() if k in keys} metadata.update( _remove_ld_keys(only_meta, self.metadata_context, False)) resource._store_metadata = wrap_dict(metadata)
def building(): type_ = "Building" name = "The Empire State Building" description = "The Empire State Building is a 102-story landmark in New York City." image = "http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg" geo = {"latitude": "40.75"} return Resource(type=type_, name=name, description=description, image=image, geo=geo)
def add_metadata(resource: Resource): metadata = { "_self": resource.id, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/unconstrained.json", "_project": "https://nexus/org/prj", "_rev": 1, "_deprecated": False, "_createdAt": "2019-03-28T13:40:38.934Z", "_createdBy": "https://nexus/u1", "_updatedAt": "2019-03-28T13:40:38.934Z", "_updatedBy": "https://nexus/u1", "_incoming": "https:/nexus/incoming", "_outgoing": "https://nexux/outgoing" } resource._synchronized = True resource._validated = True resource._store_metadata = wrap_dict(metadata)
def test_collect_values(): simple = Resource(type="Experiment", url="file.gz") r = collect_values(simple, "url") assert simple.url in r, "url should be in the list" deep = Resource(type="Experiment", level1=Resource(level2=Resource(url="file.gz"))) r = collect_values(deep, "level1.level2.url") assert deep.level1.level2.url in r, "url should be in the list" files = [Resource(type="Experiment", url=f"file{i}") for i in range(3)] files.append(Resource(type="Experiment", contentUrl=f"file3")) r = collect_values(files, "url") assert ["file0", "file1", "file2"] == r, "three elements should be in the list" r = collect_values(files, "contentUrl") assert ["file3"] == r, "one element should be in the list" data_set = Resource(type="Dataset", hasPart=files) r = collect_values(data_set, "hasPart.contentUrl") assert ["file3"] == r, "one element should be in the list" r = collect_values(data_set, "hasPart.url") assert ["file0", "file1", "file2"] == r, "three elements should be in the list" r = collect_values(data_set, "fake.path") assert len(r) == 0 with pytest.raises(ValueError): collect_values(None, "hasPart.url", ValueError)
def test_download(config): simple = Resource(type="Experiment", url="file.gz") with pytest.raises(DownloadingError): forge = KnowledgeGraphForge(config) forge._store.download(simple, "fake.path", "./", overwrite=False, cross_bucket=False)
def add_derivation(self, resource: Resource, versioned: bool = True, **kwargs) -> None: """Add information on the derivation of an entity resulting in the dataset.""" keep = ["id", "type", "name"] entity = self._forge.reshape(resource, keep, versioned) derivation = Resource(type="Derivation", entity=entity, **kwargs) _set(self, "derivation", derivation)
def _elastic(self, query: str, limit: int, offset: int = None) -> List[Resource]: try: response = requests.post( self.service.elastic_endpoint["endpoint"], data=query, headers=self.service.headers_elastic) response.raise_for_status() except Exception as e: raise QueryingError(e) else: results = response.json() return [Resource(**{k: v for k, v in hit.items()}) for hit in results["hits"]['hits']]
def _deprecate_one(self, resource: Resource) -> None: rid = resource.id try: record = self.service.deprecate(rid) except StoreLibrary.RecordMissing: raise DeprecationError("resource not found") except StoreLibrary.RecordDeprecated: raise DeprecationError("resource already deprecated") else: resource._store_metadata = wrap_dict(record["metadata"])
def _from_json(data: Union[Any, List[Any]], na: List[Any]) -> Any: if isinstance(data, List): return [_from_json(x, na) for x in data] elif isinstance(data, Dict): properties = { k: _from_json(v, na) for k, v in data.items() if v not in na } return Resource(**properties) else: return data
def _update_one(self, resource: Resource) -> None: data = as_json(resource, expanded=False, store_metadata=False, model_context=None, metadata_context=None, context_resolver=None) try: record = self.service.update(data) except StoreLibrary.RecordMissing: raise UpdatingError("resource not found") except StoreLibrary.RecordDeprecated: raise UpdatingError("resource is deprecated") else: resource._store_metadata = wrap_dict(record["metadata"])
def test_execute_lazy_actions(): fun = lambda x: x ra = Resource(pa1="pa1", pa2=Resource(pb1="pb1"), pa3=LazyAction(fun, "pa3 executed"), pa4=Resource(pc1=LazyAction(fun, "pc1 executed"), pc2="pc2"), pa5=[ LazyAction(fun, "pa5[0] executed"), 123, Resource(pd1=LazyAction(fun, "pd1 executed")), "string" ]) la = collect_lazy_actions(ra) execute_lazy_actions(ra, la) assert ra.pa1 == "pa1" assert ra.pa2.pb1 == "pb1" assert ra.pa3 == "pa3 executed" assert ra.pa4.pc1 == "pc1 executed" assert ra.pa4.pc2 == "pc2" assert ra.pa5[0] == "pa5[0] executed" assert ra.pa5[1] == 123 assert ra.pa5[2].pd1 == "pd1 executed" assert ra.pa5[3] == "string"
def _freeze_one(self, resource: Resource) -> None: # Notify of failures with exception FreezingError including a message. # Use self.versioned_id_template.format(x=resource) to freeze IDs. for _, v in resource.__dict__.items(): if isinstance(v, List): for x in v: if isinstance(x, Resource): self._freeze_one(x) elif isinstance(v, Resource): self._freeze_one(v) if hasattr(resource, "id"): resource.id = self.versioned_id_template.format(x=resource)
def _register_one(self, resource: Resource, schema_id: str) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context) try: response = nexus.resources.create(org_label=self.organisation, project_label=self.project, data=data, schema_id=schema_id) except nexus.HTTPError as e: raise RegistrationError(_error_message(e)) else: resource.id = response['@id'] # If resource had no context, update it with the one provided by the store. if not hasattr(resource, "context"): resource.context = data["@context"] self.service.sync_metadata(resource, response)
def _sparql(self, query: str, limit: int, offset: int = None) -> List[Resource]: s_offset = "" if offset is None else f"OFFSET {offset}" s_limit = "" if limit is None else f"LIMIT {limit}" query = f"{query} {s_limit} {s_offset}" try: response = requests.post( self.service.sparql_endpoint["endpoint"], data=query, headers=self.service.headers_sparql) response.raise_for_status() except Exception as e: raise QueryingError(e) else: data = response.json() # FIXME workaround to parse a CONSTRUCT query, this fix depends on # https://github.com/BlueBrain/nexus/issues/1155 _, q_comp = Query.parseString(query) if q_comp.name == "ConstructQuery": subject_triples = {} for r in data["results"]["bindings"]: subject = r['subject']['value'] s = f"<{r['subject']['value']}>" p = f"<{r['predicate']['value']}>" if r["object"]["type"] == "uri": o = f"<{r['object']['value']}>" else: if "datatype" in r["object"]: o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" else: o = f"\"{r['object']['value']}\"" if subject in subject_triples: subject_triples[subject] += f"\n{s} {p} {o} . " else: subject_triples[subject] = f"{s} {p} {o} . " def triples_to_resource(iri, triples): graph = Graph().parse(data=triples, format="nt") data_expanded = json.loads(graph.serialize(format="json-ld").decode("utf-8")) frame = {"@id": iri} data_framed = jsonld.frame(data_expanded, frame) context = self.model_context or self.context compacted = jsonld.compact(data_framed, context.document) resource = from_jsonld(compacted) resource.context = context.iri if context.is_http_iri() else context.document["@context"] return resource return [triples_to_resource(s, t) for s, t in subject_triples.items()] else: # SELECT QUERY results = data["results"]["bindings"] return [Resource(**{k: v["value"] for k, v in x.items()}) for x in results]
def test_from_jsonld(self, building, model_context, building_jsonld): building.context = model_context.document["@context"] payload = building_jsonld(building, "compacted", False, None) resource = from_jsonld(payload) assert resource == building payload_with_atvalue = deepcopy(payload) payload_with_atvalue["status"] = { "@type": "xsd:string", "@value": "opened" } resource_with_atvalue = from_jsonld(payload_with_atvalue) assert "value" not in LD_KEYS.keys() assert hasattr(resource_with_atvalue, "status") assert resource_with_atvalue.status == Resource.from_json({"type":"xsd:string", "@value":"opened"})
def reserved_attribute_error(): with pytest.raises(NotImplementedError): Resource(_validated=True)
def nresource(): return Resource(type="Entity", contribution=Resource(type="Contribution"))
def resource(): return Resource(type="Entity")
def nested_resource(): contributions = [Resource(title=f"contribution {i}") for i in range(3)] return Resource(type="Agent", name="someone", contributions=contributions)
def resource(valid: bool, index: int = 0) -> Resource: rid = str(uuid4()) r = Resource(type="Person", id=rid) if valid: r.name = f"resource {index}" return r