def test_no_context(self, form, store_metadata, valid_resource): with pytest.raises(NotSupportedError): as_jsonld(valid_resource, form=form, store_metadata=store_metadata, model_context=None, metadata_context=None, context_resolver=None)
def test_from_graph(self, building, organization, building_jsonld, model_context, store_metadata, metadata_context): store_metadata = False id = "http://test/1234" id_uri = term.URIRef(id) graph = Graph() graph.add((id_uri, RDF.type, term.URIRef("http://schema.org/Building"))) graph.add((id_uri,term.URIRef("http://schema.org/name"),term.Literal("The Empire State Building"))) graph.add((id_uri,term.URIRef("http://schema.org/description"),term.Literal("The Empire State Building is a 102-story landmark in New York City."))) graph.add((id_uri,term.URIRef("http://schema.org/image"),term.URIRef("http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg"))) bNode = term.BNode() graph.add((id_uri,term.URIRef("http://schema.org/geo"),bNode)) graph.add((bNode,term.URIRef("http://schema.org/latitude"),term.Literal("40.75"))) results = from_graph(graph) assert isinstance(results, Resource) building.id = id building.context = model_context.document["@context"] expected = building_jsonld(building, "expanded", store_metadata, None) result_jsonld = as_jsonld(results, form="expanded", store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) assert result_jsonld == expected graph.remove((id_uri, RDF.type, term.URIRef("http://schema.org/Building"))) results = from_graph(graph) assert len(results) == 3 graph.add((id_uri, RDF.type, term.URIRef("http://schema.org/Building"))) graph.add((term.URIRef("http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg"), RDF.type, term.URIRef("http://schema.org/Image"))) results = from_graph(graph, type=["http://schema.org/Building","http://schema.org/Image"]) assert len(results) == 2 assert results[0].type is not None assert results[1].type is not None result_0 = as_jsonld(results[0], form="expanded", store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) result_1 = as_jsonld(results[1], form="expanded", store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) results = [result_0, result_1] assert set(["http://schema.org/Building","http://schema.org/Image"]) == {result["@type"] for result in results} frame = { "@type": ['http://schema.org/Image'], "@embed": True } results = from_graph(graph, frame=frame) assert isinstance(results, Resource) expected = {'@type': 'http://schema.org/Image', '@id': 'http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg'} result_jsonld = as_jsonld(results, form="expanded", store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) assert result_jsonld == expected
def create_tasks(semaphore, session, loop, data, batch_action, f_callback, error): futures = [] schema_id = kwargs.get("schema_id") schema_id = "_" if schema_id is None else quote_plus(schema_id) for resource in data: params = deepcopy(kwargs.get("params", {})) if batch_action == batch_action.CREATE: context = self.model_context or self.context payload = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.resolve_context, na=nan) url = f"{self.url_resources}/{schema_id}" prepared_request = loop.create_task( queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload, params)) if batch_action == batch_action.UPDATE: url = "/".join((self.url_resources, schema_id, quote_plus(resource.id))) params["rev"] = resource._store_metadata._rev payload = as_jsonld(resource, "compacted", False, model_context=self.model_context, metadata_context=None, context_resolver=self.resolve_context, na=nan) prepared_request = loop.create_task( queue(hdrs.METH_PUT, semaphore, session, url, resource, error, payload, params)) if batch_action == batch_action.TAG: url = "/".join((self.url_resources, "_", quote_plus(resource.id), "tags")) rev = resource._store_metadata._rev params["rev"] = rev payload = {"tag": kwargs.get("tag"), "rev": rev} prepared_request = loop.create_task( queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload, params)) if batch_action == batch_action.DEPRECATE: url = "/".join((self.url_resources, "_", quote_plus(resource.id))) params["rev"] = resource._store_metadata._rev prepared_request = loop.create_task( queue(hdrs.METH_DELETE, semaphore, session, url, resource, error, params=params)) if batch_action == BatchAction.FETCH: resource_org, resource_prj = resource.project.split("/")[-2:] resource_url = "/".join((self.endpoint, "resources", quote_plus(resource_org), quote_plus(resource_prj))) url = "/".join((resource_url, "_", quote_plus(resource.id))) prepared_request = loop.create_task( queue(hdrs.METH_GET, semaphore, session, url, resource, error)) if f_callback: prepared_request.add_done_callback(f_callback) futures.append(prepared_request) return futures
def as_jsonld(self, data: Union[Resource, List[Resource]], form: str = Form.COMPACTED.value, store_metadata: bool = False) -> Union[Dict, List[Dict]]: return as_jsonld(data, form, store_metadata, self._model.context(), self._store.metadata_context, self._model.resolve_context)
def _load_shape(self, resource_id): if resource_id not in self._imported: try: shape = self.context_store.retrieve(resource_id, version=None, cross_bucket=False) except RetrievalError as e: print(e, resource_id) # failed, don't try to load again self._imported.append(resource_id) else: json_dict = as_jsonld(shape, form="compacted", store_metadata=False, model_context=None, metadata_context=None, context_resolver=self.context_store. service.resolve_context) # this double conversion was due blank nodes were not "regenerated" with json-ld temp_graph = Graph().parse(data=json.dumps(json_dict), format="json-ld") self._graph.parse(data=temp_graph.serialize(format="n3"), format="n3") self._imported.append(resource_id) if hasattr(shape, "imports"): for dependency in shape.imports: self._load_shape(self.context.expand(dependency))
def _register_one(self, resource: Resource, schema_id: str) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context, na=nan) try: schema = quote_plus(schema_id) if schema_id else "_" url_base = f"{self.service.url_resources}/{schema}" url = f"{url_base}/{quote_plus(data['@id'])}" if hasattr( data, "@id") else url_base params_register = copy.deepcopy( self.service.params.get("register", None)) response = requests.post(url, headers=self.service.headers, data=json.dumps(data, ensure_ascii=True), params=params_register) response.raise_for_status() except nexus.HTTPError as e: raise RegistrationError(_error_message(e)) else: response_json = response.json() resource.id = response_json['@id'] # If resource had no context, update it with the one provided by the store. if not hasattr(resource, "context"): resource.context = data["@context"] self.service.sync_metadata(resource, response_json)
def test_unregistered_resource_model_context(self, building, model_context, building_jsonld, form, store_metadata, metadata_context): expected = building_jsonld(building, form, store_metadata, model_context.document["@context"]) result = as_jsonld(building, form=form, store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) assert expected == result
def test_unregistered_resource_with_context(self, building_with_context, building_jsonld, form, store_metadata, metadata_context): expected = building_jsonld(building_with_context, form, store_metadata, None) result = as_jsonld(building_with_context, form=form, store_metadata=store_metadata, model_context=None, metadata_context=metadata_context, context_resolver=None) assert expected == result
def test_nested_resources(self, organization, organization_jsonld_compacted, store_metadata, model_context, metadata_context): compacted = organization_jsonld_compacted(organization, store_metadata=store_metadata) result = as_jsonld(organization, form=Form.COMPACTED.value, store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) compacted["founder"]["@type"] = sorted(compacted["founder"]["@type"]) result["founder"]["@type"] = sorted(result["founder"]["@type"]) assert compacted == result
def test_registered_resource_with_context(self, building_with_context, make_registered, building_jsonld, form, store_metadata, store_metadata_value, metadata_context): base = urljoin('file:', pathname2url(os.getcwd())) registered_resource = make_registered(building_with_context, store_metadata_value, base) expected = building_jsonld(registered_resource, form, store_metadata, None) result = as_jsonld(registered_resource, form=form, store_metadata=store_metadata, model_context=None, metadata_context=metadata_context, context_resolver=None, na=None) assert expected == result
def test_nested_resources(self, organization, organization_jsonld_compacted, store_metadata, model_context, metadata_context): compacted = organization_jsonld_compacted( organization, store_metadata=store_metadata) result = as_jsonld(organization, form=Form.COMPACTED.value, store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None) assert compacted == result
def _update_one(self, resource: Resource, schema_id: str) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context, na=nan) params_update = copy.deepcopy(self.service.params.get("update", {})) params_update["rev"] = resource._store_metadata._rev schema = quote_plus(schema_id) if schema_id else "_" url = f"{self.service.url_resources}/{schema}/{quote_plus(resource.id)}" try: response = requests.put(url, headers=self.service.headers, data=json.dumps(data, ensure_ascii=True), params=params_update) response.raise_for_status() except HTTPError as e: raise UpdatingError(_error_message(e)) else: self.service.sync_metadata(resource, response.json())
def as_json(data: Union[Resource, List[Resource]], expanded: bool, store_metadata: bool, model_context: Optional[Context], metadata_context: Optional[Context], context_resolver: Optional[Callable]) -> Union[Dict, List[Dict]]: if expanded: return as_jsonld(data, "expanded", store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=context_resolver) else: if isinstance(data, Resource): return _as_json(data, store_metadata) else: return [_as_json(x, store_metadata) for x in data]
def test_as_graph(self, building, organization, building_jsonld, model_context,store_metadata, metadata_context): store_metadata = False building.id = "http://test/1234" building.context = model_context.document["@context"] data = building_jsonld(building, "compacted", store_metadata, None) expected = Graph().parse(data=json.dumps(data), format="json-ld") result = as_graph(building, store_metadata, model_context, None, None) _assert_same_graph(result, expected) organization_jsonld = as_jsonld(organization, form="compacted", store_metadata=store_metadata, model_context=model_context, metadata_context=metadata_context, context_resolver=None, na=None) expected = Graph() expected.parse(data=json.dumps(data), format="json-ld") expected.parse(data=json.dumps(organization_jsonld), format="json-ld") result = as_graph([building, organization], store_metadata, model_context, None, None) _assert_same_graph(result, expected)
def _update_one(self, resource: Resource) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context) rev = {"rev": resource._store_metadata._rev} url = f"{self.service.url_resources}/_/{quote_plus(resource.id)}" try: response = requests.put(url, headers=self.service.headers, data=json.dumps(data, ensure_ascii=True), params=rev) response.raise_for_status() except HTTPError as e: raise UpdatingError(_error_message(e)) else: self.service.sync_metadata(resource, response.json())
def _register_one(self, resource: Resource, schema_id: str) -> None: context = self.model_context or self.context data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.service.resolve_context) try: response = nexus.resources.create(org_label=self.organisation, project_label=self.project, data=data, schema_id=schema_id) except nexus.HTTPError as e: raise RegistrationError(_error_message(e)) else: resource.id = response['@id'] # If resource had no context, update it with the one provided by the store. if not hasattr(resource, "context"): resource.context = data["@context"] self.service.sync_metadata(resource, response)
def create_tasks(semaphore, session, data, batch_action, f_callback, error): futures = [] for resource in data: if batch_action == batch_action.CREATE: context = self.model_context or self.context payload = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None, context_resolver=self.resolve_context) schema_id = kwargs.get("schema_id") schema_id = "_" if schema_id is None else quote_plus( schema_id) url = f"{self.url_resources}/{schema_id}" prepared_request = asyncio.create_task( queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload)) if batch_action == batch_action.UPDATE: url = "/".join( (self.url_resources, "_", quote_plus(resource.id))) params = {"rev": resource._store_metadata._rev} payload = as_jsonld(resource, "compacted", False, model_context=self.model_context, metadata_context=None, context_resolver=self.resolve_context) prepared_request = asyncio.create_task( queue(hdrs.METH_PUT, semaphore, session, url, resource, error, payload, params)) if batch_action == batch_action.TAG: url = "/".join((self.url_resources, "_", quote_plus(resource.id), "tags")) rev = resource._store_metadata._rev params = {"rev": rev} payload = {"tag": kwargs.get("tag"), "rev": rev} prepared_request = asyncio.create_task( queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload, params)) if batch_action == batch_action.DEPRECATE: url = "/".join( (self.url_resources, "_", quote_plus(resource.id))) params = {"rev": resource._store_metadata._rev} prepared_request = asyncio.create_task( queue(hdrs.METH_DELETE, semaphore, session, url, resource, error, params=params)) if batch_action == BatchAction.FETCH: url = "/".join( (self.url_resources, "_", quote_plus(resource.id))) prepared_request = asyncio.create_task( queue(hdrs.METH_GET, semaphore, session, url, resource, error)) if f_callback: prepared_request.add_done_callback(f_callback) futures.append(prepared_request) return futures