def test_no_context(self, form, store_metadata, valid_resource):
with pytest.raises(NotSupportedError):
as_jsonld(valid_resource,
form=form,
store_metadata=store_metadata,
model_context=None,
metadata_context=None,
context_resolver=None)
def test_from_graph(self, building, organization, building_jsonld, model_context, store_metadata, metadata_context):
store_metadata = False
id = "http://test/1234"
id_uri = term.URIRef(id)
graph = Graph()
graph.add((id_uri, RDF.type, term.URIRef("http://schema.org/Building")))
graph.add((id_uri,term.URIRef("http://schema.org/name"),term.Literal("The Empire State Building")))
graph.add((id_uri,term.URIRef("http://schema.org/description"),term.Literal("The Empire State Building is a 102-story landmark in New York City.")))
graph.add((id_uri,term.URIRef("http://schema.org/image"),term.URIRef("http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg")))
bNode = term.BNode()
graph.add((id_uri,term.URIRef("http://schema.org/geo"),bNode))
graph.add((bNode,term.URIRef("http://schema.org/latitude"),term.Literal("40.75")))
results = from_graph(graph)
assert isinstance(results, Resource)
building.id = id
building.context = model_context.document["@context"]
expected = building_jsonld(building, "expanded", store_metadata, None)
result_jsonld = as_jsonld(results, form="expanded", store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
assert result_jsonld == expected
graph.remove((id_uri, RDF.type, term.URIRef("http://schema.org/Building")))
results = from_graph(graph)
assert len(results) == 3
graph.add((id_uri, RDF.type, term.URIRef("http://schema.org/Building")))
graph.add((term.URIRef("http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg"), RDF.type, term.URIRef("http://schema.org/Image")))
results = from_graph(graph, type=["http://schema.org/Building","http://schema.org/Image"])
assert len(results) == 2
assert results[0].type is not None
assert results[1].type is not None
result_0 = as_jsonld(results[0], form="expanded", store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
result_1 = as_jsonld(results[1], form="expanded", store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
results = [result_0, result_1]
assert set(["http://schema.org/Building","http://schema.org/Image"]) == {result["@type"] for result in results}
frame = {
"@type": ['http://schema.org/Image'],
"@embed": True
}
results = from_graph(graph, frame=frame)
assert isinstance(results, Resource)
expected = {'@type': 'http://schema.org/Image', '@id': 'http://www.civil.usherbrooke.ca/cours/gci215a/empire-state-building.jpg'}
result_jsonld = as_jsonld(results, form="expanded", store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
assert result_jsonld == expected
def create_tasks(semaphore, session, loop, data, batch_action, f_callback, error):
futures = []
schema_id = kwargs.get("schema_id")
schema_id = "_" if schema_id is None else quote_plus(schema_id)
for resource in data:
params = deepcopy(kwargs.get("params", {}))
if batch_action == batch_action.CREATE:
context = self.model_context or self.context
payload = as_jsonld(resource, "compacted", False,
model_context=context, metadata_context=None,
context_resolver=self.resolve_context, na=nan)
url = f"{self.url_resources}/{schema_id}"
prepared_request = loop.create_task(
queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload, params))
if batch_action == batch_action.UPDATE:
url = "/".join((self.url_resources, schema_id, quote_plus(resource.id)))
params["rev"] = resource._store_metadata._rev
payload = as_jsonld(resource, "compacted", False,
model_context=self.model_context,
metadata_context=None,
context_resolver=self.resolve_context, na=nan)
prepared_request = loop.create_task(
queue(hdrs.METH_PUT, semaphore, session, url, resource, error, payload,
params))
if batch_action == batch_action.TAG:
url = "/".join((self.url_resources, "_", quote_plus(resource.id), "tags"))
rev = resource._store_metadata._rev
params["rev"] = rev
payload = {"tag": kwargs.get("tag"), "rev": rev}
prepared_request = loop.create_task(
queue(hdrs.METH_POST, semaphore, session, url, resource, error, payload,
params))
if batch_action == batch_action.DEPRECATE:
url = "/".join((self.url_resources, "_", quote_plus(resource.id)))
params["rev"] = resource._store_metadata._rev
prepared_request = loop.create_task(
queue(hdrs.METH_DELETE, semaphore, session, url, resource, error,
params=params))
if batch_action == BatchAction.FETCH:
resource_org, resource_prj = resource.project.split("/")[-2:]
resource_url = "/".join((self.endpoint, "resources", quote_plus(resource_org), quote_plus(resource_prj)))
url = "/".join((resource_url, "_", quote_plus(resource.id)))
prepared_request = loop.create_task(
queue(hdrs.METH_GET, semaphore, session, url, resource, error))
if f_callback:
prepared_request.add_done_callback(f_callback)
futures.append(prepared_request)
return futures
def as_jsonld(self,
data: Union[Resource, List[Resource]],
form: str = Form.COMPACTED.value,
store_metadata: bool = False) -> Union[Dict, List[Dict]]:
return as_jsonld(data, form, store_metadata, self._model.context(),
self._store.metadata_context,
self._model.resolve_context)
def _load_shape(self, resource_id):
if resource_id not in self._imported:
try:
shape = self.context_store.retrieve(resource_id,
version=None,
cross_bucket=False)
except RetrievalError as e:
print(e, resource_id)
# failed, don't try to load again
self._imported.append(resource_id)
else:
json_dict = as_jsonld(shape,
form="compacted",
store_metadata=False,
model_context=None,
metadata_context=None,
context_resolver=self.context_store.
service.resolve_context)
# this double conversion was due blank nodes were not "regenerated" with json-ld
temp_graph = Graph().parse(data=json.dumps(json_dict),
format="json-ld")
self._graph.parse(data=temp_graph.serialize(format="n3"),
format="n3")
self._imported.append(resource_id)
if hasattr(shape, "imports"):
for dependency in shape.imports:
self._load_shape(self.context.expand(dependency))
def _register_one(self, resource: Resource, schema_id: str) -> None:
context = self.model_context or self.context
data = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.service.resolve_context,
na=nan)
try:
schema = quote_plus(schema_id) if schema_id else "_"
url_base = f"{self.service.url_resources}/{schema}"
url = f"{url_base}/{quote_plus(data['@id'])}" if hasattr(
data, "@id") else url_base
params_register = copy.deepcopy(
self.service.params.get("register", None))
response = requests.post(url,
headers=self.service.headers,
data=json.dumps(data, ensure_ascii=True),
params=params_register)
response.raise_for_status()
except nexus.HTTPError as e:
raise RegistrationError(_error_message(e))
else:
response_json = response.json()
resource.id = response_json['@id']
# If resource had no context, update it with the one provided by the store.
if not hasattr(resource, "context"):
resource.context = data["@context"]
self.service.sync_metadata(resource, response_json)
def test_unregistered_resource_model_context(self, building, model_context, building_jsonld,
form, store_metadata, metadata_context):
expected = building_jsonld(building, form, store_metadata, model_context.document["@context"])
result = as_jsonld(building, form=form, store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
assert expected == result
def test_unregistered_resource_with_context(self, building_with_context, building_jsonld,
form, store_metadata, metadata_context):
expected = building_jsonld(building_with_context, form, store_metadata, None)
result = as_jsonld(building_with_context, form=form, store_metadata=store_metadata,
model_context=None, metadata_context=metadata_context,
context_resolver=None)
assert expected == result
def test_nested_resources(self, organization, organization_jsonld_compacted,
store_metadata, model_context, metadata_context):
compacted = organization_jsonld_compacted(organization, store_metadata=store_metadata)
result = as_jsonld(organization, form=Form.COMPACTED.value, store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
compacted["founder"]["@type"] = sorted(compacted["founder"]["@type"])
result["founder"]["@type"] = sorted(result["founder"]["@type"])
assert compacted == result
def test_registered_resource_with_context(self, building_with_context, make_registered,
building_jsonld, form, store_metadata,
store_metadata_value, metadata_context):
base = urljoin('file:', pathname2url(os.getcwd()))
registered_resource = make_registered(building_with_context, store_metadata_value, base)
expected = building_jsonld(registered_resource, form, store_metadata, None)
result = as_jsonld(registered_resource, form=form, store_metadata=store_metadata,
model_context=None, metadata_context=metadata_context,
context_resolver=None, na=None)
assert expected == result
def test_nested_resources(self, organization,
organization_jsonld_compacted, store_metadata,
model_context, metadata_context):
compacted = organization_jsonld_compacted(
organization, store_metadata=store_metadata)
result = as_jsonld(organization,
form=Form.COMPACTED.value,
store_metadata=store_metadata,
model_context=model_context,
metadata_context=metadata_context,
context_resolver=None)
assert compacted == result
def _update_one(self, resource: Resource, schema_id: str) -> None:
context = self.model_context or self.context
data = as_jsonld(resource, "compacted", False, model_context=context, metadata_context=None,
context_resolver=self.service.resolve_context, na=nan)
params_update = copy.deepcopy(self.service.params.get("update", {}))
params_update["rev"] = resource._store_metadata._rev
schema = quote_plus(schema_id) if schema_id else "_"
url = f"{self.service.url_resources}/{schema}/{quote_plus(resource.id)}"
try:
response = requests.put(url, headers=self.service.headers,
data=json.dumps(data, ensure_ascii=True), params=params_update)
response.raise_for_status()
except HTTPError as e:
raise UpdatingError(_error_message(e))
else:
self.service.sync_metadata(resource, response.json())
def as_json(data: Union[Resource, List[Resource]], expanded: bool,
store_metadata: bool, model_context: Optional[Context],
metadata_context: Optional[Context],
context_resolver: Optional[Callable]) -> Union[Dict, List[Dict]]:
if expanded:
return as_jsonld(data,
"expanded",
store_metadata,
model_context=model_context,
metadata_context=metadata_context,
context_resolver=context_resolver)
else:
if isinstance(data, Resource):
return _as_json(data, store_metadata)
else:
return [_as_json(x, store_metadata) for x in data]
def test_as_graph(self, building, organization, building_jsonld, model_context,store_metadata, metadata_context):
store_metadata = False
building.id = "http://test/1234"
building.context = model_context.document["@context"]
data = building_jsonld(building, "compacted", store_metadata, None)
expected = Graph().parse(data=json.dumps(data), format="json-ld")
result = as_graph(building, store_metadata, model_context, None, None)
_assert_same_graph(result, expected)
organization_jsonld = as_jsonld(organization, form="compacted", store_metadata=store_metadata,
model_context=model_context, metadata_context=metadata_context,
context_resolver=None, na=None)
expected = Graph()
expected.parse(data=json.dumps(data), format="json-ld")
expected.parse(data=json.dumps(organization_jsonld), format="json-ld")
result = as_graph([building, organization], store_metadata, model_context, None, None)
_assert_same_graph(result, expected)
def _update_one(self, resource: Resource) -> None:
context = self.model_context or self.context
data = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.service.resolve_context)
rev = {"rev": resource._store_metadata._rev}
url = f"{self.service.url_resources}/_/{quote_plus(resource.id)}"
try:
response = requests.put(url,
headers=self.service.headers,
data=json.dumps(data, ensure_ascii=True),
params=rev)
response.raise_for_status()
except HTTPError as e:
raise UpdatingError(_error_message(e))
else:
self.service.sync_metadata(resource, response.json())
def _register_one(self, resource: Resource, schema_id: str) -> None:
context = self.model_context or self.context
data = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.service.resolve_context)
try:
response = nexus.resources.create(org_label=self.organisation,
project_label=self.project,
data=data,
schema_id=schema_id)
except nexus.HTTPError as e:
raise RegistrationError(_error_message(e))
else:
resource.id = response['@id']
# If resource had no context, update it with the one provided by the store.
if not hasattr(resource, "context"):
resource.context = data["@context"]
self.service.sync_metadata(resource, response)
def create_tasks(semaphore, session, data, batch_action, f_callback,
error):
futures = []
for resource in data:
if batch_action == batch_action.CREATE:
context = self.model_context or self.context
payload = as_jsonld(resource,
"compacted",
False,
model_context=context,
metadata_context=None,
context_resolver=self.resolve_context)
schema_id = kwargs.get("schema_id")
schema_id = "_" if schema_id is None else quote_plus(
schema_id)
url = f"{self.url_resources}/{schema_id}"
prepared_request = asyncio.create_task(
queue(hdrs.METH_POST, semaphore, session, url,
resource, error, payload))
if batch_action == batch_action.UPDATE:
url = "/".join(
(self.url_resources, "_", quote_plus(resource.id)))
params = {"rev": resource._store_metadata._rev}
payload = as_jsonld(resource,
"compacted",
False,
model_context=self.model_context,
metadata_context=None,
context_resolver=self.resolve_context)
prepared_request = asyncio.create_task(
queue(hdrs.METH_PUT, semaphore, session, url, resource,
error, payload, params))
if batch_action == batch_action.TAG:
url = "/".join((self.url_resources, "_",
quote_plus(resource.id), "tags"))
rev = resource._store_metadata._rev
params = {"rev": rev}
payload = {"tag": kwargs.get("tag"), "rev": rev}
prepared_request = asyncio.create_task(
queue(hdrs.METH_POST, semaphore, session, url,
resource, error, payload, params))
if batch_action == batch_action.DEPRECATE:
url = "/".join(
(self.url_resources, "_", quote_plus(resource.id)))
params = {"rev": resource._store_metadata._rev}
prepared_request = asyncio.create_task(
queue(hdrs.METH_DELETE,
semaphore,
session,
url,
resource,
error,
params=params))
if batch_action == BatchAction.FETCH:
url = "/".join(
(self.url_resources, "_", quote_plus(resource.id)))
prepared_request = asyncio.create_task(
queue(hdrs.METH_GET, semaphore, session, url, resource,
error))
if f_callback:
prepared_request.add_done_callback(f_callback)
futures.append(prepared_request)
return futures
