def from_jsonld(req):
logger.debug("before transformation data: {}".format(req))
try:
frame = json.load(open("src/lib/fosf.jsonld"))
except:
frame = json.load(open("lib/fosf.jsonld"))
logger.debug("used frame: {}".format(frame))
done = jsonld.frame(req, frame)
logger.debug("after framing data: {}".format(done))
done["title"] = done["name"]
del done["name"]
done["description"] = done["description"].replace("\n", " ")
try:
del done["@context"]
del done["@id"]
del done["@type"]
except:
pass
data = {"data": {"type": "nodes", "attributes": done}}
logger.debug("after transformation data: {}".format(data))
return data
def rdf_to_json(format, path: Path) -> dict:
try:
import rdflib
from pyld import jsonld
except ImportError:
eprint("In order to process TTL manifests, you need RDFlib and PyLD:")
eprint(" python -m pip install rdflib PyLD")
return None
g = rdflib.Graph()
with path.open() as f:
g.load(f, format=format)
nq = g.serialize(format="ntriples").decode("utf-8")
extended = jsonld.from_rdf(nq)
with DIR.joinpath("manifest-context.jsonld").open() as f:
frame = json.load(f)
manifest = jsonld.frame(extended, frame)
# ugly hack to "relativize" IRIs
manifest = json.dumps(manifest)
manifest = manifest.replace(f'file://{path.absolute()}#', "#")
manifest = manifest.replace(f'file://{path.parent.absolute()}/', "")
manifest = json.loads(manifest)
return manifest
def do_frame(doc, context):
# frame a document
# see: https://json-ld.org/spec/latest/json-ld-framing/#introduction
framed = jsonld.frame(doc, context)
# document transformed into a particular tree structure per the given frame
print("FRAMED")
print(json.dumps(framed, indent=2))
def __init__(self, graph):
self.graph = jsonld.frame(graph, {
'@context': {
'@vocab': 'http://www.bbc.co.uk/ontologies/webmodules/'
},
'@type': 'WebModule'
})
def frame_and_write(graph, context, frame, output_dir, graph_fixer = None ):
# Take an RDF graph, convert it to JSON-LD with a compaction context, frame, and an optional graph fixer func
fp = os.path.join("frames/", frame)
with open(fp) as f:
frm = json.load(f)
print("Loading graph as JSON-LD")
# with requests.get(context) as r:
# if r.status_code == 200:
# context = json.loads(r.content)
# print("Context is",context)
# del context["@context"]["@version"]
empty_ctx = { "@context": {} }
js = json.loads(graph.serialize(format = 'json-ld', context = empty_ctx ))
#js = jsonld.compact(js, context)
print("Framing entities")
entities = jsonld.frame( js,frm )
if graph_fixer:
entities = graph_fixer(entities)
print("Writing entities")
write_jsonld(entities, output_dir)
def _test_remote_context_with(
self, frame_doc, frame_context_doc, out_doc):
input_ = json.loads(FRAME_0001_IN)
def fake_loader(url):
if url == 'http://example.com/frame.json':
return {
'contextUrl': 'http://example.com/frame-context.json',
'document': frame_doc,
'documentUrl': url
}
elif url == 'http://example.com/frame-context.json':
return {
'contextUrl': None,
'document': frame_context_doc,
'documentUrl': url
}
else:
raise Exception("Unknown URL: {}".format(url))
options = {
'documentLoader': fake_loader
}
framed = jsonld.frame(
input_, 'http://example.com/frame.json', options=options)
self.assertEqual(framed, json.loads(out_doc))
def jsonld(self,
frame=None,
options=None,
context=None,
removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if context is None:
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
def write_resources(self,
graph_id=None,
resourceinstanceids=None,
**kwargs):
super(RdfWriter,
self).write_resources(graph_id=graph_id,
resourceinstanceids=resourceinstanceids,
**kwargs)
g = self.get_rdf_graph()
value = g.serialize(format="nquads").decode("utf-8")
# print(f"Got graph: {value}")
js = from_rdf(value, {
"format": "application/nquads",
"useNativeTypes": True
})
assert len(
resourceinstanceids
) == 1 # currently, this should be limited to a single top resource
archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
resource_inst_uri = archesproject[reverse(
"resources", args=[resourceinstanceids[0]]).lstrip("/")]
context = self.graph_model.jsonldcontext
framing = {
"@omitDefault": True,
"@omitGraph": False,
"@id": str(resource_inst_uri)
}
if context:
framing["@context"] = context
js = frame(js, framing)
try:
context = JSONDeserializer().deserialize(context)
except ValueError:
if context == "":
context = {}
context = {"@context": context}
except AttributeError:
context = {"@context": {}}
# Currently omitGraph is not processed by pyLd, but data is compacted
# simulate omitGraph:
if "@graph" in js and len(js["@graph"]) == 1:
# merge up
for (k, v) in list(js["@graph"][0].items()):
js[k] = v
del js["@graph"]
out = json.dumps(js, indent=kwargs.get("indent", None), sort_keys=True)
dest = StringIO(out)
full_file_name = os.path.join("{0}.{1}".format(self.file_name,
"jsonld"))
return [{"name": full_file_name, "outputfile": dest}]
def jsonld(self, frame=None, options=None,
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if context is None:
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
def rdf2es(self, string, bibo):
"""
Does the really interesting stuff: Transformation of the
triples by subject and indexing in ES
:param string: The RDF triples as a concatenated string.
:param bibo: Is subject a bibo:Document?
:return: Body for ES indexing
"""
g = Graph().parse(data=string)
jldstr = g.serialize(format='json-ld',
indent=4)
if bibo:
esdoc = jsonld.compact(loads(jldstr.decode('utf-8')), self.loadjson(self.frame))
doctype = 'document'
else:
esdoc = loads(jldstr.decode('utf-8'))
esdoc = jsonld.frame(esdoc, self.loadjson(self.frame))['@graph'][0]
esdoc['@context'] = self.loadjson(self.frame)['@context']
doctype = 'bibliographicResource'
docid = re.findall('\w{9}', esdoc['@id'])[0]
if self.filemode:
bulkfile = [{'index': {'_index': self.index, '_type': doctype, '_id': docid}}, esdoc]
return bulkfile
else:
esdoc.update({'_index': self.index, '_type': doctype, '_id': docid})
return esdoc
def post_framing():
print(request.data)
with open("ifcJsonData.json", "r") as f:
doc = json.load(f)
frame = json.load(request.data)
framed = jsonld.frame(doc, frame)
return framed
def jams(uid, voice=''):
#n.b. voice not actually used here; used clientside
rdf_file = "{0}/rdf/{1}.ttl".format(basedir, uid)
if not os.path.isfile(rdf_file):
abort(404) # file not found
g = Graph().parse(rdf_file, format="turtle")
raw_json = json.loads(g.serialize(format="json-ld", indent=2))
framed = jsonld.frame(raw_json, {
"@context": {
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"cnt": "http://www.w3.org/2011/content#",
"oa": "http://www.w3.org/ns/oa#",
"meldterm": "http://meld.linkedmusic.org/terms/",
"manifest": "http://meld.linkedmusic.org/manifestations/",
"leitmotif": "http://meld.linkedmusic.org/leitmotif/",
"frbr": "http://purl.org/vocab/frbr/core#",
"fabio": "http://purl.org/spar/fabio/",
"dct": "http://purl.org/dc/terms/"
},
"@type": "meldterm:topLevel"
},
options={"compactArrays": False})
best = best_mimetype()
if best == "text/html":
return render_template("dynameld.html",
annotations=json.dumps(framed, indent=2))
elif best == "application/json":
return json.dumps(framed, indent=2)
else:
return g.serialize(format=best)
def loader(data: Union[str, dict], _: FileInfo) -> Optional[dict]:
"""
Process an RDF graph or a JSON-LD string. We do this by using pyld_jsonld_from_rdflib_graph to emit a JSON-LD
string and then process it with jsonld.frame.
:param data: Graph or JSON-LD string
:return: Dictionary to load into the target class
"""
if isinstance(data, str):
if fmt != 'json-ld':
g = Graph()
g.parse(data=data, format=fmt)
data = pyld_jsonld_from_rdflib_graph(g)
if not isinstance(data, dict):
# TODO: Add a context processor to the source w/ CONTEXTS_PARAM_TYPE
# TODO: figure out what to do base options below
# TODO: determine whether jsonld.frame can handle something other than string input
data_as_dict = jsonld.frame(data, contexts)
else:
data_as_dict = data
typ = data_as_dict.pop('@type', None)
# TODO: remove this when we get the linkml issue fixed
if not typ:
typ = data_as_dict.pop('type', None)
if typ and typ != target_class.class_name:
# TODO: connect this up with the logging facility or warning?
print(
f"Warning: input type mismatch. Expected: {target_class.__name__}, Actual: {typ}"
)
return json_clean(data_as_dict)
def serialize_in_json(g, uri):
context = build_graph_context(g)
cg = skolemize(g)
ted_nquads = cg.serialize(format='nquads')
ld = jsonld.from_rdf(ted_nquads)
type = list(cg.objects(uri, RDF.type)).pop()
ld = jsonld.frame(ld, {'context': context, '@type': str(type)})
return json.dumps(jsonld.compact(ld, context), indent=3, sort_keys=True)
def convert(anno):
# check we're already parsed
if not type(anno) == dict:
anno = json.loads(anno)
# rdf = expand(anno)
reframed = frame(anno, annoframe)
outjs = compact(reframed, contextURI)
return outjs
def from_jsonld(jsonld_data):
if jsonld_data is None:
return
try:
frame = json.load(open("src/lib/fdatasafe.jsonld"))
except:
frame = json.load(open("lib/fdatasafe.jsonld"))
done = jsonld.frame(jsonld_data, frame)
logger.debug("after framing: {}".format(done))
done["titles"] = [{"title": done["name"]}]
del done["name"]
done["publicationYear"] = {
"dateTime": done["datePublished"],
"dateTimeScheme": "COMPLETE_DATE",
"dateType": "Submitted"
}
del done["datePublished"]
if not isinstance(done["creator"], list):
done["creator"] = [done["creator"]]
done["creators"] = []
for creator in done["creator"]:
data = {"entityType": "Personal", "entityName": creator["name"]}
data.update(creator)
done["creators"].append(data)
del done["creator"]
done["publisher"] = {
"entityName": done["creators"][0]["affiliation"]["name"],
"entityType": "Organizational"
}
if done["resource"].find("/") > 0:
typ, subtyp = tuple(done["resource"].split("/", 1))
done["resource"] = typ
done["resourceType"] = subtyp
done["description"] = done["description"].replace("\n", "<br>")
done["descriptions"] = [done["description"]]
del done["description"]
logger.debug("after transforming: {}".format(done))
try:
del done["@context"]
del done["@id"]
del done["@type"]
except:
pass
return done
def get_lens_for(iri: AnyUrl, via: AnyUrl) -> models.Lens:
"""Fetch a Lens description for further execution."""
universe = graph()
universe.parse(
data=cached_http_get('http://*****:*****@context': {
'@vocab': 'https://iolanta.tech/',
'sparql': {
'@type': '@id',
'@container': '@set',
},
'frame': {
'@type': '@id'
},
'from-named': {
'@type': '@id'
},
},
'@id': via,
}
lens_data = jsonld.frame(
input_=lens_graph,
frame=frame,
)
try:
frame = json.loads(cached_http_get(lens_data['frame']))
except KeyError as err:
raise Exception(
f'Could not obtain the frame URL from: '
f'{json.dumps(lens_data, indent=2)}', ) from err
return models.Lens(frame=frame,
queries=list(map(get_sparql_query,
lens_data['sparql'])))
def write_resources(self,
graph_id=None,
resourceinstanceids=None,
**kwargs):
super(RdfWriter,
self).write_resources(graph_id=graph_id,
resourceinstanceids=resourceinstanceids,
**kwargs)
g = self.get_rdf_graph()
value = g.serialize(format='nquads')
js = from_rdf(value, {
'format': 'application/nquads',
'useNativeTypes': True
})
assert len(
resourceinstanceids
) == 1 # currently, this should be limited to a single top resource
archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
resource_inst_uri = archesproject[reverse(
'resources', args=[resourceinstanceids[0]]).lstrip('/')]
context = self.graph_model.jsonldcontext
framing = {
"@omitDefault": True,
"@omitGraph": False,
"@id": str(resource_inst_uri)
}
if context:
framing["@context"] = context
js = frame(js, framing)
try:
context = JSONDeserializer().deserialize(context)
except ValueError:
if context == '':
context = {}
context = {"@context": context}
except AttributeError:
context = {"@context": {}}
# Currently omitGraph is not processed by pyLd, but data is compacted
# simulate omitGraph:
if '@graph' in js and len(js['@graph']) == 1:
# merge up
for (k, v) in js['@graph'][0].items():
js[k] = v
del js['@graph']
out = json.dumps(js, indent=kwargs.get('indent', None), sort_keys=True)
dest = StringIO(out)
full_file_name = os.path.join('{0}.{1}'.format(self.file_name,
'jsonld'))
return [{'name': full_file_name, 'outputfile': dest}]
def parse(self, stream, media_type=None, parser_context=None):
data = super(JSONLDParser, self).parse(stream, media_type, parser_context)
iri = "http://%s/project/ldpcontainer/%s"%(settings.LDP_DOMAIN, parser_context["kwargs"]["pk"])
for obj in jsonld.frame(data['@graph'], data['@context'])['@graph']:
if obj.get("@id") =='./':
del obj["@id"]
return obj
if obj.get("@id") == iri:
return obj
def triples_to_resource(iri, triples):
graph = Graph().parse(data=triples, format="nt")
data_expanded = json.loads(graph.serialize(format="json-ld").decode("utf-8"))
frame = {"@id": iri}
data_framed = jsonld.frame(data_expanded, frame)
context = self.model_context or self.context
compacted = jsonld.compact(data_framed, context.document)
resource = from_jsonld(compacted)
resource.context = context.iri if context.is_http_iri() else context.document["@context"]
return resource
def FrameGraph(graph, frame=_FullFrame):
serialized = graph.serialize(format='json-ld')
json_val = json.loads(serialized)
json_val = {'@context': _Context, '@graph': AsList(json_val)}
framed = jsonld.frame(json_val, frame, {'embed': '@always'})
framed['@context'] = 'http://schema.org'
for items in framed['@graph']:
framed.update(items)
del framed['@graph']
return framed
def serialize_graph(g, format=TURTLE, frame=None):
if format == TURTLE:
return g.serialize(format='turtle')
context = build_graph_context(g)
cg = skolemize(g)
ted_nquads = cg.serialize(format='nquads')
ld = jsonld.from_rdf(ted_nquads)
if frame is not None:
ld = jsonld.frame(ld, {'context': context, '@type': str(frame)})
return json.dumps(jsonld.compact(ld, context), indent=3, sort_keys=True)
def put(self, request, **kwargs):
data = request.data
topic = self.get_object()
url = request.build_absolute_uri(topic.get_absolute_url())
framed = jsonld.frame(data, {'@id': url})
topic.ld = framed['@graph'][0] if framed['@graph'] else {}
# TODO: make revision
topic.save()
return Response(topic.ld)
def add_BD_fields(jsonld_str, esdoc):
expanded = jsonld.expand(json.loads(jsonld_str))
logger.info("EXPANDED: "+json.dumps(expanded, indent=2))
framed = jsonld.frame(expanded, dts_jsonld_frame)
logger.info("FRAMED: "+json.dumps(expanded, indent=2))
for field, obj in dts_fields.items():
# logger.info("obj: "+obj)
esdoc[field] = []
# append all the matching values to the ES field
for val in [match.value for match in obj['expr'].find(framed)]:
esdoc[field].append(val)
def _normalize(schema_obj):
framed = jsonld.frame(schema_obj, DEFAULT_FRAME)
compacted = jsonld.compact(framed,
DEFAULT_CONTEXT,
options=dict(graph=True))
graph = compacted["@graph"]
graph = [
ReorderingVisitor(add_type=True).visit_schema(schema_obj)
for schema_obj in graph
]
compacted["@graph"] = graph if len(graph) > 1 else graph[0]
return compacted
def add_BD_fields(jsonld_str, esdoc):
expanded = jsonld.expand(json.loads(jsonld_str))
logger.info("EXPANDED: "+json.dumps(expanded, indent=2))
framed = jsonld.frame(expanded, dts_jsonld_frame)
logger.info("FRAMED: "+json.dumps(expanded, indent=2))
for field, obj in dts_fields.items():
# logger.info("obj: "+obj)
esdoc[field] = []
# append all the matching values to the ES field
for val in [match.value for match in obj['expr'].find(framed)]:
esdoc[field].append(val)
def rdf_to_jsonld(self, rdf, fmt):
g = ConjunctiveGraph()
g.parse(data=rdf, format=fmt)
out = g.serialize(format='json-ld')
j2 = json.loads(out)
j2 = {"@context": context_js, "@graph": j2}
framed = frame(j2, frame_js)
out = compact(framed, context_js)
# recursively clean blank node ids
#out = self._clean_bnode_ids(out)
return out
def build_json(self, graph_id=None, resourceinstanceids=None, **kwargs):
# Build the JSON separately serializing it, so we can use internally
super(RdfWriter,
self).write_resources(graph_id=graph_id,
resourceinstanceids=resourceinstanceids,
**kwargs)
g = self.get_rdf_graph()
value = g.serialize(format="nquads").decode("utf-8")
js = from_rdf(value, {
"format": "application/nquads",
"useNativeTypes": True
})
assert len(
resourceinstanceids
) == 1 # currently, this should be limited to a single top resource
archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
resource_inst_uri = archesproject[reverse(
"resources", args=[resourceinstanceids[0]]).lstrip("/")]
context = self.graph_model.jsonldcontext
framing = {
"@omitDefault": True,
"@omitGraph": False,
"@id": str(resource_inst_uri)
}
if context:
framing["@context"] = context
js = frame(js, framing)
try:
context = JSONDeserializer().deserialize(context)
except ValueError:
if context == "":
context = {}
context = {"@context": context}
except AttributeError:
context = {"@context": {}}
# Currently omitGraph is not processed by pyLd, but data is compacted
# simulate omitGraph:
if "@graph" in js and len(js["@graph"]) == 1:
# merge up
for (k, v) in list(js["@graph"][0].items()):
js[k] = v
del js["@graph"]
return js
def create_jsonLD(graph_data, filter_frame):
"""Create JSON-LD output for the given subject."""
graph = ConjunctiveGraph()
graph.parse(data=graph_data, format="turtle")
try:
# pyld likes nquads, by default
expanded = jsonld.from_rdf(graph.serialize(format="nquads"))
framed = jsonld.frame(expanded, json.loads(filter_frame))
result = json.dumps(framed, indent=1, sort_keys=True)
app_logger.info('Serialized as JSON-LD compact with the frame.')
return result
except Exception as error:
app_logger.error('JSON-LD frame failed with error: {0}'.format(error))
return error
def frame_result(jld, result):
frame = {
"@context": {
"xsd": "http://www.w3.org/2001/XMLSchema#",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
'tc': 'https://rdf.lodgeit.net.au/testcase/'
},
#'@type':'https://rdf.lodgeit.net.au/tau_testcase_parser/Result'
'@id': franz_term_to_pyld(result)['value']
}
data = jsonld.frame(jld, frame, {'omitGraph': False, 'embed': '@always'})
#print(json.dumps(data,indent=4))
return data
def get(self, request, *args, **kwargs):
base_url = "http://" + request.get_host()
graph = Graph("Hyperdjango", identifier=URIRef(base_url))
graph.open(configuration=self.models.__name__)
r = graph.query(
"""
PREFIX schema: <http://schema.org/>
CONSTRUCT
WHERE {
?x ?p ?y .
}
""",
initBindings={"x": URIRef(base_url + request.path)},
)
if len(r) == 0:
return HttpResponse(status=404)
g = Graph()
for r_ in r:
g.add(r_)
mimetype, rdf_format = FORMAT.decide(
request.META.get("HTTP_ACCEPT", "application/ld+json"),
g.context_aware,
)
body = g.serialize(format=rdf_format, publicID=base_url).decode("utf8")
if rdf_format == "json-ld":
body = json.dumps(
jsonld.compact(
jsonld.frame(
json.loads(body),
{"@id": base_url + request.path},
{
"base": base_url,
},
),
{
"@vocab": base_url + "/",
"@base": base_url,
},
))
return HttpResponse(body, content_type=mimetype)
def apply_lens(iri: AnyUrl, lens: models.Lens) -> dict:
"""Apply lens to given IRI and get the JSON-LD data."""
result = rdflib.Graph()
for query in lens.queries:
result += graph_by_query(query)
# TODO can serialize() return a dict?
jsonld_subgraph = json.loads(result.serialize(format='json-ld', ))
jsonld_subgraph = jsonld.frame(
input_=jsonld_subgraph,
frame=lens.frame,
)
return jsonld_subgraph
def frameAndCompact(input, frame, context):
orderStruct = QueryUtils.getOrderFromFrame(frame)
QueryUtils.addDatatypes(input)
QueryUtils.dirtyCompactFullIRIs(input,
QueryUtils.invertContext(context))
framedResults = jsonld.frame(input, frame)
compactedResults = jsonld.compact(framedResults, context)
if "@graph" in compactedResults:
results = compactedResults["@graph"]
else:
if "@context" in compactedResults:
del compactedResults["@context"]
results = compactedResults
QueryUtils.executeOrder(results, orderStruct)
return results
def convert(input_path, output_path):
skos = jsonld.from_rdf(get_skos(input_path).decode('unicode_escape').encode('utf-8','ignore'))
context = {
"@vocab": "http://www.w3.org/2004/02/skos/core#",
"name": {
"@id": "http://www.w3.org/2004/02/skos/core#prefLabel",
"@container": "@set"
},
"alternateName": {
"@id": "http://www.w3.org/2004/02/skos/core#altLabel",
"@container": "@set"
},
"narrower": {
"@id": "http://www.w3.org/2004/02/skos/core#narrower",
"@container": "@set"
},
"description": {
"@id": "http://purl.org/dc/terms/description",
"@container": "@set"
},
"scopeNote": {
"@container": "@set"
},
"notation": {
"@container": "@set"
},
"publisher": "http://purl.org/dc/terms/publisher",
"title": "http://purl.org/dc/terms/title",
"preferredNamespacePrefix": "http://purl.org/vocab/vann/preferredNamespacePrefix",
"preferredNamespaceUri": "http://purl.org/vocab/vann/preferredNamespaceUri",
"source": "http://purl.org/dc/terms/source"
}
frame = {
"@context": context,
"@type": "ConceptScheme",
"@explicit": True,
"hasTopConcept": {
"@type": "Concept",
"narrower": {
"@type": "Concept"
}
}
}
framed = jsonld.compact(jsonld.frame(skos, frame), context)
del framed['@context']
with open(output_path, 'w') as output_file:
json.dump(framed, output_file, indent=2, ensure_ascii=False)
print "Wrote data for " + input_path + " to " + output_path
def _rdf_to_jsonld(self, b):
fmt = request.headers['Content-Type']
if fmt in self.rdflib_format_map:
rdftype = self.rdflib_format_map[fmt]
g = Graph()
g.parse(data=b, format=rdftype)
out = g.serialize(format='json-ld')
# AND THIS IS WHERE IT GETS CRAAAAAZEEEE...
# aka rdflib doesn't do framing so we re-re-parse it
j2 = json.loads(out)
j2 = {"@context": self.default_context, "@graph": j2}
framed = frame(j2, self.annoframe)
out = compact(framed, self.default_context)
# recursively clean blank node ids
out = self._clean_bnode_ids(out)
return out
def _rdf_to_jsonld(self, b):
fmt = request.headers['Content-Type']
if self.rdflib_format_map.has_key(fmt):
rdftype = self.rdflib_format_map[fmt]
g = Graph()
g.parse(data=b, format=rdftype)
out = g.serialize(format='json-ld')
# AND THIS IS WHERE IT GETS CRAAAAAZEEEE...
# aka rdflib doesn't do framing so we re-re-parse it
j2 = json.loads(out)
j2 = {"@context": self.default_context, "@graph": j2}
framed = frame(j2, self.annoframe)
out = compact(framed, self.default_context)
# recursively clean blank node ids
out = self._clean_bnode_ids(out)
return out
def _create_ld(self):
"""Create JSON-LD output for the given subject."""
graph = self._merge_graphs()
# result = None
try:
# pyld likes nquads, by default
expanded = pyld_jsonld_from_rdflib_graph(graph)
framed = jsonld.frame(expanded, json.loads(self.ld_frame))
result = json.dumps(framed, indent=1, sort_keys=True)
app_logger.info('Serialized as JSON-LD compact with the frame.')
except Exception as error:
app_logger.error(
'JSON-LD frame failed with error: {0}'.format(error))
raise
finally:
return result
def get_jsonld(self, context, new_context={}, format="full"):
"""Return the JSON-LD serialization.
:param: context the context to use for raw publishing; each SmartJsonLD
instance is expected to have a default context associated.
:param: new_context the context to use for formatted publishing,
usually supplied by the client; used by the 'compacted', 'framed',
and 'normalized' formats.
:param: format the publishing format; can be 'full', 'inline',
'compacted', 'expanded', 'flattened', 'framed' or 'normalized'.
Note that 'full' and 'inline' are synonims, referring to the
document form which includes the context; for more information see:
[http://www.w3.org/TR/json-ld/]
"""
from pyld import jsonld
if isinstance(context, six.string_types):
ctx = self.get_context(context)
elif isinstance(context, dict):
ctx = context
else:
raise TypeError('JSON-LD context must be a string or dictionary')
try:
doc = self.translate(context, ctx)
except NotImplementedError:
# model does not require translation
doc = self.dumps(clean=True)
doc["@context"] = ctx
if format in ["full", "inline"]:
return doc
if format == "compacted":
return jsonld.compact(doc, new_context)
elif format == "expanded":
return jsonld.expand(doc)
elif format == "flattened":
return jsonld.flatten(doc)
elif format == "framed":
return jsonld.frame(doc, new_context)
elif format == "normalized":
return jsonld.normalize(doc, new_context)
raise ValueError('Invalid JSON-LD serialization format')
def main(self):
print('PyLD Unit Tests')
print('Use -h or --help to view options.')
# add program options
self.parser.add_option('-f', '--file', dest='file',
help='The single test file to run', metavar='FILE')
self.parser.add_option('-d', '--directory', dest='directory',
help='The directory full of test files', metavar='DIR')
self.parser.add_option('-e', '--earl', dest='earl',
help='The filename to write the EARL report to', metavar='EARL')
self.parser.add_option('-v', '--verbose', dest='verbose',
action='store_true', default=False,
help='Prints verbose test data')
# parse options
(self.options, args) = self.parser.parse_args()
# check if file or directory were specified
if self.options.file == None and self.options.directory == None:
raise Exception('No test file or directory specified.')
# check if file was specified, exists, and is file
if self.options.file is not None:
if (os.path.exists(self.options.file) and
os.path.isfile(self.options.file)):
# add manifest file to the file list
self.manifest_files.append(os.path.abspath(self.options.file))
else:
raise Exception('Invalid test file: "%s"' % self.options.file)
# check if directory was specified, exists and is dir
if self.options.directory is not None:
if (os.path.exists(self.options.directory) and
os.path.isdir(self.options.directory)):
# load manifest files from test directory
for test_dir, dirs, files in os.walk(self.options.directory):
for manifest in files:
# add all .jsonld manifest files to the file list
if (manifest.find('manifest') != -1 and
manifest.endswith('.jsonld')):
self.manifest_files.append(
join(test_dir, manifest))
else:
raise Exception('Invalid test directory: "%s"' %
self.options.directory)
# see if any manifests have been specified
if len(self.manifest_files) == 0:
raise Exception('No manifest files found.')
passed = 0
failed = 0
total = 0
# run the tests from each manifest file
for manifest_file in self.manifest_files:
test_dir = os.path.dirname(manifest_file)
manifest = json.load(open(manifest_file, 'r'))
count = 1
for test in manifest['sequence']:
# skip unsupported types
skip = True
test_type = test['@type']
for tt in test_type:
if tt in SKIP_TEST_TYPES:
skip = True
break
if tt in TEST_TYPES:
skip = False
if skip:
# print 'Skipping test: "%s" ...' % test['name']
continue
print('JSON-LD/%s %04d/%s...' % (
manifest['name'], count, test['name']), end=' ')
total += 1
count += 1
# read input file
with open(join(test_dir, test['input'])) as f:
if test['input'].endswith('.jsonld'):
input = json.load(f)
else:
input = f.read().decode('utf8')
# read expect file
with open(join(test_dir, test['expect'])) as f:
if test['expect'].endswith('.jsonld'):
expect = json.load(f)
else:
expect = f.read().decode('utf8')
result = None
# JSON-LD options
options = {
'base': 'http://json-ld.org/test-suite/tests/' +
test['input'],
'useNativeTypes': True
}
success = False
try:
if 'jld:ExpandTest' in test_type:
result = jsonld.expand(input, options)
elif 'jld:CompactTest' in test_type:
ctx = json.load(open(join(test_dir, test['context'])))
result = jsonld.compact(input, ctx, options)
elif 'jld:FlattenTest' in test_type:
result = jsonld.flatten(input, None, options)
elif 'jld:FrameTest' in test_type:
frame = json.load(open(join(test_dir, test['frame'])))
result = jsonld.frame(input, frame, options)
elif 'jld:FromRDFTest' in test_type:
result = jsonld.from_rdf(input, options)
elif 'jld:ToRDFTest' in test_type:
options['format'] = 'application/nquads'
result = jsonld.to_rdf(input, options)
elif 'jld:NormalizeTest' in test_type:
options['format'] = 'application/nquads'
result = jsonld.normalize(input, options)
# check the expected value against the test result
success = deep_compare(expect, result)
if success:
passed += 1
print('PASS')
else:
failed += 1
print('FAIL')
if not success or self.options.verbose:
print('Expect:', json.dumps(expect, indent=2))
print('Result:', json.dumps(result, indent=2))
except jsonld.JsonLdError as e:
print('\nError: ', e)
failed += 1
print('FAIL')
# add EARL report assertion
EARL['subjectOf'].append({
'@type': 'earl:Assertion',
'earl:assertedBy': EARL['doap:developer']['@id'],
'earl:mode': 'earl:automatic',
'earl:test': ('http://json-ld.org/test-suite/tests/' +
os.path.basename(manifest_file) + test.get('@id', '')),
'earl:result': {
'@type': 'earl:TestResult',
'dc:date': datetime.datetime.utcnow().isoformat(),
'earl:outcome': ('earl:' + 'passed' if success else
'failed')
}
})
if self.options.earl:
f = open(self.options.earl, 'w')
f.write(json.dumps(EARL, indent=2))
f.close()
print('Done. Total:%d Passed:%d Failed:%d' % (total, passed, failed))
#add the thesis to the temporary graph
# print thesisXML.serialize(format='n3')
tmpgraph = rdflib.Graph(g.store, osuNs['theses'])
tmpgraph.parse(data=thesisXML.serialize(format='xml'))
for o in tmpgraph.objects():
if isinstance(o, rdflib.URIRef):
query = "DESCRIBE <" + o + ">"
endpoint.setQuery(query)
desc = endpoint.query().convert()
tmpgraph.parse(data=desc.serialize(format='xml'))
j = jsonld.compact(jsonld.from_rdf(g.serialize(format='nquads')), contexts)
if 0 in j['@graph']:
try:
outfile.write(json.dumps(j['@graph'][0], indent=1))
j = jsonld.frame(j, json.load(urllib2.urlopen('http://achelo.us/thesis_frame.jsonld')))
outfile_f.write(json.dumps(j['@graph'][0], indent=1))
elastic.index(j['@graph'][0], 'theses', 'thesis')
except Exception as e:
print e
print json.dumps(j['@graph'])
outfile_e.write(json.dumps(j))
continue
if (count % 100) == 0:
print count
outfile.close()
outfile_f.close()
def frame(self, frame=None, options=None):
if frame is None:
frame = self._frame
if options is None:
options = {}
return jsonld.frame(self, frame, options)
def serialize_json(graph, resource, _binding=None):
"""I serialize an RDF graph as JSON-LD.
I serialize 'graph' in JSON-LD, using a frame if available.
See :func:`rdfrest.serializer.serialize_rdf_xml` for prototype
documentation.
"""
is_obsels_graph = False
local_graph = Graph()
local_graph += graph
json_obj = []
base_uri = resource.uri
for p in XREVS:
rev_p = URIRef("x-rev:%s" % p)
for s, o in local_graph.subject_objects(p):
local_graph.remove((s, p, o))
local_graph.add((o, rev_p, s))
if (graph.value(predicate=RDF.type,
object=KTBS.StoredTraceObsels) is not None) or \
(graph.value(predicate=RDF.type,
object=KTBS.ComputedTraceObsels) is not None):
is_obsels_graph = True
trace_uri = graph.value(object=base_uri,
predicate=KTBS.hasObselCollection)
trace = resource.factory(trace_uri)
trace_model = trace.get_model_uri()
cache = {}
for s, p, o in local_graph:
sdict = cache.get(s)
if sdict is None:
sdict = cache[s] = node2jld(s)
if p == RDF.type:
p = u"@type"
o = uri2iri(o)
else:
p = uri2iri(p)
o = node2jld(o)
oobj = sdict.get(p)
if oobj is None:
sdict[p] = o
else:
if not isinstance(oobj, list):
oobj = [oobj]
sdict[p] = oobj
oobj.append(o)
json_obj = list(cache.itervalues())
object_types = cache.get(base_uri, {}).get("@type")
if not isinstance(object_types, list):
object_types = [object_types]
context_dict = {}
context_dict.update(CONTEXT)
ktbs_frame = None
for o in object_types:
if o in KTBS_FRAME_TYPES:
if is_obsels_graph:
if "m" not in context_dict:
context_dict["m"] = trace_model
ktbs_frame = {
u"@context": context_dict,
u"@type": KTBS_FRAME_TYPES[o],
u"obsels": [] # Only used for @obsels
}
else:
ktbs_frame = {
u"@context": CONTEXT,
u"@type": KTBS_FRAME_TYPES[o],
}
break
if frame is not None:
json_obj = frame(json_obj, ktbs_frame)
if is_obsels_graph:
context_list = json_obj["@context"] = []
context_list.append(CONTEXT_URI)
context_list.append({"m": trace_model})
json_obj[u'obsels'].sort(key=lambda d: d[u'begin'])
else:
json_obj["@context"] = CONTEXT_URI
if __debug__:
return dumps(json_obj, indent=4)
else:
return dumps(json_obj)
}
}
def format_json(d):
return json.dumps(d, sort_keys=True, indent=4, separators=(',', ': '))
print "Input:"
print format_json(library)
print
print "Frame:",
print format_json(frame)
print
print "Framed:"
framed = jsonld.frame(library, frame)
print format_json(framed)
print
friendly_context = {
"creator": "http://purl.org/dc/elements/1.1/creator",
"title": "http://purl.org/dc/elements/1.1/title",
"description": "http://purl.org/dc/elements/1.1/description",
"contains": "http://example.org/vocab#contains",
}
print "Framed, compacted:"
compacted = jsonld.compact(framed, context)
print format_json(compacted)
print "Framed, compacted with friendly context"
def main(self):
print "PyLD TestRunner"
print "Use -h or --help to view options."
# add program options
self.parser.add_option("-f", "--file", dest="file",
help="The single test file to run", metavar="FILE")
self.parser.add_option("-d", "--directory", dest="directory",
help="The directory full of test files", metavar="DIR")
self.parser.add_option("-v", "--verbose", dest="verbose",
action="store_true", default=False, help="Prints verbose test data")
# parse options
(self.options, args) = self.parser.parse_args()
# check if file or directory were specified
if self.options.file == None and self.options.directory == None:
print "No test file or directory specified."
return
# check if file was specified, exists and is file
if self.options.file != None:
if (os.path.exists(self.options.file) and
os.path.isfile(self.options.file)):
# add test file to the file list
self.testfiles.append(os.path.abspath(self.options.file))
self.testdir = os.path.dirname(self.options.file)
else:
print "Invalid test file."
return
# check if directory was specified, exists and is dir
if self.options.directory != None:
if (os.path.exists(self.options.directory) and
os.path.isdir(self.options.directory)):
# load test files from test directory
for self.testdir, dirs, files in os.walk(self.options.directory):
for testfile in files:
# add all .test files to the file list
if testfile.endswith(".test"):
self.testfiles.append(join(self.testdir, testfile))
else:
print "Invalid test directory."
return
# see if any tests have been specified
if len(self.testfiles) == 0:
print "No tests found."
return
# FIXME:
#self.testFiles.sort()
run = 0
passed = 0
failed = 0
# run the tests from each test file
for testfile in self.testfiles:
# test group in test file
testgroup = json.load(open(testfile, 'r'))
count = 1
for test in testgroup['tests']:
print 'Test: %s %04d/%s...' % (
testgroup['group'], count, test['name']),
run += 1
count += 1
# open the input and expected result json files
inputFd = open(join(self.testdir, test['input']))
expectFd = open(join(self.testdir, test['expect']))
inputJson = json.load(inputFd)
expectJson = json.load(expectFd)
resultJson = None
testType = test['type']
if testType == 'normalize':
resultJson = jsonld.normalize(inputJson)
elif testType == 'expand':
resultJson = jsonld.expand(inputJson)
elif testType == 'compact':
contextFd = open(join(self.testdir, test['context']))
contextJson = json.load(contextFd)
resultJson = jsonld.compact(contextJson, inputJson)
elif testType == 'frame':
frameFd = open(join(self.testdir, test['frame']))
frameJson = json.load(frameFd)
resultJson = jsonld.frame(inputJson, frameJson)
else:
print "Unknown test type."
# check the expected value against the test result
if expectJson == resultJson:
passed += 1
print 'PASS'
if self.options.verbose:
print 'Expect:', json.dumps(expectJson, indent=4)
print 'Result:', json.dumps(resultJson, indent=4)
else:
failed += 1
print 'FAIL'
print 'Expect:', json.dumps(expectJson, indent=4)
print 'Result:', json.dumps(resultJson, indent=4)
print "Tests run: %d, Tests passed: %d, Tests Failed: %d" % (run, passed, failed)
def main(self):
print 'PyLD Unit Tests'
print 'Use -h or --help to view options.'
# add program options
self.parser.add_option('-f', '--file', dest='file',
help='The single test file to run', metavar='FILE')
self.parser.add_option('-d', '--directory', dest='directory',
help='The directory full of test files', metavar='DIR')
self.parser.add_option('-v', '--verbose', dest='verbose',
action='store_true', default=False,
help='Prints verbose test data')
# parse options
(self.options, args) = self.parser.parse_args()
# check if file or directory were specified
if self.options.file == None and self.options.directory == None:
raise Exception('No test file or directory specified.')
# check if file was specified, exists, and is file
if self.options.file is not None:
if (os.path.exists(self.options.file) and
os.path.isfile(self.options.file)):
# add manifest file to the file list
self.manifest_files.append(os.path.abspath(self.options.file))
else:
raise Exception('Invalid test file: "%s"' % self.options.file)
# check if directory was specified, exists and is dir
if self.options.directory is not None:
if (os.path.exists(self.options.directory) and
os.path.isdir(self.options.directory)):
# load manifest files from test directory
for test_dir, dirs, files in os.walk(self.options.directory):
for manifest in files:
# add all .jsonld manifest files to the file list
if (manifest.find('manifest') != -1 and
manifest.endswith('.jsonld')):
self.manifest_files.append(
join(test_dir, manifest))
else:
raise Exception('Invalid test directory: "%s"' %
self.options.directory)
# see if any manifests have been specified
if len(self.manifest_files) == 0:
raise Exception('No manifest files found.')
passed = 0
failed = 0
total = 0
# run the tests from each manifest file
for manifest_file in self.manifest_files:
test_dir = os.path.dirname(manifest_file)
manifest = json.load(open(manifest_file, 'r'))
count = 1
for test in manifest['sequence']:
# skip unsupported types
skip = True
test_type = test['@type']
for tt in TEST_TYPES:
if tt in test_type:
skip = False
break
if skip:
print 'Skipping test: "%s" ...' % test['name']
continue
print 'JSON-LD/%s %04d/%s...' % (
manifest['name'], count, test['name']),
total += 1
count += 1
# read input file
with open(join(test_dir, test['input'])) as f:
if test['input'].endswith('.jsonld'):
input = json.load(f)
else:
input = f.read().decode('utf8')
# read expect file
with open(join(test_dir, test['expect'])) as f:
if test['expect'].endswith('.jsonld'):
expect = json.load(f)
else:
expect = f.read().decode('utf8')
result = None
# JSON-LD options
options = {
'base': 'http://json-ld.org/test-suite/tests/' +
test['input']}
try:
if 'jld:NormalizeTest' in test_type:
options['format'] = 'application/nquads'
result = jsonld.normalize(input, options)
elif 'jld:ExpandTest' in test_type:
result = jsonld.expand(input, options)
elif 'jld:CompactTest' in test_type:
ctx = json.load(open(join(test_dir, test['context'])))
result = jsonld.compact(input, ctx, options)
elif 'jld:FrameTest' in test_type:
frame = json.load(open(join(test_dir, test['frame'])))
result = jsonld.frame(input, frame, options)
elif 'jld:FromRDFTest' in test_type:
result = jsonld.from_rdf(input, options)
elif 'jld:ToRDFTest' in test_type:
options['format'] = 'application/nquads'
result = jsonld.to_rdf(input, options)
# check the expected value against the test result
success = deep_compare(expect, result)
if success:
passed += 1
print 'PASS'
else:
failed += 1
print 'FAIL'
if not success or self.options.verbose:
print 'Expect:', json.dumps(expect, indent=2)
print 'Result:', json.dumps(result, indent=2)
except jsonld.JsonLdError as e:
print '\nError: ', e
failed += 1
print 'FAIL'
print 'Done. Total:%d Passed:%d Failed:%d' % (total, passed, failed)
from pyld import jsonld import sys import json graph = json.load(sys.stdin) frame = json.load(open(sys.argv[1])) print json.dumps(jsonld.frame(graph, frame), indent=2)
from pyld.jsonld import compact, frame, to_rdf, from_rdf
import json, urllib, pprint
framejs = json.load(urllib.urlopen('http://iiif.io/api/presentation/2/manifest_frame.json'))
manifestjs = json.load(urllib.urlopen('http://iiif.io/api/presentation/2.0/example/manifest1.json'))
manifestjs = json.load(urllib.urlopen('http://localhost:4000/api/presentation/2.0/example/test_case.json'))
contextjs = json.load(urllib.urlopen('http://iiif.io/api/presentation/2/context.json'))
contextURI = "http://iiif.io/api/presentation/2/context.json"
# Convert example JSON-LD to RDF
#rdf = to_rdf(manifestjs)
# Convert back to JSON-LD
#manifestjs2 = from_rdf(rdf)
# Frame the JSON-LD
framed = frame(manifestjs, framejs)
# Compact it
compacted = compact(framed, contextjs)
# Except we want just the URI in the output
compacted = compact(framed, contextURI)
# And print it
pprint.pprint(compacted)
# expand a document, removing its context
# see: http://json-ld.org/spec/latest/json-ld/#expanded-document-form
expanded = jsonld.expand(compacted)
print(json.dumps(expanded, indent=2))
# Output:
# {
# "http://schema.org/image": [{"@id": "http://manu.sporny.org/images/manu.png"}],
# "http://schema.org/name": [{"@value": "Manu Sporny"}],
# "http://schema.org/url": [{"@id": "http://manu.sporny.org/"}]
# }
# expand using URLs
jsonld.expand('http://example.org/doc')
# flatten a document
# see: http://json-ld.org/spec/latest/json-ld/#flattened-document-form
flattened = jsonld.flatten(doc)
# all deep-level trees flattened to the top-level
# frame a document
# see: http://json-ld.org/spec/latest/json-ld-framing/#introduction
framed = jsonld.frame(doc, frame)
# document transformed into a particular tree structure per the given frame
# normalize a document
normalized = jsonld.normalize(doc, {'format': 'application/nquads'})
# normalized is a string that is a canonical representation of the document
# that can be used for hashing
