def runTest(self):
testfile = self.testbase + ".htm"
resultsf = self.testbase + ".ttl"
self.failIf(not os.path.isfile(resultsf),
"missing expected results file.")
store1 = RGraph()
store1.load(resultsf, publicID=self.pubId, format="n3")
pcontents = store1.serialize(format='nt')
pg = Graph()
for a, b, c in store1:
pg.triples.add(tuple(map(self.nodeToString, (a, b, c))))
#print tuple(map(self.nodeToString, (a,b,c)))
store2 = RGraph()
store2.load(testfile, publicID=self.pubId, format="rdfa")
qcontents = store2.serialize(format='nt')
qg = Graph()
for a, b, c in store2:
qg.triples.add(tuple(map(self.nodeToString, (a, b, c))))
self.failIf(
not hash(pg) == hash(qg),
"In %s: results do not match.\n%s\n\n%s" %
(self.shortDescription(), pcontents, qcontents))
def load(self, url):
src = VOCAB_SOURCE_MAP.get(str(url), url)
if os.path.isfile(url):
context_id = create_input_source(url).getPublicId()
last_vocab_mtime = self.mtime_map.get(url)
vocab_mtime = os.stat(url).st_mtime
if not last_vocab_mtime or last_vocab_mtime < vocab_mtime:
logger.debug("Parse file: '%s'", url)
self.mtime_map[url] = vocab_mtime
# use CG as workaround for json-ld always loading as dataset
graph = ConjunctiveGraph()
graph.parse(src, format=guess_format(src))
self.graph.remove_context(context_id)
for s, p, o in graph:
self.graph.add((s, p, o, context_id))
return graph
else:
context_id = url
if any(self.graph.triples((None, None, None), context=context_id)):
logger.debug("Using context <%s>" % context_id)
return self.graph.get_context(context_id)
cache_path = self.get_fs_path(url)
if os.path.exists(cache_path):
logger.debug("Load local copy of <%s> from '%s'", context_id, cache_path)
return self.graph.parse(cache_path, format='turtle', publicID=context_id)
else:
logger.debug("Fetching <%s> to '%s'", context_id, cache_path)
graph = self.graph.parse(src,
format='rdfa' if url.endswith('html') else None)
with open(cache_path, 'w') as f:
graph.serialize(f, format='turtle')
return graph
def add_mediator(params):
#Write user metadata and save the rdf file
graph = Graph()
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
uri = URIRef("http://vocab.ox.ac.uk/owner/uuid:%s"%uuid.uuid4())
graph.add((uri, namespaces['foaf']['firstName'], Literal(params['firstname'])))
graph.add((uri, namespaces['foaf']['lastName'], Literal(params['lastname'])))
graph.add((uri, namespaces['foaf']['mbox'], Literal(params['email'])))
graph.add((uri, namespaces['foaf']['account'], Literal(params['username'])))
if 'title' in params and params['title']:
graph.add((uri, namespaces['foaf']['title'], Literal(params['title'])))
if 'department' in params and params['department']:
department = params['department'].split(';')
for d in department:
graph.add((uri, namespaces['dcterms']['isPartOf'], Literal(d.strip())))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']), 'w', 'utf-8')
f.write(rdf_str)
f.close()
graph2 = Graph()
graph2.parse(ag.mediatorslist)
for prefix, url in namespaces.iteritems():
graph2.bind(prefix, URIRef(url))
graph2.add((uri, namespaces['foaf']['account'], Literal(params['username'])))
rdf_str = None
rdf_str = graph2.serialize()
f = codecs.open(ag.mediatorslist, 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
def update():
"""
Update the library with new articles.
"""
graph = ConjunctiveGraph()
# load the existing graph
library = 'data/articles.rdf'
graph.load(library)
feeds = {
"http://www3.interscience.wiley.com/rss/journal/118485807": "wiley.xsl",
"http://phg.sagepub.com/rss/current.xml": "sage.xsl",
"http://www.informaworld.com/ampp/rss~content=t713446924": "infoworld.xsl",
"http://www.informaworld.com/ampp/rss~content=t788352614": "infoworld.xsl",
"http://www.envplan.com/rss.cgi?journal=D": "envplan.xsl",
"http://www.envplan.com/rss.cgi?journal=A": "envplan.xsl",
"http://cgj.sagepub.com/rss/current.xml": "sage.xsl"
}
for feed, stylesheet in feeds.iteritems():
# grab the feed and transform it
print "grabbing ", feed
new = StringIO.StringIO(feed_transform(feed, stylesheet))
# merge the new triples into the graph
graph.parse(new)
new.close()
graph.serialize(library, format='pretty-xml')
def transpose_to_rdf(doc, con, text, context_included, name, f):
g = ConjunctiveGraph()
if text not in con:
if not context_included:
if mp:
with lock:
get_context(con, text)
else:
get_context(con, text)
if not args.debug:
opener = open
if ".bz" in name:
opener = bz2.open
if context_included:
g.parse(data=json.dumps(doc), format='json-ld')
else:
g.parse(data=json.dumps(doc), format='json-ld', context=con[text])
with opener(name, "at") as fd:
print(str(g.serialize(format='nt').decode('utf-8').rstrip()),
file=fd)
else:
if context_included:
g.parse(data=json.dumps(doc), format='json-ld')
else:
g.parse(data=json.dumps(doc), format='json-ld', context=con[text])
print(str(g.serialize(format=f).decode('utf-8').rstrip()))
def out(self,file) :
# RDF output
RDF = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
RDFS = Namespace("http://www.w3.org/2000/01/rdf-schema#")
GT = Namespace("http://purl.org/ontology/flat/")
FOAF = Namespace("http://xmlns.com/foaf/0.1/")
DC = Namespace("http://purl.org/dc/elements/1.1/")
WGS = Namespace("http://www.w3.org/2003/01/geo/wgs84_pos#")
graph = ConjunctiveGraph()
flat = URIRef("#flat")
p = BNode()
e = URIRef(self.email)
i = URIRef(self.image)
graph.add((flat,RDF.type,GT['Flat']))
graph.add((flat,FOAF['based_near'],p))
graph.add((p,RDFS.label,Literal(self.place)))
graph.add((p,DC['title'],Literal(self.location)))
graph.add((p,WGS['lat'],Literal(self.lat)))
graph.add((p,WGS['long'],Literal(self.lng)))
graph.add((flat,FOAF['mbox'],e))
graph.add((flat,FOAF['depiction'],i))
graph.add((flat,DC['title'],Literal(self.title)))
graph.add((flat,DC['description'],Literal(self.description)))
print graph.serialize(destination=file,format='xml')
def serialize_demo():
try:
store = ConjunctiveGraph()
store.parse(FILE, format='xml')
print store.serialize(format='xml')
except OSError:
print "Cannot read file '%s'" % FILE
def out(self, file):
# RDF output
RDF = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
RDFS = Namespace("http://www.w3.org/2000/01/rdf-schema#")
GT = Namespace("http://purl.org/ontology/flat/")
FOAF = Namespace("http://xmlns.com/foaf/0.1/")
DC = Namespace("http://purl.org/dc/elements/1.1/")
WGS = Namespace("http://www.w3.org/2003/01/geo/wgs84_pos#")
graph = ConjunctiveGraph()
flat = URIRef("#flat")
p = BNode()
e = URIRef(self.email)
i = URIRef(self.image)
graph.add((flat, RDF.type, GT['Flat']))
graph.add((flat, FOAF['based_near'], p))
graph.add((p, RDFS.label, Literal(self.place)))
graph.add((p, DC['title'], Literal(self.location)))
graph.add((p, WGS['lat'], Literal(self.lat)))
graph.add((p, WGS['long'], Literal(self.lng)))
graph.add((flat, FOAF['mbox'], e))
graph.add((flat, FOAF['depiction'], i))
graph.add((flat, DC['title'], Literal(self.title)))
graph.add((flat, DC['description'], Literal(self.description)))
print graph.serialize(destination=file, format='xml')
def serialize_demo():
try:
store = ConjunctiveGraph()
store.parse(FILE, format='xml')
print store.serialize(format='xml')
except OSError:
print "Cannot read file '%s'" % FILE
def process_tools_by_id(id="SPROUT"):
"""
Go through all bio.tools entries and produce an RDF graph representation (BioSchemas / JSON-LD).
"""
tool_files = get_biotools_files_in_repo()
for tool_file in tool_files:
if id in tool_file:
print(tool_file)
tool = json.load(open(tool_file))
if "biotoolsID" in tool.keys():
tool_id = tool["biotoolsID"]
tpe_id = tool_id.lower()
print(tool_id)
print(tpe_id)
directory = os.path.join("..", "..", "data", tpe_id)
dest = os.path.join(directory, tpe_id + ".bioschemas.jsonld")
jsonld = rdfize(tool)
temp_graph = ConjunctiveGraph()
temp_graph.parse(data=jsonld, format="json-ld")
temp_graph.serialize(format="json-ld",
auto_compact=True,
destination=os.path.join(
directory,
tpe_id + ".bioschemas.jsonld"))
print(f'generated markup at {dest}')
def __store_in_file(self, cur_g, cur_file_path, context_path):
# Note: the following lines from here and until 'cur_json_ld' are a sort of hack for including all
# the triples of the input graph into the final stored file. Some how, some of them are not written
# in such file otherwise - in particular the provenance ones.
new_g = ConjunctiveGraph()
for s, p, o in cur_g.triples((None, None, None)):
g_iri = None
for g_context in cur_g.contexts((s, p, o)):
g_iri = g_context.identifier
break
new_g.addN([(s, p, o, g_iri)])
if not self.nt and not self.nq and context_path:
cur_json_ld = json.loads(
new_g.serialize(
format="json-ld",
context=self.__get_context(context_path)).decode("utf-8"))
if isinstance(cur_json_ld, dict):
cur_json_ld["@context"] = context_path
else: # it is a list
for item in cur_json_ld:
item["@context"] = context_path
with open(cur_file_path, "w") as f:
json.dump(cur_json_ld, f, indent=4, ensure_ascii=False)
elif self.nt:
new_g.serialize(cur_file_path, format="nt11", encoding="utf-8")
elif self.nq:
new_g.serialize(cur_file_path, format="nquads", encoding="utf-8")
self.repok.add_sentence("File '%s' added." % cur_file_path)
def fusion(ontologies, output):
global mode
# Definition of namespaces
# Uncomment if needed
# NS_owl = Namespace("http://www.w3.org/2002/07/owl#")
# NS_rdfs = Namespace("http://www.w3.org/2000/01/rdf-schema#")
# NS_xsd = Namespace("http://www.w3.org/2001/XMLSchema#")
# NS_rdf = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
# NS_mcf = Namespace("http://www.mycorporisfabrica.org/ontology/mcf.owl#")
# Final graph creation
gMerge = ConjunctiveGraph()
myPrint("Beginning additions...\n\n")
for ontology in ontologies:
gAdd = ConjunctiveGraph()
if mode == 2 or mode == 3:
myPrint("\tParsing ontology "+ontology+"...\n")
gAdd.parse(ontology, format=guess_format(ontology))
if mode == 2 or mode == 3:
myPrint("\tAdding ontology "+ontology+", "+str(len(gAdd))+ " triples...\n")
gMerge = gMerge + gAdd
if mode == 2 or mode == 3:
myPrint("\tOntology "+ontology+" added !\n")
myPrint("\tNew size of merged ontology : "+str(len(gMerge))+" triples\n\n")
myPrint("Additions complete !\n")
myPrint("Final size of merged ontology : "+str(len(gMerge))+" triples\n\n")
myPrint("Saving the ontology in turtle format...\n")
# Saving the merged ontology in turtle
gMerge.serialize(output, format="turtle")
myPrint("Saving done !\n\n")
def get_rdf():
rdf = ConjunctiveGraph()
for key in COUNTRIES.keys():
print "parsing %s" % COUNTRIES[key]
rdf.parse(COUNTRIES[key])
print "serialize"
rdf.serialize("countries.rdf")
def test_exclude_xhtml(self):
ns = "http://www.w3.org/1999/xhtml/vocab#"
kg = ConjunctiveGraph()
kg.add((
BNode(),
URIRef("http://www.w3.org/1999/xhtml/vocab#role"),
URIRef("http://www.w3.org/1999/xhtml/vocab#button"),
))
print(kg.serialize(format="turtle"))
q_xhtml = ('SELECT * WHERE { ?s ?p ?o . FILTER (strstarts(str(?p), "' +
ns + '"))}')
print(q_xhtml)
res = kg.query(q_xhtml)
self.assertEquals(len(res), 1)
q_del = (
'DELETE {?s ?p ?o} WHERE { ?s ?p ?o . FILTER (strstarts(str(?p), "'
+ ns + '"))}')
kg.update(q_del)
print(kg.serialize(format="turtle"))
res = kg.query(q_xhtml)
self.assertEquals(len(res), 0)
def to_rdf_etree(sources):
graph = ConjunctiveGraph()
for source in sources:
graph.load(source, format=guess_format(source))
io = StringIO()
graph.serialize(io, format="pretty-xml")
io.seek(0)
return etree.parse(io)
def get(self, id, format):
# Content negotiation
if format is None:
accept = self.request.headers.get('Accept').lower()
if "application/rdf+xml" in accept:
self.set_status(303)
self.set_header("Location", self.base_uri + "/signs/" + id + ".rdf")
else:
self.set_status(303)
self.set_header("Location", self.base_uri + "/signs/" + id + ".json")
self.finish()
# invalid format
elif format != ".json" and format != ".rdf":
self.write_error(401, message="Format %s not supported" % format)
self.finish()
# call socrata
else:
crime = yield tornado.gen.Task(SocrataLookup.get_crime, id)
# Connection still good after calling socrata
if not self.request.connection.stream.closed():
if not crime:
self.write_error(404, message="crime not found")
self.finish()
else:
crime = models.Crime(crime, self.base_uri)
# JSON
if format == ".json":
self.set_header("Content-Type", "application/json")
self.write(json.dumps(crime))
# RDF
elif format == ".rdf" or format == ".nt" or format == ".ttl":
graph = ConjunctiveGraph()
graph.bind('geo', GEO)
graph.bind('dcterms', DCTERMS)
graph.bind('dbpediaowl', DBPEDIAOWL)
graph.bind('parksafecrime', PARKSAFECRIME)
crimeURIRef = URIRef(crime['uri'])
graph.add((crimeURIRef, RDF.type, DBPEDIAOWL['event']))
graph.add((crimeURIRef, GEO['lat'], Literal(crime['latitude'])))
graph.add((crimeURIRef, GEO['lon'], Literal(crime['longitude'])))
graph.add((crimeURIRef, PARKSAFECRIME['description'], Literal(crime['description'])))
graph.add((crimeURIRef, PARKSAFECRIME['date'], Literal(crime['date'])))
graph.add((crimeURIRef, PARKSAFECRIME['block'], Literal(crime['block'])))
graph.add((crimeURIRef, PARKSAFECRIME['type'], Literal(crime['type'])))
if format == ".rdf":
self.set_header("Content-Type", "application/rdf+xml")
self.write(graph.serialize())
elif format == ".nt":
self.set_header("Content-Type", "text/plain")
self.write(graph.serialize(format='nt'))
else:
self.set_header("Content-Type", "text/turtle")
self.write(graph.serialize(format='turtle'))
self.finish()
class Store:
def __init__(self):
self.graph = ConjunctiveGraph()
if os.path.exists(storefn):
self.graph.load(storeuri, format='n3')
self.graph.bind('dc', 'http://purl.org/dc/elements/1.1/')
self.graph.bind('foaf', 'http://xmlns.com/foaf/0.1/')
self.graph.bind('imdb',
'http://www.csd.abdn.ac.uk/~ggrimnes/dev/imdb/IMDB#')
self.graph.bind('rev', 'http://purl.org/stuff/rev#')
def save(self):
self.graph.serialize(storeuri, format='n3')
def who(self, who=None):
if who is not None:
name, email = (r_who.match(who).group(1),
r_who.match(who).group(2))
self.graph.add(
(URIRef(storeuri), DC['title'], Literal(title % name)))
self.graph.add(
(URIRef(storeuri + '#author'), RDF.type, FOAF['Person']))
self.graph.add(
(URIRef(storeuri + '#author'), FOAF['name'], Literal(name)))
self.graph.add(
(URIRef(storeuri + '#author'), FOAF['mbox'], Literal(email)))
self.save()
else:
return self.graph.objects(URIRef(storeuri + '#author'),
FOAF['name'])
def new_movie(self, movie):
movieuri = URIRef('http://www.imdb.com/title/tt%s/' % movie.movieID)
self.graph.add((movieuri, RDF.type, IMDB['Movie']))
self.graph.add((movieuri, DC['title'], Literal(movie['title'])))
self.graph.add((movieuri, IMDB['year'], Literal(int(movie['year']))))
self.save()
def new_review(self, movie, date, rating, comment=None):
review = BNode(
) # @@ humanize the identifier (something like #rev-$date)
movieuri = URIRef('http://www.imdb.com/title/tt%s/' % movie.movieID)
self.graph.add(
(movieuri, REV['hasReview'], URIRef('%s#%s' % (storeuri, review))))
self.graph.add((review, RDF.type, REV['Review']))
self.graph.add((review, DC['date'], Literal(date)))
self.graph.add((review, REV['maxRating'], Literal(5)))
self.graph.add((review, REV['minRating'], Literal(0)))
self.graph.add((review, REV['reviewer'], URIRef(storeuri + '#author')))
self.graph.add((review, REV['rating'], Literal(rating)))
if comment is not None:
self.graph.add((review, REV['text'], Literal(comment)))
self.save()
def movie_is_in(self, uri):
return (URIRef(uri), RDF.type, IMDB['Movie']) in self.graph
class Store:
def __init__(self):
self.graph = ConjunctiveGraph()
if os.path.exists(storefn):
self.graph.load(storeuri, format="n3")
self.graph.bind("dc", DC)
self.graph.bind("foaf", FOAF)
self.graph.bind("imdb", IMDB)
self.graph.bind("rev", "http://purl.org/stuff/rev#")
def save(self):
self.graph.serialize(storeuri, format="n3")
def who(self, who=None):
if who is not None:
name, email = (r_who.match(who).group(1),
r_who.match(who).group(2))
self.graph.add(
(URIRef(storeuri), DC["title"], Literal(title % name)))
self.graph.add(
(URIRef(storeuri + "#author"), RDF.type, FOAF["Person"]))
self.graph.add(
(URIRef(storeuri + "#author"), FOAF["name"], Literal(name)))
self.graph.add(
(URIRef(storeuri + "#author"), FOAF["mbox"], Literal(email)))
self.save()
else:
return self.graph.objects(URIRef(storeuri + "#author"),
FOAF["name"])
def new_movie(self, movie):
movieuri = URIRef("http://www.imdb.com/title/tt%s/" % movie.movieID)
self.graph.add((movieuri, RDF.type, IMDB["Movie"]))
self.graph.add((movieuri, DC["title"], Literal(movie["title"])))
self.graph.add((movieuri, IMDB["year"], Literal(int(movie["year"]))))
self.save()
def new_review(self, movie, date, rating, comment=None):
review = BNode(
) # @@ humanize the identifier (something like #rev-$date)
movieuri = URIRef("http://www.imdb.com/title/tt%s/" % movie.movieID)
self.graph.add(
(movieuri, REV["hasReview"], URIRef("%s#%s" % (storeuri, review))))
self.graph.add((review, RDF.type, REV["Review"]))
self.graph.add((review, DC["date"], Literal(date)))
self.graph.add((review, REV["maxRating"], Literal(5)))
self.graph.add((review, REV["minRating"], Literal(0)))
self.graph.add((review, REV["reviewer"], URIRef(storeuri + "#author")))
self.graph.add((review, REV["rating"], Literal(rating)))
if comment is not None:
self.graph.add((review, REV["text"], Literal(comment)))
self.save()
def movie_is_in(self, uri):
return (URIRef(uri), RDF.type, IMDB["Movie"]) in self.graph
def dump(self, f):
assert self.alldata, ".load() not called"
graph = ConjunctiveGraph()
for triple in self.preamble():
graph.add(triple)
for triple in self.classes():
graph.add(triple)
for spell in self.alldata:
for triple in spell:
graph.add(triple)
graph.serialize(destination=f, format='n3')
graph.serialize(destination=sys.stdout, format='n3')
def dump():
query = '''SELECT ?x ?y ?z ?g where { GRAPH ?g {?x ?y ?z}}'''
data = get_query(query)
if data:
g = ConjunctiveGraph()
for q in data:
if q['z']['type'] == 'uri':
g.addN([(URIRef(q['x']['value']), URIRef(q['y']['value']),
URIRef(q['z']['value']), URIRef(q['g']['value']))])
else:
g.addN([(URIRef(q['x']['value']), URIRef(q['y']['value']),
Literal(q['z']['value']), URIRef(q['g']['value']))])
g.serialize("dump/dump.nq", format="nquads")
def test_base_url(selfself):
# from https://github.com/RDFLib/rdflib/issues/1003
rdf_triples_base = """
@prefix category: <http://example.org/> .
@prefix dct: <http://purl.org/dc/terms/> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
@base <http://example.org/> .
<> a skos:ConceptScheme ;
dct:creator <https://creator.com> ;
dct:description "Test Description"@en ;
dct:source <nick> ;
dct:title "Title"@en .
"""
kg = ConjunctiveGraph()
kg.parse(data=rdf_triples_base, format="turtle")
print(kg.serialize(format="turtle"))
rdf_triples_NO_base = """
@prefix category: <http://example.org/> .
@prefix dct: <http://purl.org/dc/terms/> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
<> a skos:ConceptScheme ;
dct:creator <https://creator.com> ;
dct:description "Test Description"@en ;
dct:source <nick> ;
dct:title "Title"@en .
"""
kg = ConjunctiveGraph()
kg.parse(data=rdf_triples_NO_base, format="turtle")
print(kg.serialize(format="turtle"))
# from scratch
kg2 = ConjunctiveGraph()
kg2.add(
(
URIRef("http://fair-checker/example/qs"),
URIRef("http://value"),
Literal("2"),
)
)
print(kg2.serialize(format="turtle", base="http://fair-checker/example/"))
kg3 = ConjunctiveGraph()
kg3.parse(
data="@base <http://example.org/> . <> a <http://example.org/Class> .",
format="turtle",
)
kg3 = kg3 + kg2
print(kg3.serialize(format="turtle", base="http://fair-checker/example/"))
def trig(test):
g = ConjunctiveGraph()
try:
base = "http://www.w3.org/2013/TriGTests/" + split_uri(test.action)[1]
g.parse(test.action, publicID=base, format="trig")
if not test.syntax:
raise AssertionError("Input shouldn't have parsed!")
if test.result: # eval test
res = ConjunctiveGraph()
res.parse(test.result, format="nquads")
if verbose:
both, first, second = graph_diff(g, res)
if not first and not second:
return
print("===============================")
print("TriG")
print(g.serialize(format="nquads"))
print("===============================")
print("NQuads")
print(res.serialize(format="nquads"))
print("===============================")
print("Diff:")
# print "%d triples in both"%len(both)
print("TriG Only:")
for t in first:
print(t)
print("--------------------")
print("NQuads Only")
for t in second:
print(t)
raise Exception("Graphs do not match!")
assert isomorphic(
g, res
), "graphs must be the same, expected\n%s\n, got\n%s" % (
g.serialize(),
res.serialize(),
)
except:
if test.syntax:
raise
class Store:
def __init__(self):
self.graph = ConjunctiveGraph()
if os.path.exists(storefn):
self.graph.load(storeuri, format='n3')
self.graph.bind('dc', DC)
self.graph.bind('foaf', FOAF)
self.graph.bind('imdb', IMDB)
self.graph.bind('rev', 'http://purl.org/stuff/rev#')
def save(self):
self.graph.serialize(storeuri, format='n3')
def who(self, who=None):
if who is not None:
name, email = (r_who.match(who).group(1), r_who.match(who).group(2))
self.graph.add((URIRef(storeuri), DC['title'], Literal(title % name)))
self.graph.add((URIRef(storeuri + '#author'), RDF.type, FOAF['Person']))
self.graph.add((URIRef(storeuri + '#author'),
FOAF['name'], Literal(name)))
self.graph.add((URIRef(storeuri + '#author'),
FOAF['mbox'], Literal(email)))
self.save()
else:
return self.graph.objects(URIRef(storeuri + '#author'), FOAF['name'])
def new_movie(self, movie):
movieuri = URIRef('http://www.imdb.com/title/tt%s/' % movie.movieID)
self.graph.add((movieuri, RDF.type, IMDB['Movie']))
self.graph.add((movieuri, DC['title'], Literal(movie['title'])))
self.graph.add((movieuri, IMDB['year'], Literal(int(movie['year']))))
self.save()
def new_review(self, movie, date, rating, comment=None):
review = BNode() # @@ humanize the identifier (something like #rev-$date)
movieuri = URIRef('http://www.imdb.com/title/tt%s/' % movie.movieID)
self.graph.add((movieuri, REV['hasReview'], URIRef('%s#%s' % (storeuri, review))))
self.graph.add((review, RDF.type, REV['Review']))
self.graph.add((review, DC['date'], Literal(date)))
self.graph.add((review, REV['maxRating'], Literal(5)))
self.graph.add((review, REV['minRating'], Literal(0)))
self.graph.add((review, REV['reviewer'], URIRef(storeuri + '#author')))
self.graph.add((review, REV['rating'], Literal(rating)))
if comment is not None:
self.graph.add((review, REV['text'], Literal(comment)))
self.save()
def movie_is_in(self, uri):
return (URIRef(uri), RDF.type, IMDB['Movie']) in self.graph
class PatchableGraph(GraphEditApi):
"""
Master graph that you modify with self.patch, and we get the
updates to all current listeners.
"""
def __init__(self):
self._graph = ConjunctiveGraph()
self._observers = []
def serialize(self, to, **kw):
return self._graph.serialize(to, **kw)
def patch(self, p):
if p.isNoop():
return
patchQuads(self._graph,
deleteQuads=p.delQuads,
addQuads=p.addQuads,
perfect=False) # true?
for ob in self._observers:
ob(patchAsJson(p))
def asJsonLd(self):
return graphAsJson(self._graph)
def addObserver(self, onPatch):
self._observers.append(onPatch)
def removeObserver(self, onPatch):
try:
self._observers.remove(onPatch)
except ValueError:
pass
class Serializer(PythonSerializer):
"""
Convert a queryset to RDF
"""
internal_use_only = False
def end_serialization(self):
FOAF = Namespace('http://xmlns.com/foaf/0.1/')
DC = Namespace('http://purl.org/dc/elements/1.1/')
self.graph = ConjunctiveGraph()
self.options.pop('stream', None)
fields = filter(None, self.options.pop('fields','').split(','))
meta = None
subject = None
for object in self.objects:
if not fields:
fields = object['fields'].keys()
newmeta = object['model']
if newmeta != meta:
meta = newmeta
subject = BNode('%s.%s'%(FOAF[newmeta],object['pk']))
self.graph.add((subject,FOAF['pk'],Literal(object['pk'])))
for k in fields:
if k:
self.graph.add((subject,FOAF[k],Literal(object['fields'][k])))
def getvalue(self):
if callable(getattr(self.graph, 'serialize', None)):
return self.graph.serialize()
def _test_serializer(inputpath, expectedpath, context, serpar):
test_tree, test_graph = _load_test_data(inputpath, expectedpath, context)
if isinstance(test_tree, ConjunctiveGraph):
expected = test_tree.serialize(format="json-ld")
else:
expected = _to_json(_to_ordered(test_tree))
if test_graph is not None:
# toRdf, expected are nquads
result_tree = to_tree(test_graph, context_data=context)
result = _to_json(_to_ordered(result_tree))
elif inputpath.startswith('fromRdf'):
# fromRdf, expected in json-ld
g = ConjunctiveGraph()
data = open(p.join(test_dir, inputpath), 'rb').read()
g.parse(data=data, format="nquads", context=context)
result = g.serialize(format="json-ld", base=context)
else:
# json
f = open(p.join(test_dir, inputpath), 'rb')
result = json.load(f)[0]
f.close()
if isinstance(result, ConjunctiveGraph):
assert isomorphic(result, expected), \
"Expected graph of %s:\n%s\nGot graph of %s:\n %s" % (
expected.serialize(format='n3'),
result.serialize(format='n3'))
else:
assert jsonld_compare(expected, result) == True, \
"Expected JSON:\n%s\nGot:\n%s" % (expected, result)
def get_rdf_template(item_uri, item_id):
g = ConjunctiveGraph(identifier=item_uri)
g.bind('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')
g.bind('dcterms', 'http://purl.org/dc/terms/')
g.add((URIRef(item_uri), URIRef('http://purl.org/dc/terms/identifier'), Literal(item_id)))
data2 = g.serialize(format='xml', encoding="utf-8") + '\n'
return data2
class Topic(object):
def __init__(self, entity_name, entity_id):
'''
Constructor
'''
# Get the event page and compute its id
self.entity_id = entity_id
self.resource = LDES[self.entity_id]
# Create the graph
self.graph = ConjunctiveGraph()
self.graph.bind('swc', SWC)
self.graph.bind('cfp', CFP)
self.graph.bind('ical', ICAL)
self.graph.bind('foaf', FOAF)
self.graph.bind('dct', DCT)
self.graph.bind('lode', LODE)
# Declare the type of the resource
self.graph.add((self.resource, RDF.type, SIOCT['Tag']))
self.graph.add((self.named_graph(), DCT['modified'], Literal(datetime.now())))
def get_rdf_data(self):
return self.graph.serialize()
def named_graph(self):
return URIRef(NAMED_GRAPHS_BASE + self.entity_id + '.rdf')
def process(self, record, entity_url):
# Get the document
document = BeautifulSoup(urllib2.urlopen("http://eventseer.net" + entity_url).read())
del document
def computeHybridOntology( ff, topConcepts ):
""" computes the hybrid ontology
@param[in] ff list of input ontologies
@param[in] topConcepts concepts which are required to participate in
every hybrid ontology relation
@returns a hybrid ontology which contains all relations found in the
ontologies ff between concepts listed in the topConcept list
"""
g = ConjunctiveGraph()
allTopConcepts = set( topConcepts )
usedTopConcepts = set()
for f in ff:
for s, p, o in extractSPO( XCL2RDF.toRDF(open(f).read() ) ):
if s in allTopConcepts and o in allTopConcepts:
g.add( (getUrl(s), NS_RDFS['label'], Literal(s)) )
g.add( (getUrl(p), NS_RDFS['label'], Literal(p)) )
g.add( (getUrl(o), NS_RDFS['label'], Literal(o)) )
g.add( (getUrl(s), getUrl(p), getUrl(o)) )
usedTopConcepts.add( s )
usedTopConcepts.add( o )
_addUseCaseSpecificUnusedConcepts(g)
with open("hybrid-graph.rdf", "w") as f:
f.write( g.serialize() )
unusedConcepts = allTopConcepts.difference( usedTopConcepts )
print "# of unused concepts: %d" % len( unusedConcepts )
print ", ".join( list(unusedConcepts) )
def view(name=None, format=None, view=None):
current_app.db.store.nsBindings = {}
entity, content_type = current_app.get_entity_uri(name, format)
resource = current_app.get_resource(entity)
# 'view' is the default view
fileid = resource.value(current_app.NS.whyis.hasFileID)
if fileid is not None and 'view' not in request.args:
fileid = fileid.value
f = None
if current_app.nanopub_depot.exists(fileid):
f = current_app.nanopub_depot.get(fileid)
elif current_app.file_depot.exists(fileid):
f = current_app.file_depot.get(fileid)
if f is not None:
fsa = FileServeApp(f, current_app.config["file_archive"].get("cache_max_age",3600*24*7))
return fsa
if content_type is None:
content_type = request.headers['Accept'] if 'Accept' in request.headers else 'text/turtle'
#print entity
fmt = sadi.mimeparse.best_match([mt for mt in list(DATA_FORMATS.keys()) if mt is not None],content_type)
if 'view' in request.args or fmt in HTML_MIME_TYPES:
return current_app.render_view(resource)
elif fmt in DATA_FORMATS:
output_graph = ConjunctiveGraph()
result, status, headers = current_app.render_view(resource, view='describe')
output_graph.parse(data=result, format="json-ld")
return output_graph.serialize(format=DATA_FORMATS[fmt]), 200, {'Content-Type':content_type}
#elif 'view' in request.args or sadi.mimeparse.best_match(htmls, content_type) in htmls:
else:
return current_app.render_view(resource)
def _process_data(self, document):
'''
Creates the RDF graph describing the topic
@param document: the DOM document of the topic
'''
# Create the graph
graph = ConjunctiveGraph()
graph.bind('swc', SWC)
graph.bind('cfp', CFP)
graph.bind('ical', ICAL)
graph.bind('foaf', FOAF)
graph.bind('dct', DCT)
graph.bind('lode', LODE)
# Init the event
topic_event = LDES[self.get_resource_name()]
graph.add((topic_event, RDF.type, SIOCT['Tag']))
# Get the label
try:
label = document.find(id='inner_left').find('h1').text
graph.add((topic_event, RDFS.label, Literal(label)))
except:
pass
# Set the last modification date
graph.add(
(self.get_named_graph(), DCT['modified'], Literal(datetime.now())))
# Save the data
self.rdf_data = graph.serialize()
def _test_serializer(inputpath, expectedpath, context, serpar):
test_tree, test_graph = _load_test_data(inputpath, expectedpath, context)
if isinstance(test_tree, ConjunctiveGraph):
expected = test_tree.serialize(format="json-ld")
else:
expected = _to_json(_to_ordered(test_tree))
if test_graph is not None:
# toRdf, expected are nquads
result_tree = to_tree(test_graph, context_data=context)
result = _to_json(_to_ordered(result_tree))
elif inputpath.startswith('fromRdf'):
# fromRdf, expected in json-ld
g = ConjunctiveGraph()
data = open(p.join(test_dir, inputpath), 'rb').read()
g.parse(data=data, format="nquads", context=context)
result = g.serialize(format="json-ld", base=context)
else:
# json
f = open(p.join(test_dir, inputpath), 'rb')
result = json.load(f)[0]
f.close()
if isinstance(result, ConjunctiveGraph):
assert isomorphic(result, expected), \
"Expected graph of %s:\n%s\nGot graph of %s:\n %s" % (
expected.serialize(format='n3'),
result.serialize(format='n3'))
else:
assert jsonld_compare(expected, result) == True, \
"Expected JSON:\n%s\nGot:\n%s" % (expected, result)
def serialize(self, add, delete):
commit = Namespace("urn:commit:" + str(uuid.uuid1()) + ":")
eccrev = Namespace("https://vocab.eccenca.com/revision/")
g = ConjunctiveGraph()
namespace_manager = NamespaceManager(g)
namespace_manager.bind('eccrev', eccrev, override=False)
g.add((commit.term(""), RDF.type, eccrev.Commit))
graphUris = set(delete.keys()) | set(add.keys())
for graphUri in graphUris:
if (graphUri in delete.keys() and len(delete[graphUri]) > 0) or (graphUri in add.keys() and len(add[graphUri]) > 0):
revision = Namespace("urn:revision:" + str(uuid.uuid1()) + ":")
g.add((commit.term(""), eccrev.hasRevision, revision.term("")))
g.add((revision.term(""), RDF.type, eccrev.Revision))
if str(graphUri) != 'http://quitdiff.default/':
g.add((revision.term(""), eccrev.hasRevisionGraph, graphUri))
if graphUri in delete.keys() and len(delete[graphUri]) > 0:
deleteGraphName = revision.term(":delete")
g.add((revision.term(""), eccrev.deltaDelete, deleteGraphName))
for triple in delete[graphUri]:
g.add(triple + (deleteGraphName,))
if graphUri in add.keys() and len(add[graphUri]) > 0:
insertGraphName = revision.term(":insert")
g.add((revision.term(""), eccrev.deltaInsert, insertGraphName))
for triple in add[graphUri]:
g.add(triple + (insertGraphName,))
return g.serialize(format="trig").decode("utf-8")
def serialize(
self, destination=None, format='turtle', base=None, encoding=None
):
for prefix in self.prefixes:
mapped_iri = self.curie_map[prefix]
self.bind(prefix, Namespace(mapped_iri))
return ConjunctiveGraph.serialize(self, destination, format)
def update_mediator(params):
#Write user metadata and save the rdf file
if not ('username' in params and params['username']):
return False
det = get_mediator_details(params['username'])
graph = Graph()
graph.parse(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']))
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
uri = URIRef(det['uri'])
if 'firstname' in params and params['firstname']:
graph.remove((uri, namespaces['foaf']['firstName'], None))
graph.add((uri, namespaces['foaf']['firstName'], Literal(params['firstname'])))
if 'lastname' in params and params['lastname']:
graph.remove((uri, namespaces['foaf']['lastName'], None))
graph.add((uri, namespaces['foaf']['lastName'], Literal(params['lastname'])))
if 'email' in params and params['email']:
graph.remove((uri, namespaces['foaf']['mbox'], None))
graph.add((uri, namespaces['foaf']['mbox'], Literal(params['email'])))
if 'title' in params and params['title']:
graph.remove((uri, namespaces['foaf']['title'], None))
graph.add((uri, namespaces['foaf']['title'], Literal(params['title'])))
if 'department' in params and params['department']:
graph.remove((uri, namespaces['dcterms']['isPartOf'], None))
department = params['department'].split(';')
for d in department:
graph.add((uri, namespaces['dcterms']['isPartOf'], Literal(d.strip())))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']), 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
def changesets(self):
"""
Return an RDF serialized representation of the current changes
in the buffer, and empty the buffer.
"""
if not self._changesets:
return
g = Graph()
last = None
for s in self._changesets:
### Create the changeset for this subject, with a unique ID
csuuid = uuid.uuid3(uuid.NAMESPACE_OID, "%s/%s" % (
str(s),
datetime.now().strftime("%s")
))
csuri = BNode()
g.add((csuri, RDF.type, CS["ChangeSet"]))
g.add((csuri, CS["subjectOfChange"], s))
g.add((csuri, CS["createdDate"], Literal(datetime.now())))
g.add((csuri, CS["creatorName"], Literal(USER_AGENT)))
g.add((csuri, CS["changeReason"], Literal("Auto-Generated %s" % csuuid)))
### If we are storing version information we need to link the changes
if self.versioned:
if last:
g.add((csuri, CS["preceedingChangeSet"], last))
last = csuri
for op, p, o in self._changesets[s]:
opnode = BNode()
g.add((csuri, CS[op], opnode))
g.add((opnode, RDF.type, RDF.Statement))
g.add((opnode, RDF.subject, s))
g.add((opnode, RDF.predicate, p))
g.add((opnode, RDF.object, o))
self._changesets = {}
return g.serialize()
def serialize( # rdflib version
self,
destination=None,
format='turtle',
base=None,
encoding=None):
return ConjunctiveGraph.serialize(self, destination, format)
def computeHybridOntology( ff, topConcepts ):
""" computes the hybrid ontology
@param[in] ff list of input ontologies
@param[in] topConcepts concepts which are required to participate in
every hybrid ontology relation
@returns a hybrid ontology which contains all relations found in the
ontologies ff between concepts listed in the topConcept list
"""
g = ConjunctiveGraph()
allTopConcepts = set( topConcepts )
usedTopConcepts = set()
for f in ff:
for s, p, o in extractSPO( XCL2RDF.toRDF(open(f).read() ) ):
if s in allTopConcepts and o in allTopConcepts:
g.add( (getUrl(s), NS_RDFS['label'], Literal(s)) )
g.add( (getUrl(p), NS_RDFS['label'], Literal(p)) )
g.add( (getUrl(o), NS_RDFS['label'], Literal(o)) )
g.add( (getUrl(s), getUrl(p), getUrl(o)) )
usedTopConcepts.add( s )
usedTopConcepts.add( o )
_addUseCaseSpecificUnusedConcepts(g)
with open("hybrid-graph.rdf", "w") as f:
f.write( g.serialize() )
unusedConcepts = allTopConcepts.difference( usedTopConcepts )
print("# of unused concepts: %d" % len(unusedConcepts))
print(", ".join( list(unusedConcepts) ))
def main():
store = SPARQLStore(SPARQL_ENDPOINT)
g = ConjunctiveGraph(store=store)
# g.bind("sg", "http://www.springernature.com/scigraph/ontologies/core/")
# get a few articles
q1 = g.query(ALL_ARTICLES_IDS_SAMPLE)
for row in q1:
print("Article URI:", str(row[0]))
# extract more article info
for row in q1:
try:
with time_limit(MAX_TIMEOUT):
raw = g.query(ARTICLE_INFO_QUERY % str(row[0]))
g1 = ConjunctiveGraph()
g1.parse(data=raw.serialize())
# create JSON-LD
context = {
"@vocab": "http://elastic-index.scigraph.com/",
"@language": "en"
}
print(g1.serialize(format='json-ld', context=context,
indent=4))
print("======")
except TimeoutException, msg:
error = "Timed out!"
print(error)
except Exception, e:
error = "Exception: %s" % e
print(error)
def change_status(vocabprefix, uri, predicate, message, action):
if not action in ['add', 'remove']:
return False
vocab_uri = URIRef(uri)
vocabdir = os.path.join(ag.vocabulariesdir, vocabprefix)
vocabstatusfile = os.path.join(vocabdir, "status.rdf")
if not os.path.isfile(vocabstatusfile):
return False
graph = Graph()
graph.parse(vocabstatusfile)
predicate = predicate.split(':')
ns = predicate[0]
term = predicate[1]
if message and (message.startswith('http://') or message.startswith('file://')):
message = URIRef(message)
elif message:
message = Literal(message)
if action == 'add':
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
graph.add((vocab_uri, namespaces[ns][term], message))
elif action == 'remove':
graph.remove((vocab_uri, namespaces[ns][term], message))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(vocabstatusfile, 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
def check_serialize_parse(fpath, infmt, testfmt, verbose=False):
g = ConjunctiveGraph()
_parse_or_report(verbose, g, fpath, format=infmt)
if verbose:
for t in g:
print t
print "========================================"
print "Parsed OK!"
s = g.serialize(format=testfmt)
if verbose:
print s
g2 = ConjunctiveGraph()
_parse_or_report(verbose, g2, data=s, format=testfmt)
if verbose:
print g2.serialize()
crapCompare(g,g2)
def do_test_parser(suite_base, cat, num, inputpath, expectedpath, context,
options):
input_uri = suite_base + inputpath
input_obj = _load_json(inputpath)
old_nodeid = W3CNTriplesParser.nodeid
# monkey patch nodeid fn in NTriplesParser
W3CNTriplesParser.nodeid = _preserving_nodeid
try:
expected_graph = _load_nquads(expectedpath)
finally:
W3CNTriplesParser.nodeid = old_nodeid
result_graph = ConjunctiveGraph()
requested_version = options.get("specVersion")
version = DEFAULT_PARSER_VERSION
if requested_version:
if requested_version == "json-ld-1.1":
version = 1.1
elif requested_version == "json-ld-1.0":
version = 1.0
to_rdf(
input_obj,
result_graph,
context_data=context,
base=options.get("base", input_uri),
version=version,
generalized_rdf=options.get("produceGeneralizedRdf", False),
)
assert isomorphic(result_graph,
expected_graph), "Expected:\n%s\nGot:\n%s" % (
expected_graph.serialize(),
result_graph.serialize(),
)
def check_serialize_parse(fpath, infmt, testfmt, verbose=False):
g = ConjunctiveGraph()
_parse_or_report(verbose, g, fpath, format=infmt)
if verbose:
for t in g:
print(t)
print("========================================")
print("Parsed OK!")
s = g.serialize(format=testfmt)
if verbose:
print(s)
g2 = ConjunctiveGraph()
_parse_or_report(verbose, g2, data=s, format=testfmt)
if verbose:
print(g2.serialize())
crapCompare(g, g2)
def to_rdf_graph(self):
g = ConjunctiveGraph()
post_uri = URIRef(POST_URI + self.slug)
g.add((SITE_URI, RDF.type, SMOB["Hub"]))
g.add((post_uri, RDF.type, SIOCT["MicroblogPost"]))
g.add((post_uri, SIOC["has_container"], URIRef(self.has_container())))
g.add((post_uri, DCT["created"], Literal(self.created)))
#',datatype=_XSD_NS.date)
# g.add((post_uri, SIOC["has_creator"], URIRef(self.has_creator())))
# g.add((post_uri, FOAF["maker"], URIRef(self.maker())))
g.add((post_uri, SIOC["content"], Literal(self.content)))
g.add((post_uri, DCT["title"], Literal(self.title)))
if self.reply_of:
g.add((post_uri, SIOC["reply_of"], URIRef(self.reply_of.uri())))
# opo_uri = URIRef(self.opo_uri())
# g.add((opo_uri, RDF.type, OPO["OnlinePresence"]))
# g.add((opo_uri, OPO["declaredOn"], URIRef(self.has_creator())))
# g.add((opo_uri, OPO["declaredBy"], URIRef(self.maker())))
# g.add((opo_uri, OPO["StartTime"], Literal(self.created)))
# g.add((opo_uri, OPO["customMessage"], post_uri))
if self.location:
g.add((opo_uri, OPO["currentLocation"], URIRef(self.location_uri)))
g.add((URIRef(self.location_uri), RDFS.label, Literal(self.location_label)))
# g.add(())
rdf = g.serialize(format="nt")
logging.debug("Post.to_rdf_graph(), rdf")
logging.debug(rdf)
return rdf
def render(self,
data1,
media_type=None,
renderer_context=None,
format_1=None,
binary=False,
store=False,
named_graph=None):
if type(data1) == dict:
data1 = [data1]
if format_1 is not None:
self.format = format_1
cidoc = Namespace("http://www.cidoc-crm.org/cidoc-crm/")
geo = Namespace("http://www.opengis.net/ont/geosparql#")
if not store:
store = IOMemory()
if named_graph:
uri_entities = URIRef(named_graph)
else:
uri_entities = URIRef(f'{base_uri}/entities#')
g = Graph(store, identifier=uri_entities)
g.bind('cidoc', cidoc, override=False)
g.bind('geo', geo, override=False)
g.bind('owl', OWL, override=False)
ns = {'cidoc': cidoc, 'geo': geo}
if type(data1) == list:
for data in data1:
g, ent = self.ent_func[data['entity_type']](g,
ns,
data,
drill_down=True)
elif type(data1) == str:
directory = os.fsencode(data1)
for fn in os.listdir(directory):
with open(os.path.join(directory, fn), 'rb') as inf:
data2 = pickle.load(inf)
for data in data2:
g, ent = self.ent_func[data['entity_type']](
g, ns, data, drill_down=True)
g_prov = Graph(
store,
identifier=URIRef('https://omnipot.acdh.oeaw.ac.at/provenance'))
g_prov.bind('dct', DCTERMS, override=False)
g_prov.bind('void', VOID, override=False)
g_prov.add((uri_entities, DCTERMS.title,
Literal(PROJECT_METADATA['title'], lang=lang)))
g_prov.add((uri_entities, DCTERMS.description,
Literal(PROJECT_METADATA['description'], lang=lang)))
g_prov.add((uri_entities, DCTERMS.creator,
Literal(PROJECT_METADATA['author'], lang=lang)))
g_prov.add(
(uri_entities, DCTERMS.publisher, Literal('ACDH-OeAW', lang=lang)))
g_prov.add((uri_entities, DCTERMS.source, URIRef(base_uri)))
g_prov.add((uri_entities, DCTERMS.created,
Literal(str(date.today()), datatype=XSD.date)))
g_prov, g = generateVoID(g, dataset=uri_entities, res=g_prov)
g_all = ConjunctiveGraph(store=store)
if binary:
return g_all, store
return g_all.serialize(format=self.format.split('+')[-1]), store
def _process_data(self, document):
'''
Creates the RDF graph describing the topic
@param document: the DOM document of the topic
'''
# Create the graph
graph = ConjunctiveGraph()
graph.bind('swc', SWC)
graph.bind('cfp', CFP)
graph.bind('ical', ICAL)
graph.bind('foaf', FOAF)
graph.bind('dct', DCT)
graph.bind('lode', LODE)
# Init the event
topic_event = LDES[self.get_resource_name()]
graph.add((topic_event, RDF.type, SIOCT['Tag']))
# Get the label
try:
label = document.find(id='inner_left').find('h1').text
graph.add((topic_event, RDFS.label, Literal(label)))
except:
pass
# Set the last modification date
graph.add((self.get_named_graph(), DCT['modified'], Literal(datetime.now())))
# Save the data
self.rdf_data = graph.serialize()
def extract_rdf_extruct(self, url) -> ConjunctiveGraph:
while True:
try:
response = requests.get(url=url, timeout=10)
break
except SSLError:
time.sleep(5)
except requests.exceptions.Timeout:
print("Timeout, retrying")
time.sleep(5)
except requests.exceptions.ConnectionError as e:
print(e)
print("ConnectionError, retrying...")
time.sleep(10)
self.status_code = response.status_code
html_source = response.content
data = extruct.extract(
html_source, syntaxes=["microdata", "rdfa", "json-ld"], errors="ignore"
)
kg = ConjunctiveGraph()
base_path = Path(__file__).parent.parent # current directory
static_file_path = str((base_path / "static/data/jsonldcontext.json").resolve())
if "json-ld" in data.keys():
for md in data["json-ld"]:
if "@context" in md.keys():
if ("https://schema.org" in md["@context"]) or (
"http://schema.org" in md["@context"]
):
md["@context"] = static_file_path
kg.parse(data=json.dumps(md, ensure_ascii=False), format="json-ld")
if "rdfa" in data.keys():
for md in data["rdfa"]:
if "@context" in md.keys():
if ("https://schema.org" in md["@context"]) or (
"http://schema.org" in md["@context"]
):
md["@context"] = static_file_path
kg.parse(data=json.dumps(md, ensure_ascii=False), format="json-ld")
if "microdata" in data.keys():
for md in data["microdata"]:
if "@context" in md.keys():
if ("https://schema.org" in md["@context"]) or (
"http://schema.org" in md["@context"]
):
md["@context"] = static_file_path
kg.parse(data=json.dumps(md, ensure_ascii=False), format="json-ld")
logging.debug(kg.serialize(format="turtle"))
kg.namespace_manager.bind("sc", URIRef("http://schema.org/"))
kg.namespace_manager.bind("bsc", URIRef("https://bioschemas.org/"))
kg.namespace_manager.bind("dct", URIRef("http://purl.org/dc/terms/"))
return kg
def save_item(self, item):
'''
Convert the item into a graph and put the graph into the triple store
'''
# Delete the previous triples associated to that resource
#conn = httplib.HTTPConnection(self.url)
#conn.request("DELETE", "/data/%s" % item.get_resource())
#conn.close()
# Generate the new graph
graph = ConjunctiveGraph()
for (key, values) in item.get_metadata():
if type(values) == type([]):
for value in values:
graph.add((item.get_resource(), key, value))
else:
graph.add((item.get_resource(), key, values))
# Save it
#print graph.serialize()
headers = {'Accept': '*/*', 'Content-Type': 'application/rdf+xml'}
conn = httplib.HTTPConnection(self.url)
conn.request("PUT",
"/data/%s" % item.get_resource(),
body=graph.serialize(),
headers=headers)
conn.getresponse()
conn.close()
def getPropFile(fname):
g = ConjunctiveGraph(identifier=URIRef(ads_baseurl))
bindgraph(g)
recordstree=ElementTree.parse(fname)
rootnode=recordstree.getroot()
xobj=XMLObj(recordstree)
trec={}
trec['propname']=rootnode.attrib['name']
trec['propid']=rootnode.attrib['id']
trec['title']=xobj.title
trec['category']=xobj.category
#we used a proposalType here, as this is somewhat different from justscienceprocess. add to ontology
trec['abstract']=xobj.abstract
trec['pi']=[xobj.elementAttribute('pi', 'last'),xobj.elementAttribute('pi', 'first')]
#print trec
propuri=getPropURI(trec['propid'])
#This is FALSE. TODO..fix to ads normed name or lookitup How? Blanknode? WOW.
qplabel=trec['pi'][0]+'_'+trec['pi'][1]
fullname=trec['pi'][0]+', '+trec['pi'][1]
auth_uri = uri_agents["PersonName/"+qplabel+"/"+str(uuid.uuid4())]
gdadd(g, auth_uri, [
a, agent.PersonName,
agent.fullName, Literal(fullname)
])
gadd(g, propuri, a, adsbase.ObservationProposal)
gdadd(g, propuri, [
adsobsv.observationProposalId, Literal(trec['propid']),
adsobsv.observationProposalType, Literal("CHANDRA/"+trec['category']),
adsbase.principalInvestigator, auth_uri,
adsbase.title, Literal(trec['title'])
]
)
serializedstuff=g.serialize(format='xml')
return serializedstuff
def ConvertToRDFN3 (filename, destinationFileName):
_graph = ConjunctiveGraph()
_graph.parse(filename, format="nt")
of = open(destinationFileName, "wb")
of.write(_graph.serialize(format="n3"))
of.close()
def test_serialize(self):
g = ConjunctiveGraph()
uri1 = URIRef("http://example.org/mygraph1")
uri2 = URIRef("http://example.org/mygraph2")
bob = URIRef(u"urn:bob")
likes = URIRef(u"urn:likes")
pizza = URIRef(u"urn:pizza")
g.get_context(uri1).add((bob, likes, pizza))
g.get_context(uri2).add((bob, likes, pizza))
s = g.serialize(format="nquads")
self.assertEqual(
len([x for x in s.split("\n".encode("latin-1")) if x.strip()]), 2
)
g2 = ConjunctiveGraph()
g2.parse(data=s, format="nquads")
self.assertEqual(len(g), len(g2))
self.assertEqual(
sorted(x.identifier for x in g.contexts()),
sorted(x.identifier for x in g2.contexts()),
)
def _get_entity_data(qids):
sparql = open(os.path.join(SPARQL_DIR, 'entities.rq'), 'r').read()
sparql = sparql.replace('VALUES (?item) {}',
f'VALUES (?item) {{ ({") (".join(qids)}) }}')
context = json.loads(
open(os.path.join(SPARQL_DIR, 'entities_context.json'), 'r').read())
for _ in range(3):
resp = requests.post('https://query.wikidata.org/sparql',
headers={
'Accept': 'text/plain',
'Content-type':
'application/x-www-form-urlencoded',
'User-agent': 'JSTOR Labs python client'
},
data='query=%s' % quote(sparql))
if resp.status_code == 200:
# Convert N-Triples to json-ld using json-ld context
graph = Graph()
graph.parse(data=resp.text, format='nt')
_jsonld = json.loads(
str(
graph.serialize(format='json-ld',
context=context,
indent=None), 'utf-8'))
if '@graph' not in _jsonld:
_context = _jsonld.pop('@context')
_jsonld = {'@context': _context, '@graph': [_jsonld]}
return _jsonld
logger.debug(
f'_get_entity_data: resp_code={resp.status_code} msg=${resp.text}')
def __store_graph(cur_g, rdf_iri_string, d_dir):
try:
res_dir, dest_file = \
find_paths(rdf_iri_string, args.base + os.sep, "https://w3id.org/oc/corpus/", 10000, 1000)
dest_dir = res_dir.replace(args.base + os.sep, d_dir + os.sep)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
cur_file = dest_file.replace(res_dir, dest_dir)
if os.path.exists(cur_file):
c_graph = __load_graph(cur_file)
else:
c_graph = ConjunctiveGraph()
c_graph.remove_context(c_graph.get_context(cur_g.identifier))
c_graph.addN([item + (cur_g.identifier,) for item in list(cur_g)])
with open(dest_file.replace(res_dir, dest_dir), "w") as f:
cur_json_ld = json.loads(c_graph.serialize(format="json-ld", context=context_json))
cur_json_ld["@context"] = context_path
json.dump(cur_json_ld, f, indent=4)
# repok.add_sentence("File '%s' added." % cur_file)
return dest_file
except Exception as e:
reperr.add_sentence("[5] It was impossible to store the RDF statements in %s. %s" %
(dest_file, str(e)))
def parse(filename):
parser = xml.sax.make_parser()
handler = FahrplanHandler()
parser.setContentHandler(handler)
parser.parse(filename)
graph = ConjunctiveGraph()
handler.conference.add_to_graph(graph)
print(graph.serialize())
def ConvertToRDFXML (filename,destinationFileName):
_graph = ConjunctiveGraph()
_graph.parse(filename, format="nt")
_graph.triples((None, None, None))
of = open(destinationFileName, "wb")
of.write(_graph.serialize(format="pretty-xml"))
of.close()
def test03(self):
ingraph = to_rdf(json.loads(test03_in), ConjunctiveGraph())
outgraph = ConjunctiveGraph().parse(
data=ingraph.serialize(format="xml"), format="xml")
assert isomorphic(outgraph, ingraph), \
"Expected graph of %s:\n%s\nGot graph of %s:\n %s" % (
len(outgraph), outgraph.serialize(),
len(ingraph), ingraph.serialize())
