def get_rdf():
rdf = ConjunctiveGraph()
for key in COUNTRIES.keys():
print "parsing %s" % COUNTRIES[key]
rdf.parse(COUNTRIES[key])
print "serialize"
rdf.serialize("countries.rdf")
def fusion(ontologies, output):
global mode
# Definition of namespaces
# Uncomment if needed
# NS_owl = Namespace("http://www.w3.org/2002/07/owl#")
# NS_rdfs = Namespace("http://www.w3.org/2000/01/rdf-schema#")
# NS_xsd = Namespace("http://www.w3.org/2001/XMLSchema#")
# NS_rdf = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
# NS_mcf = Namespace("http://www.mycorporisfabrica.org/ontology/mcf.owl#")
# Final graph creation
gMerge = ConjunctiveGraph()
myPrint("Beginning additions...\n\n")
for ontology in ontologies:
gAdd = ConjunctiveGraph()
if mode == 2 or mode == 3:
myPrint("\tParsing ontology "+ontology+"...\n")
gAdd.parse(ontology, format=guess_format(ontology))
if mode == 2 or mode == 3:
myPrint("\tAdding ontology "+ontology+", "+str(len(gAdd))+ " triples...\n")
gMerge = gMerge + gAdd
if mode == 2 or mode == 3:
myPrint("\tOntology "+ontology+" added !\n")
myPrint("\tNew size of merged ontology : "+str(len(gMerge))+" triples\n\n")
myPrint("Additions complete !\n")
myPrint("Final size of merged ontology : "+str(len(gMerge))+" triples\n\n")
myPrint("Saving the ontology in turtle format...\n")
# Saving the merged ontology in turtle
gMerge.serialize(output, format="turtle")
myPrint("Saving done !\n\n")
class Config(object):
def __init__(self, masterGraph, hubHost):
self.etcd = etcd3.client(host=hubHost, port=9022)
self.masterGraph = masterGraph
self.hubHost = hubHost
self.configGraph = ConjunctiveGraph()
self.boards = []
self.etcPrefix = 'pi/'
self.reread()
deferToThread(self.watchEtcd)
def watchEtcd(self):
events, cancel = self.etcd.watch_prefix(self.etcPrefix)
reactor.addSystemEventTrigger('before', 'shutdown', cancel)
for ev in events:
log.info('%s changed', ev.key)
reactor.callFromThread(self.configChanged)
def configChanged(self):
self.cancelRead()
self.rereadLater = reactor.callLater(.1, self.reread)
def cancelRead(self):
if getattr(self, 'rereadLater', None):
self.rereadLater.cancel()
self.rereadLater = None
@STATS.configReread.time()
def reread(self):
self.rereadLater = None
log.info('read config')
self.configGraph = ConjunctiveGraph()
for v, md in self.etcd.get_prefix(self.etcPrefix):
log.info(' read file %r', md.key)
self.configGraph.parse(StringInputSource(v), format='n3')
self.configGraph.bind('', ROOM)
self.configGraph.bind('rdf', RDF)
# config graph is too noisy; maybe make it a separate resource
#masterGraph.patch(Patch(addGraph=self.configGraph))
self.setupBoards()
def setupBoards(self):
thisHost = Literal(hostname)
for row in self.configGraph.query(
'SELECT ?board WHERE { ?board a :PiBoard; :hostname ?h }',
initBindings=dict(h=thisHost)):
thisBoard = row.board
break
else:
log.warn("config had no board for :hostname %s. Waiting for config update." %
thisHost)
self.boards = []
return
log.info("found config for board %r" % thisBoard)
self.boards = [Board(self.configGraph, self.masterGraph, thisBoard, self.hubHost)]
self.boards[0].startPolling()
def _user_graph(self, uri):
userGraph = Graph()
try:
userGraph.parse(uri)
except Exception, e:
u = "http://www.w3.org/2007/08/pyRdfa/extract?space-preserve=true&uri=" + uri
userGraph.parse(u, identifier=uri)
def load(self, url):
src = VOCAB_SOURCE_MAP.get(str(url), url)
if os.path.isfile(url):
context_id = create_input_source(url).getPublicId()
last_vocab_mtime = self.mtime_map.get(url)
vocab_mtime = os.stat(url).st_mtime
if not last_vocab_mtime or last_vocab_mtime < vocab_mtime:
logger.debug("Parse file: '%s'", url)
self.mtime_map[url] = vocab_mtime
# use CG as workaround for json-ld always loading as dataset
graph = ConjunctiveGraph()
graph.parse(src, format=guess_format(src))
self.graph.remove_context(context_id)
for s, p, o in graph:
self.graph.add((s, p, o, context_id))
return graph
else:
context_id = url
if any(self.graph.triples((None, None, None), context=context_id)):
logger.debug("Using context <%s>" % context_id)
return self.graph.get_context(context_id)
cache_path = self.get_fs_path(url)
if os.path.exists(cache_path):
logger.debug("Load local copy of <%s> from '%s'", context_id, cache_path)
return self.graph.parse(cache_path, format='turtle', publicID=context_id)
else:
logger.debug("Fetching <%s> to '%s'", context_id, cache_path)
graph = self.graph.parse(src,
format='rdfa' if url.endswith('html') else None)
with open(cache_path, 'w') as f:
graph.serialize(f, format='turtle')
return graph
def update_mediator(params):
#Write user metadata and save the rdf file
if not ('username' in params and params['username']):
return False
det = get_mediator_details(params['username'])
graph = Graph()
graph.parse(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']))
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
uri = URIRef(det['uri'])
if 'firstname' in params and params['firstname']:
graph.remove((uri, namespaces['foaf']['firstName'], None))
graph.add((uri, namespaces['foaf']['firstName'], Literal(params['firstname'])))
if 'lastname' in params and params['lastname']:
graph.remove((uri, namespaces['foaf']['lastName'], None))
graph.add((uri, namespaces['foaf']['lastName'], Literal(params['lastname'])))
if 'email' in params and params['email']:
graph.remove((uri, namespaces['foaf']['mbox'], None))
graph.add((uri, namespaces['foaf']['mbox'], Literal(params['email'])))
if 'title' in params and params['title']:
graph.remove((uri, namespaces['foaf']['title'], None))
graph.add((uri, namespaces['foaf']['title'], Literal(params['title'])))
if 'department' in params and params['department']:
graph.remove((uri, namespaces['dcterms']['isPartOf'], None))
department = params['department'].split(';')
for d in department:
graph.add((uri, namespaces['dcterms']['isPartOf'], Literal(d.strip())))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']), 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
def describe(self, s_or_po, initBindings={}, initNs={}):
"""
Executes a SPARQL describe of resource
:param s_or_po: is either
* a subject ... should be a URIRef
* a tuple of (predicate,object) ... pred should be inverse functional
* a describe query string
:param initBindings: A mapping from a Variable to an RDFLib term (used
as initial bindings for SPARQL query)
:param initNs: A mapping from a namespace prefix to a namespace
"""
if isinstance(s_or_po, str):
query = s_or_po
if initNs:
prefixes = ''.join(["prefix %s: <%s>\n" % (p, n)
for p, n in initNs.items()])
query = prefixes + query
elif isinstance(s_or_po, URIRef) or isinstance(s_or_po, BNode):
query = "describe %s" % (s_or_po.n3())
else:
p, o = s_or_po
query = "describe ?s where {?s %s %s}" % (p.n3(), o.n3())
query = dict(query=query)
url = self.url + "?" + urlencode(query)
req = Request(url)
req.add_header('Accept', 'application/rdf+xml')
log.debug("opening url: %s\n with headers: %s" %
(req.get_full_url(), req.header_items()))
subgraph = ConjunctiveGraph()
subgraph.parse(urlopen(req))
return subgraph
def test(request, ttype, test):
# Get the request_host (without the port)
request_path = request.path
request_host = 'http://'+request.get_host()+request_path.replace(request.path_info,'')
# print request_host
if ttype in ['publications', 'mascc'] :
if test in [re.search(r'ttl/(.*)\.ttl',n).group(1) for n in glob('/var/www/semweb.cs.vu.nl/plugins/ttl/*.ttl')] :
filename = "/var/www/semweb.cs.vu.nl/plugins/ttl/{}.ttl".format(test)
cg = ConjunctiveGraph()
cg.parse(filename, format='n3')
# print "Request received"
if ttype == 'publications' :
response = HttpResponseRedirect(request_host+'/publications/{}'.format(urllib.quote(cg.serialize(format='turtle'),safe='')))
elif ttype == 'mascc' :
response = HttpResponseRedirect(request_host+'/mascc/{}'.format(urllib.quote(cg.serialize(format='turtle'),safe='')))
else :
response = HttpResponseBadRequest()
else :
response = HttpResponseNotFound()
elif ttype == 'bad' :
response = HttpResponseRedirect(request_host+'/mascc/bad_request')
else :
response = HttpResponseNotFound()
return response
def change_status(vocabprefix, uri, predicate, message, action):
if not action in ['add', 'remove']:
return False
vocab_uri = URIRef(uri)
vocabdir = os.path.join(ag.vocabulariesdir, vocabprefix)
vocabstatusfile = os.path.join(vocabdir, "status.rdf")
if not os.path.isfile(vocabstatusfile):
return False
graph = Graph()
graph.parse(vocabstatusfile)
predicate = predicate.split(':')
ns = predicate[0]
term = predicate[1]
if message and (message.startswith('http://') or message.startswith('file://')):
message = URIRef(message)
elif message:
message = Literal(message)
if action == 'add':
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
graph.add((vocab_uri, namespaces[ns][term], message))
elif action == 'remove':
graph.remove((vocab_uri, namespaces[ns][term], message))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(vocabstatusfile, 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
def add_mediator(params):
#Write user metadata and save the rdf file
graph = Graph()
for prefix, url in namespaces.iteritems():
graph.bind(prefix, URIRef(url))
uri = URIRef("http://vocab.ox.ac.uk/owner/uuid:%s"%uuid.uuid4())
graph.add((uri, namespaces['foaf']['firstName'], Literal(params['firstname'])))
graph.add((uri, namespaces['foaf']['lastName'], Literal(params['lastname'])))
graph.add((uri, namespaces['foaf']['mbox'], Literal(params['email'])))
graph.add((uri, namespaces['foaf']['account'], Literal(params['username'])))
if 'title' in params and params['title']:
graph.add((uri, namespaces['foaf']['title'], Literal(params['title'])))
if 'department' in params and params['department']:
department = params['department'].split(';')
for d in department:
graph.add((uri, namespaces['dcterms']['isPartOf'], Literal(d.strip())))
rdf_str = None
rdf_str = graph.serialize()
f = codecs.open(os.path.join(ag.mediatorsdir, '%s.rdf'%params['username']), 'w', 'utf-8')
f.write(rdf_str)
f.close()
graph2 = Graph()
graph2.parse(ag.mediatorslist)
for prefix, url in namespaces.iteritems():
graph2.bind(prefix, URIRef(url))
graph2.add((uri, namespaces['foaf']['account'], Literal(params['username'])))
rdf_str = None
rdf_str = graph2.serialize()
f = codecs.open(ag.mediatorslist, 'w', 'utf-8')
f.write(rdf_str)
f.close()
return True
class OntoInspector(object):
"""Class that includes methods for querying an RDFS/OWL ontology"""
def __init__(self, uri, language=""):
super(OntoInspector, self).__init__()
self.rdfGraph = ConjunctiveGraph()
try:
self.rdfGraph.parse(uri, format="xml")
except:
try:
self.rdfGraph.parse(uri, format="n3")
except:
raise exceptions.Error("Could not parse the file! Is it a valid RDF/OWL ontology?")
finally:
# let's cache some useful info for faster access
self.baseURI = self.get_OntologyURI() or uri
self.allclasses = self.__getAllClasses(classPredicate)
self.toplayer = self.__getTopclasses()
self.tree = self.__getTree()
def get_OntologyURI(self, ....):
# todo
pass
def update():
"""
Update the library with new articles.
"""
graph = ConjunctiveGraph()
# load the existing graph
library = 'data/articles.rdf'
graph.load(library)
feeds = {
"http://www3.interscience.wiley.com/rss/journal/118485807": "wiley.xsl",
"http://phg.sagepub.com/rss/current.xml": "sage.xsl",
"http://www.informaworld.com/ampp/rss~content=t713446924": "infoworld.xsl",
"http://www.informaworld.com/ampp/rss~content=t788352614": "infoworld.xsl",
"http://www.envplan.com/rss.cgi?journal=D": "envplan.xsl",
"http://www.envplan.com/rss.cgi?journal=A": "envplan.xsl",
"http://cgj.sagepub.com/rss/current.xml": "sage.xsl"
}
for feed, stylesheet in feeds.iteritems():
# grab the feed and transform it
print "grabbing ", feed
new = StringIO.StringIO(feed_transform(feed, stylesheet))
# merge the new triples into the graph
graph.parse(new)
new.close()
graph.serialize(library, format='pretty-xml')
def projects(self):
context = aq_inner(self.context)
_projectType = _mcltype.Project
# title
_title = _terms.title
# abbriviation
_abbrName = _schema.abbreviatedName
# description
_description = _terms.description
# Temporary rdf read
rdfDataSource = "https://edrn-dev.jpl.nasa.gov/ksdb/publishrdf/?rdftype=project"
graph = ConjunctiveGraph()
graph.parse(URLInputSource(rdfDataSource))
statements = _parseRDF(graph)
projects = []
for uri, i in statements.items():
project = dict(url=uri, title="", abbrName="", description="")
if _title in i:
project["title"] = unicode(i[_title][0])
if _abbrName in i:
project["abbrName"] = unicode(i[_abbrName][0])
if _description in i:
project["description"] = strip_tags(unicode(i[_description][0]))
projects.append(project)
return projects
def construct(self, strOrTriple, initBindings={}, initNs={}):
"""
Executes a SPARQL Construct
:param strOrTriple: can be either
* a string in which case it it considered a CONSTRUCT query
* a triple in which case it acts as the rdflib `triples((s,p,o))`
:param initBindings: A mapping from a Variable to an RDFLib term (used as initial bindings for SPARQL query)
:param initNs: A mapping from a namespace prefix to a namespace
:returns: an instance of rdflib.ConjuctiveGraph('IOMemory')
"""
if isinstance(strOrTriple, str):
query = strOrTriple
if initNs:
prefixes = ''.join(["prefix %s: <%s>\n"%(p,n) for p,n in initNs.items()])
query = prefixes + query
else:
s,p,o = strOrTriple
t='%s %s %s'%((s and s.n3() or '?s'),(p and p.n3() or '?p'),(o and o.n3() or '?o'))
query='construct {%s} where {%s}'%(t,t)
query = dict(query=query)
url = self.url+"?"+urlencode(query)
req = Request(url)
req.add_header('Accept','application/rdf+xml')
log.debug("Request url: %s\n with headers: %s" % (req.get_full_url(), req.header_items()))
subgraph = ConjunctiveGraph('IOMemory')
subgraph.parse(urlopen(req))
return subgraph
def test_flowcells_index_rdfa(self):
model = ConjunctiveGraph()
response = self.client.get(reverse('flowcell_index'))
self.assertEqual(response.status_code, 200)
model.parse(data=smart_text(response.content), format='rdfa')
add_default_schemas(model)
inference = Infer(model)
errmsgs = list(inference.run_validation())
self.assertEqual(len(errmsgs), 0, errmsgs)
body = """prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
prefix libns: <http://jumpgate.caltech.edu/wiki/LibraryOntology#>
select ?flowcell
where {
?flowcell a libns:IlluminaFlowcell .
}"""
bindings = set(['flowcell'])
count = 0
for r in model.query(body):
count += 1
self.assertEqual(count, len(FlowCell.objects.all()))
def _test_serializer(inputpath, expectedpath, context, serpar):
test_tree, test_graph = _load_test_data(inputpath, expectedpath, context)
if isinstance(test_tree, ConjunctiveGraph):
expected = test_tree.serialize(format="json-ld")
else:
expected = _to_json(_to_ordered(test_tree))
if test_graph is not None:
# toRdf, expected are nquads
result_tree = to_tree(test_graph, context_data=context)
result = _to_json(_to_ordered(result_tree))
elif inputpath.startswith('fromRdf'):
# fromRdf, expected in json-ld
g = ConjunctiveGraph()
data = open(p.join(test_dir, inputpath), 'rb').read()
g.parse(data=data, format="nquads", context=context)
result = g.serialize(format="json-ld", base=context)
else:
# json
f = open(p.join(test_dir, inputpath), 'rb')
result = json.load(f)[0]
f.close()
if isinstance(result, ConjunctiveGraph):
assert isomorphic(result, expected), \
"Expected graph of %s:\n%s\nGot graph of %s:\n %s" % (
expected.serialize(format='n3'),
result.serialize(format='n3'))
else:
assert jsonld_compare(expected, result) == True, \
"Expected JSON:\n%s\nGot:\n%s" % (expected, result)
def serialize_demo():
try:
store = ConjunctiveGraph()
store.parse(FILE, format='xml')
print store.serialize(format='xml')
except OSError:
print "Cannot read file '%s'" % FILE
def __call__(self, url, **kwargs):
if not url:
return []
graph = ConjunctiveGraph()
graph.parse(url)
output = {}
for subject, predicate, context in graph:
key = self.strip(subject)
prop = self.strip(predicate)
value = self.defrag(context)
output.setdefault(key, {
'label': key,
'uri': unicode(subject)
})
if prop in output[key]:
old = output[key][prop]
if not isinstance(old, list):
output[key][prop] = [old]
output[key][prop].append(value)
else:
output[key][prop] = value
return output.values()
def initialize(config_file):
print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime())
sys.stdout.flush()
config = __import__(config_file)
try:
g = ConjunctiveGraph(config.graph_store, config.graph_identifier)
g.open(config.db_configstring, create=True)
if config.input_file != None:
print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), config.input_file)
sys.stdout.flush()
g.parse(config.input_file, format=config.input_format)
g.commit()
else:
dir_list = os.listdir(config.input_dir)
for file_name in dir_list:
print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()) ,file_name)
sys.stdout.flush()
g.parse(config.input_dir + '/' + file_name, format=config.input_format)
g.commit()
except Exception as e:
traceback.print_exc()
print e
print '"%s" not found, or incorrect RDF serialization.' % config.input_file
sys.stdout.flush()
exit(-1)
return g, config
def patchFromJson(j):
body = json.loads(j)['patch']
a = ConjunctiveGraph()
a.parse(StringInputSource(json.dumps(body['adds'])), format='json-ld')
d = ConjunctiveGraph()
d.parse(StringInputSource(json.dumps(body['deletes'])), format='json-ld')
return Patch(addGraph=a, delGraph=d)
def partsites(self):
context = aq_inner(self.context)
_partsiteType = _mcltype.FundedSite
# title
_title = _terms.title
# description
_description = _terms.description
# Temporary rdf read
rdfDataSource = "https://edrn-dev.jpl.nasa.gov/ksdb/publishrdf/?rdftype=fundedsite"
graph = ConjunctiveGraph()
graph.parse(URLInputSource(rdfDataSource))
statements = _parseRDF(graph)
partsites = []
for uri, i in statements.items():
partsite = dict(url=uri, title="", description="")
if _title in i:
partsite["title"] = unicode(i[_title][0])
if _description in i:
partsite["description"] = strip_tags(unicode(i[_description][0]))
partsites.append(partsite)
return partsites
class FOAF(callbacks.Privmsg):
DATAFILE = "/var/www/rc98.net/zoia.rdf"
def __init__(self, irc):
self.g = Graph()
# self.g.parse('http://rc98.net/zoia.rdf')
self.g.parse(self.DATAFILE, format="xml")
self.uri = rdflib.URIRef("http://www.code4lib.org/id/zoia")
self.FOAF = Namespace("http://xmlns.com/foaf/0.1/")
super(callbacks.Plugin, self).__init__(irc)
def _uri_of_user(self, nick):
result = self.g.query(
"""
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
SELECT ?uri WHERE
{<http://www.code4lib.org/id/zoia> foaf:knows ?uri . ?uri foaf:nick ?nick .}
""",
initBindings={"nick": nick},
)
if len(result) > 0:
userURI = list(result)[0][0]
return userURI
else:
return None
def _user_graph(self, uri):
userGraph = Graph()
try:
userGraph.parse(uri)
except Exception, e:
u = "http://www.w3.org/2007/08/pyRdfa/extract?space-preserve=true&uri=" + uri
userGraph.parse(u, identifier=uri)
return userGraph
def get_graph(self, with_mappings=False, include_mapping_target=False, acceptance=False, target_uri=None):
"""Get Graph instance of this EDMRecord.
:param target_uri: target_uri if you want a sub-selection of the whole graph
:param acceptance: if the acceptance data should be listed
:param include_mapping_target: Boolean also include the mapping target triples in graph
:param with_mappings: Boolean integrate the ProxyMapping into the graph
"""
rdf_string = self.source_rdf
if acceptance and self.acceptance_rdf:
rdf_string = self.acceptance_rdf
graph = ConjunctiveGraph(identifier=self.named_graph)
graph.namespace_manager = namespace_manager
graph.parse(data=rdf_string, format='nt')
if with_mappings:
proxy_resources, graph = ProxyResource.update_proxy_resource_uris(self.dataset, graph)
self.proxy_resources.add(*proxy_resources)
for proxy_resource in proxy_resources:
graph = graph + proxy_resource.to_graph(include_mapping_target=include_mapping_target)
if target_uri and not target_uri.endswith("/about") and target_uri != self.document_uri:
g = Graph(identifier=URIRef(self.named_graph))
subject = URIRef(target_uri)
for p, o in graph.predicate_objects(subject=subject):
g.add((subject, p, o))
graph = g
return graph
def __load_graph(file_p, tmp_dir=None):
errors = ""
current_graph = ConjunctiveGraph()
if tmp_dir is not None:
file_path = tmp_dir + os.sep + "tmp_rdf_file.rdf"
shutil.copyfile(file_p, file_path)
else:
file_path = file_p
try:
with open(file_path) as f:
json_ld_file = json.load(f)
if isinstance(json_ld_file, dict):
json_ld_file = [json_ld_file]
for json_ld_resource in json_ld_file:
# Trick to force the use of a pre-loaded context if the format
# specified is JSON-LD
cur_context = json_ld_resource["@context"]
json_ld_resource["@context"] = context_json
current_graph.parse(data=json.dumps(json_ld_resource), format="json-ld")
return current_graph
except Exception as e:
errors = " | " + str(e) # Try another format
if tmp_dir is not None:
os.remove(file_path)
raise IOError("[1]", "It was impossible to handle the format used for storing the file '%s'%s" %
(file_path, errors))
def rdfFromText(self, text):
"""Take text, return an RDF graph."""
postdata = {}
postdata['licenseID'] = self.api_key
postdata['paramsXML'] = ' '.join(['<c:params xmlns:c="http://s.opencalais.com/1/pred/"'
,'xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">'
,'<c:processingDirectives c:contentType="text/raw"'
,'c:outputFormat="text/xml"'
,'c:enableMetadataType="GenericRelations,SocialTags">'
,'</c:processingDirectives>'
,'<c:userDirectives c:allowDistribution="false"'
,'c:allowSearch="false"'
,'c:externalID="{0}"'.format(uuid.uuid4())
,'c:submitter="{0}">'.format(self.app_name)
,'</c:userDirectives>'
,'<c:externalMetadata></c:externalMetadata>'
,'</c:params>'])
postdata['content'] = text
poststring = urllib.urlencode(postdata)
data = self.post_data("{0}".format(self.api_url), poststring, timeout=60*5)
graph = Graph()
inpt = StringInputSource(data)
try: graph.parse(inpt, 'xml')
except:
print data
raise
return graph
def check(kws):
cg = ConjunctiveGraph()
cg.parse(**kws)
for g in cg.contexts():
gid = g.identifier
assert isinstance(gid, Identifier)
def ConvertToRDFN3 (filename, destinationFileName):
_graph = ConjunctiveGraph()
_graph.parse(filename, format="nt")
of = open(destinationFileName, "wb")
of.write(_graph.serialize(format="n3"))
of.close()
def test_remove_period(self):
with open(filepath('test-patch-remove-period.json')) as f:
patch1 = f.read()
with self.client as client:
res = client.patch(
'/d/',
data=patch1,
content_type='application/json',
headers={'Authorization': 'Bearer '
+ 'NTAwNWViMTgtYmU2Yi00YWMwLWIwODQtMDQ0MzI4OWIzMzc4'})
patch_url = urlparse(res.headers['Location']).path
res = client.post(
patch_url + 'merge',
buffered=True,
headers={'Authorization': 'Bearer '
+ 'ZjdjNjQ1ODQtMDc1MC00Y2I2LThjODEtMjkzMmY1ZGFhYmI4'})
self.assertEqual(res.status_code, http.client.NO_CONTENT)
removed_entities = database.get_removed_entity_keys()
self.assertEqual(removed_entities, set(['p0trgkvwbjd']))
res = client.get('/trgkvwbjd',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.GONE)
res = client.get('/trgkvwbjd.json',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.GONE)
res = client.get('/trgkvwbjd?version=0',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.NOT_FOUND)
res = client.get('/trgkvwbjd.json?version=0',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.NOT_FOUND)
res = client.get('/trgkvwbjd?version=1',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.OK)
res = client.get('/trgkvwbjd.json?version=1',
headers={'Accept': 'application/json'},
follow_redirects=True)
self.assertEqual(res.status_code, http.client.OK)
res = client.get('/history.jsonld?inline-context')
self.assertEqual(
res.headers['Cache-Control'],
'public, max-age=0')
self.assertEqual(
res.headers['X-Accel-Expires'],
'{}'.format(cache.MEDIUM_TIME))
g = ConjunctiveGraph()
g.parse(format='json-ld', data=res.get_data(as_text=True))
generated = list(g.objects(subject=HOST['h#change-2'],
predicate=PROV.generated))
self.assertEqual(len(generated), 1)
self.assertIn(HOST['d?version=2'], generated)
def ConvertToRDFXML (filename,destinationFileName):
_graph = ConjunctiveGraph()
_graph.parse(filename, format="nt")
_graph.triples((None, None, None))
of = open(destinationFileName, "wb")
of.write(_graph.serialize(format="pretty-xml"))
of.close()
def test_parse_distinct_bnode_context(self):
g = ConjunctiveGraph()
g.parse(self.data, format="nquads", bnode_context=dict())
s1 = set(g.subjects())
self.data.seek(0)
g.parse(self.data, format="nquads", bnode_context=dict())
s2 = set(g.subjects())
self.assertNotEqual(set(), s2 - s1)
def test_parse_shared_bnode_context(self):
bnode_ctx = dict()
g = ConjunctiveGraph()
h = ConjunctiveGraph()
g.parse(self.data, format="nquads", bnode_context=bnode_ctx)
self.data.seek(0)
h.parse(self.data, format="nquads", bnode_context=bnode_ctx)
self.assertEqual(set(h.subjects()), set(g.subjects()))
def get_mediator_account(user_uuid):
uri = URIRef("http://vocab.ox.ac.uk/owner/%s"%user_uuid)
graph = Graph()
graph.parse(ag.mediatorslist)
for account in graph.objects(uri, namespaces['foaf']['account']):
if account:
return account
return False
def getRDFStatements(self):
u'''Parse the main and additional RDF data sources and return a dict {uri → {predicate → [objects]}}'''
context = aq_inner(self.context)
statements = {}
graph = ConjunctiveGraph()
graph.parse(URLInputSource(context.rdfDataSource))
self.addGraphToStatements(graph, statements)
return statements
def generate_example(self, egtext, resource):
# Yes really...
exec(egtext)
# Now in scope should be a top resource
factory.pipe_scoped_contexts = False
factory.toFile(top, compact=False)
factory.pipe_scoped_contexts = True
jsstr = factory.toString(top, compact=False, collapse=80)
factory.pipe_scoped_contexts = False
js = factory.toJSON(top)
# Generate all our serializations
nq = to_rdf(js, {"format": "application/nquads"})
g = ConjunctiveGraph()
for ns in ['crm', 'dc', 'schema', 'dcterms', 'skos', 'la']:
g.bind(ns, ctxt[ns])
g.parse(data=nq, format="nt")
out = g.serialize(format="turtle")
fp = js['id'][len(factory.base_url):]
fp2 = fp + ".ttl"
fh = open(os.path.join(factory.base_dir, fp2), 'w')
fh.write(out)
fh.close()
# And build mermaid description
mermaid = self.build_mermaid(js)
# Build index references
self.traverse(js, top.id, resource)
# And return the JSON plus links, to be substed by the top level filter
raw = top.id + ".json"
self.example_list.append(raw)
rawq = urllib.quote(raw).replace('/', "%2F")
playground = "http://json-ld.org/playground-dev/#startTab=tab-expanded©Context=true&json-ld=%s" % rawq
turtle = top.id + ".ttl"
turtle_play = "http://cdn.rawgit.com/niklasl/ldtr/v0.2.2/demo/?edit=true&url=%s" % turtle
egid = fp.replace('/', '_')
resp = """
<a id="%s"></a>
```json
%s
```
<div class="mermaid">
%s
</div>
Other Representations: [JSON-LD (Raw)](%s) |
[JSON-LD (Playground)](%s) |
[Turtle (Raw)](%s) |
[Turtle (Styled)](%s)
""" % (egid, jsstr, mermaid, raw, playground, turtle, turtle_play)
return resp
def bundle_to_graph(bundle):
g = ConjunctiveGraph()
g.parse("links.json", format="json-ld")
for file in bundle['bundle']['files']:
if "dcp-type=\"metadata/" in file['content-type']:
print(file['content-type'])
g = add_to_graph(g, file['uuid'])
# then get the links.json from the bundle
return g
def testOrderBy(self):
graph = ConjunctiveGraph(plugin.get('IOMemory', Store)())
graph.parse(StringIO(test_data), format="n3")
results = graph.query(test_query)
self.failUnless(False not in [
r[0] == a
for r, a in zip(results, ['Alice', 'Bob', 'Charlie', 'Dave'])
])
def check_n3(rdffile):
if not os.path.isfile(rdffile):
return False
graph = Graph()
try:
graph.parse(rdffile, format="n3")
return True
except:
return False
def check_rdf(rdffile):
if not os.path.isfile(rdffile):
return False
graph = Graph()
try:
graph.parse(rdffile)
return True
except:
return False
def get_ref_vocabs():
reflist = {}
if not os.path.isfile(ag.vocabulariesref):
return reflist
graph = Graph()
graph.parse(ag.vocabulariesref)
for s, p, o in graph.triples((None, namespaces['dcterms']['isVersionOf'], None)):
reflist[str(s)] = str(o)
return reflist
def _load_test_expectedpath(inputpath, expectedpath, context):
if '.jsonld' in expectedpath:
test_graph = None
elif '.nq' in expectedpath:
test_graph = ConjunctiveGraph()
test_graph.parse(expectedpath, format='nquads', publicId=context)
else:
raise Exception("expectedpath %s" % expectedpath)
return test_graph
class TestSparqlJsonResults(unittest.TestCase):
def setUp(self):
self.graph = ConjunctiveGraph()
self.graph.parse(StringIO(test_data), format="n3")
def test_base_ref(self):
rt = self.graph.query(test_query).serialize("python")
self.failUnless(rt[0] == Literal("Alice"),
"Expected:\n 'Alice' \nGot:\n %s" % rt)
def run_query(source, query_file, context=None):
graph = ConjunctiveGraph()
fmt = ('json-ld' if source.endswith('.jsonld') else guess_format(source))
graph.parse(source, format=fmt, context=context)
with open(query_file) as fp:
query_text = fp.read().decode('utf-8')
result = graph.query(query_text)
for row in result:
print('\t'.join(t.n3() if t else 'UNDEF' for t in row).encode('utf-8'))
def labelsforpath(pFilePath):
# if file name is the same as an image instance already present in the database, don't read file:
p = BDR[Path(pFilePath).stem]
model = ConjunctiveGraph()
model.parse(str(pFilePath), format="trig")
res = {}
for _, _, o in model.triples((p, SKOS.prefLabel, None)):
res[o.language] = o.value
return res
def test_parse_shared_bnode_context_same_graph(self):
bnode_ctx = dict()
g = ConjunctiveGraph()
g.parse(self.data_obnodes, format="nquads", bnode_context=bnode_ctx)
o1 = set(g.objects())
self.data_obnodes.seek(0)
g.parse(self.data_obnodes, format="nquads", bnode_context=bnode_ctx)
o2 = set(g.objects())
self.assertEqual(o1, o2)
def test_others(metadata_file):
with CSVW(csv_path="tests/datatypes.others.csv",
metadata_path=metadata_file) as csvw:
rdf_output = csvw.to_rdf()
g = ConjunctiveGraph()
g.parse(data=rdf_output, format="turtle")
triples_to_look_for = [
(NS['custom_pred'], "someformatteddata",
URIRef("https://www.datatypes.org#mycustomdatatypedefinition")),
(NS["anyURI"], "https://www.sampleuri.org", XSD.anyURI),
(NS["base64Binary"], "0FB8", XSD.base64Binary),
(NS['boolean1'], True, XSD.boolean),
(NS['boolean2'], False, XSD.boolean),
(NS['boolean3'], True, XSD.boolean),
(NS['boolean4'], False, XSD.boolean),
(NS['integer'], -3, XSD.integer),
(NS['long'], -1231235555, XSD.long),
(NS['int'], 3, XSD.int),
(NS['short'], -1231, XSD.short),
(NS['byte'], 45, XSD.byte),
(NS['nonNegativeInteger'], 111, XSD.nonNegativeInteger),
(NS['positiveInteger'], 123456, XSD.positiveInteger),
(NS['unsignedLong'], 3456, XSD.unsignedLong),
(NS['unsignedInt'], 7890000, XSD.unsignedInt),
(NS['unsignedShort'], 65000, XSD.unsignedShort),
(NS['unsignedByte'], 254, XSD.unsignedByte),
(NS['nonPositiveInteger'], -123, XSD.nonPositiveInteger),
(NS['negativeInteger'], -34500000, XSD.negativeInteger),
(NS['decimal'], "+3.5", XSD.decimal),
(NS['double'], "4268.22752E11", XSD.double),
(NS['float'], "+24.3e-3", XSD.float),
(NS['duration'], "P2Y6M5DT12H35M30S", XSD.duration),
(NS['dayTimeDuration'], "P1DT2H", XSD.dayTimeDuration),
(NS['yearMonthDuration'], "P0Y20M", XSD.yearMonthDuration),
(NS['gDay'], "---02", XSD.gDay),
(NS['gMonth'], "--04", XSD.gMonth),
(NS['gMonthDay'], "--04-12", XSD.gMonthDay),
(NS['gYear'], "2004", XSD.gYear),
(NS['gYearMonth'], "2004-04", XSD.gYearMonth),
(NS['hexBinary'], "0FB8", XSD.hexBinary),
(NS['QName'], "myElement", XSD.QName),
(NS['normalizedString'], "This is a normalized string!",
XSD.normalizedString),
(NS['token'], "token", XSD.token),
(NS['language'], "en", XSD.language),
(NS['Name'], "_my.Element", XSD.Name),
(NS['NMTOKEN'], "123_456", XSD.NMTOKEN),
(NS['xml'], "<a>bla</a>", RDF.XMLLiteral),
(NS['html'], "<div><p>xyz</p></div>", RDF.HTML),
(NS['json'], "{}", CSVW_NS.JSON),
]
for pred, lit_val, lit_type in triples_to_look_for:
lit = Literal(lit_val, datatype=lit_type)
assert len(list(g.triples(
(NS['event/1'], pred, lit)))) == 1, "Failed for {}".format(pred)
def fmaRDF(self, rdf):
print "Checking RDF file..."
if os.path.exists(rdf):
tempGraph = Graph()
tempGraph.parse(rdf)
self.fmaGraph = tempGraph
else:
print "Cannot find rdf file " + str(rdf) + " . Exiting!"
sys.exit()
def test_parse_distinct_bnode_contexts_between_graphs(self):
g = ConjunctiveGraph()
h = ConjunctiveGraph()
g.parse(self.data, format="nquads")
s1 = set(g.subjects())
self.data.seek(0)
h.parse(self.data, format="nquads")
s2 = set(h.subjects())
self.assertNotEqual(s1, s2)
def _load_nquads(source):
graph = ConjunctiveGraph()
with open(source) as f:
if PY3:
data = f.read()
else:
data = f.read().decode('utf-8')
graph.parse(data=data, format='nquads')
return graph
def nquads(test):
g = ConjunctiveGraph()
try:
g.parse(test.action, format="nquads")
if not test.syntax:
raise AssertionError("Input shouldn't have parsed!")
except:
if test.syntax:
raise
def build_graph(self):
COW(mode='convert', files=[os.path.join(self.upload_folder, self.path)])
self.logger.debug("Convert finished")
try:
with open(os.path.join(self.upload_folder, self.path + '.nq')) as nquads_file:
g = ConjunctiveGraph()
g.parse(nquads_file, format='nquads')
return g
except IOError:
raise IOError("COW could not generate any RDF output. Please check the syntax of your CSV and JSON files and try again.")
def fmaURL(self, url):
print "Checking URL..."
try:
tempGraph = Graph()
fmaURL = urlopen(url)
tempGraph.parse(fmaURL)
self.fmaGraph = tempGraph
except:
print "Cannot open " + str(url) + ". Exiting."
sys.exit()
def test_empty():
csvw = CSVW(csv_path="tests/empty.csv",
metadata_path="tests/empty.csv-metadata.json")
rdf_output = csvw.to_rdf()
g = ConjunctiveGraph()
g.parse(data=rdf_output, format="turtle")
assert len(g) == 0
def test01(self):
# tree, graph, base=None, context_data=None
g = ConjunctiveGraph()
ingraph = to_rdf(json.loads(test01_in), g)
outgraph = ConjunctiveGraph()
outgraph.parse(data=test01_out, format="nquads")
assert isomorphic(outgraph, ingraph), \
"Expected graph of %s:\n%s\nGot graph of %s:\n %s" % (
len(outgraph), outgraph.serialize(),
len(ingraph), ingraph.serialize())
def _get_graphs(self):
graph = ConjunctiveGraph()
try:
graph.parse(f'http://{os.environ["WIFI_PORT_80_TCP_ADDR"]}/graph/wifi', format='trig')
# graph.parse('http://bang:10006/graph/timebank', format='trig')
except Exception as e:
plog(f"failed to fetch graph(s): {e!r}")
plog(f'fetched {len(graph)} statements from timebank and wifi')
return graph
def json_to_rdf(self, js, fmt=None):
d2 = self._mk_rdflib_jsonld(js)
js = json.dumps(d2)
g = ConjunctiveGraph()
g.parse(data=js, format='json-ld')
if fmt:
out = g.serialize(format=fmt)
return out
else:
return g
def rdf_data(rdfobject, f):
input = ConjunctiveGraph()
input.open("store2", create=True)
input.parse(data=rdfobject, format=f)
#print(input.serialize(format='json-ld', auto_compact=True, indent=4))
for s, p, o in input:
g.add((s, p, o))
input.close()
def _load_test_inputpath(inputpath, expectedpath, context):
if '.jsonld' in inputpath:
f = open(inputpath, 'rb')
test_tree = json.load(StringIO(f.read().decode('utf-8')) if PY3 else f)
f.close()
elif '.nq' in inputpath:
test_tree = ConjunctiveGraph()
test_tree.parse(inputpath, format='nquads')
else:
raise Exception("Unable to load test_data from %s" % inputpath)
return test_tree
def load_nquads_file(path_to_nquad_file):
"""
load rdf file in nquads format
:param str path_to_nquad_file: path to rdf file in nquad format
:rtype: rdflib.graph.ConjunctiveGraph
:return: nquad
"""
g = ConjunctiveGraph()
with open(path_to_nquad_file, "rb") as infile:
g.parse(infile, format="nquads")
return g
def __init__(self, raw_result):
self._detect_fails(raw_result)
rdf = Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
c = Namespace('http://s.opencalais.com/1/pred/')
g = Graph()
self.graph = g
g.parse(StringIO(raw_result.decode('utf-8').encode('utf-8')))
self.categories = [row for row in g.query(CATEGORY_QUERY['SPARQL'])]
self.entities = [row for row in g.query(ENTITY_QUERY['SPARQL'])]
def process_document(self, session, doc):
fmt = self.get_setting(session, 'format', '')
data = doc.get_raw(session)
graph = Graph()
inpt = StringInputSource(data)
if fmt:
graph.parse(inpt, fmt)
else:
graph.parse(inpt)
rec = GraphRecord(graph)
return rec
