def model_from_uri(uri=None, **opts):
if RDF is None:
raise ConnectionError(
'Redland support is not available, install from librdf.org')
options = []
for key, val in opts.items():
options.append("%s='%s'" % (key, val))
options = ','.join(options)
if uri == None or uri == 'memory':
options += ",hash-type='memory'"
store = RDF.Storage(storage_name='hashes',
name='memory_store',
options_string=options)
elif uri.startswith('sqlite://'):
file = uri[9:]
try:
store = RDF.Storage(storage_name='sqlite',
name=file,
options_string=options)
except RDF.RedlandError:
raise ConnectionError("Can't connect to %s" % uri)
elif uri.startswith('bdb://'):
file = uri[6:]
if not os.path.isfile(file+'-sp2o.db') and not 'new=' in options:
options += ",new='yes'"
options += ",hash-type='bdb'"
store = RDF.Storage(storage_name='hashes',
name=file,
options_string=options)
elif uri.startswith('mysql://'):
uri = uri[8:]
userpass, hostdb = uri.split('@')
user, pwd = userpass.split(':')
host, db = hostdb.split('/')
if ':' in host:
host, port = host.split(':')
else:
port = '3306'
if '#' in db:
db, name = db.split('#')
else:
name = 'main'
options += (",host='%s',port='%s',database='%s',user='******',"
"password='******'" % (host, port, db, user, pwd))
try:
store = RDF.Storage(storage_name='mysql',
name=name,
options_string=options)
except RDF.RedlandError:
raise ConnectionError("Can't connect to %s" % uri)
else:
raise ConnectionError('Unknown dburi: %s' % uri)
return RDF.Model(store)
def __init__(self, name):
storage_type, options = self.get_store_options(name)
# workaround: sqlite doesn't support 'dir' so prepend directory to the name
if storage_type == 'sqlite':
name = os.path.abspath(os.path.join(config.CATALOG_DATA_DIR, name))
self._store = RDF.Storage(storage_name=storage_type, name=name, options_string=options)
self._model = RDF.Model(self._store)
def new_storage():
'''Create a new in-memory Redland store for storing the RDF model.'''
storage = RDF.Storage(storage_name="hashes",
name="test",
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
raise CDAOError("Creation of new RDF.Storage failed.")
return storage
def createModel():
storage = RDF.Storage(
storage_name="hashes",
name="geolink",
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
raise Exception("new RDF.Storage failed")
model = RDF.Model(storage)
if model is None:
raise Exception("new RDF.model failed")
return model
def export_volume(self, volume_urn, fd):
""" Serialize a suitable properties file for the
volume_urn. We include all the objects which are contained in
the volume.
"""
storage = RDF.Storage(storage_name='hashes',
name='X',
options_string="new='yes',hash-type='memory',dir='.'")
model = RDF.Model(storage)
for urn in aff4.oracle.resolve_list(volume_urn, AFF4_CONTAINS):
try:
urn_id = self.get_id_by_urn(urn)
except ValueError: continue
self.export_model(urn, model)
self.export_model(volume_urn, model)
serializer = RDF.Serializer("turtle")
fd.write(serializer.serialize_model_to_string(model))
def __init__(self, namespaces={}):
# Initialize variables
name = "db-{0!s}".format(random.randint(1, 999999))
self.db = RDF.Storage(storage_name="hashes",
name=name,
options_string="hash-type='bdb'")
self.model = RDF.Model(self.db)
self.serializer = RDF.Serializer(name="turtle")
self.ns = {}
# Define basic namespaces
basicNamespaces = {
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
}
# Extend basic namespaces with those provided in parameters
basicNamespaces.update(namespaces)
self.addNamespaces(basicNamespaces)
atexit.register(self.clearTempFiles)
def init_model(*filenames):
"""Input: An on-disk path (filenames) to start from.
Output: A model with those suckers parsed."""
for filename in filenames: # filenames, not URIs
die_unless(
':/' not in filename,
"You passed in something that " + "looks like a URI; blowing up")
storage = RDF.Storage(
storage_name="hashes",
name="test",
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
raise "new RDF.Storage failed"
model = RDF.Model(storage)
if model is None:
raise "new RDF.Model failed"
parser = RDF.Parser('raptor')
for filename in filenames:
filename_uri = RDF.Uri(string="file:" + filename)
parser.parse_into_model(model, filename_uri)
return model
def check_version(self):
import copy
import RDF
if self.software == 'FusionForge':
response = urllib2.urlopen(self.url)
the_page = response.read()
# hack to suppress news, which tend to mess with the validity of the document
# <div class="one-news ... <!-- class="one-news" -->
the_page = the_page.replace("\n", '')
myMassage = [
(re.compile('<div class="one-news.*<!-- class="one-news" -->'),
lambda match: '')
]
myNewMassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
myNewMassage.extend(myMassage)
soup = BeautifulSoup(the_page, markupMassage=myNewMassage)
version = None
divs = soup.findAll('div', id='ft')
for div in divs:
text = div.text
running_version = re.compile(
'This site is running FusionForge version (.*)')
m = running_version.match(text)
version = m.group(1)
break
if version:
self.version = version
# The following will apply only for FF > 5.1 which includes RDFa
myMassage = [(
re.compile('(<body[^>]*>)(.*)(<div id="ft")'),
lambda match: match.group(1) + match.group(3) +
' xmlns:planetforge="http://coclico-project.org/ontology/planetforge#"'
)]
myNewMassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
myNewMassage.extend(myMassage)
soup = BeautifulSoup(the_page, markupMassage=myNewMassage)
storage = RDF.Storage(
storage_name="hashes",
name="test",
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
raise "new RDF.Storage failed"
# RDF.debug(1)
model = RDF.Model(storage)
if model is None:
raise "new RDF.model failed"
#parser=RDF.Parser(name='raptor')
# parser=RDF.Parser(name='rdfa', mime_type='application/xhtml+xml')
parser = RDF.Parser(name='rdfa')
if parser is None:
raise "Failed to create RDF.Parser raptor"
def error_handler(code, level, facility, message, line, column,
byte, file, uri):
print 'error_handler', code, level, facility, message, line, column, byte, file, uri
pass
#parser.parse_into_model(model, self.url, handler=error_handler)
#response = urllib2.urlopen(self.url)
#the_page = response.read()
the_page = soup.prettify()
# print the_page
parser.parse_string_into_model(model,
the_page,
self.url,
handler=error_handler)
print "Printing all statements"
for s in model.as_stream():
print "Statement:", s
q = RDF.Query("SELECT ?p ?o WHERE (<" + self.url + "> ?p ?o)")
print "Querying for page meta-data"
for result in q.execute(model):
print "{"
for k in result:
print " " + k + " = " + str(result[k])
print "}"
import RDF
storage = RDF.Storage(storage_name="memory", name="test", options_string="")
# options_string="new='yes',hash-type='bdb',dir='.'")
if storage is None:
raise "new RDF.storage failed"
model = RDF.Model(storage)
if model is None:
raise "new RDF.model failed"
statement = RDF.Statement(RDF.Uri("http://www.dajobe.org/"),
RDF.Uri("http://purl.org/dc/elements/1.1/creator"),
RDF.Node("Dave Beckett"))
if statement is None:
raise "new RDF.statement failed"
model.add_statement(statement)
print("printing all model statements")
# Match against an empty statement - find everything
statement = RDF.Statement(subject=None, predicate=None, object=None)
for s in model.find_statements(statement):
print(" found statement: %s" % (s, ))
# Use any rdf/xml parser that is available
parser = RDF.Parser(name="rdfxml", mime_type="application/rdf+xml")
if parser is None:
raise "Could not find any rdf/xml parser"
#!/usr/bin/env python
import RDF, os, sys, re
from catalog.config import config
from catalog.store import MainStore
for name in ['public', 'works']:
storage_type, options = MainStore.get_store_options(name)
# workaround: sqlite doesn't support 'dir' so prepend directory to the name
if storage_type == 'sqlite':
name = os.path.abspath(os.path.join(config.CATALOG_DATA_DIR, name))
sys.stdout.write(
'Creating {type} store: {name}\nUsing: {options}\n'.format(
type=storage_type,
name=name,
options=re.sub("password='******']+'", "password='******'", options)))
options = options + "new='true'"
store = RDF.Storage(storage_name=storage_type,
name=name,
options_string=options)
model = RDF.Model(store)
model.sync()
# 1. GNU Lesser General Public License (LGPL) V2.1 or any newer version
# 2. GNU General Public License (GPL) V2 or any newer version
# 3. Apache License, V2.0 or any newer version
#
# You may not use this file except in compliance with at least one of
# the above three licenses.
#
# See LICENSE.html or LICENSE.txt at the top of this package for the
# full license terms.
#
#
import RDF
storage = RDF.Storage(storage_name="hashes",
name="test",
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
raise Exception("new RDF.Storage failed")
#RDF.debug(1)
model = RDF.Model(storage)
if model is None:
raise Exception("new RDF.model failed")
statement = RDF.Statement(RDF.Uri("http://www.dajobe.org/"),
RDF.Uri("http://purl.org/dc/elements/1.1/creator"),
RDF.Node("Dave Beckett"))
if statement is None:
raise Exception("new RDF.Statement failed")
def procfile(rdf_file):
global rdf_format
if rdf_format is None:
head, tail = ntpath.splitext(rdf_file)
if tail.lower() == '.rdf' or tail.lower() == '.owl':
rdf_format = 'rdfxml'
elif tail.lower() == '.nt':
rdf_format = 'ntriples'
elif tail.lower() == '.ttl':
rdf_format = 'turtle'
elif tail.lower() == '.rss':
rdf_format = 'rss-tag-soup'
else:
rdf_format = 'unknown'
if debug:
RDF.debug(1)
try:
storage = RDF.Storage(
storage_name='hashes',
name='rdf_storage',
options_string="new='yes',hash-type='memory',dir='.'")
if storage is None:
sys.exit('RDF.Storage creation failed>')
model = RDF.Model(storage)
if model is None:
sys.exit('RDF.Model creation failed>')
sys.stderr.write('Parsing file {} as {}\n'.format(
rdf_file, rdf_format))
uri = RDF.Uri(string='file:' + rdf_file)
parser = RDF.Parser(rdf_format)
if parser is None:
sys.exit('RDF.Parser({}) creation failed>'.format(rdf_format))
except:
exc = traceback.format_exception_only(sys.exc_info()[0],
sys.exc_info()[1])
lineno = sys.exc_info()[-1].tb_lineno
errmsg = 'RDF creation(%d): %s' % (lineno, clean(exc[-1]))
sys.exit(errmsg)
try:
count = 0
for s in parser.parse_as_stream(uri):
if debug:
sys.stderr.write(
'---------------------------------------- {}\n'.format(
count + 1))
sys.stderr.write(str(s.subject) + '\n')
sys.stderr.write(str(s.predicate) + '\n')
sys.stderr.write(str(s.object) + '\n')
model.add_statement(s)
count += 1
if count >= max_count:
break
if debug:
sys.stderr.write('----------------------------------------\n')
except:
exc = traceback.format_exception_only(sys.exc_info()[0],
sys.exc_info()[1])
lineno = sys.exc_info()[-1].tb_lineno
errmsg = 'Parse stream(%d): %s' % (lineno, clean(exc[-1]))
if re.search(r'RedlandError:..Adding statement failed',
errmsg,
flags=re.IGNORECASE):
sys.stderr.write(errmsg + '\n')
else:
sys.exit(errmsg)
sys.stderr.write('Found {} statements\n'.format(count))
ns = parser.namespaces_seen() # {str: URI}
for n in defns:
if n not in ns:
ns[n] = defns[n]
for n in ns:
ns[n] = str(ns[n])
if debug:
sys.stderr.write('@prefix {}: <{}> .\n'.format(n, ns[n]))
print('@startuml')
# print('scale max 100000')
# Populate objects with their attributes
objects = {}
alias = {}
for s in model:
# Subject
subj, subj_lit = get_subject(s, ns)
ox = subj
if ox not in objects:
objects[ox] = []
alias[ox] = simplify(ox)
# Predicate
pred, pred_lit = get_predicate(s, ns)
# Object
obj, obj_lit = get_object(s, ns)
if pred != 'a' and not obj_lit:
ox = obj
if ox not in objects:
objects[ox] = []
alias[ox] = simplify(ox)
# Add attributes
attrs = objects[ox]
if pred == 'a':
val = '{} {}'.format(pred, obj)
if val not in attrs:
attrs.append(val)
elif obj_lit:
obj_lang = s.object.literal[1]
obj_type = s.object.literal[2]
if sel_lang is not None:
if obj_lang is None or obj_lang == sel_lang:
val = '{} "{}"'.format(pred, obj)
if add_lit_type and obj_type is not None:
val += ' [{}]'.format(replns(obj_type, ns))
if val not in attrs:
attrs.append(val)
else:
val = '{} "{}"'.format(pred, obj)
if add_lit_type and obj_type is not None:
val += ' [{}]'.format(replns(obj_type, ns))
if val not in attrs:
attrs.append(val)
objects[ox] = attrs
# Print objects and their aliases at the beginning
for o in objects:
# Print object's alias if needed
if o != alias[o]:
if o.startswith('_:'): # blank node
sys.stdout.write('object " " as {}\n'.format(alias[o]))
else:
sys.stdout.write('object "{}" as {}\n'.format(o, alias[o]))
# Print object's attributes if any
if len(objects[o]):
sys.stdout.write('object {} {{\n'.format(alias[o]))
attrs = objects[o]
for a in attrs:
sys.stdout.write(' {}\n'.format(a))
sys.stdout.write('}\n')
# Print connections between objects
for s in model:
subj, subj_lit = get_subject(s, ns)
pred, pred_lit = get_predicate(s, ns)
obj, obj_lit = get_object(s, ns)
if pred == 'a':
continue
elif obj_lit:
continue
sys.stdout.write('{}'.format(alias[subj]))
sys.stdout.write(' --> ')
sys.stdout.write('{} : {}'.format(alias[obj], pred))
sys.stdout.write('\n')
print('@enduml')
return 0
# adapted from http://blog.literarymachine.net/?p=5
import RDF
# Create a new MySQL storage. The second parameter is NOT the
# name of the MySQL database to use, but the name of the
# triplestore. This makes it possible to create several
# triplestores within one database. The third parameter is
# a string containing the options for the actual MySQL database.
# They should speak for themselves, except for "new='yes'". If
# this option is given, the necessary table structure is created and
# any existing triples are dropped. You probably only want to use
# it in some kind of setup or installation procedure.
storage = RDF.Storage(storage_name="mysql", name="www.dajobe.org",
options_string="new='yes',host='localhost',database='tests',user='******',password='******'");
if storage is None:
raise Exception("new RDF.Storage failed")
#RDF.debug(1)
model=RDF.Model(storage)
if model is None:
raise Exception("new RDF.model failed")
test_file='../data/dc.rdf'
print "Parsing URI (file)", test_file
uri=RDF.Uri(string="file:"+test_file)
parser=RDF.Parser('raptor')
