def model_from_uri(uri=None, **opts): if RDF is None: raise ConnectionError( 'Redland support is not available, install from librdf.org') options = [] for key, val in opts.items(): options.append("%s='%s'" % (key, val)) options = ','.join(options) if uri == None or uri == 'memory': options += ",hash-type='memory'" store = RDF.Storage(storage_name='hashes', name='memory_store', options_string=options) elif uri.startswith('sqlite://'): file = uri[9:] try: store = RDF.Storage(storage_name='sqlite', name=file, options_string=options) except RDF.RedlandError: raise ConnectionError("Can't connect to %s" % uri) elif uri.startswith('bdb://'): file = uri[6:] if not os.path.isfile(file+'-sp2o.db') and not 'new=' in options: options += ",new='yes'" options += ",hash-type='bdb'" store = RDF.Storage(storage_name='hashes', name=file, options_string=options) elif uri.startswith('mysql://'): uri = uri[8:] userpass, hostdb = uri.split('@') user, pwd = userpass.split(':') host, db = hostdb.split('/') if ':' in host: host, port = host.split(':') else: port = '3306' if '#' in db: db, name = db.split('#') else: name = 'main' options += (",host='%s',port='%s',database='%s',user='******'," "password='******'" % (host, port, db, user, pwd)) try: store = RDF.Storage(storage_name='mysql', name=name, options_string=options) except RDF.RedlandError: raise ConnectionError("Can't connect to %s" % uri) else: raise ConnectionError('Unknown dburi: %s' % uri) return RDF.Model(store)
def __init__(self, name): storage_type, options = self.get_store_options(name) # workaround: sqlite doesn't support 'dir' so prepend directory to the name if storage_type == 'sqlite': name = os.path.abspath(os.path.join(config.CATALOG_DATA_DIR, name)) self._store = RDF.Storage(storage_name=storage_type, name=name, options_string=options) self._model = RDF.Model(self._store)
def new_storage(): '''Create a new in-memory Redland store for storing the RDF model.''' storage = RDF.Storage(storage_name="hashes", name="test", options_string="new='yes',hash-type='memory',dir='.'") if storage is None: raise CDAOError("Creation of new RDF.Storage failed.") return storage
def createModel(): storage = RDF.Storage( storage_name="hashes", name="geolink", options_string="new='yes',hash-type='memory',dir='.'") if storage is None: raise Exception("new RDF.Storage failed") model = RDF.Model(storage) if model is None: raise Exception("new RDF.model failed") return model
def export_volume(self, volume_urn, fd): """ Serialize a suitable properties file for the volume_urn. We include all the objects which are contained in the volume. """ storage = RDF.Storage(storage_name='hashes', name='X', options_string="new='yes',hash-type='memory',dir='.'") model = RDF.Model(storage) for urn in aff4.oracle.resolve_list(volume_urn, AFF4_CONTAINS): try: urn_id = self.get_id_by_urn(urn) except ValueError: continue self.export_model(urn, model) self.export_model(volume_urn, model) serializer = RDF.Serializer("turtle") fd.write(serializer.serialize_model_to_string(model))
def __init__(self, namespaces={}): # Initialize variables name = "db-{0!s}".format(random.randint(1, 999999)) self.db = RDF.Storage(storage_name="hashes", name=name, options_string="hash-type='bdb'") self.model = RDF.Model(self.db) self.serializer = RDF.Serializer(name="turtle") self.ns = {} # Define basic namespaces basicNamespaces = { "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "xsd": "http://www.w3.org/2001/XMLSchema#", } # Extend basic namespaces with those provided in parameters basicNamespaces.update(namespaces) self.addNamespaces(basicNamespaces) atexit.register(self.clearTempFiles)
def init_model(*filenames): """Input: An on-disk path (filenames) to start from. Output: A model with those suckers parsed.""" for filename in filenames: # filenames, not URIs die_unless( ':/' not in filename, "You passed in something that " + "looks like a URI; blowing up") storage = RDF.Storage( storage_name="hashes", name="test", options_string="new='yes',hash-type='memory',dir='.'") if storage is None: raise "new RDF.Storage failed" model = RDF.Model(storage) if model is None: raise "new RDF.Model failed" parser = RDF.Parser('raptor') for filename in filenames: filename_uri = RDF.Uri(string="file:" + filename) parser.parse_into_model(model, filename_uri) return model
def check_version(self): import copy import RDF if self.software == 'FusionForge': response = urllib2.urlopen(self.url) the_page = response.read() # hack to suppress news, which tend to mess with the validity of the document # <div class="one-news ... <!-- class="one-news" --> the_page = the_page.replace("\n", '') myMassage = [ (re.compile('<div class="one-news.*<!-- class="one-news" -->'), lambda match: '') ] myNewMassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE) myNewMassage.extend(myMassage) soup = BeautifulSoup(the_page, markupMassage=myNewMassage) version = None divs = soup.findAll('div', id='ft') for div in divs: text = div.text running_version = re.compile( 'This site is running FusionForge version (.*)') m = running_version.match(text) version = m.group(1) break if version: self.version = version # The following will apply only for FF > 5.1 which includes RDFa myMassage = [( re.compile('(<body[^>]*>)(.*)(<div id="ft")'), lambda match: match.group(1) + match.group(3) + ' xmlns:planetforge="http://coclico-project.org/ontology/planetforge#"' )] myNewMassage = copy.copy(BeautifulSoup.MARKUP_MASSAGE) myNewMassage.extend(myMassage) soup = BeautifulSoup(the_page, markupMassage=myNewMassage) storage = RDF.Storage( storage_name="hashes", name="test", options_string="new='yes',hash-type='memory',dir='.'") if storage is None: raise "new RDF.Storage failed" # RDF.debug(1) model = RDF.Model(storage) if model is None: raise "new RDF.model failed" #parser=RDF.Parser(name='raptor') # parser=RDF.Parser(name='rdfa', mime_type='application/xhtml+xml') parser = RDF.Parser(name='rdfa') if parser is None: raise "Failed to create RDF.Parser raptor" def error_handler(code, level, facility, message, line, column, byte, file, uri): print 'error_handler', code, level, facility, message, line, column, byte, file, uri pass #parser.parse_into_model(model, self.url, handler=error_handler) #response = urllib2.urlopen(self.url) #the_page = response.read() the_page = soup.prettify() # print the_page parser.parse_string_into_model(model, the_page, self.url, handler=error_handler) print "Printing all statements" for s in model.as_stream(): print "Statement:", s q = RDF.Query("SELECT ?p ?o WHERE (<" + self.url + "> ?p ?o)") print "Querying for page meta-data" for result in q.execute(model): print "{" for k in result: print " " + k + " = " + str(result[k]) print "}"
import RDF storage = RDF.Storage(storage_name="memory", name="test", options_string="") # options_string="new='yes',hash-type='bdb',dir='.'") if storage is None: raise "new RDF.storage failed" model = RDF.Model(storage) if model is None: raise "new RDF.model failed" statement = RDF.Statement(RDF.Uri("http://www.dajobe.org/"), RDF.Uri("http://purl.org/dc/elements/1.1/creator"), RDF.Node("Dave Beckett")) if statement is None: raise "new RDF.statement failed" model.add_statement(statement) print("printing all model statements") # Match against an empty statement - find everything statement = RDF.Statement(subject=None, predicate=None, object=None) for s in model.find_statements(statement): print(" found statement: %s" % (s, )) # Use any rdf/xml parser that is available parser = RDF.Parser(name="rdfxml", mime_type="application/rdf+xml") if parser is None: raise "Could not find any rdf/xml parser"
#!/usr/bin/env python import RDF, os, sys, re from catalog.config import config from catalog.store import MainStore for name in ['public', 'works']: storage_type, options = MainStore.get_store_options(name) # workaround: sqlite doesn't support 'dir' so prepend directory to the name if storage_type == 'sqlite': name = os.path.abspath(os.path.join(config.CATALOG_DATA_DIR, name)) sys.stdout.write( 'Creating {type} store: {name}\nUsing: {options}\n'.format( type=storage_type, name=name, options=re.sub("password='******']+'", "password='******'", options))) options = options + "new='true'" store = RDF.Storage(storage_name=storage_type, name=name, options_string=options) model = RDF.Model(store) model.sync()
# 1. GNU Lesser General Public License (LGPL) V2.1 or any newer version # 2. GNU General Public License (GPL) V2 or any newer version # 3. Apache License, V2.0 or any newer version # # You may not use this file except in compliance with at least one of # the above three licenses. # # See LICENSE.html or LICENSE.txt at the top of this package for the # full license terms. # # import RDF storage = RDF.Storage(storage_name="hashes", name="test", options_string="new='yes',hash-type='memory',dir='.'") if storage is None: raise Exception("new RDF.Storage failed") #RDF.debug(1) model = RDF.Model(storage) if model is None: raise Exception("new RDF.model failed") statement = RDF.Statement(RDF.Uri("http://www.dajobe.org/"), RDF.Uri("http://purl.org/dc/elements/1.1/creator"), RDF.Node("Dave Beckett")) if statement is None: raise Exception("new RDF.Statement failed")
def procfile(rdf_file): global rdf_format if rdf_format is None: head, tail = ntpath.splitext(rdf_file) if tail.lower() == '.rdf' or tail.lower() == '.owl': rdf_format = 'rdfxml' elif tail.lower() == '.nt': rdf_format = 'ntriples' elif tail.lower() == '.ttl': rdf_format = 'turtle' elif tail.lower() == '.rss': rdf_format = 'rss-tag-soup' else: rdf_format = 'unknown' if debug: RDF.debug(1) try: storage = RDF.Storage( storage_name='hashes', name='rdf_storage', options_string="new='yes',hash-type='memory',dir='.'") if storage is None: sys.exit('RDF.Storage creation failed>') model = RDF.Model(storage) if model is None: sys.exit('RDF.Model creation failed>') sys.stderr.write('Parsing file {} as {}\n'.format( rdf_file, rdf_format)) uri = RDF.Uri(string='file:' + rdf_file) parser = RDF.Parser(rdf_format) if parser is None: sys.exit('RDF.Parser({}) creation failed>'.format(rdf_format)) except: exc = traceback.format_exception_only(sys.exc_info()[0], sys.exc_info()[1]) lineno = sys.exc_info()[-1].tb_lineno errmsg = 'RDF creation(%d): %s' % (lineno, clean(exc[-1])) sys.exit(errmsg) try: count = 0 for s in parser.parse_as_stream(uri): if debug: sys.stderr.write( '---------------------------------------- {}\n'.format( count + 1)) sys.stderr.write(str(s.subject) + '\n') sys.stderr.write(str(s.predicate) + '\n') sys.stderr.write(str(s.object) + '\n') model.add_statement(s) count += 1 if count >= max_count: break if debug: sys.stderr.write('----------------------------------------\n') except: exc = traceback.format_exception_only(sys.exc_info()[0], sys.exc_info()[1]) lineno = sys.exc_info()[-1].tb_lineno errmsg = 'Parse stream(%d): %s' % (lineno, clean(exc[-1])) if re.search(r'RedlandError:..Adding statement failed', errmsg, flags=re.IGNORECASE): sys.stderr.write(errmsg + '\n') else: sys.exit(errmsg) sys.stderr.write('Found {} statements\n'.format(count)) ns = parser.namespaces_seen() # {str: URI} for n in defns: if n not in ns: ns[n] = defns[n] for n in ns: ns[n] = str(ns[n]) if debug: sys.stderr.write('@prefix {}: <{}> .\n'.format(n, ns[n])) print('@startuml') # print('scale max 100000') # Populate objects with their attributes objects = {} alias = {} for s in model: # Subject subj, subj_lit = get_subject(s, ns) ox = subj if ox not in objects: objects[ox] = [] alias[ox] = simplify(ox) # Predicate pred, pred_lit = get_predicate(s, ns) # Object obj, obj_lit = get_object(s, ns) if pred != 'a' and not obj_lit: ox = obj if ox not in objects: objects[ox] = [] alias[ox] = simplify(ox) # Add attributes attrs = objects[ox] if pred == 'a': val = '{} {}'.format(pred, obj) if val not in attrs: attrs.append(val) elif obj_lit: obj_lang = s.object.literal[1] obj_type = s.object.literal[2] if sel_lang is not None: if obj_lang is None or obj_lang == sel_lang: val = '{} "{}"'.format(pred, obj) if add_lit_type and obj_type is not None: val += ' [{}]'.format(replns(obj_type, ns)) if val not in attrs: attrs.append(val) else: val = '{} "{}"'.format(pred, obj) if add_lit_type and obj_type is not None: val += ' [{}]'.format(replns(obj_type, ns)) if val not in attrs: attrs.append(val) objects[ox] = attrs # Print objects and their aliases at the beginning for o in objects: # Print object's alias if needed if o != alias[o]: if o.startswith('_:'): # blank node sys.stdout.write('object " " as {}\n'.format(alias[o])) else: sys.stdout.write('object "{}" as {}\n'.format(o, alias[o])) # Print object's attributes if any if len(objects[o]): sys.stdout.write('object {} {{\n'.format(alias[o])) attrs = objects[o] for a in attrs: sys.stdout.write(' {}\n'.format(a)) sys.stdout.write('}\n') # Print connections between objects for s in model: subj, subj_lit = get_subject(s, ns) pred, pred_lit = get_predicate(s, ns) obj, obj_lit = get_object(s, ns) if pred == 'a': continue elif obj_lit: continue sys.stdout.write('{}'.format(alias[subj])) sys.stdout.write(' --> ') sys.stdout.write('{} : {}'.format(alias[obj], pred)) sys.stdout.write('\n') print('@enduml') return 0
# adapted from http://blog.literarymachine.net/?p=5 import RDF # Create a new MySQL storage. The second parameter is NOT the # name of the MySQL database to use, but the name of the # triplestore. This makes it possible to create several # triplestores within one database. The third parameter is # a string containing the options for the actual MySQL database. # They should speak for themselves, except for "new='yes'". If # this option is given, the necessary table structure is created and # any existing triples are dropped. You probably only want to use # it in some kind of setup or installation procedure. storage = RDF.Storage(storage_name="mysql", name="www.dajobe.org", options_string="new='yes',host='localhost',database='tests',user='******',password='******'"); if storage is None: raise Exception("new RDF.Storage failed") #RDF.debug(1) model=RDF.Model(storage) if model is None: raise Exception("new RDF.model failed") test_file='../data/dc.rdf' print "Parsing URI (file)", test_file uri=RDF.Uri(string="file:"+test_file) parser=RDF.Parser('raptor')