def initialize(config_file): print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime()) sys.stdout.flush() config = __import__(config_file) try: g = ConjunctiveGraph(config.graph_store, config.graph_identifier) g.open(config.db_configstring, create=True) if config.input_file != None: print '[%s] Parsing %s...' % (strftime( "%a, %d %b %Y %H:%M:%S", localtime()), config.input_file) sys.stdout.flush() g.parse(config.input_file, format=config.input_format) g.commit() else: dir_list = os.listdir(config.input_dir) for file_name in dir_list: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), file_name) sys.stdout.flush() g.parse(config.input_dir + '/' + file_name, format=config.input_format) g.commit() except Exception as e: traceback.print_exc() print e print '"%s" not found, or incorrect RDF serialization.' % config.input_file sys.stdout.flush() exit(-1) return g, config
def initialize(config_file): print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime()) sys.stdout.flush() config = __import__(config_file) try: g = ConjunctiveGraph(config.graph_store, config.graph_identifier) g.open(config.db_configstring, create=True) if config.input_file != None: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), config.input_file) sys.stdout.flush() g.parse(config.input_file, format=config.input_format) g.commit() else: dir_list = os.listdir(config.input_dir) for file_name in dir_list: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()) ,file_name) sys.stdout.flush() g.parse(config.input_dir + '/' + file_name, format=config.input_format) g.commit() except Exception as e: traceback.print_exc() print e print '"%s" not found, or incorrect RDF serialization.' % config.input_file sys.stdout.flush() exit(-1) return g, config
def example_1(): """Creates a ConjunctiveGraph and performs some BerkeleyDB tasks with it """ path = mktemp() # Declare we are using a BerkeleyDB Store graph = ConjunctiveGraph("BerkeleyDB") # Open previously created store, or create it if it doesn't exist yet # (always doesn't exist in this example as using temp file location) rt = graph.open(path, create=False) if rt == NO_STORE: # There is no underlying BerkeleyDB infrastructure, so create it print("Creating new DB") graph.open(path, create=True) else: print("Using existing DB") assert rt == VALID_STORE, "The underlying store is corrupt" print("Triples in graph before add:", len(graph)) print("(will always be 0 when using temp file for DB)") # Now we'll add some triples to the graph & commit the changes EG = Namespace("http://example.net/test/") graph.bind("eg", EG) graph.add((EG["pic:1"], EG.name, Literal("Jane & Bob"))) graph.add((EG["pic:2"], EG.name, Literal("Squirrel in Tree"))) graph.commit() print("Triples in graph after add:", len(graph)) print("(should be 2)") # display the graph in Turtle print(graph.serialize()) # close when done, otherwise BerkeleyDB will leak lock entries. graph.close() graph = None # reopen the graph graph = ConjunctiveGraph("BerkeleyDB") graph.open(path, create=False) print("Triples still in graph:", len(graph)) print("(should still be 2)") graph.close() # Clean up the temp folder to remove the BerkeleyDB database files... for f in os.listdir(path): os.unlink(path + "/" + f) os.rmdir(path)
def ConvertToSQLLITE (filename,destinationFileName): _graph = ConjunctiveGraph() _graph.parse(filename, format="nt") sql = ConjunctiveGraph('SQLite') sql.open(destinationFileName, create=True) for t in _graph.triples((None,None,None)): sql.add(t) sql.commit() sql.close()
class DeepGraphStore(): store_name = 'SQLite' def __init__(self, create=False, parse=None): self.parse = parse self.create = create self.graph = None def setUp(self): self.path = "" + random_file_generating() self.graph = Graph(store=self.store_name) self.graph.open(self.path, create=self.create) if self.create: if not self.parse: self.graph.parse("http://njh.me/foaf.rdf", format='xml') else: self.graph.parse(self.parse) self.graph.commit() def open(self, path): self.graph = ConjunctiveGraph(self.store_name) self.path = path self.graph.open(self.path, create=False) def query(self, sparql_query): return self.graph.query(sparql_query) def parse(self, path_to_file_): self.graph.parse(path_to_file_) def load(self, triples): self.graph.load(triples) def close(self): self.graph.close() def size(self): size = self.graph.__len__() size = len(self.graph) # self.close() return size
class Command(BaseCommand): args = "<path_to_skos_file path_to_skos_file>..." help = "import skos ref in rdflib alchemy store" def __init__(self): super(Command, self).__init__() self.ident = "jocondelab" #'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. #'NAME': '', # Or path to database file if using sqlite3. # The following settings are not used with sqlite3: #'USER': '', #'PASSWORD': '', #'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP. #'PORT': '' db_settings = connections['default'].settings_dict sa_db_settings = { 'engine': 'postgresql+psycopg2' if db_settings['ENGINE'] == "django.db.backends.postgresql_psycopg2" else db_settings['ENGINE'], 'user': db_settings['USER'], 'password': db_settings['PASSWORD'], 'port': db_settings['PORT'] if db_settings['PORT'] else "5432", 'host': db_settings['HOST'] if db_settings['HOST'] else "localhost", 'name': db_settings['NAME'] } self.connect_config = "%(engine)s://%(user)s:%(password)s@%(host)s:%(port)s/%(name)s"%sa_db_settings self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident) self.graph = ConjunctiveGraph(self.store, identifier=self.ident) self.graph.open(self.connect_config, create=True) def handle(self, *args, **options): #import pydevd #@UnresolvedImport #pydevd.settrace(suspend=True) for skos_path, public_id in zip(args[::2],args[1::2]): filepath = os.path.abspath(skos_path) self.stdout.write("Importing %s" % filepath) self.graph.parse(filepath, publicID=public_id, format='xml') self.stdout.write("graph size %d" % len(self.graph)) self.graph.commit() self.graph.close() self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident) self.graph = ConjunctiveGraph(self.store, identifier=self.ident) self.graph.open(self.connect_config, create=False) self.stdout.write("correct alt labels") litteral_statements = self.store.tables['literal_statements'] with self.store.engine.connect() as connection: q = litteral_statements.select().where(litteral_statements.c.predicate == "http://www.w3.org/2004/02/skos/core#altLabel") for row in connection.execute(q): if row['object'] and row['object'] != row['object'].strip(): u_q = litteral_statements.update().where(and_( litteral_statements.c.subject == row['subject'], litteral_statements.c.predicate == row['predicate'], litteral_statements.c.object == row['object'], litteral_statements.c.context == row['context'], litteral_statements.c.termComb == row['termcomb'], litteral_statements.c.objLanguage == row['objlanguage'], litteral_statements.c.objDatatype == row['objdatatype'] )).values(object = row['object'].strip() ) #u_q_compiled = u_q.compile() #self.stdout.write("UPDATE QUERY for %s : %s : %s - %s" % (row['subject'], row['object'], str(u_q_compiled), repr(u_q_compiled.params))) connection.execute(u_q) self.stdout.write("graph size %d" % len(self.graph)) self.stdout.write("graph contexts %s" % repr([g for g in self.graph.contexts()]))
class ManifestHelper(object): def __init__(self, uri=None): self.uri = None if uri: self.uri = uri self.reset() def reset(self): self.g = None if self.uri: self.g = ConjunctiveGraph(identifier=self.uri) else: self.g = ConjunctiveGraph() self.namespaces = {} self.urihelper = URIHelper(self.namespaces) #add defaults for prefix, ns in NAMESPACES.iteritems(): self.add_namespace(prefix, ns) def from_string(self, textfile, format="xml", encoding="utf-8"): self.reset() self.g.parse(textfile, format) return def triple_exists(self, s, p, o): if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']: return False if s == '*': s = None if p == '*': p = None if o == '*': o = None if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None: s = self.urihelper.get_uriref(s) if not isinstance(p, URIRef) and not p == None: p = self.urihelper.parse_uri(p) if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode) and not o == None: if not isinstance(o, basestring): o = unicode(o) o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True) count = 0 for ans_s, ans_p, ans_o in self.g.triples((s, p, o)): count += 1 if count > 0: return True else: return False def list_objects(self, s, p): objects = [] if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']: return objects if s == '*': s = None if p == '*': p = None if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None: s = self.urihelper.get_uriref(s) if not isinstance(p, URIRef) and not p == None: p = self.urihelper.parse_uri(p) for o in self.g.objects(s, p): objects.append(o) return objects def add_triple(self, s, p, o): if not isinstance(s, URIRef) and not isinstance(s, BNode): s = self.urihelper.get_uriref(s) if not isinstance(p, URIRef): p = self.urihelper.parse_uri(p) if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode): if not isinstance(o, basestring): o = unicode(o) o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True) self.g.add((s, p, o)) self.g.commit() return def add_namespace(self, prefix, uri): if not isinstance (prefix, basestring): raise TypeError('Add namespace: prefix is not of type string or unicode') if not isinstance(uri, (URIRef, Namespace)): if not isinstance(uri, basestring): raise TypeError('Add namespace: namespace is not of type string or unicode') if not isinstance(prefix, unicode): prefix = unicode(prefix) if isinstance(uri, basestring) and not isinstance(uri, unicode): uri = unicode(uri) self.namespaces[prefix] = self.urihelper.get_namespace(uri) if prefix not in self.urihelper.namespaces: self.urihelper.namespaces[prefix] = self.urihelper.get_namespace(uri) self.g.bind(prefix, self.namespaces[prefix]) return def del_namespace(self, prefix, ns): if prefix in self.namespaces: del self.namespaces[prefix] return def del_triple(self, s, p, o=None): if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']: return if s == '*': s = None if p == '*': p = None if o == '*': o = None if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None: s = self.urihelper.get_uriref(s) if not isinstance(p, URIRef) and not p == None: p = self.urihelper.parse_uri(p) if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode) and not o == None: if not isinstance(o, basestring): o = unicode(o) o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True) self.g.remove((s, p, o)) return def get_graph(self): return self.g def to_string(self, format="xml"): if type(self.g).__name__ in ['ConjunctiveGraph', 'Graph'] and len(self.g)>0: self.g.commit() ans_str = self.g.serialize(format=format, encoding="utf-8")+"\n" return ans_str else: return u'<?xml version="1.0" encoding="UTF-8"?>\n'
''' from rdflib import ConjunctiveGraph, plugin from rdflib.store import Store, VALID_STORE from rdflib import URIRef from api import BASE_GRAPH_URI, _get_mysql_config_string store = plugin.get('MySQL', Store)(identifier='rdfstore') rt = store.open(_get_mysql_config_string(), create=True) assert rt == VALID_STORE,"The underlying store is corrupted" citg = ConjunctiveGraph(store, identifier=URIRef(BASE_GRAPH_URI)) ''' citg = ConjunctiveGraph('MySQL', identifier=URIRef(BASE_GRAPH_URI)) rt = citg.open(_get_mysql_config_string(), create=True) assert rt == VALID_STORE,"The underlying store is corrupted" ''' citg.commit() citg.close() store.close() print "Successfully initialized database"
from config import DevelopmentConfig from rdflib_sqlalchemy.store import SQLAlchemy import os.path import sys registerplugins() app = Flask(__name__) app.config.from_object(DevelopmentConfig) parser = argparse.ArgumentParser(description='Input File') parser.add_argument("-f", dest="filename", required=True, help="input file to be parsed", metavar="FILE") args = parser.parse_args() if not os.path.exists(args.filename): print("Invalid file: exiting.") sys.exit(-1) DB_URI = app.config.get('DB_URI', None) IDENTIFIER = app.config.get('IDENTIFIER', None) uri = Literal(DB_URI) store = SQLAlchemy(identifier=IDENTIFIER, configuration=uri) graph = ConjunctiveGraph(store) graph.parse(source=args.filename, format='text/turtle', publicID=IDENTIFIER) graph.commit() print("Created new database '{}'".format(app.config.get("POSTGRES_DB")))
class MemoryStore: """A class that combines and syncronieses n-quad files and an in-memory quad store. This class contains information about all graphs, their corresponding URIs and pathes in the file system. For every Graph (context of Quad-Store) exists a FileReference object (n-quad) that enables versioning (with git) and persistence. """ def __init__(self): """Initialize a new MemoryStore instance.""" logger = logging.getLogger('quit.core.MemoryStore') logger.debug('Create an instance of MemoryStore') self.store = ConjunctiveGraph(identifier='default') return def getgraphuris(self): """Method to get all available named graphs. Returns: A list containing all graph uris found in store. """ graphs = [] for graph in self.store.contexts(): if isinstance(graph, BNode) or str(graph.identifier) == 'default': pass else: graphs.append(graph.identifier) return graphs def getgraphcontent(self, graphuri): """Get the serialized content of a named graph. Args: graphuri: The URI of a named graph. Returns: content: A list of strings where each string is a quad. """ data = [] context = self.store.get_context(URIRef(graphuri)) triplestring = context.serialize(format='nt').decode('UTF-8') # Since we have triples here, we transform them to quads by adding the graphuri # TODO This might cause problems if ' .\n' will be part of a literal. # Maybe a regex would be a better solution triplestring = triplestring.replace(' .\n', ' <' + graphuri + '> .\n') data = triplestring.splitlines() data.remove('') return data def getstoreobject(self): """Get the conjunctive graph object. Returns: graph: A list of strings where each string is a quad. """ def graphexists(self, graphuri): """Ask if a named graph FileReference object for a named graph URI. Args: graphuri: A string containing the URI of a named graph Returns: True or False """ if self.store.get_context(URIRef(graphuri)) is None: return False else: return True def addfile(self, filename, serialization): """Add a file to the store. Args: filename: A String for the path to the file. serialization: A String containg the RDF format Raises: ValueError if the given file can't be parsed as nquads. """ try: self.store.parse(source=filename, format=serialization) except Exception as e: logger.debug(e) logger.debug("Could not import file: {}. " + "Make sure the file exists and contains data in {}". format(filename, serialization)) def addquads(self, quads): """Add quads to the MemoryStore. Args: quads: Rdflib.quads that should be added to the MemoryStore. """ self.store.addN(quads) self.store.commit() def query(self, querystring): """Execute a SPARQL select query. Args: querystring: A string containing a SPARQL ask or select query. Returns: The SPARQL result set """ return self.store.query(querystring) def update(self, querystring, versioning=True): """Execute a SPARQL update query and update the store. This method executes a SPARQL update query and updates and commits all affected files. Args: querystring: A string containing a SPARQL upate query. """ # methods of rdflib ConjunciveGraph if versioning: actions = evalUpdate(self.store, querystring) self.store.update(querystring) return actions else: self.store.update(querystring) return return def removequads(self, quads): """Remove quads from the MemoryStore. Args: quads: Rdflib.quads that should be removed to the MemoryStore. """ self.store.remove((quads)) self.store.commit() return def exit(self): """Execute actions on API shutdown.""" return
def export(self, rlist): rset = set() for rid in rlist: rset.add(rid) rtype = self.get_property(rid, RDF.type) # Get resources (2-level for Collections) if rtype == PIMO['Collection']: for s, p, o in self.ask.get_triples(URIRef(rid)): if type(o) == URIRef: if o.startswith('vazaar'): rset.add(o) for s2, p2, o2 in self.ask.get_triples(URIRef(o)): if type(o2) == URIRef: if o2.startswith('vazaar'): rset.add(o2) else: for s, p, o in self.ask.get_triples(URIRef(rid)): if type(o) == URIRef: if o.startswith('vazaar'): rset.add(o) # Add user info user_owner_pim_uri = self.app.cfgmgr.get_value('PIMO', 'PIM') user_owner_uri = self.ask.get_owner(user_owner_pim_uri) user_owner_email_uri = self.ask.get_owner_email(user_owner_uri) rset.add(user_owner_pim_uri) rset.add(user_owner_uri) rset.add(user_owner_email_uri) # Get triples from selected resources triples = [] for rid in rset: triples += self.ask.get_triples(URIRef(rid)) # Build graph g = ConjunctiveGraph() #Bind namespaces to graph for ns in NSBINDINGS: g.bind(ns, NSBINDINGS[ns]) # Add triples from selected resources for triple in triples: g.add(triple) g.commit() # Build export environment pid = self.ask.get_pim_id() # get PersonalInformationModel backup_dir = LPATH['TMP'] now = datetime.now() bdate = '%4d%02d%02d' % (now.year, now.month, now.day) btime = '%02d%02d%02d' % (now.hour, now.minute, now.second) temp_export_dir = LPATH['TMP'] + '/' + bdate + btime relative_export_dir = bdate + btime temp_metadata_dir = temp_export_dir + '/metadata' temp_data_dir = temp_export_dir + '/data' metadata_file = temp_metadata_dir + '/' + pid[9:] + '.rdf' targz_file = backup_dir + '/' + bdate + btime + '.vzr' command = 'cd %s; tar cfz %s %s' % (LPATH['TMP'], targz_file, relative_export_dir) os.makedirs(temp_export_dir) os.makedirs(temp_metadata_dir) os.makedirs(temp_data_dir) # Copy selected resources contents to temporal backup directory for rid in rset: rfile = rid[9:] src = LPATH['RESOURCES'] + '/' + rfile dst = temp_data_dir + '/' + rfile try: shutil.copy(src, dst) except Exception, error: pass #self.log.error('shutil: %s' % error)