def initialize(config_file):
print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S",
localtime())
sys.stdout.flush()
config = __import__(config_file)
try:
g = ConjunctiveGraph(config.graph_store, config.graph_identifier)
g.open(config.db_configstring, create=True)
if config.input_file != None:
print '[%s] Parsing %s...' % (strftime(
"%a, %d %b %Y %H:%M:%S", localtime()), config.input_file)
sys.stdout.flush()
g.parse(config.input_file, format=config.input_format)
g.commit()
else:
dir_list = os.listdir(config.input_dir)
for file_name in dir_list:
print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S",
localtime()), file_name)
sys.stdout.flush()
g.parse(config.input_dir + '/' + file_name,
format=config.input_format)
g.commit()
except Exception as e:
traceback.print_exc()
print e
print '"%s" not found, or incorrect RDF serialization.' % config.input_file
sys.stdout.flush()
exit(-1)
return g, config
def initialize(config_file):
print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime())
sys.stdout.flush()
config = __import__(config_file)
try:
g = ConjunctiveGraph(config.graph_store, config.graph_identifier)
g.open(config.db_configstring, create=True)
if config.input_file != None:
print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), config.input_file)
sys.stdout.flush()
g.parse(config.input_file, format=config.input_format)
g.commit()
else:
dir_list = os.listdir(config.input_dir)
for file_name in dir_list:
print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()) ,file_name)
sys.stdout.flush()
g.parse(config.input_dir + '/' + file_name, format=config.input_format)
g.commit()
except Exception as e:
traceback.print_exc()
print e
print '"%s" not found, or incorrect RDF serialization.' % config.input_file
sys.stdout.flush()
exit(-1)
return g, config
def example_1():
"""Creates a ConjunctiveGraph and performs some BerkeleyDB tasks with it
"""
path = mktemp()
# Declare we are using a BerkeleyDB Store
graph = ConjunctiveGraph("BerkeleyDB")
# Open previously created store, or create it if it doesn't exist yet
# (always doesn't exist in this example as using temp file location)
rt = graph.open(path, create=False)
if rt == NO_STORE:
# There is no underlying BerkeleyDB infrastructure, so create it
print("Creating new DB")
graph.open(path, create=True)
else:
print("Using existing DB")
assert rt == VALID_STORE, "The underlying store is corrupt"
print("Triples in graph before add:", len(graph))
print("(will always be 0 when using temp file for DB)")
# Now we'll add some triples to the graph & commit the changes
EG = Namespace("http://example.net/test/")
graph.bind("eg", EG)
graph.add((EG["pic:1"], EG.name, Literal("Jane & Bob")))
graph.add((EG["pic:2"], EG.name, Literal("Squirrel in Tree")))
graph.commit()
print("Triples in graph after add:", len(graph))
print("(should be 2)")
# display the graph in Turtle
print(graph.serialize())
# close when done, otherwise BerkeleyDB will leak lock entries.
graph.close()
graph = None
# reopen the graph
graph = ConjunctiveGraph("BerkeleyDB")
graph.open(path, create=False)
print("Triples still in graph:", len(graph))
print("(should still be 2)")
graph.close()
# Clean up the temp folder to remove the BerkeleyDB database files...
for f in os.listdir(path):
os.unlink(path + "/" + f)
os.rmdir(path)
def ConvertToSQLLITE (filename,destinationFileName):
_graph = ConjunctiveGraph()
_graph.parse(filename, format="nt")
sql = ConjunctiveGraph('SQLite')
sql.open(destinationFileName, create=True)
for t in _graph.triples((None,None,None)):
sql.add(t)
sql.commit()
sql.close()
class DeepGraphStore():
store_name = 'SQLite'
def __init__(self, create=False, parse=None):
self.parse = parse
self.create = create
self.graph = None
def setUp(self):
self.path = "" + random_file_generating()
self.graph = Graph(store=self.store_name)
self.graph.open(self.path, create=self.create)
if self.create:
if not self.parse:
self.graph.parse("http://njh.me/foaf.rdf", format='xml')
else:
self.graph.parse(self.parse)
self.graph.commit()
def open(self, path):
self.graph = ConjunctiveGraph(self.store_name)
self.path = path
self.graph.open(self.path, create=False)
def query(self, sparql_query):
return self.graph.query(sparql_query)
def parse(self, path_to_file_):
self.graph.parse(path_to_file_)
def load(self, triples):
self.graph.load(triples)
def close(self):
self.graph.close()
def size(self):
size = self.graph.__len__()
size = len(self.graph)
# self.close()
return size
class DeepGraphStore():
store_name = 'SQLite'
def __init__(self, create=False, parse=None):
self.parse = parse
self.create = create
self.graph = None
def setUp(self):
self.path = "" + random_file_generating()
self.graph = Graph(store=self.store_name)
self.graph.open(self.path, create=self.create)
if self.create:
if not self.parse:
self.graph.parse("http://njh.me/foaf.rdf", format='xml')
else:
self.graph.parse(self.parse)
self.graph.commit()
def open(self, path):
self.graph = ConjunctiveGraph(self.store_name)
self.path = path
self.graph.open(self.path, create=False)
def query(self, sparql_query):
return self.graph.query(sparql_query)
def parse(self, path_to_file_):
self.graph.parse(path_to_file_)
def load(self, triples):
self.graph.load(triples)
def close(self):
self.graph.close()
def size(self):
size = self.graph.__len__()
size = len(self.graph)
# self.close()
return size
class Command(BaseCommand):
args = "<path_to_skos_file path_to_skos_file>..."
help = "import skos ref in rdflib alchemy store"
def __init__(self):
super(Command, self).__init__()
self.ident = "jocondelab"
#'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
#'NAME': '', # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
#'USER': '',
#'PASSWORD': '',
#'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
#'PORT': ''
db_settings = connections['default'].settings_dict
sa_db_settings = {
'engine': 'postgresql+psycopg2' if db_settings['ENGINE'] == "django.db.backends.postgresql_psycopg2" else db_settings['ENGINE'],
'user': db_settings['USER'],
'password': db_settings['PASSWORD'],
'port': db_settings['PORT'] if db_settings['PORT'] else "5432",
'host': db_settings['HOST'] if db_settings['HOST'] else "localhost",
'name': db_settings['NAME']
}
self.connect_config = "%(engine)s://%(user)s:%(password)s@%(host)s:%(port)s/%(name)s"%sa_db_settings
self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident)
self.graph = ConjunctiveGraph(self.store, identifier=self.ident)
self.graph.open(self.connect_config, create=True)
def handle(self, *args, **options):
#import pydevd #@UnresolvedImport
#pydevd.settrace(suspend=True)
for skos_path, public_id in zip(args[::2],args[1::2]):
filepath = os.path.abspath(skos_path)
self.stdout.write("Importing %s" % filepath)
self.graph.parse(filepath, publicID=public_id, format='xml')
self.stdout.write("graph size %d" % len(self.graph))
self.graph.commit()
self.graph.close()
self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident)
self.graph = ConjunctiveGraph(self.store, identifier=self.ident)
self.graph.open(self.connect_config, create=False)
self.stdout.write("correct alt labels")
litteral_statements = self.store.tables['literal_statements']
with self.store.engine.connect() as connection:
q = litteral_statements.select().where(litteral_statements.c.predicate == "http://www.w3.org/2004/02/skos/core#altLabel")
for row in connection.execute(q):
if row['object'] and row['object'] != row['object'].strip():
u_q = litteral_statements.update().where(and_(
litteral_statements.c.subject == row['subject'],
litteral_statements.c.predicate == row['predicate'],
litteral_statements.c.object == row['object'],
litteral_statements.c.context == row['context'],
litteral_statements.c.termComb == row['termcomb'],
litteral_statements.c.objLanguage == row['objlanguage'],
litteral_statements.c.objDatatype == row['objdatatype']
)).values(object = row['object'].strip() )
#u_q_compiled = u_q.compile()
#self.stdout.write("UPDATE QUERY for %s : %s : %s - %s" % (row['subject'], row['object'], str(u_q_compiled), repr(u_q_compiled.params)))
connection.execute(u_q)
self.stdout.write("graph size %d" % len(self.graph))
self.stdout.write("graph contexts %s" % repr([g for g in self.graph.contexts()]))
class ManifestHelper(object):
def __init__(self, uri=None):
self.uri = None
if uri:
self.uri = uri
self.reset()
def reset(self):
self.g = None
if self.uri:
self.g = ConjunctiveGraph(identifier=self.uri)
else:
self.g = ConjunctiveGraph()
self.namespaces = {}
self.urihelper = URIHelper(self.namespaces)
#add defaults
for prefix, ns in NAMESPACES.iteritems():
self.add_namespace(prefix, ns)
def from_string(self, textfile, format="xml", encoding="utf-8"):
self.reset()
self.g.parse(textfile, format)
return
def triple_exists(self, s, p, o):
if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']:
return False
if s == '*':
s = None
if p == '*':
p = None
if o == '*':
o = None
if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None:
s = self.urihelper.get_uriref(s)
if not isinstance(p, URIRef) and not p == None:
p = self.urihelper.parse_uri(p)
if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode) and not o == None:
if not isinstance(o, basestring):
o = unicode(o)
o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True)
count = 0
for ans_s, ans_p, ans_o in self.g.triples((s, p, o)):
count += 1
if count > 0:
return True
else:
return False
def list_objects(self, s, p):
objects = []
if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']:
return objects
if s == '*':
s = None
if p == '*':
p = None
if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None:
s = self.urihelper.get_uriref(s)
if not isinstance(p, URIRef) and not p == None:
p = self.urihelper.parse_uri(p)
for o in self.g.objects(s, p):
objects.append(o)
return objects
def add_triple(self, s, p, o):
if not isinstance(s, URIRef) and not isinstance(s, BNode):
s = self.urihelper.get_uriref(s)
if not isinstance(p, URIRef):
p = self.urihelper.parse_uri(p)
if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode):
if not isinstance(o, basestring):
o = unicode(o)
o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True)
self.g.add((s, p, o))
self.g.commit()
return
def add_namespace(self, prefix, uri):
if not isinstance (prefix, basestring):
raise TypeError('Add namespace: prefix is not of type string or unicode')
if not isinstance(uri, (URIRef, Namespace)):
if not isinstance(uri, basestring):
raise TypeError('Add namespace: namespace is not of type string or unicode')
if not isinstance(prefix, unicode):
prefix = unicode(prefix)
if isinstance(uri, basestring) and not isinstance(uri, unicode):
uri = unicode(uri)
self.namespaces[prefix] = self.urihelper.get_namespace(uri)
if prefix not in self.urihelper.namespaces:
self.urihelper.namespaces[prefix] = self.urihelper.get_namespace(uri)
self.g.bind(prefix, self.namespaces[prefix])
return
def del_namespace(self, prefix, ns):
if prefix in self.namespaces:
del self.namespaces[prefix]
return
def del_triple(self, s, p, o=None):
if not type(self.g).__name__ in ['ConjunctiveGraph', 'Graph']:
return
if s == '*':
s = None
if p == '*':
p = None
if o == '*':
o = None
if not isinstance(s, URIRef) and not isinstance(s, BNode) and not s == None:
s = self.urihelper.get_uriref(s)
if not isinstance(p, URIRef) and not p == None:
p = self.urihelper.parse_uri(p)
if not isinstance(o, URIRef) and not isinstance(o, Literal) and not isinstance(o, BNode) and not o == None:
if not isinstance(o, basestring):
o = unicode(o)
o = self.urihelper.parse_uri(o, return_Literal_not_Exception=True)
self.g.remove((s, p, o))
return
def get_graph(self):
return self.g
def to_string(self, format="xml"):
if type(self.g).__name__ in ['ConjunctiveGraph', 'Graph'] and len(self.g)>0:
self.g.commit()
ans_str = self.g.serialize(format=format, encoding="utf-8")+"\n"
return ans_str
else:
return u'<?xml version="1.0" encoding="UTF-8"?>\n'
'''
from rdflib import ConjunctiveGraph, plugin
from rdflib.store import Store, VALID_STORE
from rdflib import URIRef
from api import BASE_GRAPH_URI, _get_mysql_config_string
store = plugin.get('MySQL', Store)(identifier='rdfstore')
rt = store.open(_get_mysql_config_string(), create=True)
assert rt == VALID_STORE,"The underlying store is corrupted"
citg = ConjunctiveGraph(store, identifier=URIRef(BASE_GRAPH_URI))
'''
citg = ConjunctiveGraph('MySQL', identifier=URIRef(BASE_GRAPH_URI))
rt = citg.open(_get_mysql_config_string(), create=True)
assert rt == VALID_STORE,"The underlying store is corrupted"
'''
citg.commit()
citg.close()
store.close()
print "Successfully initialized database"
from config import DevelopmentConfig
from rdflib_sqlalchemy.store import SQLAlchemy
import os.path
import sys
registerplugins()
app = Flask(__name__)
app.config.from_object(DevelopmentConfig)
parser = argparse.ArgumentParser(description='Input File')
parser.add_argument("-f", dest="filename", required=True,
help="input file to be parsed", metavar="FILE")
args = parser.parse_args()
if not os.path.exists(args.filename):
print("Invalid file: exiting.")
sys.exit(-1)
DB_URI = app.config.get('DB_URI', None)
IDENTIFIER = app.config.get('IDENTIFIER', None)
uri = Literal(DB_URI)
store = SQLAlchemy(identifier=IDENTIFIER, configuration=uri)
graph = ConjunctiveGraph(store)
graph.parse(source=args.filename, format='text/turtle', publicID=IDENTIFIER)
graph.commit()
print("Created new database '{}'".format(app.config.get("POSTGRES_DB")))
class MemoryStore:
"""A class that combines and syncronieses n-quad files and an in-memory quad store.
This class contains information about all graphs, their corresponding URIs and
pathes in the file system. For every Graph (context of Quad-Store) exists a
FileReference object (n-quad) that enables versioning (with git) and persistence.
"""
def __init__(self):
"""Initialize a new MemoryStore instance."""
logger = logging.getLogger('quit.core.MemoryStore')
logger.debug('Create an instance of MemoryStore')
self.store = ConjunctiveGraph(identifier='default')
return
def getgraphuris(self):
"""Method to get all available named graphs.
Returns:
A list containing all graph uris found in store.
"""
graphs = []
for graph in self.store.contexts():
if isinstance(graph, BNode) or str(graph.identifier) == 'default':
pass
else:
graphs.append(graph.identifier)
return graphs
def getgraphcontent(self, graphuri):
"""Get the serialized content of a named graph.
Args:
graphuri: The URI of a named graph.
Returns:
content: A list of strings where each string is a quad.
"""
data = []
context = self.store.get_context(URIRef(graphuri))
triplestring = context.serialize(format='nt').decode('UTF-8')
# Since we have triples here, we transform them to quads by adding the graphuri
# TODO This might cause problems if ' .\n' will be part of a literal.
# Maybe a regex would be a better solution
triplestring = triplestring.replace(' .\n', ' <' + graphuri + '> .\n')
data = triplestring.splitlines()
data.remove('')
return data
def getstoreobject(self):
"""Get the conjunctive graph object.
Returns:
graph: A list of strings where each string is a quad.
"""
def graphexists(self, graphuri):
"""Ask if a named graph FileReference object for a named graph URI.
Args:
graphuri: A string containing the URI of a named graph
Returns:
True or False
"""
if self.store.get_context(URIRef(graphuri)) is None:
return False
else:
return True
def addfile(self, filename, serialization):
"""Add a file to the store.
Args:
filename: A String for the path to the file.
serialization: A String containg the RDF format
Raises:
ValueError if the given file can't be parsed as nquads.
"""
try:
self.store.parse(source=filename, format=serialization)
except Exception as e:
logger.debug(e)
logger.debug("Could not import file: {}. " +
"Make sure the file exists and contains data in {}".
format(filename, serialization))
def addquads(self, quads):
"""Add quads to the MemoryStore.
Args:
quads: Rdflib.quads that should be added to the MemoryStore.
"""
self.store.addN(quads)
self.store.commit()
def query(self, querystring):
"""Execute a SPARQL select query.
Args:
querystring: A string containing a SPARQL ask or select query.
Returns:
The SPARQL result set
"""
return self.store.query(querystring)
def update(self, querystring, versioning=True):
"""Execute a SPARQL update query and update the store.
This method executes a SPARQL update query and updates and commits all affected files.
Args:
querystring: A string containing a SPARQL upate query.
"""
# methods of rdflib ConjunciveGraph
if versioning:
actions = evalUpdate(self.store, querystring)
self.store.update(querystring)
return actions
else:
self.store.update(querystring)
return
return
def removequads(self, quads):
"""Remove quads from the MemoryStore.
Args:
quads: Rdflib.quads that should be removed to the MemoryStore.
"""
self.store.remove((quads))
self.store.commit()
return
def exit(self):
"""Execute actions on API shutdown."""
return
def export(self, rlist):
rset = set()
for rid in rlist:
rset.add(rid)
rtype = self.get_property(rid, RDF.type)
# Get resources (2-level for Collections)
if rtype == PIMO['Collection']:
for s, p, o in self.ask.get_triples(URIRef(rid)):
if type(o) == URIRef:
if o.startswith('vazaar'):
rset.add(o)
for s2, p2, o2 in self.ask.get_triples(URIRef(o)):
if type(o2) == URIRef:
if o2.startswith('vazaar'):
rset.add(o2)
else:
for s, p, o in self.ask.get_triples(URIRef(rid)):
if type(o) == URIRef:
if o.startswith('vazaar'):
rset.add(o)
# Add user info
user_owner_pim_uri = self.app.cfgmgr.get_value('PIMO', 'PIM')
user_owner_uri = self.ask.get_owner(user_owner_pim_uri)
user_owner_email_uri = self.ask.get_owner_email(user_owner_uri)
rset.add(user_owner_pim_uri)
rset.add(user_owner_uri)
rset.add(user_owner_email_uri)
# Get triples from selected resources
triples = []
for rid in rset:
triples += self.ask.get_triples(URIRef(rid))
# Build graph
g = ConjunctiveGraph()
#Bind namespaces to graph
for ns in NSBINDINGS:
g.bind(ns, NSBINDINGS[ns])
# Add triples from selected resources
for triple in triples:
g.add(triple)
g.commit()
# Build export environment
pid = self.ask.get_pim_id() # get PersonalInformationModel
backup_dir = LPATH['TMP']
now = datetime.now()
bdate = '%4d%02d%02d' % (now.year, now.month, now.day)
btime = '%02d%02d%02d' % (now.hour, now.minute, now.second)
temp_export_dir = LPATH['TMP'] + '/' + bdate + btime
relative_export_dir = bdate + btime
temp_metadata_dir = temp_export_dir + '/metadata'
temp_data_dir = temp_export_dir + '/data'
metadata_file = temp_metadata_dir + '/' + pid[9:] + '.rdf'
targz_file = backup_dir + '/' + bdate + btime + '.vzr'
command = 'cd %s; tar cfz %s %s' % (LPATH['TMP'], targz_file, relative_export_dir)
os.makedirs(temp_export_dir)
os.makedirs(temp_metadata_dir)
os.makedirs(temp_data_dir)
# Copy selected resources contents to temporal backup directory
for rid in rset:
rfile = rid[9:]
src = LPATH['RESOURCES'] + '/' + rfile
dst = temp_data_dir + '/' + rfile
try:
shutil.copy(src, dst)
except Exception, error:
pass #self.log.error('shutil: %s' % error)
