def validate_sparql_endpoint(form, field): try: g = ConjunctiveGraph('SPARQLStore') g.open(field.data) g.query('SELECT * WHERE { ?s ?p ?o } LIMIT 1') except: raise ValidationError('This is not a valid SPARQL endpoint.')
def __init__(self, endpoint): graph = ConjunctiveGraph('SPARQLStore') graph.open(endpoint) graph.namespace_manager = ns_mgr self.graph = graph self.default_graph = \ 'http://vitro.mannlib.cornell.edu/default/vitro-kb-2'
def initialize(config_file): print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime()) sys.stdout.flush() config = __import__(config_file) try: g = ConjunctiveGraph(config.graph_store, config.graph_identifier) g.open(config.db_configstring, create=True) if config.input_file != None: print '[%s] Parsing %s...' % (strftime( "%a, %d %b %Y %H:%M:%S", localtime()), config.input_file) sys.stdout.flush() g.parse(config.input_file, format=config.input_format) g.commit() else: dir_list = os.listdir(config.input_dir) for file_name in dir_list: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), file_name) sys.stdout.flush() g.parse(config.input_dir + '/' + file_name, format=config.input_format) g.commit() except Exception as e: traceback.print_exc() print e print '"%s" not found, or incorrect RDF serialization.' % config.input_file sys.stdout.flush() exit(-1) return g, config
def test_sqlalchemy_obj_language(): logger.info(f'Python version: {python_version}') logger.info(f'RDFLib version: {rdflib_version}') logger.info(f'RDFLib-SQLAlchemy version: {rdflib_sqlalchemy_version}') identifier = URIRef('local://test_sqlalchemy_obj_language/') store = plugin.get( 'SQLAlchemy', Store, )(identifier=identifier, ) graph = ConjunctiveGraph( store=store, identifier=identifier, ) graph.open('sqlite:///', create=True) triple = ( URIRef('https://foo'), RDFS.comment, Literal('', lang='en'), ) return triple in graph
def create(self, conjunctive=False, gid=None, loader=None, format=None): self.__lock.acquire() uuid_lock = None cached = False p = r.pipeline(transaction=True) p.multi() try: uuid = shortuuid.uuid() if conjunctive: if 'persist' in app.config['STORE']: g = ConjunctiveGraph('Sleepycat') g.open('store/resources/{}'.format(uuid), create=True) else: g = ConjunctiveGraph() g.store.graph_aware = False self.__graph_dict[g] = uuid self.__uuid_dict[uuid] = g return g else: g = resources_cache.get_context(uuid) try: if gid is not None: st_uuid = r.hget(self.__gids_key, gid) if st_uuid is not None: cached = True uuid = st_uuid uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() g = self.__uuid_dict[uuid] uuid_lock.release() else: post_ts = dt.now() elapsed = (post_ts - self.__last_creation_ts).total_seconds() throttling = (1.0 / GRAPH_THROTTLING) - elapsed if throttling > 0: sleep(throttling) temp_key = '{}:cache:{}'.format(AGENT_ID, uuid) counter_key = '{}:cnt'.format(temp_key) ttl = MIN_CACHE_TIME + int(10 * random()) ttl_ts = calendar.timegm((dt.now() + datetime.timedelta(ttl)).timetuple()) if st_uuid is None: p.delete(counter_key) p.sadd(self.__cache_key, uuid) p.hset(self.__gids_key, uuid, gid) p.hset(self.__gids_key, gid, uuid) self.__last_creation_ts = dt.now() p.incr(counter_key) p.set(temp_key, ttl_ts) p.expire(temp_key, ttl) uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() except Exception, e: log.error(e.message) traceback.print_exc() self.__graph_dict[g] = uuid self.__uuid_dict[uuid] = g
def calculate_edges((offset_limit, config)): g = ConjunctiveGraph(config['graph_store'], config['graph_identifier']) g.open(config['db_configstring'], create=False) engine = create_engine(config['alchemy_configstring']) Session = sessionmaker(bind=engine) session = Session() results = session.query(Node).filter_by(node_type="subject").order_by( Node.node_id.desc()).offset(offset_limit[0]).limit( offset_limit[1]).all() for result in results: query = "SELECT ?p ?o WHERE { <%s> ?p ?o }" % result.node_uri items = g.query(query) for item in items: if str(item[0] ) != "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": neighbors = session.query(Node).filter_by( node_uri=item[1]).all() if len(neighbors) > 0: for neighbor in neighbors: result.add_neighbors(str(item[0]), neighbor) session.commit() g.close() session.close()
def initialize(config_file): print '[%s] Initializing...' % strftime("%a, %d %b %Y %H:%M:%S", localtime()) sys.stdout.flush() config = __import__(config_file) try: g = ConjunctiveGraph(config.graph_store, config.graph_identifier) g.open(config.db_configstring, create=True) if config.input_file != None: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), config.input_file) sys.stdout.flush() g.parse(config.input_file, format=config.input_format) g.commit() else: dir_list = os.listdir(config.input_dir) for file_name in dir_list: print '[%s] Parsing %s...' % (strftime("%a, %d %b %Y %H:%M:%S", localtime()) ,file_name) sys.stdout.flush() g.parse(config.input_dir + '/' + file_name, format=config.input_format) g.commit() except Exception as e: traceback.print_exc() print e print '"%s" not found, or incorrect RDF serialization.' % config.input_file sys.stdout.flush() exit(-1) return g, config
def query_handler(source_id, query, context): # Attempt to cnonicalize this object by round-tripping decoding and re-encoding query = json.loads(json.dumps(query, sort_keys=True)) rdf_store = context["rdf_store"] graph = ConjunctiveGraph(rdf_store) graph.open(DB_URI) results = graph.query( "select ?subject ?predicate ?object where {?subject ?predicate ?object}" ) # Inelegant, but works: serialize to JSON string, then re-parse statements = json.loads(results.serialize(format="json")) graph.close() context = { "@context": { "@vocab": "http://facta.kendra.io/vocab#", "kv": "http://facta.kendra.io/vocab#", "kendra": "http://kendra.io/types#", "kuid": "http://kendra.io/uuid#", "schema": "http://schema.org/", "xsd": "http://www.w3.org/2001/XMLSchema#" } } compacted_graph, contained = s2j.result_data_to_jsonld( statements, context) result = s2j.extract_salient_results( compacted_graph, contained, ["kendra:InclusionRelationship", "kendra:TextSelection"]) result = s2j.quicksearch(result, query) return {"result": result}
def open(self): # XXX: If we have a source that's read only, should we need to set the # store separately?? g0 = ConjunctiveGraph('SPARQLUpdateStore') g0.open(tuple(self.conf['rdf.store_conf'])) self.graph = g0 return self.graph
def test_sqlalchemy_luuu(): logger.info(f'Python version: {python_version}') logger.info(f'RDFLib version: {rdflib_version}') logger.info(f'RDFLib-SQLAlchemy version: {rdflib_sqlalchemy_version}') identifier = URIRef('local://test_sqlalchemy_luuu/') store = plugin.get( 'SQLAlchemy', Store, )(identifier=identifier, ) graph = ConjunctiveGraph( store=store, identifier=identifier, ) graph.open('sqlite:///', create=True) graph.add([ Literal('https://example.org'), # <--- sic! RDF.type, SDO.WebSite, URIRef('https://example.org/about/'), ])
def open(self): import logging # XXX: If we have a source that's read only, should we need to set the store separately?? g0 = ConjunctiveGraph('Sleepycat') self.conf['rdf.store'] = 'Sleepycat' g0.open(self.conf['rdf.store_conf'],create=True) self.graph = g0 logging.debug("Opened SleepyCatSource")
def get_uri_types(uri, lang): g = ConjunctiveGraph('SPARQLStore') g.open(get_dbpedia_endpoint(lang)) #print uri #print len(list( g.triples(( URIRef(uri), URIRef('http://dbpedia.org/ontology/country'), URIRef('http://es.dbpedia.org/resource/España') )) )) return [ str(typ) for typ in g.objects(URIRef(uri), RDF.type) ]
def open(self): import logging # XXX: If we have a source that's read only, should we need to set the store separately?? g0 = ConjunctiveGraph('Sleepycat') self.conf['rdf.store'] = 'Sleepycat' g0.open(self.conf['rdf.store_conf'], create=True) self.graph = g0 logging.debug("Opened SleepyCatSource")
def example_1(): """Creates a ConjunctiveGraph and performs some BerkeleyDB tasks with it """ path = mktemp() # Declare we are using a BerkeleyDB Store graph = ConjunctiveGraph("BerkeleyDB") # Open previously created store, or create it if it doesn't exist yet # (always doesn't exist in this example as using temp file location) rt = graph.open(path, create=False) if rt == NO_STORE: # There is no underlying BerkeleyDB infrastructure, so create it print("Creating new DB") graph.open(path, create=True) else: print("Using existing DB") assert rt == VALID_STORE, "The underlying store is corrupt" print("Triples in graph before add:", len(graph)) print("(will always be 0 when using temp file for DB)") # Now we'll add some triples to the graph & commit the changes EG = Namespace("http://example.net/test/") graph.bind("eg", EG) graph.add((EG["pic:1"], EG.name, Literal("Jane & Bob"))) graph.add((EG["pic:2"], EG.name, Literal("Squirrel in Tree"))) graph.commit() print("Triples in graph after add:", len(graph)) print("(should be 2)") # display the graph in Turtle print(graph.serialize()) # close when done, otherwise BerkeleyDB will leak lock entries. graph.close() graph = None # reopen the graph graph = ConjunctiveGraph("BerkeleyDB") graph.open(path, create=False) print("Triples still in graph:", len(graph)) print("(should still be 2)") graph.close() # Clean up the temp folder to remove the BerkeleyDB database files... for f in os.listdir(path): os.unlink(path + "/" + f) os.rmdir(path)
def get_triple_store(persist_mode=False, base='store', path='', **kwargs): if persist_mode: prepare_store_path(base, path) graph = ConjunctiveGraph('Sleepycat', identifier=path) graph.open('{}/{}'.format(base, path), create=True) else: graph = ConjunctiveGraph() graph.store.graph_aware = False return graph
def open(self): conn = sqlite3.connect(self.conf['sqldb']) cur = conn.cursor() # first step, grab all entities and add them to the graph n = self.conf['rdf.namespace'] cur.execute("SELECT DISTINCT ID, Entity FROM tblentity") g0 = ConjunctiveGraph(self.conf['rdf.store']) g0.open(self.conf['rdf.store_conf'], create=True) for r in cur.fetchall(): # first item is a number -- needs to be converted to a string first = str(r[0]) # second item is text second = str(r[1]) # This is the backbone of any RDF graph. The unique # ID for each entity is encoded as a URI and every other piece of # knowledge about that entity is connected via triples to that URI # In this case, we connect the common name of that entity to the # root URI via the RDFS label property. g0.add((n[first], RDFS.label, Literal(second))) # second step, get the relationships between them and add them to the # graph cur.execute( "SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship" ) gi = '' i = 0 for r in cur.fetchall(): # all items are numbers -- need to be converted to a string first = str(r[0]) second = str(r[1]) third = str(r[2]) prov = str(r[3]) ui = self.conf['molecule_name'](prov) gi = Graph(g0.store, ui) gi.add((n[first], n[second], n[third])) g0.add([ui, RDFS.label, Literal(str(i))]) if (prov != ''): g0.add([ui, n[u'text_reference'], Literal(prov)]) i = i + 1 cur.close() conn.close() self.graph = g0
def rdf_data(rdfobject, f): input = ConjunctiveGraph() input.open("store2", create=True) input.parse(data=rdfobject, format=f) #print(input.serialize(format='json-ld', auto_compact=True, indent=4)) for s, p, o in input: g.add((s, p, o)) input.close()
class SQLSource(RDFSource): def __init__(self, *args, **kwargs): super(SQLSource, self).__init__(*args, **kwargs) self.conf['rdf.store'] = self.store_name def open(self): try: from rdflib_sqlalchemy import registerplugins except ImportError: raise OpenFailError( 'The rdflib-sqlalchemy package is not installed.' ' You may need to install one of "sqlite_source", "mysql_source", or' ' "postgresql_source" extra for owmeta_core.' ' For example, change "owmeta_core" in your setup.py or' ' requirements.txt to "owmeta_core[sqlite_source]" and reinstall' ) registerplugins() store = plugin.get("SQLAlchemy", Store)(**self._initargs()) self.graph = ConjunctiveGraph(store) cfg = self._openconfig() self.graph.open(cfg, create=True) def _initargs(self): a = self._initargs_augment() if not a or not isinstance(a, dict): return dict() return a def _openconfig(self): c = self.conf['rdf.store_conf'] if isinstance(c, dict): c = dict(c) url = c.pop('url', None) if not url: raise OpenFailError( 'A "url" argument must be provided in config dict') c.pop('init_args', None) self.url = self._openurl(url) c['url'] = self.url return c else: self.url = self._openurl(c) return self.url def _openurl(self, url): return url def _initargs_augment(self): c = self.conf['rdf.store_conf'] if isinstance(c, dict): initargs = self.conf['rdf.store_conf'].get('init_args', None) if initargs: return dict(initargs)
def get(self): conn = sqlite3.connect(self.conf['sqldb']) cur = conn.cursor() #first step, grab all entities and add them to the graph n = self.conf['rdf.namespace'] cur.execute("SELECT DISTINCT ID, Entity FROM tblentity") g0 = ConjunctiveGraph(self.conf['rdf.store']) g0.open(self.conf['rdf.store_conf'], create=True) for r in cur.fetchall(): #first item is a number -- needs to be converted to a string first = str(r[0]) #second item is text second = str(r[1]) # This is the backbone of any RDF graph. The unique # ID for each entity is encoded as a URI and every other piece of # knowledge about that entity is connected via triples to that URI # In this case, we connect the common name of that entity to the # root URI via the RDFS label property. g0.add( (n[first], RDFS.label, Literal(second)) ) #second step, get the relationships between them and add them to the graph cur.execute("SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship") gi = '' i = 0 for r in cur.fetchall(): #all items are numbers -- need to be converted to a string first = str(r[0]) second = str(r[1]) third = str(r[2]) prov = str(r[3]) ui = self.conf['molecule_name'](prov) gi = Graph(g0.store, ui) gi.add( (n[first], n[second], n[third]) ) g0.add([ui, RDFS.label, Literal(str(i))]) if (prov != ''): g0.add([ui, n[u'text_reference'], Literal(prov)]) i = i + 1 cur.close() conn.close() return g0
def get_db(triplestore_type): if triplestore_type == 'memory': g = Triplestore._create_db() elif triplestore_type == 'pickle': # Load pickled Graph object from disk. Check the time. If time has passed specified duration, then # re-harvest data. if os.path.isfile(Config.triplestore_path_pickle): with open(Config.triplestore_path_pickle, 'rb') as f: g = pickle.load(f) for date in g.objects(Triplestore.THIS_GRAPH, DCTERMS.created): now = datetime.now() now -= timedelta(hours=Config.store_hours, minutes=Config.store_minutes) if now > date.toPython(): g = Triplestore._create_pickle_disk() else: g = Triplestore._create_pickle_disk() elif triplestore_type == 'sleepycat': # TODO: Re-harvest like 'pickle'. if hasattr(Config, 'g'): # Config has a Graph object, reuse it and open the persistent store. g = Config.g rt = g.open(Config.triplestore_path_sleepy_cat, create=False) else: # If this is the initial load and Config does not have a Graph object in memory, then create it. g = ConjunctiveGraph('Sleepycat') rt = g.open(Config.triplestore_path_sleepy_cat, create=False) if rt == NO_STORE: g.open(Config.triplestore_path_sleepy_cat, create=True) Triplestore._add_triples(g) else: assert rt == VALID_STORE, 'The underlying store is corrupt' # elif triplestore_type == 'sparql': # if os.path.isfile(Config.triplestore_path_pickle): # with open(Config.triplestore_path_pickle, 'rb') as f: # g = pickle.load(f) # else: # sparql = SPARQLWrapper(Config.sparql_endpoint) # sparql.setQuery("""DESCRIBE * WHERE { # ?s ?p ?o . # }""") # sparql.setReturnFormat(N3) # results = sparql.query().convert() # g = Graph().parse(data=results, format='n3') # with open(Config.triplestore_path_pickle, 'wb') as f: # pickle.dump(g, f) else: raise InvalidTriplestoreType( 'Expected one of: [memory, pickle, sleepycat]. Instead got {}'. format(triplestore_type)) return g
class ConfigTest(unittest.TestCase): def setUp(self): self.store = plugin.get("Elasticsearch", Store)() self.graph = ConjunctiveGraph(self.store) def tearDown(self): self.graph.close() def test_success(self): with patch.object(requests.Session, 'get') as p: self.graph.open('http://localhost:9200/collection', create=True) p.assert_called_with('http://localhost:9200/collection')
def blogs(): g = ConjunctiveGraph("Sleepycat") g.open("store") for person, blog in g.subject_objects(predicate=w.Blog): name = g.value(subject=person, predicate=w.Name) for title, feed_url in discover_feeds(blog): if title: title = "%s (%s)" % (name, title) else: title = name logging.info("found %s <%s>" % (title, feed_url)) yield title, feed_url g.close()
def generate_config_file(data_source, rdf_data=None, rdf_format=None, sparql_url=None, sparql_graph=None): current_task.update_state(state='PROGRESS', meta={'progress_percent': 15, 'progress_msg': 'Reading provided RDF data...'}) try: if data_source == 'rdf': data_graph = Graph() data_graph.parse(format=rdf_format, data=rdf_data) else: g = ConjunctiveGraph('SPARQLStore') g.open(sparql_url) data_graph = g.get_context(sparql_graph) if sparql_graph else g except Exception, e: raise Exception("An error occurred while trying to read provided data source: %s" % str(e))
def ConvertToSQLLITE (filename,destinationFileName): _graph = ConjunctiveGraph() _graph.parse(filename, format="nt") sql = ConjunctiveGraph('SQLite') sql.open(destinationFileName, create=True) for t in _graph.triples((None,None,None)): sql.add(t) sql.commit() sql.close()
def create(self, conjunctive=False): uuid = shortuuid.uuid() if conjunctive: if 'persist' in app.config['STORE']: g = ConjunctiveGraph('Sleepycat') g.open('store/query/{}'.format(uuid), create=True) g.store.graph_aware = False else: g = ConjunctiveGraph() g.store.graph_aware = False else: g = query.get_context(uuid) self.__graph_dict[g] = uuid return g
def post(g, URLendpoint, graphURI): # eseguo la post del grafo g nel database specificato output = { 'risposta': [] } endpoint = ConjunctiveGraph ('SPARQLUpdateStore') endpoint.open(URLendpoint) enddata = """INSERT DATA { GRAPH <"""+graphURI+"""> { %s } }""" % g.serialize(format="nt") endpoint.update(enddata) output['risposta'].append(g.serialize(format="turtle"))
def get_triple_store(persist_mode=False, base='store', path='', **kwargs): if persist_mode: full_path = prepare_store_path(base, path) graph = ConjunctiveGraph('Sleepycat', identifier=path) try: graph.open(full_path, create=True) except Exception as e: rmtree(full_path) raise EnvironmentError(e.message) else: graph = ConjunctiveGraph() graph.store.graph_aware = False return graph
def __init__(self, percolationdir="~/.percolation/"): percolationdir = os.path.expanduser(percolationdir) if not os.path.isdir(percolationdir): os.mkdir(percolationdir) dbdir = percolationdir+"sleepydb/" if not os.path.isdir(dbdir): os.mkdir(dbdir) percolation_graph = ConjunctiveGraph(store="Sleepycat") try: percolation_graph.open(dbdir, create=False) except: # get exception type (?) percolation_graph.open(dbdir, create=True) P.percolation_graph = percolation_graph self.percolation_graph = percolation_graph P.percolation_server = self
def __init__(self, percolationdir="~/.percolation/"): percolationdir = os.path.expanduser(percolationdir) if not os.path.isdir(percolationdir): os.mkdir(percolationdir) dbdir = percolationdir + "sleepydb/" if not os.path.isdir(dbdir): os.mkdir(dbdir) percolation_graph = ConjunctiveGraph(store="Sleepycat") try: percolation_graph.open(dbdir, create=False) except: # get exception type (?) percolation_graph.open(dbdir, create=True) P.percolation_graph = percolation_graph self.percolation_graph = percolation_graph P.percolation_server = self
def get_dbpedia_resource_triples(uri, lang): g = ConjunctiveGraph('SPARQLStore') g.open(get_dbpedia_endpoint(lang)) triples = [] for p, o in g.predicate_objects(URIRef(uri)): if isinstance(o, Literal): triples.append({ 'p': unicode(p), 'o': unicode(o), 'lang': o.language }) return triples
def setup_conjunctive(): rdf_store_ident = URIRef("rdflib_explorer") # Note that this DBURI does not specify host or username: instead, connection is # made via UNIX domain sockets, which are self-authenticating # To use this, this code needs to be run as the user in question # Note also that pip install is in general user-local dburi = Literal('postgresql+psycopg2:///kendraio_facta') registerplugins() store = plugin.get("SQLAlchemy", Store)(identifier=rdf_store_ident) graph = ConjunctiveGraph(store) graph.open(dburi, create=True) return graph, rdf_store_ident
def example_2(): """Loads a number of SKOS vocabularies from GitHub into a BerkeleyDB-backed graph stored in the local folder 'gsq_vocabs' Should print out the number of triples after each load, e.g.: 177 248 289 379 421 628 764 813 965 1381 9666 9719 ... """ from urllib.request import urlopen, Request from urllib.error import HTTPError import json import base64 g = ConjunctiveGraph("BerkeleyDB") g.open("gsg_vocabs", create=True) # gsq_vocabs = "https://api.github.com/repos/geological-survey-of-queensland/vocabularies/git/trees/master" gsq_vocabs = "https://api.github.com/repos/geological-survey-of-queensland/vocabularies/git/trees/cd7244d39337c1f4ef164b1cf1ea1f540a7277db" try: res = urlopen( Request(gsq_vocabs, headers={"Accept": "application/json"})) except HTTPError as e: return e.code, str(e), None data = res.read() encoding = res.info().get_content_charset('utf-8') j = json.loads(data.decode(encoding)) for v in j["tree"]: # process the element in GitHub result if it's a Turtle file if v["path"].endswith(".ttl"): # for each file, call it by URL, decode it and parse it into the graph r = urlopen(v['url']) content = json.loads(r.read().decode())["content"] g.parse(data=base64.b64decode(content).decode(), format="turtle") print(len(g)) print("loading complete")
class DefaultSource(RDFSource): """ Reads from and queries against a configured database. The default configuration. The database store is configured with:: "rdf.source" = "default" "rdf.store" = <your rdflib store name here> "rdf.store_conf" = <your rdflib store configuration here> Leaving unconfigured simply gives an in-memory data store. """ def open(self): self.graph = ConjunctiveGraph(self.conf['rdf.store']) self.graph.open(self.conf['rdf.store_conf'],create=True)
class DefaultSource(RDFSource): """ Reads from and queries against a configured database. The default configuration. The database store is configured with:: "rdf.source" = "default" "rdf.store" = <your rdflib store name here> "rdf.store_conf" = <your rdflib store configuration here> Leaving unconfigured simply gives an in-memory data store. """ def open(self): self.graph = ConjunctiveGraph(self.conf['rdf.store']) self.graph.open(self.conf['rdf.store_conf'], create=True)
def get_uri(uri): '''Returns all triples relevant to the URI. In other words, those, which have URI as a Resource. If there is no data in the triplestore, tries to harvest data. ''' global ds # dirty ds.commit() cg = ConjunctiveGraph(store=STORE) cg.open(DATAPATH, create=False) get_data(uri, ds) triple_table = query_union(cg, uri) return triple_table
def db_graph(url): create=True if url.lower().startswith('sqlalchemy:'): db = ConjunctiveGraph('SQLAlchemy') openstr = url[11:] log.info("abs path %s" % openstr) db.open(openstr,create=create) elif url.lower().startswith('sqlite://'): db = ConjunctiveGraph('SQLite') openstr = url[url.index('://')+3:] create=not os.path.exists(openstr) log.info("abs path %s" % openstr) db.open(openstr,create=create) elif url.lower().startswith('virtuoso'): from virtuoso.vstore import Virtuoso store = Virtuoso("DSN=VOS;UID=dba;PWD=dba;WideAsUTF16=N") db = ConjunctiveGraph(store, 'virtuoso') return db
class SQLATestCase(unittest.TestCase): identifier = URIRef("rdflib_test") dburi = Literal("sqlite://") def setUp(self): self.store = plugin.get( "SQLAlchemy", Store)(identifier=self.identifier) self.graph = ConjunctiveGraph(self.store, identifier=self.identifier) self.graph.open(self.dburi, create=True) def tearDown(self): self.graph.destroy(self.dburi) try: self.graph.close() except: pass def test_registerplugins(self): # I doubt this is quite right for a fresh pip installation, # this test is mainly here to fill a coverage gap. registerplugins() self.assert_(plugin.get("SQLAlchemy", Store) is not None) p = plugin._plugins self.assert_(("SQLAlchemy", Store) in p, p) del p[("SQLAlchemy", Store)] plugin._plugins = p registerplugins() self.assert_(("SQLAlchemy", Store) in p, p) def test_namespaces(self): self.assert_(list(self.graph.namespaces()) != []) def test_contexts_without_triple(self): self.assert_(list(self.graph.contexts()) == []) def test_contexts_with_triple(self): statemnt = (michel, likes, pizza) self.assert_(self.graph.contexts(triple=statemnt) != []) def test__len(self): self.assert_(self.store.__len__() == 0) def test__remove_context(self): self.store._remove_context(self.identifier)
def _create_graph(self): """the dump is huge and in-memory parsing consumes too much RAM use Sleepcat storage to store graph during parsing """ sleepycat_path = os.path.join(self.tempdir, "sleepycat") log.msg("create graph with sleepycat store '{0}'".format(sleepycat_path)) if not os.path.exists(sleepycat_path): log.msg("create '{0}'".format(sleepycat_path)) os.makedirs(sleepycat_path) graph = ConjunctiveGraph(store="Sleepycat") graph.open(sleepycat_path, create=True) """ rt = graph.open(sleepycat_path, create=False) if rt == NO_STORE: graph.open(sleepycat_path, create=True) else: assert rt == VALID_STORE, "The underlying store is corrupt" """ return graph
def open(self): if not self.graph: self.graph = True import glob # Check the ages of the files. Read the more recent one. g0 = ConjunctiveGraph(store=self.conf['rdf.store']) database_store = self.conf['rdf.store_conf'] source_file = self.conf['rdf.serialization'] file_format = self.conf['rdf.serialization_format'] # store_time only works for stores that are on the local # machine. try: store_time = modification_date(database_store) # If the store is newer than the serialization # get the newest file in the store for x in glob.glob(database_store + "/*"): mod = modification_date(x) if store_time < mod: store_time = mod except: store_time = DT.min trix_time = modification_date(source_file) g0.open(database_store, create=True) if store_time > trix_time: # just use the store pass else: # delete the database and read in the new one # read in the serialized format import warnings # Filters a warning from rdflib not closing its files from a # parse warnings.filterwarnings( 'ignore', ".*unclosed file <_io.BufferedReader .*") g0.parse(source_file, format=file_format) self.graph = g0 return self.graph
class DeepGraphStore(): store_name = 'SQLite' def __init__(self, create=False, parse=None): self.parse = parse self.create = create self.graph = None def setUp(self): self.path = "" + random_file_generating() self.graph = Graph(store=self.store_name) self.graph.open(self.path, create=self.create) if self.create: if not self.parse: self.graph.parse("http://njh.me/foaf.rdf", format='xml') else: self.graph.parse(self.parse) self.graph.commit() def open(self, path): self.graph = ConjunctiveGraph(self.store_name) self.path = path self.graph.open(self.path, create=False) def query(self, sparql_query): return self.graph.query(sparql_query) def parse(self, path_to_file_): self.graph.parse(path_to_file_) def load(self, triples): self.graph.load(triples) def close(self): self.graph.close() def size(self): size = self.graph.__len__() size = len(self.graph) # self.close() return size
def __init__(self, location, repository, inmemory=False): super(RDFLibStore, self).__init__(location, repository) self.inmemory = inmemory self.closed = False graphid = URIRef("file://" + self.repository) g = ConjunctiveGraph(store=self._storeid(), identifier=graphid) if os.path.exists(self.location): g.open(self.location, create=False) else: g.open(self.location, create=True) l = logging.getLogger(__name__) if inmemory: l.debug("Loading store into memory") ig = ConjunctiveGraph(identifier=graphid) ig.addN(g.quads()) g.close() self.graph = ig else: l.debug("Using on-disk store") self.graph = g
def calculate_edges((offset_limit, config)): g = ConjunctiveGraph(config['graph_store'], config['graph_identifier']) g.open(config['db_configstring'], create=False) engine = create_engine(config['alchemy_configstring']) Session = sessionmaker(bind=engine) session = Session() results = session.query(Node).filter_by(node_type="subject").order_by(Node.node_id.desc()).offset(offset_limit[0]).limit(offset_limit[1]).all() for result in results: query = "SELECT ?p ?o WHERE { <%s> ?p ?o }" % result.node_uri items = g.query(query) for item in items: if str(item[0]) != "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": neighbors = session.query(Node).filter_by(node_uri=item[1]).all() if len(neighbors) > 0: for neighbor in neighbors: result.add_neighbors(str(item[0]), neighbor) session.commit() g.close() session.close()
def build_item_data_graph(item_type, item_properties, item_data_graph_path, endpoint_url, max_num_objects, create=True): item_data_graph = ConjunctiveGraph("Sleepycat") item_data_graph.open(item_data_graph_path, create=create) for item_property in item_properties: item_data_query = RETRIEVE_ITEM_PROPERTIES_QUERY % ( item_type, item_type, item_type, item_type, item_type, item_property) sparql_client = SPARQLWrapper(endpoint_url, returnFormat=JSON) sparql_client.setTimeout(604800) sparql_client.setQuery(item_data_query) results = sparql_client.queryAndConvert() num_bindings = len(results["results"]["bindings"]) added_triples = defaultdict(lambda: defaultdict(lambda: 0)) for i, binding in enumerate(results["results"]["bindings"]): print("[{}/{}]".format(i + 1, num_bindings)) subject = URIRef(binding["s"]["value"]) predicate = URIRef(binding["p"]["value"]) if binding["o"]["type"] == "literal": object = Literal(binding["o"]["value"], datatype=binding["o"]["datatype"]) else: object = URIRef(binding["o"]["value"]) if max_num_objects is not None: if added_triples[subject][predicate] < max_num_objects: triple = (subject, predicate, object) added_triples[subject][predicate] += 1 item_data_graph.add(triple) else: triple = (subject, predicate, object) item_data_graph.add(triple) item_data_graph.close()
def open(self, create=False): db_settings = connections['default'].settings_dict sa_db_settings = { 'engine': 'postgresql+psycopg2' if db_settings['ENGINE'] == "django.db.backends.postgresql_psycopg2" else db_settings['ENGINE'], 'user': db_settings['USER'], 'password': db_settings['PASSWORD'], 'port': db_settings['PORT'] if db_settings['PORT'] else "5432", 'host': db_settings['HOST'] if db_settings['HOST'] else "localhost", 'name': db_settings['NAME'] } connect_config = "%(engine)s://%(user)s:%(password)s@%(host)s:%(port)s/%(name)s"%sa_db_settings return ConjunctiveGraph.open(self, connect_config, create=create)
class Triple_store_api(abstract_api): endpt = '' def __init__(self, usecache=False): super(Triple_store_api, self).__init__(usecache=usecache) self.g = None self.ssg = None self.opener = None self.maxattempts = 3 self.alphamap = None self.setup_sparql() def setup_sparql(self): self.g = ConjunctiveGraph(store='SPARQLStore') self.g.open(self.endpt) @memoize() def query(self, sparql): self._wait() if self.g == None: self.setup_sparql() attempt = 0 response = None while response == None and attempt < self.maxattempts: try: attempt += 1 response = self.g.query(sparql) except: traceback.print_exc() self._wait(self.delaytime**attempt) if response == None: raise IOError("Cannot query SPARQL endpoint") return response
def open(self): if not self.graph: self.graph = True import glob # Check the ages of the files. Read the more recent one. g0 = ConjunctiveGraph(store=self.conf['rdf.store']) database_store = self.conf['rdf.store_conf'] source_file = self.conf['rdf.serialization'] file_format = self.conf['rdf.serialization_format'] # store_time only works for stores that are on the local # machine. try: store_time = modification_date(database_store) # If the store is newer than the serialization # get the newest file in the store for x in glob.glob(database_store + "/*"): mod = modification_date(x) if store_time < mod: store_time = mod except: store_time = DT.min trix_time = modification_date(source_file) g0.open(database_store, create=True) if store_time > trix_time: # just use the store pass else: # delete the database and read in the new one # read in the serialized format g0.parse(source_file, format=file_format) self.graph = g0 return self.graph
class Command(BaseCommand): args = "<path_to_skos_file path_to_skos_file>..." help = "import skos ref in rdflib alchemy store" def __init__(self): super(Command, self).__init__() self.ident = "jocondelab" #'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. #'NAME': '', # Or path to database file if using sqlite3. # The following settings are not used with sqlite3: #'USER': '', #'PASSWORD': '', #'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP. #'PORT': '' db_settings = connections['default'].settings_dict sa_db_settings = { 'engine': 'postgresql+psycopg2' if db_settings['ENGINE'] == "django.db.backends.postgresql_psycopg2" else db_settings['ENGINE'], 'user': db_settings['USER'], 'password': db_settings['PASSWORD'], 'port': db_settings['PORT'] if db_settings['PORT'] else "5432", 'host': db_settings['HOST'] if db_settings['HOST'] else "localhost", 'name': db_settings['NAME'] } self.connect_config = "%(engine)s://%(user)s:%(password)s@%(host)s:%(port)s/%(name)s"%sa_db_settings self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident) self.graph = ConjunctiveGraph(self.store, identifier=self.ident) self.graph.open(self.connect_config, create=True) def handle(self, *args, **options): #import pydevd #@UnresolvedImport #pydevd.settrace(suspend=True) for skos_path, public_id in zip(args[::2],args[1::2]): filepath = os.path.abspath(skos_path) self.stdout.write("Importing %s" % filepath) self.graph.parse(filepath, publicID=public_id, format='xml') self.stdout.write("graph size %d" % len(self.graph)) self.graph.commit() self.graph.close() self.store = plugin.get("SQLAlchemy", Store)(identifier=self.ident) self.graph = ConjunctiveGraph(self.store, identifier=self.ident) self.graph.open(self.connect_config, create=False) self.stdout.write("correct alt labels") litteral_statements = self.store.tables['literal_statements'] with self.store.engine.connect() as connection: q = litteral_statements.select().where(litteral_statements.c.predicate == "http://www.w3.org/2004/02/skos/core#altLabel") for row in connection.execute(q): if row['object'] and row['object'] != row['object'].strip(): u_q = litteral_statements.update().where(and_( litteral_statements.c.subject == row['subject'], litteral_statements.c.predicate == row['predicate'], litteral_statements.c.object == row['object'], litteral_statements.c.context == row['context'], litteral_statements.c.termComb == row['termcomb'], litteral_statements.c.objLanguage == row['objlanguage'], litteral_statements.c.objDatatype == row['objdatatype'] )).values(object = row['object'].strip() ) #u_q_compiled = u_q.compile() #self.stdout.write("UPDATE QUERY for %s : %s : %s - %s" % (row['subject'], row['object'], str(u_q_compiled), repr(u_q_compiled.params))) connection.execute(u_q) self.stdout.write("graph size %d" % len(self.graph)) self.stdout.write("graph contexts %s" % repr([g for g in self.graph.contexts()]))
class ContextTestCase(unittest.TestCase): storetest = True identifier = URIRef("rdflib_test") michel = URIRef(u'michel') tarek = URIRef(u'tarek') bob = URIRef(u'bob') likes = URIRef(u'likes') hates = URIRef(u'hates') pizza = URIRef(u'pizza') cheese = URIRef(u'cheese') c1 = URIRef(u'context-1') c2 = URIRef(u'context-2') def setUp(self, uri='sqlite://', storename=None): store = plugin.get(storename, Store)(identifier=self.identifier) self.graph = ConjunctiveGraph(store, identifier=self.identifier) self.graph.open(uri, create=True) def tearDown(self, uri='sqlite://'): self.graph.destroy(uri) try: self.graph.close() except: pass def get_context(self, identifier): assert isinstance(identifier, URIRef) or \ isinstance(identifier, BNode), type(identifier) return Graph(store=self.graph.store, identifier=identifier, namespace_manager=self) def addStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.add((tarek, likes, pizza)) graph.add((tarek, likes, cheese)) graph.add((michel, likes, pizza)) graph.add((michel, likes, cheese)) graph.add((bob, likes, cheese)) graph.add((bob, hates, pizza)) graph.add((bob, hates, michel)) # gasp! def removeStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.remove((tarek, likes, pizza)) graph.remove((tarek, likes, cheese)) graph.remove((michel, likes, pizza)) graph.remove((michel, likes, cheese)) graph.remove((bob, likes, cheese)) graph.remove((bob, hates, pizza)) graph.remove((bob, hates, michel)) # gasp! def addStuffInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! # add to default context self.graph.add(triple) # add to context 1 graph = Graph(self.graph.store, c1) graph.add(triple) # add to context 2 graph = Graph(self.graph.store, c2) graph.add(triple) def testConjunction(self): self.addStuffInMultipleContexts() triple = (self.pizza, self.likes, self.pizza) # add to context 1 graph = Graph(self.graph.store, self.c1) graph.add(triple) # print("Graph", graph.identifier, graph.serialize(format="nt")) # print("Selfgraph", self.graph.identifier, # self.graph.serialize(format="nt")) self.assertEquals(len(self.graph.store), len(graph.store)) def testAdd(self): self.addStuff() def testRemove(self): self.addStuff() self.removeStuff() def testLenInOneContext(self): c1 = self.c1 # make sure context is empty self.graph.remove_context(self.get_context(c1)) graph = Graph(self.graph.store, c1) oldLen = len(self.graph) for i in range(0, 10): graph.add((BNode(), self.hates, self.hates)) self.assertEquals(len(graph), oldLen + 10) self.assertEquals(len(self.get_context(c1)), oldLen + 10) self.graph.remove_context(self.get_context(c1)) self.assertEquals(len(self.graph), oldLen) self.assertEquals(len(graph), 0) def testLenInMultipleContexts(self): oldLen = len(self.graph.store) print("Original", oldLen, self.graph.store) self.addStuffInMultipleContexts() newLen = len(self.graph.store) print("MultipleContexts", newLen, self.graph.store) # addStuffInMultipleContexts is adding the same triple to # three different contexts. So it's only + 1 print("No context", len(list(self.graph.triples((None, None, None))))) print("Context context-1", len( list(self.graph.triples((None, None, None), context=self.c1)))) print("Context context-2", len( list(self.graph.triples((None, None, None), context=self.c2)))) self.assertEquals(len(self.graph.store), oldLen + 1, [self.graph.store, oldLen + 1]) graph = Graph(self.graph.store, self.c1) self.assertEquals(len(graph.store), oldLen + 1, [graph.store, oldLen + 1]) def testRemoveInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() # triple should be still in store after removing it from c1 + c2 self.assert_(triple in self.graph) graph = Graph(self.graph.store, c1) graph.remove(triple) self.assert_(triple in self.graph) graph = Graph(self.graph.store, c2) graph.remove(triple) self.assert_(triple in self.graph) self.graph.remove(triple) # now gone! self.assert_(triple not in self.graph) # add again and see if remove without context removes all triples! self.addStuffInMultipleContexts() self.graph.remove(triple) self.assert_(triple not in self.graph) def testContexts(self): triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() def cid(c): if (PY3 and not isinstance(c,(str, bytes))) or not isinstance(c, basestring): return c.identifier return c self.assert_(self.c1 in list(map(cid, self.graph.contexts()))) self.assert_(self.c2 in list(map(cid, self.graph.contexts()))) contextList = list(map(cid, list(self.graph.contexts(triple)))) self.assert_(self.c1 in contextList) self.assert_(self.c2 in contextList) def testRemoveContext(self): c1 = self.c1 self.addStuffInMultipleContexts() self.assertEquals(len(Graph(self.graph.store, c1)), 1) self.assertEquals(len(self.get_context(c1)), 1) self.graph.remove_context(self.get_context(c1)) self.assert_(self.c1 not in self.graph.contexts()) def testRemoveAny(self): Any = None self.addStuffInMultipleContexts() self.graph.remove((Any, Any, Any)) self.assertEquals(len(self.graph), 0) def testTriples(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 asserte = self.assertEquals triples = self.graph.triples graph = self.graph c1graph = Graph(self.graph.store, c1) c1triples = c1graph.triples Any = None self.addStuff() # unbound subjects with context asserte(len(list(c1triples((Any, likes, pizza)))), 2) asserte(len(list(c1triples((Any, hates, pizza)))), 1) asserte(len(list(c1triples((Any, likes, cheese)))), 3) asserte(len(list(c1triples((Any, hates, cheese)))), 0) # unbound subjects without context, same results! asserte(len(list(triples((Any, likes, pizza)))), 2) asserte(len(list(triples((Any, hates, pizza)))), 1) asserte(len(list(triples((Any, likes, cheese)))), 3) asserte(len(list(triples((Any, hates, cheese)))), 0) # unbound objects with context asserte(len(list(c1triples((michel, likes, Any)))), 2) asserte(len(list(c1triples((tarek, likes, Any)))), 2) asserte(len(list(c1triples((bob, hates, Any)))), 2) asserte(len(list(c1triples((bob, likes, Any)))), 1) # unbound objects without context, same results! asserte(len(list(triples((michel, likes, Any)))), 2) asserte(len(list(triples((tarek, likes, Any)))), 2) asserte(len(list(triples((bob, hates, Any)))), 2) asserte(len(list(triples((bob, likes, Any)))), 1) # unbound predicates with context asserte(len(list(c1triples((michel, Any, cheese)))), 1) asserte(len(list(c1triples((tarek, Any, cheese)))), 1) asserte(len(list(c1triples((bob, Any, pizza)))), 1) asserte(len(list(c1triples((bob, Any, michel)))), 1) # unbound predicates without context, same results! asserte(len(list(triples((michel, Any, cheese)))), 1) asserte(len(list(triples((tarek, Any, cheese)))), 1) asserte(len(list(triples((bob, Any, pizza)))), 1) asserte(len(list(triples((bob, Any, michel)))), 1) # unbound subject, objects with context asserte(len(list(c1triples((Any, hates, Any)))), 2) asserte(len(list(c1triples((Any, likes, Any)))), 5) # unbound subject, objects without context, same results! asserte(len(list(triples((Any, hates, Any)))), 2) asserte(len(list(triples((Any, likes, Any)))), 5) # unbound predicates, objects with context asserte(len(list(c1triples((michel, Any, Any)))), 2) asserte(len(list(c1triples((bob, Any, Any)))), 3) asserte(len(list(c1triples((tarek, Any, Any)))), 2) # unbound predicates, objects without context, same results! asserte(len(list(triples((michel, Any, Any)))), 2) asserte(len(list(triples((bob, Any, Any)))), 3) asserte(len(list(triples((tarek, Any, Any)))), 2) # unbound subjects, predicates with context asserte(len(list(c1triples((Any, Any, pizza)))), 3) asserte(len(list(c1triples((Any, Any, cheese)))), 3) asserte(len(list(c1triples((Any, Any, michel)))), 1) # unbound subjects, predicates without context, same results! asserte(len(list(triples((Any, Any, pizza)))), 3) asserte(len(list(triples((Any, Any, cheese)))), 3) asserte(len(list(triples((Any, Any, michel)))), 1) # all unbound with context asserte(len(list(c1triples((Any, Any, Any)))), 7) # all unbound without context, same result! asserte(len(list(triples((Any, Any, Any)))), 7) for c in [graph, self.get_context(c1)]: # unbound subjects asserte(set(c.subjects(likes, pizza)), set((michel, tarek))) asserte(set(c.subjects(hates, pizza)), set((bob,))) asserte(set(c.subjects(likes, cheese)), set([tarek, bob, michel])) asserte(set(c.subjects(hates, cheese)), set()) # unbound objects asserte(set(c.objects(michel, likes)), set([cheese, pizza])) asserte(set(c.objects(tarek, likes)), set([cheese, pizza])) asserte(set(c.objects(bob, hates)), set([michel, pizza])) asserte(set(c.objects(bob, likes)), set([cheese])) # unbound predicates asserte(set(c.predicates(michel, cheese)), set([likes])) asserte(set(c.predicates(tarek, cheese)), set([likes])) asserte(set(c.predicates(bob, pizza)), set([hates])) asserte(set(c.predicates(bob, michel)), set([hates])) asserte(set( c.subject_objects(hates)), set([(bob, pizza), (bob, michel)])) asserte(set(c.subject_objects(likes)), set([(tarek, cheese), (michel, cheese), (michel, pizza), (bob, cheese), (tarek, pizza)])) asserte(set(c.predicate_objects( michel)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.predicate_objects(bob)), set([(likes, cheese), (hates, pizza), (hates, michel)])) asserte(set(c.predicate_objects( tarek)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.subject_predicates( pizza)), set([(bob, hates), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(cheese)), set([( bob, likes), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(michel)), set([(bob, hates)])) asserte(set(c), set([ (bob, hates, michel), (bob, likes, cheese), (tarek, likes, pizza), (michel, likes, pizza), (michel, likes, cheese), (bob, hates, pizza), (tarek, likes, cheese)])) # remove stuff and make sure the graph is empty again self.removeStuff() asserte(len(list(c1triples((Any, Any, Any)))), 0) asserte(len(list(triples((Any, Any, Any)))), 0)
from rdflib import ConjunctiveGraph, Graph from rdflib import plugin from rdflib.store import Store from rdflib.store import VALID_STORE from rdflib import Literal from rdflib import Namespace from rdflib import URIRef from pprint import pprint from rdflib import RDF, RDFS, OWL, XSD from globals import DEFAULT_GRAPH_URI, SCHEMA_GRAPH_URI, DB, _get_postgresql_config_string citg = ConjunctiveGraph('PostgreSQL', identifier=URIRef(DEFAULT_GRAPH_URI)) rt = citg.open(_get_postgresql_config_string(), create=False) assert rt == VALID_STORE,"The underlying store is corrupted" print("Triples in graph: {0}".format(len(citg))) initNs = { 'cit': Namespace(SCHEMA_GRAPH_URI), 'rdf': RDF, 'owl': OWL, 'rdfs': RDFS } # CONTEXTS for c in citg.contexts():
txy_list.append((t, x, y)) accident_url_list.append(ident) yield from accident_coverage_triples(txy_list, accident_url_list) parser = ArgumentParser() parser.add_argument( '--tweets', type=FileType('r'), default='data/tweets.json' ) parser.add_argument( '--streets', type=FileType('r'), default='data/streets.json' ) parser.add_argument( '--out', type=FileType('wb'), default='data/accidents.ttl' ) if __name__ == '__main__': args = parser.parse_args() streets = load(args.streets) tweets = load(args.tweets) print('Loaded {} tweets and {} streets'.format(len(tweets), len(streets))) fuseki = ConjunctiveGraph(store='SPARQLUpdateStore') fuseki.open(('http://localhost:3030/accidents/query', 'http://localhost:3030/accidents/update')) default = 'urn:x-arq:DefaultGraph' add_namespaces(fuseki) fuseki.addN((s,p,o,default) for s,p,o in build_graph(tweets)) fuseki.close()
class ContextTestCase(unittest.TestCase): store = 'default' slow = True tmppath = None def setUp(self): try: self.graph = ConjunctiveGraph(store=self.store) except ImportError: raise SkipTest( "Dependencies for store '%s' not available!" % self.store) if self.store == "SQLite": _, self.tmppath = mkstemp( prefix='test', dir='/tmp', suffix='.sqlite') else: self.tmppath = mkdtemp() self.graph.open(self.tmppath, create=True) self.michel = URIRef(u'michel') self.tarek = URIRef(u'tarek') self.bob = URIRef(u'bob') self.likes = URIRef(u'likes') self.hates = URIRef(u'hates') self.pizza = URIRef(u'pizza') self.cheese = URIRef(u'cheese') self.c1 = URIRef(u'context-1') self.c2 = URIRef(u'context-2') # delete the graph for each test! self.graph.remove((None, None, None)) def tearDown(self): self.graph.close() if os.path.isdir(self.tmppath): shutil.rmtree(self.tmppath) else: os.remove(self.tmppath) def addStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.add((tarek, likes, pizza)) graph.add((tarek, likes, cheese)) graph.add((michel, likes, pizza)) graph.add((michel, likes, cheese)) graph.add((bob, likes, cheese)) graph.add((bob, hates, pizza)) graph.add((bob, hates, michel)) # gasp! def removeStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.remove((tarek, likes, pizza)) graph.remove((tarek, likes, cheese)) graph.remove((michel, likes, pizza)) graph.remove((michel, likes, cheese)) graph.remove((bob, likes, cheese)) graph.remove((bob, hates, pizza)) graph.remove((bob, hates, michel)) # gasp! def addStuffInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! # add to default context self.graph.add(triple) # add to context 1 graph = Graph(self.graph.store, c1) graph.add(triple) # add to context 2 graph = Graph(self.graph.store, c2) graph.add(triple) def testConjunction(self): if self.store == "SQLite": raise SkipTest("Skipping known issue with __len__") self.addStuffInMultipleContexts() triple = (self.pizza, self.likes, self.pizza) # add to context 1 graph = Graph(self.graph.store, self.c1) graph.add(triple) self.assertEqual(len(self.graph), len(graph)) def testAdd(self): self.addStuff() def testRemove(self): self.addStuff() self.removeStuff() def testLenInOneContext(self): c1 = self.c1 # make sure context is empty self.graph.remove_context(self.graph.get_context(c1)) graph = Graph(self.graph.store, c1) oldLen = len(self.graph) for i in range(0, 10): graph.add((BNode(), self.hates, self.hates)) self.assertEqual(len(graph), oldLen + 10) self.assertEqual(len(self.graph.get_context(c1)), oldLen + 10) self.graph.remove_context(self.graph.get_context(c1)) self.assertEqual(len(self.graph), oldLen) self.assertEqual(len(graph), 0) def testLenInMultipleContexts(self): if self.store == "SQLite": raise SkipTest("Skipping known issue with __len__") oldLen = len(self.graph) self.addStuffInMultipleContexts() # addStuffInMultipleContexts is adding the same triple to # three different contexts. So it's only + 1 self.assertEqual(len(self.graph), oldLen + 1) graph = Graph(self.graph.store, self.c1) self.assertEqual(len(graph), oldLen + 1) def testRemoveInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() # triple should be still in store after removing it from c1 + c2 self.assertTrue(triple in self.graph) graph = Graph(self.graph.store, c1) graph.remove(triple) self.assertTrue(triple in self.graph) graph = Graph(self.graph.store, c2) graph.remove(triple) self.assertTrue(triple in self.graph) self.graph.remove(triple) # now gone! self.assertTrue(triple not in self.graph) # add again and see if remove without context removes all triples! self.addStuffInMultipleContexts() self.graph.remove(triple) self.assertTrue(triple not in self.graph) def testContexts(self): triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() def cid(c): return c.identifier self.assertTrue(self.c1 in map(cid, self.graph.contexts())) self.assertTrue(self.c2 in map(cid, self.graph.contexts())) contextList = list(map(cid, list(self.graph.contexts(triple)))) self.assertTrue(self.c1 in contextList, (self.c1, contextList)) self.assertTrue(self.c2 in contextList, (self.c2, contextList)) def testRemoveContext(self): c1 = self.c1 self.addStuffInMultipleContexts() self.assertEqual(len(Graph(self.graph.store, c1)), 1) self.assertEqual(len(self.graph.get_context(c1)), 1) self.graph.remove_context(self.graph.get_context(c1)) self.assertTrue(self.c1 not in self.graph.contexts()) def testRemoveAny(self): Any = None self.addStuffInMultipleContexts() self.graph.remove((Any, Any, Any)) self.assertEqual(len(self.graph), 0) def testTriples(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 asserte = self.assertEqual triples = self.graph.triples graph = self.graph c1graph = Graph(self.graph.store, c1) c1triples = c1graph.triples Any = None self.addStuff() # unbound subjects with context asserte(len(list(c1triples((Any, likes, pizza)))), 2) asserte(len(list(c1triples((Any, hates, pizza)))), 1) asserte(len(list(c1triples((Any, likes, cheese)))), 3) asserte(len(list(c1triples((Any, hates, cheese)))), 0) # unbound subjects without context, same results! asserte(len(list(triples((Any, likes, pizza)))), 2) asserte(len(list(triples((Any, hates, pizza)))), 1) asserte(len(list(triples((Any, likes, cheese)))), 3) asserte(len(list(triples((Any, hates, cheese)))), 0) # unbound objects with context asserte(len(list(c1triples((michel, likes, Any)))), 2) asserte(len(list(c1triples((tarek, likes, Any)))), 2) asserte(len(list(c1triples((bob, hates, Any)))), 2) asserte(len(list(c1triples((bob, likes, Any)))), 1) # unbound objects without context, same results! asserte(len(list(triples((michel, likes, Any)))), 2) asserte(len(list(triples((tarek, likes, Any)))), 2) asserte(len(list(triples((bob, hates, Any)))), 2) asserte(len(list(triples((bob, likes, Any)))), 1) # unbound predicates with context asserte(len(list(c1triples((michel, Any, cheese)))), 1) asserte(len(list(c1triples((tarek, Any, cheese)))), 1) asserte(len(list(c1triples((bob, Any, pizza)))), 1) asserte(len(list(c1triples((bob, Any, michel)))), 1) # unbound predicates without context, same results! asserte(len(list(triples((michel, Any, cheese)))), 1) asserte(len(list(triples((tarek, Any, cheese)))), 1) asserte(len(list(triples((bob, Any, pizza)))), 1) asserte(len(list(triples((bob, Any, michel)))), 1) # unbound subject, objects with context asserte(len(list(c1triples((Any, hates, Any)))), 2) asserte(len(list(c1triples((Any, likes, Any)))), 5) # unbound subject, objects without context, same results! asserte(len(list(triples((Any, hates, Any)))), 2) asserte(len(list(triples((Any, likes, Any)))), 5) # unbound predicates, objects with context asserte(len(list(c1triples((michel, Any, Any)))), 2) asserte(len(list(c1triples((bob, Any, Any)))), 3) asserte(len(list(c1triples((tarek, Any, Any)))), 2) # unbound predicates, objects without context, same results! asserte(len(list(triples((michel, Any, Any)))), 2) asserte(len(list(triples((bob, Any, Any)))), 3) asserte(len(list(triples((tarek, Any, Any)))), 2) # unbound subjects, predicates with context asserte(len(list(c1triples((Any, Any, pizza)))), 3) asserte(len(list(c1triples((Any, Any, cheese)))), 3) asserte(len(list(c1triples((Any, Any, michel)))), 1) # unbound subjects, predicates without context, same results! asserte(len(list(triples((Any, Any, pizza)))), 3) asserte(len(list(triples((Any, Any, cheese)))), 3) asserte(len(list(triples((Any, Any, michel)))), 1) # all unbound with context asserte(len(list(c1triples((Any, Any, Any)))), 7) # all unbound without context, same result! asserte(len(list(triples((Any, Any, Any)))), 7) for c in [graph, self.graph.get_context(c1)]: # unbound subjects asserte(set(c.subjects(likes, pizza)), set((michel, tarek))) asserte(set(c.subjects(hates, pizza)), set((bob,))) asserte(set(c.subjects(likes, cheese)), set([tarek, bob, michel])) asserte(set(c.subjects(hates, cheese)), set()) # unbound objects asserte(set(c.objects(michel, likes)), set([cheese, pizza])) asserte(set(c.objects(tarek, likes)), set([cheese, pizza])) asserte(set(c.objects(bob, hates)), set([michel, pizza])) asserte(set(c.objects(bob, likes)), set([cheese])) # unbound predicates asserte(set(c.predicates(michel, cheese)), set([likes])) asserte(set(c.predicates(tarek, cheese)), set([likes])) asserte(set(c.predicates(bob, pizza)), set([hates])) asserte(set(c.predicates(bob, michel)), set([hates])) asserte(set( c.subject_objects(hates)), set([(bob, pizza), (bob, michel)])) asserte( set(c.subject_objects(likes)), set( [(tarek, cheese), (michel, cheese), (michel, pizza), (bob, cheese), (tarek, pizza)])) asserte(set(c.predicate_objects( michel)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.predicate_objects(bob)), set([(likes, cheese), (hates, pizza), (hates, michel)])) asserte(set(c.predicate_objects( tarek)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.subject_predicates( pizza)), set([(bob, hates), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(cheese)), set([( bob, likes), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(michel)), set([(bob, hates)])) asserte(set(c), set( [(bob, hates, michel), (bob, likes, cheese), (tarek, likes, pizza), (michel, likes, pizza), (michel, likes, cheese), (bob, hates, pizza), (tarek, likes, cheese)])) # remove stuff and make sure the graph is empty again self.removeStuff() asserte(len(list(c1triples((Any, Any, Any)))), 0) asserte(len(list(triples((Any, Any, Any)))), 0)
def sparql_store(query_url, update_url): gs = ConjunctiveGraph('SPARQLUpdateStore') gs.open((query_url, update_url)) return gs.store
class ContextTestCase(unittest.TestCase): store_name = 'default' path = None storetest = True create = True michel = URIRef(u'michel') tarek = URIRef(u'tarek') bob = URIRef(u'bob') likes = URIRef(u'likes') hates = URIRef(u'hates') pizza = URIRef(u'pizza') cheese = URIRef(u'cheese') c1 = URIRef(u'context-1') c2 = URIRef(u'context-2') def setUp(self): self.graph = ConjunctiveGraph(store=self.store_name) self.graph.destroy(self.path) if isinstance(self.path, type(None)): if self.store_name == "SQLite": self.path = mkstemp(prefix='test',dir='/tmp') else: self.path = mkdtemp(prefix='test',dir='/tmp') self.graph.open(self.path, create=self.create) def tearDown(self): self.graph.destroy(self.path) try: self.graph.close() except: pass import os if hasattr(self,'path') and self.path is not None: if os.path.exists(self.path): if os.path.isdir(self.path): for f in os.listdir(self.path): os.unlink(self.path+'/'+f) os.rmdir(self.path) elif len(self.path.split(':')) == 1: os.unlink(self.path) else: os.remove(self.path) def get_context(self, identifier): assert isinstance(identifier, URIRef) or \ isinstance(identifier, BNode), type(identifier) return Graph(store=self.graph.store, identifier=identifier, namespace_manager=self) def addStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.add((tarek, likes, pizza)) graph.add((tarek, likes, cheese)) graph.add((michel, likes, pizza)) graph.add((michel, likes, cheese)) graph.add((bob, likes, cheese)) graph.add((bob, hates, pizza)) graph.add((bob, hates, michel)) # gasp! def removeStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 graph = Graph(self.graph.store, c1) graph.remove((tarek, likes, pizza)) graph.remove((tarek, likes, cheese)) graph.remove((michel, likes, pizza)) graph.remove((michel, likes, cheese)) graph.remove((bob, likes, cheese)) graph.remove((bob, hates, pizza)) graph.remove((bob, hates, michel)) # gasp! def addStuffInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! # add to default context self.graph.add(triple) # add to context 1 graph = Graph(self.graph.store, c1) graph.add(triple) # add to context 2 graph = Graph(self.graph.store, c2) graph.add(triple) def testConjunction(self): self.addStuffInMultipleContexts() triple = (self.pizza, self.likes, self.pizza) # add to context 1 graph = Graph(self.graph.store, self.c1) graph.add(triple) self.assertEquals(len(self.graph), len(graph)) def testAdd(self): self.addStuff() def testRemove(self): self.addStuff() self.removeStuff() def testLenInOneContext(self): c1 = self.c1 # make sure context is empty self.graph.remove_context(self.get_context(c1)) graph = Graph(self.graph.store, c1) oldLen = len(self.graph) for i in range(0, 10): graph.add((BNode(), self.hates, self.hates)) self.assertEquals(len(graph), oldLen + 10) self.assertEquals(len(self.get_context(c1)), oldLen + 10) self.graph.remove_context(self.get_context(c1)) self.assertEquals(len(self.graph), oldLen) self.assertEquals(len(graph), 0) def testLenInMultipleContexts(self): oldLen = len(self.graph) self.addStuffInMultipleContexts() # addStuffInMultipleContexts is adding the same triple to # three different contexts. So it's only + 1 self.assertEquals(len(self.graph), oldLen + 1) graph = Graph(self.graph.store, self.c1) self.assertEquals(len(graph), oldLen + 1) def testRemoveInMultipleContexts(self): c1 = self.c1 c2 = self.c2 triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() # triple should be still in store after removing it from c1 + c2 self.assert_(triple in self.graph) graph = Graph(self.graph.store, c1) graph.remove(triple) self.assert_(triple in self.graph) graph = Graph(self.graph.store, c2) graph.remove(triple) self.assert_(triple in self.graph) self.graph.remove(triple) # now gone! self.assert_(triple not in self.graph) # add again and see if remove without context removes all triples! self.addStuffInMultipleContexts() self.graph.remove(triple) self.assert_(triple not in self.graph) def testContexts(self): triple = (self.pizza, self.hates, self.tarek) # revenge! self.addStuffInMultipleContexts() def cid(c): if not isinstance(c, basestring): return c.identifier return c self.assert_(self.c1 in map(cid, self.graph.contexts())) self.assert_(self.c2 in map(cid, self.graph.contexts())) contextList = map(cid, list(self.graph.contexts(triple))) self.assert_(self.c1 in contextList) self.assert_(self.c2 in contextList) def testRemoveContext(self): c1 = self.c1 self.addStuffInMultipleContexts() self.assertEquals(len(Graph(self.graph.store, c1)), 1) self.assertEquals(len(self.get_context(c1)), 1) self.graph.remove_context(self.get_context(c1)) self.assert_(self.c1 not in self.graph.contexts()) def testRemoveAny(self): Any = None self.addStuffInMultipleContexts() self.graph.remove((Any, Any, Any)) self.assertEquals(len(self.graph), 0) def testTriples(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese c1 = self.c1 asserte = self.assertEquals triples = self.graph.triples graph = self.graph c1graph = Graph(self.graph.store, c1) c1triples = c1graph.triples Any = None self.addStuff() # unbound subjects with context asserte(len(list(c1triples((Any, likes, pizza)))), 2) asserte(len(list(c1triples((Any, hates, pizza)))), 1) asserte(len(list(c1triples((Any, likes, cheese)))), 3) asserte(len(list(c1triples((Any, hates, cheese)))), 0) # unbound subjects without context, same results! asserte(len(list(triples((Any, likes, pizza)))), 2) asserte(len(list(triples((Any, hates, pizza)))), 1) asserte(len(list(triples((Any, likes, cheese)))), 3) asserte(len(list(triples((Any, hates, cheese)))), 0) # unbound objects with context asserte(len(list(c1triples((michel, likes, Any)))), 2) asserte(len(list(c1triples((tarek, likes, Any)))), 2) asserte(len(list(c1triples((bob, hates, Any)))), 2) asserte(len(list(c1triples((bob, likes, Any)))), 1) # unbound objects without context, same results! asserte(len(list(triples((michel, likes, Any)))), 2) asserte(len(list(triples((tarek, likes, Any)))), 2) asserte(len(list(triples((bob, hates, Any)))), 2) asserte(len(list(triples((bob, likes, Any)))), 1) # unbound predicates with context asserte(len(list(c1triples((michel, Any, cheese)))), 1) asserte(len(list(c1triples((tarek, Any, cheese)))), 1) asserte(len(list(c1triples((bob, Any, pizza)))), 1) asserte(len(list(c1triples((bob, Any, michel)))), 1) # unbound predicates without context, same results! asserte(len(list(triples((michel, Any, cheese)))), 1) asserte(len(list(triples((tarek, Any, cheese)))), 1) asserte(len(list(triples((bob, Any, pizza)))), 1) asserte(len(list(triples((bob, Any, michel)))), 1) # unbound subject, objects with context asserte(len(list(c1triples((Any, hates, Any)))), 2) asserte(len(list(c1triples((Any, likes, Any)))), 5) # unbound subject, objects without context, same results! asserte(len(list(triples((Any, hates, Any)))), 2) asserte(len(list(triples((Any, likes, Any)))), 5) # unbound predicates, objects with context asserte(len(list(c1triples((michel, Any, Any)))), 2) asserte(len(list(c1triples((bob, Any, Any)))), 3) asserte(len(list(c1triples((tarek, Any, Any)))), 2) # unbound predicates, objects without context, same results! asserte(len(list(triples((michel, Any, Any)))), 2) asserte(len(list(triples((bob, Any, Any)))), 3) asserte(len(list(triples((tarek, Any, Any)))), 2) # unbound subjects, predicates with context asserte(len(list(c1triples((Any, Any, pizza)))), 3) asserte(len(list(c1triples((Any, Any, cheese)))), 3) asserte(len(list(c1triples((Any, Any, michel)))), 1) # unbound subjects, predicates without context, same results! asserte(len(list(triples((Any, Any, pizza)))), 3) asserte(len(list(triples((Any, Any, cheese)))), 3) asserte(len(list(triples((Any, Any, michel)))), 1) # all unbound with context asserte(len(list(c1triples((Any, Any, Any)))), 7) # all unbound without context, same result! asserte(len(list(triples((Any, Any, Any)))), 7) for c in [graph, self.get_context(c1)]: # unbound subjects asserte(set(c.subjects(likes, pizza)), set((michel, tarek))) asserte(set(c.subjects(hates, pizza)), set((bob,))) asserte(set(c.subjects(likes, cheese)), set([tarek, bob, michel])) asserte(set(c.subjects(hates, cheese)), set()) # unbound objects asserte(set(c.objects(michel, likes)), set([cheese, pizza])) asserte(set(c.objects(tarek, likes)), set([cheese, pizza])) asserte(set(c.objects(bob, hates)), set([michel, pizza])) asserte(set(c.objects(bob, likes)), set([cheese])) # unbound predicates asserte(set(c.predicates(michel, cheese)), set([likes])) asserte(set(c.predicates(tarek, cheese)), set([likes])) asserte(set(c.predicates(bob, pizza)), set([hates])) asserte(set(c.predicates(bob, michel)), set([hates])) asserte(set(c.subject_objects(hates)), set([(bob, pizza), (bob, michel)])) asserte(set(c.subject_objects(likes)), set([(tarek, cheese), (michel, cheese), (michel, pizza), (bob, cheese), (tarek, pizza)])) asserte(set(c.predicate_objects(michel)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.predicate_objects(bob)), set([(likes, cheese), (hates, pizza), (hates, michel)])) asserte(set(c.predicate_objects(tarek)), set([(likes, cheese), (likes, pizza)])) asserte(set(c.subject_predicates(pizza)), set([(bob, hates), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(cheese)), set([(bob, likes), (tarek, likes), (michel, likes)])) asserte(set(c.subject_predicates(michel)), set([(bob, hates)])) asserte(set(c), set([(bob, hates, michel), (bob, likes, cheese), (tarek, likes, pizza), (michel, likes, pizza), (michel, likes, cheese), (bob, hates, pizza), (tarek, likes, cheese)])) # remove stuff and make sure the graph is empty again self.removeStuff() asserte(len(list(c1triples((Any, Any, Any)))), 0) asserte(len(list(triples((Any, Any, Any)))), 0)