def add(self, triple, context=None, quoted=False):
""" Add a triple to the store.
Apply the modifications on the cache, trigger an exception if data
has already been modified on the server.
:param triple: Triple (subject, predicate, object) to add.
:param context:
:param quoted: The quoted argument is interpreted by formula-aware
stores to indicate this statement is quoted/hypothetical. It
should be an error to not specify a context and have the
quoted argument be True. It should also be an error for the
quoted argument to be True when the store is not
formula-aware.
:returns:
"""
LOG.debug("-- ProxyStore.add(triple=%s, context=%s, quoted=%s) --",
triple, context, quoted)
assert self._identifier is not None, "The store must be open."
# TODO LATER : Wrong, assert is made to test bugs
assert self._format is not None, "The store must be open."
assert quoted == False, "The store -proxyStore- is not formula-aware"
Store.add(self, triple, context, quoted)
# Instruction suivant extraite du plugin Sleepycat
# Store.add(self, (subject, predicate, object), context, quoted)
self._graph.add(triple)
def add(self, triple, context, quoted=False):
# oldlen = len(self)
Store.add(self, triple, context, quoted)
context = getattr(context, 'identifier', context)
if context is None:
context = DEFAULT
if context is not DEFAULT and context not in self.__all_contexts:
self.__all_contexts.add(context)
enctriple = self.__encodeTriple(triple)
sid, pid, oid = enctriple
self.__addTripleContext(enctriple, context, quoted)
if sid in self.__subjectIndex:
self.__subjectIndex[sid].add(enctriple)
else:
self.__subjectIndex[sid] = self.family.OO.Set((enctriple,))
if pid in self.__predicateIndex:
self.__predicateIndex[pid].add(enctriple)
else:
self.__predicateIndex[pid] = self.family.OO.Set((enctriple,))
if oid in self.__objectIndex:
self.__objectIndex[oid].add(enctriple)
else:
self.__objectIndex[oid] = self.family.OO.Set((enctriple,))
def add(self, triple, context, quoted=False):
Store.add(self, triple, context, quoted)
if context is not None:
self.__all_contexts.add(context)
enctriple = self.__encodeTriple(triple)
sid, pid, oid = enctriple
self.__addTripleContext(enctriple, context, quoted)
if sid in self.__subjectIndex:
self.__subjectIndex[sid].add(enctriple)
else:
self.__subjectIndex[sid] = set([enctriple])
if pid in self.__predicateIndex:
self.__predicateIndex[pid].add(enctriple)
else:
self.__predicateIndex[pid] = set([enctriple])
if oid in self.__objectIndex:
self.__objectIndex[oid].add(enctriple)
else:
self.__objectIndex[oid] = set([enctriple])
def remove_graph(self, graph):
if not self.graph_aware:
Store.remove_graph(self, graph)
elif graph.identifier == DATASET_DEFAULT_GRAPH_ID:
self.update("DROP DEFAULT")
else:
self.update("DROP GRAPH <%s>" % graph.identifier)
def remove_graph(self, graph):
if not self.graph_aware:
Store.remove_graph(self, graph)
elif graph.identifier == DATASET_DEFAULT_GRAPH_ID:
self.update("DROP DEFAULT")
else:
self.update(
"DROP GRAPH %s" % self.node_to_sparql(graph.identifier))
def remove_graph(self, graph):
if not self.graph_aware:
Store.remove_graph(self, graph)
else:
self.remove((None, None, None), graph)
try:
self.__all_contexts.remove(graph)
except KeyError:
pass # we didn't know this graph, no problem
def remove_graph(self, graph):
if not self.graph_aware:
Store.remove_graph(self, graph)
else:
self.remove((None, None, None), graph)
try:
self.__all_contexts.remove(graph)
except KeyError:
pass # we didn't know this graph, no problem
def add(self, triple, context, quoted=False):
"""\
Add a triple to the store of triples.
"""
# add dictionary entries for spo[s][p][p] = 1 and pos[p][o][s]
# = 1, creating the nested dictionaries where they do not yet
# exits.
Store.add(self, triple, context, quoted=quoted)
if context is not None:
self.__all_contexts.add(context)
subject, predicate, object_ = triple
spo = self.__spo
try:
po = spo[subject]
except LookupError:
po = spo[subject] = {}
try:
o = po[predicate]
except LookupError:
o = po[predicate] = {}
try:
_ = o[object_]
# This cannot be reached if (s, p, o) was not inserted before.
triple_exists = True
except KeyError:
o[object_] = 1
triple_exists = False
self.__add_triple_context(triple, triple_exists, context, quoted)
if triple_exists:
# No need to insert twice this triple.
return
pos = self.__pos
try:
os = pos[predicate]
except LookupError:
os = pos[predicate] = {}
try:
s = os[object_]
except LookupError:
s = os[object_] = {}
s[subject] = 1
osp = self.__osp
try:
sp = osp[object_]
except LookupError:
sp = osp[object_] = {}
try:
p = sp[subject]
except LookupError:
p = sp[subject] = {}
p[predicate] = 1
def remove(self, triple, context=None):
Store.remove(self, triple, context)
if context is not None:
if context == self:
context = None
# f = self.forward
r = self.reverse
if context is None:
for triple, cg in self.triples(triple):
subject, predicate, object = triple
si, pi, oi = self.identifierToInt((subject, predicate, object))
contexts = list(self.contexts(triple))
for context in contexts:
ci = r[context]
del self.cspo[ci][si][pi][oi]
del self.cpos[ci][pi][oi][si]
del self.cosp[ci][oi][si][pi]
self._removeNestedIndex(self.spo, si, pi, oi, ci)
self._removeNestedIndex(self.pos, pi, oi, si, ci)
self._removeNestedIndex(self.osp, oi, si, pi, ci)
# grr!! hafta ref-count these before you can collect
# them dumbass!
#del f[si], f[pi], f[oi]
#del r[subject], r[predicate], r[object]
else:
subject, predicate, object = triple
ci = r.get(context, None)
if ci:
for triple, cg in self.triples(triple, context):
si, pi, oi = self.identifierToInt(triple)
del self.cspo[ci][si][pi][oi]
del self.cpos[ci][pi][oi][si]
del self.cosp[ci][oi][si][pi]
try:
self._removeNestedIndex(self.spo, si, pi, oi, ci)
self._removeNestedIndex(self.pos, pi, oi, si, ci)
self._removeNestedIndex(self.osp, oi, si, pi, ci)
except KeyError:
# the context may be a quoted one in which
# there will not be a triple in spo, pos or
# osp. So ignore any KeyErrors
pass
# TODO delete references to resources in
# self.forward/self.reverse that are not in use anymore...
if subject is None and predicate is None and object is None:
# remove context
try:
ci = self.reverse[context]
del self.cspo[ci], self.cpos[ci], self.cosp[ci]
except KeyError:
# TODO: no exception when removing non-existant context?
pass
def remove(self, triple, context=None):
Store.remove(self, triple, context)
if context is not None:
if context == self:
context = None
# f = self.forward
r = self.reverse
if context is None:
for triple, cg in self.triples(triple):
subject, predicate, object = triple
si, pi, oi = self.identifierToInt((subject, predicate, object))
contexts = list(self.contexts(triple))
for context in contexts:
ci = r[context]
del self.cspo[ci][si][pi][oi]
del self.cpos[ci][pi][oi][si]
del self.cosp[ci][oi][si][pi]
self._removeNestedIndex(self.spo, si, pi, oi, ci)
self._removeNestedIndex(self.pos, pi, oi, si, ci)
self._removeNestedIndex(self.osp, oi, si, pi, ci)
# grr!! hafta ref-count these before you can collect
# them dumbass!
# del f[si], f[pi], f[oi]
# del r[subject], r[predicate], r[object]
else:
subject, predicate, object = triple
ci = r.get(context, None)
if ci:
for triple, cg in self.triples(triple, context):
si, pi, oi = self.identifierToInt(triple)
del self.cspo[ci][si][pi][oi]
del self.cpos[ci][pi][oi][si]
del self.cosp[ci][oi][si][pi]
try:
self._removeNestedIndex(self.spo, si, pi, oi, ci)
self._removeNestedIndex(self.pos, pi, oi, si, ci)
self._removeNestedIndex(self.osp, oi, si, pi, ci)
except KeyError:
# the context may be a quoted one in which
# there will not be a triple in spo, pos or
# osp. So ignore any KeyErrors
pass
# TODO delete references to resources in
# self.forward/self.reverse that are not in use anymore...
if subject is None and predicate is None and object is None:
# remove context
try:
ci = self.reverse[context]
del self.cspo[ci], self.cpos[ci], self.cosp[ci]
except KeyError:
# TODO: no exception when removing non-existant context?
pass
def remove(self, triple, context):
(subject, predicate, object) = triple
assert self.__open, "The Store must be open."
Store.remove(self, (subject, predicate, object), context)
_to_string = self._to_string
if context is not None:
if context == self:
context = None
if subject is not None \
and predicate is not None \
and object is not None \
and context is not None:
s = _to_string(subject)
p = _to_string(predicate)
o = _to_string(object)
c = _to_string(context)
value = self.__indices[0].get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
if value is not None:
self.__remove((bb(s), bb(p), bb(o)), bb(c))
self.__needs_sync = True
else:
cspo, cpos, cosp = self.__indices
index, prefix, from_key, results_from_key = self.__lookup(
(subject, predicate, object), context)
needs_sync = False
for key in index.match_prefix(prefix):
c, s, p, o = from_key(key)
if context is None:
contexts_value = index.get(key) or b("")
# remove triple from all non quoted contexts
contexts = set(contexts_value.split(b("^")))
contexts.add(b("")) # and from the conjunctive index
for c in contexts:
for i, _to_key, _ in self.__indices_info:
i.remove(_to_key((s, p, o), c))
else:
self.__remove((s, p, o), c)
needs_sync = True
if context is not None:
if subject is None and predicate is None and object is None:
# TODO: also if context becomes empty and not just on
# remove((None, None, None), c)
try:
self.__contexts.remove(bb(_to_string(context)))
# except db.DBNotFoundError, e:
# pass
except Exception as e: # pragma: NO COVER
print("%s, Failed to delete %s" % (
e, context)) # pragma: NO COVER
pass # pragma: NO COVER
self.__needs_sync = needs_sync
def remove(self, triple, context, txn=None):
assert self.__open, "The Store must be open."
Store.remove(self, triple, context)
for result in self.triples(triple):
(spo, ctx) = result
result = self.session.run(
"CALL n10s.rdf.delete.inline($rdf,'N-Triples')",
rdf=self.__serialise(spo)).single()
if (result["terminationStatus"]) == "KO":
raise Exception("Could not delete triple from Neo4j: ",
result["extraInfo"])
def add(self, triple, context=None, quoted=False):
"""
Add a triple and start indexing.
:param tuple(rdflib.Identifier) triple: Tuple of three identifiers.
:param context: Context identifier. ``None`` inserts in the default
graph.
:type context: rdflib.Identifier or None
:param bool quoted: Not used.
"""
context = self._normalize_context(context)
if context is None:
context = RDFLIB_DEFAULT_GRAPH_URI
Store.add(self, triple, context)
#logger.info('Adding triple: {}'.format(triple))
pk_trp = self._pickle(triple)
pk_s, pk_p, pk_o = [self._pickle(t) for t in triple]
#logger.debug('Adding quad: {} {}'.format(triple, context))
pk_c = self._pickle(context)
# Add new individual terms or gather keys for existing ones.
keys = [None, None, None, None]
with self.cur('th:t') as icur:
for i, pk_t in enumerate((pk_s, pk_p, pk_o, pk_c)):
thash = self._hash(pk_t)
if icur.set_key(thash):
keys[i] = bytes(icur.value())
else:
# Put new term.
with self.cur('t:st') as dcur:
keys[i] = self._append(dcur, (pk_t, ))[0]
# Index.
icur.put(thash, keys[i])
# Add context in context DB.
ck = keys[3]
with self.cur('c:') as cur:
if not cur.set_key(ck):
cur.put(ck, b'')
# Add triple:context association.
spok = b''.join(keys[:3])
with self.cur('spo:c') as dcur:
if not dcur.set_key_dup(spok, ck):
dcur.put(spok, ck)
# Index spo:c association.
with self.cur('c:spo') as icur:
icur.put(ck, spok)
self._index_triple('add', spok)
def add(self, triple, context, quoted=False):
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
Store.add(self, triple, context, quoted)
if self.inbatch:
self.tripleBuffer.append(self.__serialise(triple))
if len(self.tripleBuffer) >= self.bufferMaxSize:
self.__flushBuffer()
else:
result = self.session.run(
"CALL n10s.rdf.import.inline($rdf,'N-Triples')",
rdf=self.__serialise(triple)).single()
if (result["terminationStatus"]) == "KO":
raise Exception("Could not persist triple in Neo4j: ",
result["extraInfo"])
def add(self, xxx_todo_changeme, context, quoted=False):
if quoted or self.locked:
raise RuntimeError('nope')
Store.add(self, xxx_todo_changeme, context, quoted)
s1, p1, o1 = xxx_todo_changeme
incoming = s1, p1, o1, context
for pat in patterns:
i = self.indexes[pat]
for incoming_idx, l in enumerate(pat):
if l == 'f':
i = i[None]
else:
i = i[incoming[incoming_idx]]
i.append((xxx_todo_changeme, context))
self.quads.append((xxx_todo_changeme, context))
def test_namespaces_via_manager(self):
"""
This tests that NamespaceManager.namespaces works correctly with an
abstract Store.
"""
namespace_manager = NamespaceManager(Graph(store=Store()))
self.assertEqual(list(namespace_manager.namespaces()), [])
def add(self, triple, context, quoted=False):
"""\
Add a triple to the store of triples.
"""
# add dictionary entries for spo[s][p][p] = 1 and pos[p][o][s]
# = 1, creating the nested dictionaries where they do not yet
# exits.
Store.add(self, triple, context, quoted=quoted)
if context is not None:
self.__all_contexts.add(context)
subject, predicate, object_ = triple
self.__add_triple_context(triple, context, quoted)
spo = self.__spo
try:
po = spo[subject]
except LookupError:
po = spo[subject] = {}
try:
o = po[predicate]
except LookupError:
o = po[predicate] = {}
o[object_] = 1
pos = self.__pos
try:
os = pos[predicate]
except LookupError:
os = pos[predicate] = {}
try:
s = os[object_]
except LookupError:
s = os[object_] = {}
s[subject] = 1
osp = self.__osp
try:
sp = osp[object_]
except LookupError:
sp = osp[object_] = {}
try:
p = sp[subject]
except LookupError:
p = sp[subject] = {}
p[predicate] = 1
def __init__(self, configuration=None, identifier=None):
""" ProxyStore initialization.
Creates an empty Graph, intializes the HTTP client.
Use the defaut for internal graph storage, i.e IOMemory.
The URIref of the graph must be supplied either in identifier or
in configuration parameter. It will be checked by open().
The cache file path could be given in the configuration dictionary
(__init__ only). We have to search about the memory cache.
"""
LOG.debug("-- ProxyStore.init(configuration=%s, identifer=%s) --\n",
configuration, identifier)
self._identifier = identifier
self._format = None
self._etags = None
self._req_headers = {}
self.configuration = None
configuration = self._configuration_extraction(configuration)
self._graph = Graph()
# Most important parameter : identifier and graph address
# If not given, we can not go further
if (identifier is not None) and len(identifier) > 0:
if len(configuration) == 0:
configuration = {PS_CONFIG_URI: identifier}
# Show the network activity
if PS_CONFIG_DEBUG_HTTP in configuration.keys():
httplib2.debuglevel = 1
# File path for HTTPLIB2 cache
# As it is a file cache, it is conserved between two executions
# Should we delete the directory on application end (i.e close()) ?
if PS_CONFIG_HTTP_CACHE in configuration.keys():
self.httpserver = httplib2.Http(configuration[PS_CONFIG_HTTP_CACHE])
else:
self.httpserver = httplib2.Http(CACHE_DIR)
# Store will call open() if configuration is not None
Store.__init__(self, configuration)
def triples(self, triple, context=None):
""" Returns an iterator over all the triples (within the conjunctive
graph or just the given context) matching the given pattern.
:param triple: Triple (subject, predicate, object) to remove.
:param context: ProxyStore is not context aware but it's internal
cache IOMemory store is. Avoid context parameter.
:returns: An iterator over the triples.
"""
LOG.debug("-- ProxyStore.triples(triple=%s, context=%s) --", triple,
context)
Store.triples(self, triple) #, context=None)
self._pull()
return self._graph.store.triples(triple) #, context=None)
def triples(self, triple, context=None):
""" Returns an iterator over all the triples (within the conjunctive
graph or just the given context) matching the given pattern.
:param triple: Triple (subject, predicate, object) to remove.
:param context: ProxyStore is not context aware but it's internal
cache IOMemory store is. Avoid context parameter.
:returns: An iterator over the triples.
"""
LOG.debug("-- ProxyStore.triples(triple=%s, context=%s) --",
triple, context)
Store.triples(self, triple) #, context=None)
self._pull()
return self._graph.store.triples(triple) #, context=None)
def __init__(self, configuration=None, identifier=None):
""" ProxyStore initialization.
Creates an empty Graph, intializes the HTTP client.
Use the defaut for internal graph storage, i.e IOMemory.
The URIref of the graph must be supplied either in identifier or
in configuration parameter. It will be checked by open().
The cache file path could be given in the configuration dictionary
(__init__ only). We have to search about the memory cache.
"""
LOG.debug("-- ProxyStore.init(configuration=%s, identifer=%s) --\n",
configuration, identifier)
self._identifier = identifier
self._format = None
self._etags = None
self._req_headers = {}
self.configuration = None
configuration = self._configuration_extraction(configuration)
self._graph = Graph()
# Most important parameter : identifier and graph address
# If not given, we can not go further
if (identifier is not None) and len(identifier) > 0:
if len(configuration) == 0:
configuration = {PS_CONFIG_URI: identifier}
# Show the network activity
if PS_CONFIG_DEBUG_HTTP in configuration.keys():
httplib2.debuglevel = 1
# Use provided Http connection if any
http_cx = configuration.get(PS_CONFIG_HTTP_CX)
if http_cx is None:
http_cx = httplib2.Http()
else:
assert isinstance(http_cx, httplib2.Http)
self.httpserver = http_cx
# Store will call open() if configuration is not None
Store.__init__(self, configuration)
def add(self, triple, context, quoted=False, txn=None):
"""\
Add a triple to the store of triples.
"""
(subject, predicate, object) = triple
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
cspo, cpos, cosp = self.__indicies
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
if value is None:
self.__contexts.put(bb(c), "", txn=txn)
contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o)),
txn=txn) or "".encode("latin-1")
contexts = set(contexts_value.split("^".encode("latin-1")))
contexts.add(bb(c))
contexts_value = "^".encode("latin-1").join(contexts)
assert contexts_value is not None
cspo.put(bb("%s^%s^%s^%s^" % (c, s, p, o)), "", txn=txn)
cpos.put(bb("%s^%s^%s^%s^" % (c, p, o, s)), "", txn=txn)
cosp.put(bb("%s^%s^%s^%s^" % (c, o, s, p)), "", txn=txn)
if not quoted:
cspo.put(bb("%s^%s^%s^%s^" % ("", s, p, o)),
contexts_value,
txn=txn)
cpos.put(bb("%s^%s^%s^%s^" % ("", p, o, s)),
contexts_value,
txn=txn)
cosp.put(bb("%s^%s^%s^%s^" % ("", o, s, p)),
contexts_value,
txn=txn)
self.__needs_sync = True
def remove(self, triplepat, context=None):
Store.remove(self, triplepat, context)
context = getattr(context, 'identifier', context)
if context is None:
context = DEFAULT
defid = self._obj2id(DEFAULT)
req_cid = self._obj2id(context)
was_in_all_triples = False
for triple, contexts in self.triples(triplepat, context):
was_in_all_triples = self._is_in_all_triples
enctriple = self._encodeTriple(triple)
for cid in self._getTripleContexts(enctriple):
if context is not DEFAULT and req_cid != cid:
continue
self._removeTripleContext(enctriple, cid)
ctxs = self._getTripleContexts(enctriple, skipQuoted=True)
if defid in ctxs and (context is DEFAULT or len(ctxs) == 1):
self._removeTripleContext(enctriple, defid)
if len(self._getTripleContexts(enctriple)) == 0:
# triple has been removed from all contexts
sid, pid, oid = enctriple
self._subjectIndex[sid].remove(enctriple)
self._predicateIndex[pid].remove(enctriple)
self._objectIndex[oid].remove(enctriple)
del self._tripleContexts[enctriple]
if triplepat == (None, None, None) and \
context in self._all_contexts and \
not self.graph_aware:
# remove the whole context but not empty graphs
self._all_contexts.remove(context)
self._contextIndex.pop(req_cid)
if was_in_all_triples:
trips = self._contextIndex.get(req_cid, None)
if trips is not None:
if isinstance(triplepat[2], tuple):
raise NotImplementedError('Cannot remove ranges')
else:
self._contextIndex.pop(req_cid)
def addN(self, quads):
c1, c2, c3 = 0, 0, 0
for qgroup in grouper(quads, 10000):
encquads = list()
for q in qgroup:
ctx = _fix_ctx(q[3])
Store.add(self, q[:3], q[3], False)
encquads.append(((self._obj2id(q[0]), self._obj2id(q[1]),
self._obj2id(q[2])), self._obj2id(ctx), ctx))
for enctriple, cid, context in encquads:
if context is not DEFAULT:
self._all_contexts.add(context)
self._addTripleContext(enctriple, cid, False)
c1 += self._addN_helper(encquads, self._subjectIndex, 0)
c2 += self._addN_helper(encquads, self._predicateIndex, 1)
c3 += self._addN_helper(encquads, self._objectIndex, 2)
assert c1 == c2
assert c2 == c3
return c1
def add(self, triple, context, quoted=False, txn=None):
"""\
Add a triple to the store of triples.
"""
(subject, predicate, object) = triple
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
cspo, cpos, cosp = self.__indicies
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
if value is None:
self.__contexts.put(bb(c), "", txn=txn)
contexts_value = cspo.get(
bb("%s^%s^%s^%s^" % ("", s, p, o)), txn=txn) or b("")
contexts = set(contexts_value.split(b("^")))
contexts.add(bb(c))
contexts_value = b("^").join(contexts)
assert contexts_value is not None
cspo.put(bb("%s^%s^%s^%s^" % (c, s, p, o)), "", txn=txn)
cpos.put(bb("%s^%s^%s^%s^" % (c, p, o, s)), "", txn=txn)
cosp.put(bb("%s^%s^%s^%s^" % (c, o, s, p)), "", txn=txn)
if not quoted:
cspo.put(bb(
"%s^%s^%s^%s^" % ("", s, p, o)), contexts_value, txn=txn)
cpos.put(bb(
"%s^%s^%s^%s^" % ("", p, o, s)), contexts_value, txn=txn)
cosp.put(bb(
"%s^%s^%s^%s^" % ("", o, s, p)), contexts_value, txn=txn)
self.__needs_sync = True
def remove(self, triple, context):
"""Remove the set of triples matching the pattern from the store
:param triple: Triple (subject, predicate, object) to remove.
:param context:
:returns:
"""
# pylint: disable-msg=W0222
# Signature differs from overriden method
LOG.debug("-- ProxyStore.remove(triple=%s, context=%s) --", triple,
context)
Store.remove(self, triple, context)
if triple == (None, None, None):
self._graph = Graph()
# the default implementation of Graph is not efficient in doing
# this, so better create a new empty one
else:
self._graph.store.remove(triple)
def remove(self, triple, context):
"""Remove the set of triples matching the pattern from the store
:param triple: Triple (subject, predicate, object) to remove.
:param context:
:returns:
"""
# pylint: disable-msg=W0222
# Signature differs from overriden method
LOG.debug("-- ProxyStore.remove(triple=%s, context=%s) --",
triple, context)
Store.remove(self, triple, context)
if triple == (None, None, None):
self._graph = Graph()
# the default implementation of Graph is not efficient in doing
# this, so better create a new empty one
else:
self._graph.store.remove(triple)
def add(self, xxx_todo_changeme, context, quoted=False):
"""\
Add a triple to the store of triples.
"""
(subject, predicate, object) = xxx_todo_changeme
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
# Add the triple to the Store, triggering TripleAdded events
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject)
p = _to_string(predicate)
o = _to_string(object)
c = _to_string(context)
cspo, cpos, cosp = self.__indices
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
if value is None:
self.__contexts.set(bb(c), "")
contexts_value = cspo.get(bb(
"%s^%s^%s^%s^" % ("", s, p, o))) or b("")
contexts = set(contexts_value.split(b("^")))
contexts.add(bb(c))
contexts_value = b("^").join(contexts)
assert contexts_value != None
cspo.set(bb("%s^%s^%s^%s^" % (c, s, p, o)), "")
cpos.set(bb("%s^%s^%s^%s^" % (c, p, o, s)), "")
cosp.set(bb("%s^%s^%s^%s^" % (c, o, s, p)), "")
if not quoted:
cspo.set(bb("%s^%s^%s^%s^" % ("", s, p, o)), contexts_value)
cpos.set(bb("%s^%s^%s^%s^" % ("", p, o, s)), contexts_value)
cosp.set(bb("%s^%s^%s^%s^" % ("", o, s, p)), contexts_value)
self.__needs_sync = True
def add(self, triple, context, quoted=False):
"""\
Add a triple to the store of triples.
"""
(subject, predicate, object) = triple
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
# Add the triple to the Store, triggering TripleAdded events
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject)
p = _to_string(predicate)
o = _to_string(object)
c = _to_string(context)
cspo, cpos, cosp = self.__indices
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
if value is None:
self.__contexts.set(bb(c), "")
contexts_value = cspo.get(bb(
"%s^%s^%s^%s^" % ("", s, p, o))) or b("")
contexts = set(contexts_value.split(b("^")))
contexts.add(bb(c))
contexts_value = b("^").join(contexts)
assert contexts_value != None
cspo.set(bb("%s^%s^%s^%s^" % (c, s, p, o)), "")
cpos.set(bb("%s^%s^%s^%s^" % (c, p, o, s)), "")
cosp.set(bb("%s^%s^%s^%s^" % (c, o, s, p)), "")
if not quoted:
cspo.set(bb("%s^%s^%s^%s^" % ("", s, p, o)), contexts_value)
cpos.set(bb("%s^%s^%s^%s^" % ("", p, o, s)), contexts_value)
cosp.set(bb("%s^%s^%s^%s^" % ("", o, s, p)), contexts_value)
self.__needs_sync = True
self.__prefix.close()
self.__i2k.close()
self.__k2i.close()
self.db_env.close()
def add(self,
(subject, predicate, object),
context,
quoted=False,
txn=None):
"""\
Add a triple to the store of triples.
"""
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
cspo, cpos, cosp = self.__indicies
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
if value is None:
self.__contexts.put(bb(c), "", txn=txn)
contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o)),
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
self.update("CREATE GRAPH <%s>" % graph.identifier)
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
self.update("CREATE GRAPH <%s>" % graph.identifier)
def addN(self, quads):
Store.addN(self, quads)
for i, _to_key, _from_key in self.__indicies_info:
i.delete(_to_key((s, p, o), c), txn=txn)
if not quoted:
if contexts_value:
for i, _to_key, _from_key in self.__indicies_info:
i.put(_to_key((s, p, o), b("")), contexts_value, txn=txn)
else:
for i, _to_key, _from_key in self.__indicies_info:
try:
i.delete(_to_key((s, p, o), b("")), txn=txn)
except db.DBNotFoundError:
pass # TODO: is it okay to ignore these?
def remove(self, (subject, predicate, object), context, txn=None):
assert self.__open, "The Store must be open."
Store.remove(self, (subject, predicate, object), context)
_to_string = self._to_string
if context is not None:
if context == self:
context = None
if subject is not None \
and predicate is not None \
and object is not None \
and context is not None:
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
def remove(self, spo, context, txn=None):
subject, predicate, object = spo
assert self.__open, "The Store must be open."
Store.remove(self, (subject, predicate, object), context)
_to_string = self._to_string
if context is not None:
if context == self:
context = None
if subject is not None \
and predicate is not None \
and object is not None \
and context is not None:
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
(c, s, p, o)), txn=txn)
if value is not None:
self.__remove((bb(s), bb(p), bb(o)), bb(c), txn=txn)
self.__needs_sync = True
else:
cspo, cpos, cosp = self.__indicies
index, prefix, from_key, results_from_key = self.__lookup(
(subject, predicate, object), context, txn=txn)
cursor = index.cursor(txn=txn)
try:
current = cursor.set_range(prefix)
needs_sync = True
except db.DBNotFoundError:
current = None
needs_sync = False
cursor.close()
while current:
key, value = current
cursor = index.cursor(txn=txn)
try:
cursor.set_range(key)
# Hack to stop 2to3 converting this to next(cursor)
current = getattr(cursor, 'next')()
except db.DBNotFoundError:
current = None
cursor.close()
if key.startswith(prefix):
c, s, p, o = from_key(key)
if context is None:
contexts_value = index.get(key, txn=txn) or b("")
# remove triple from all non quoted contexts
contexts = set(contexts_value.split(b("^")))
# and from the conjunctive index
contexts.add(b(""))
for c in contexts:
for i, _to_key, _ in self.__indicies_info:
i.delete(_to_key((s, p, o), c), txn=txn)
else:
self.__remove((s, p, o), c, txn=txn)
else:
break
if context is not None:
if subject is None and predicate is None and object is None:
# TODO: also if context becomes empty and not just on
# remove((None, None, None), c)
try:
self.__contexts.delete(
bb(_to_string(context, txn=txn)), txn=txn)
except db.DBNotFoundError:
pass
self.__needs_sync = needs_sync
def destroy(self, configuration=''):
import os
path = configuration or self.homeDir
if os.path.exists(path):
for f in os.listdir(path):
os.unlink(path+'/'+f)
os.rmdir(path)
def add(self, (subject, predicate, object), context, quoted=False):
"""\
Add a triple to the store of triples.
"""
assert self.__open, "The Store must be open."
assert context != self, "Can not add triple directly to store"
# Add the triple to the Store, triggering TripleAdded events
Store.add(self, (subject, predicate, object), context, quoted)
_to_string = self._to_string
s = _to_string(subject)
p = _to_string(predicate)
o = _to_string(object)
c = _to_string(context)
cspo, cpos, cosp = self.__indices
value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
if value is None:
self.__contexts.set(bb(c), "")
contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o))) or b("")
def add(self, xxx_todo_changeme, context, quoted=False):
Store.add(self, xxx_todo_changeme, context, quoted)
s1, p1, o1 = xxx_todo_changeme
self.quads.append(((s1, p1, o1), context))
self.quoted_info.append(quoted)
def test_namespaces(self):
"""
This tests that Store.namespaces is an empty generator.
"""
store = Store()
self.assertEqual(list(store.namespaces()), [])
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
else:
self._all_contexts.add(getattr(graph, 'identifier', graph))
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
else:
self.__all_contexts.add(graph)
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
self.update(
"CREATE GRAPH %s" % self.node_to_sparql(graph.identifier))
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
self.update("CREATE GRAPH %s" %
self.node_to_sparql(graph.identifier))
def remove(self, spo, context, txn=None):
subject, predicate, object = spo
assert self.__open, "The Store must be open."
Store.remove(self, (subject, predicate, object), context)
_to_string = self._to_string
if context is not None:
if context == self:
context = None
if (
subject is not None
and predicate is not None
and object is not None
and context is not None
):
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
value = self.__indicies[0].get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
if value is not None:
self.__remove((bb(s), bb(p), bb(o)), bb(c), txn=txn)
self.__needs_sync = True
else:
cspo, cpos, cosp = self.__indicies
index, prefix, from_key, results_from_key = self.__lookup(
(subject, predicate, object), context, txn=txn
)
cursor = index.cursor(txn=txn)
try:
current = cursor.set_range(prefix)
needs_sync = True
except db.DBNotFoundError:
current = None
needs_sync = False
cursor.close()
while current:
key, value = current
cursor = index.cursor(txn=txn)
try:
cursor.set_range(key)
# Hack to stop 2to3 converting this to next(cursor)
current = getattr(cursor, "next")()
except db.DBNotFoundError:
current = None
cursor.close()
if key.startswith(prefix):
c, s, p, o = from_key(key)
if context is None:
contexts_value = index.get(key, txn=txn) or "".encode("latin-1")
# remove triple from all non quoted contexts
contexts = set(contexts_value.split("^".encode("latin-1")))
# and from the conjunctive index
contexts.add("".encode("latin-1"))
for c in contexts:
for i, _to_key, _ in self.__indicies_info:
i.delete(_to_key((s, p, o), c), txn=txn)
else:
self.__remove((s, p, o), c, txn=txn)
else:
break
if context is not None:
if subject is None and predicate is None and object is None:
# TODO: also if context becomes empty and not just on
# remove((None, None, None), c)
try:
self.__contexts.delete(
bb(_to_string(context, txn=txn)), txn=txn
)
except db.DBNotFoundError:
pass
self.__needs_sync = needs_sync
for i, _to_key, _from_key in self.__indicies_info:
i.delete(_to_key((s, p, o), c), txn=txn)
if not quoted:
if contexts_value:
for i, _to_key, _from_key in self.__indicies_info:
i.put(_to_key((s, p, o), b("")), contexts_value, txn=txn)
else:
for i, _to_key, _from_key in self.__indicies_info:
try:
i.delete(_to_key((s, p, o), b("")), txn=txn)
except db.DBNotFoundError:
pass # TODO: is it okay to ignore these?
def remove(self, (subject, predicate, object), context, txn=None):
assert self.__open, "The Store must be open."
Store.remove(self, (subject, predicate, object), context)
_to_string = self._to_string
if context is not None:
if context == self:
context = None
if subject is not None \
and predicate is not None \
and object is not None \
and context is not None:
s = _to_string(subject, txn=txn)
p = _to_string(predicate, txn=txn)
o = _to_string(object, txn=txn)
c = _to_string(context, txn=txn)
value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
def add_graph(self, graph):
if not self.graph_aware:
Store.add_graph(self, graph)
else:
self.__all_contexts.add(graph)
def add(self, triple, context, quoted=False):
"""\
Add a triple to the store.
"""
Store.add(self, triple, context, quoted)
for triple, cg in self.triples(triple, context):
# triple is already in the store.
return
subject, predicate, object = triple
f = self.forward
r = self.reverse
# assign keys for new identifiers
if subject not in r:
si = randid()
while si in f:
si = randid()
f[si] = subject
r[subject] = si
else:
si = r[subject]
if predicate not in r:
pi = randid()
while pi in f:
pi = randid()
f[pi] = predicate
r[predicate] = pi
else:
pi = r[predicate]
if object not in r:
oi = randid()
while oi in f:
oi = randid()
f[oi] = object
r[object] = oi
else:
oi = r[object]
if context not in r:
ci = randid()
while ci in f:
ci = randid()
f[ci] = context
r[context] = ci
else:
ci = r[context]
# add dictionary entries for cspo[c][s][p][o] = 1,
# cpos[c][p][o][s] = 1, and cosp[c][o][s][p] = 1, creating the
# nested {} where they do not yet exits.
self._setNestedIndex(self.cspo, ci, si, pi, oi)
self._setNestedIndex(self.cpos, ci, pi, oi, si)
self._setNestedIndex(self.cosp, ci, oi, si, pi)
if not quoted:
self._setNestedIndex(self.spo, si, pi, oi, ci)
self._setNestedIndex(self.pos, pi, oi, si, ci)
self._setNestedIndex(self.osp, oi, si, pi, ci)
def addN(self, quads):
Store.addN(self,quads)
def add(self, triple, context, quoted=False):
"""\
Add a triple to the store.
"""
Store.add(self, triple, context, quoted)
for triple, cg in self.triples(triple, context):
#triple is already in the store.
return
subject, predicate, object = triple
f = self.forward
r = self.reverse
# assign keys for new identifiers
if not r.has_key(subject):
si = randid()
while f.has_key(si):
si = randid()
f[si] = subject
r[subject] = si
else:
si = r[subject]
if not r.has_key(predicate):
pi = randid()
while f.has_key(pi):
pi = randid()
f[pi] = predicate
r[predicate] = pi
else:
pi = r[predicate]
if not r.has_key(object):
oi = randid()
while f.has_key(oi):
oi = randid()
f[oi] = object
r[object] = oi
else:
oi = r[object]
if not r.has_key(context):
ci = randid()
while f.has_key(ci):
ci = randid()
f[ci] = context
r[context] = ci
else:
ci = r[context]
# add dictionary entries for cspo[c][s][p][o] = 1,
# cpos[c][p][o][s] = 1, and cosp[c][o][s][p] = 1, creating the
# nested {} where they do not yet exits.
self._setNestedIndex(self.cspo, ci, si, pi, oi)
self._setNestedIndex(self.cpos, ci, pi, oi, si)
self._setNestedIndex(self.cosp, ci, oi, si, pi)
if not quoted:
self._setNestedIndex(self.spo, si, pi, oi, ci)
self._setNestedIndex(self.pos, pi, oi, si, ci)
self._setNestedIndex(self.osp, oi, si, pi, ci)
