Example #1
0
    def add(self, triple, context=None, quoted=False):
        """ Add a triple to the store.
            Apply the modifications on the cache, trigger an exception if data
            has already been modified on the server.
            
            :param triple: Triple (subject, predicate, object) to add.
            :param context: 
            :param quoted: The quoted argument is interpreted by formula-aware
                stores to indicate this statement is quoted/hypothetical. It
                should be an error to not specify a context and have the
                quoted argument be True. It should also be an error for the
                quoted argument to be True when the store is not
                formula-aware.

            :returns: 
        """

        LOG.debug("-- ProxyStore.add(triple=%s, context=%s, quoted=%s) --", 
                  triple, context, quoted)

        assert self._identifier is not None, "The store must be open."

        # TODO LATER : Wrong, assert is made to test bugs
        assert self._format is not None, "The store must be open."
        assert quoted == False, "The store -proxyStore- is not formula-aware"

        Store.add(self, triple, context, quoted)

        # Instruction suivant extraite du plugin Sleepycat
        # Store.add(self, (subject, predicate, object), context, quoted)
        self._graph.add(triple)
Example #2
0
    def add(self, triple, context, quoted=False):
        # oldlen = len(self)
        Store.add(self, triple, context, quoted)
        context = getattr(context, 'identifier', context)
        if context is None:
            context = DEFAULT
        if context is not DEFAULT and context not in self.__all_contexts:
            self.__all_contexts.add(context)

        enctriple = self.__encodeTriple(triple)
        sid, pid, oid = enctriple

        self.__addTripleContext(enctriple, context, quoted)

        if sid in self.__subjectIndex:
            self.__subjectIndex[sid].add(enctriple)
        else:
            self.__subjectIndex[sid] = self.family.OO.Set((enctriple,))

        if pid in self.__predicateIndex:
            self.__predicateIndex[pid].add(enctriple)
        else:
            self.__predicateIndex[pid] = self.family.OO.Set((enctriple,))

        if oid in self.__objectIndex:
            self.__objectIndex[oid].add(enctriple)
        else:
            self.__objectIndex[oid] = self.family.OO.Set((enctriple,))
Example #3
0
    def add(self, triple, context, quoted=False):
        Store.add(self, triple, context, quoted)

        if context is not None:
            self.__all_contexts.add(context)

        enctriple = self.__encodeTriple(triple)
        sid, pid, oid = enctriple

        self.__addTripleContext(enctriple, context, quoted)

        if sid in self.__subjectIndex:
            self.__subjectIndex[sid].add(enctriple)
        else:
            self.__subjectIndex[sid] = set([enctriple])

        if pid in self.__predicateIndex:
            self.__predicateIndex[pid].add(enctriple)
        else:
            self.__predicateIndex[pid] = set([enctriple])

        if oid in self.__objectIndex:
            self.__objectIndex[oid].add(enctriple)
        else:
            self.__objectIndex[oid] = set([enctriple])
Example #4
0
 def remove_graph(self, graph):
     if not self.graph_aware:
         Store.remove_graph(self, graph)
     elif graph.identifier == DATASET_DEFAULT_GRAPH_ID:
         self.update("DROP DEFAULT")
     else:
         self.update("DROP GRAPH <%s>" % graph.identifier)
Example #5
0
 def remove_graph(self, graph):
     if not self.graph_aware:
         Store.remove_graph(self, graph)
     elif graph.identifier == DATASET_DEFAULT_GRAPH_ID:
         self.update("DROP DEFAULT")
     else:
         self.update(
             "DROP GRAPH %s" % self.node_to_sparql(graph.identifier))
Example #6
0
 def remove_graph(self, graph):
     if not self.graph_aware:
         Store.remove_graph(self, graph)
     else:
         self.remove((None, None, None), graph)
         try:
             self.__all_contexts.remove(graph)
         except KeyError:
             pass  # we didn't know this graph, no problem
Example #7
0
 def remove_graph(self, graph):
     if not self.graph_aware:
         Store.remove_graph(self, graph)
     else:
         self.remove((None, None, None), graph)
         try:
             self.__all_contexts.remove(graph)
         except KeyError:
             pass  # we didn't know this graph, no problem
Example #8
0
    def add(self, triple, context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        # add dictionary entries for spo[s][p][p] = 1 and pos[p][o][s]
        # = 1, creating the nested dictionaries where they do not yet
        # exits.
        Store.add(self, triple, context, quoted=quoted)
        if context is not None:
            self.__all_contexts.add(context)
        subject, predicate, object_ = triple

        spo = self.__spo
        try:
            po = spo[subject]
        except LookupError:
            po = spo[subject] = {}
        try:
            o = po[predicate]
        except LookupError:
            o = po[predicate] = {}

        try:
            _ = o[object_]
            # This cannot be reached if (s, p, o) was not inserted before.
            triple_exists = True
        except KeyError:
            o[object_] = 1
            triple_exists = False
        self.__add_triple_context(triple, triple_exists, context, quoted)

        if triple_exists:
            # No need to insert twice this triple.
            return

        pos = self.__pos
        try:
            os = pos[predicate]
        except LookupError:
            os = pos[predicate] = {}
        try:
            s = os[object_]
        except LookupError:
            s = os[object_] = {}
        s[subject] = 1

        osp = self.__osp
        try:
            sp = osp[object_]
        except LookupError:
            sp = osp[object_] = {}
        try:
            p = sp[subject]
        except LookupError:
            p = sp[subject] = {}
        p[predicate] = 1
Example #9
0
    def remove(self, triple, context=None):
        Store.remove(self, triple, context)
        if context is not None:
            if context == self:
                context = None

        # f = self.forward
        r = self.reverse
        if context is None:
            for triple, cg in self.triples(triple):
                subject, predicate, object = triple
                si, pi, oi = self.identifierToInt((subject, predicate, object))
                contexts = list(self.contexts(triple))
                for context in contexts:
                    ci = r[context]
                    del self.cspo[ci][si][pi][oi]
                    del self.cpos[ci][pi][oi][si]
                    del self.cosp[ci][oi][si][pi]

                    self._removeNestedIndex(self.spo, si, pi, oi, ci)
                    self._removeNestedIndex(self.pos, pi, oi, si, ci)
                    self._removeNestedIndex(self.osp, oi, si, pi, ci)
                    # grr!! hafta ref-count these before you can collect
                    # them dumbass!
                    #del f[si], f[pi], f[oi]
                    #del r[subject], r[predicate], r[object]
        else:
            subject, predicate, object = triple
            ci = r.get(context, None)
            if ci:
                for triple, cg in self.triples(triple, context):
                    si, pi, oi = self.identifierToInt(triple)
                    del self.cspo[ci][si][pi][oi]
                    del self.cpos[ci][pi][oi][si]
                    del self.cosp[ci][oi][si][pi]

                    try:
                        self._removeNestedIndex(self.spo, si, pi, oi, ci)
                        self._removeNestedIndex(self.pos, pi, oi, si, ci)
                        self._removeNestedIndex(self.osp, oi, si, pi, ci)
                    except KeyError:
                        # the context may be a quoted one in which
                        # there will not be a triple in spo, pos or
                        # osp. So ignore any KeyErrors
                        pass
                    # TODO delete references to resources in
                    # self.forward/self.reverse that are not in use anymore...

            if subject is None and predicate is None and object is None:
                # remove context
                try:
                    ci = self.reverse[context]
                    del self.cspo[ci], self.cpos[ci], self.cosp[ci]
                except KeyError:
                    # TODO: no exception when removing non-existant context?
                    pass
Example #10
0
    def remove(self, triple, context=None):
        Store.remove(self, triple, context)
        if context is not None:
            if context == self:
                context = None

        # f = self.forward
        r = self.reverse
        if context is None:
            for triple, cg in self.triples(triple):
                subject, predicate, object = triple
                si, pi, oi = self.identifierToInt((subject, predicate, object))
                contexts = list(self.contexts(triple))
                for context in contexts:
                    ci = r[context]
                    del self.cspo[ci][si][pi][oi]
                    del self.cpos[ci][pi][oi][si]
                    del self.cosp[ci][oi][si][pi]

                    self._removeNestedIndex(self.spo, si, pi, oi, ci)
                    self._removeNestedIndex(self.pos, pi, oi, si, ci)
                    self._removeNestedIndex(self.osp, oi, si, pi, ci)
                    # grr!! hafta ref-count these before you can collect
                    # them dumbass!
                    # del f[si], f[pi], f[oi]
                    # del r[subject], r[predicate], r[object]
        else:
            subject, predicate, object = triple
            ci = r.get(context, None)
            if ci:
                for triple, cg in self.triples(triple, context):
                    si, pi, oi = self.identifierToInt(triple)
                    del self.cspo[ci][si][pi][oi]
                    del self.cpos[ci][pi][oi][si]
                    del self.cosp[ci][oi][si][pi]

                    try:
                        self._removeNestedIndex(self.spo, si, pi, oi, ci)
                        self._removeNestedIndex(self.pos, pi, oi, si, ci)
                        self._removeNestedIndex(self.osp, oi, si, pi, ci)
                    except KeyError:
                        # the context may be a quoted one in which
                        # there will not be a triple in spo, pos or
                        # osp. So ignore any KeyErrors
                        pass
                    # TODO delete references to resources in
                    # self.forward/self.reverse that are not in use anymore...

            if subject is None and predicate is None and object is None:
                # remove context
                try:
                    ci = self.reverse[context]
                    del self.cspo[ci], self.cpos[ci], self.cosp[ci]
                except KeyError:
                    # TODO: no exception when removing non-existant context?
                    pass
Example #11
0
    def remove(self, triple, context):
        (subject, predicate, object) = triple
        assert self.__open, "The Store must be open."
        Store.remove(self, (subject, predicate, object), context)
        _to_string = self._to_string

        if context is not None:
            if context == self:
                context = None
        if subject is not None \
                and predicate is not None \
                and object is not None \
                and context is not None:
            s = _to_string(subject)
            p = _to_string(predicate)
            o = _to_string(object)
            c = _to_string(context)
            value = self.__indices[0].get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
            if value is not None:
                self.__remove((bb(s), bb(p), bb(o)), bb(c))
                self.__needs_sync = True
        else:
            cspo, cpos, cosp = self.__indices
            index, prefix, from_key, results_from_key = self.__lookup(
                                    (subject, predicate, object), context)

            needs_sync = False
            for key in index.match_prefix(prefix):
                c, s, p, o = from_key(key)
                if context is None:
                    contexts_value = index.get(key) or b("")
                    # remove triple from all non quoted contexts
                    contexts = set(contexts_value.split(b("^")))
                    contexts.add(b(""))  # and from the conjunctive index
                    for c in contexts:
                        for i, _to_key, _ in self.__indices_info:
                            i.remove(_to_key((s, p, o), c))
                else:
                    self.__remove((s, p, o), c)
                needs_sync = True
            if context is not None:
                if subject is None and predicate is None and object is None:
                    # TODO: also if context becomes empty and not just on
                    # remove((None, None, None), c)
                    try:
                        self.__contexts.remove(bb(_to_string(context)))
                    # except db.DBNotFoundError, e:
                    #     pass
                    except Exception as e:  # pragma: NO COVER
                        print("%s, Failed to delete %s" % (
                            e, context))  # pragma: NO COVER
                        pass  # pragma: NO COVER

            self.__needs_sync = needs_sync
Example #12
0
    def remove(self, triple, context, txn=None):
        assert self.__open, "The Store must be open."
        Store.remove(self, triple, context)

        for result in self.triples(triple):
            (spo, ctx) = result
            result = self.session.run(
                "CALL n10s.rdf.delete.inline($rdf,'N-Triples')",
                rdf=self.__serialise(spo)).single()
            if (result["terminationStatus"]) == "KO":
                raise Exception("Could not delete triple from Neo4j: ",
                                result["extraInfo"])
Example #13
0
    def add(self, triple, context=None, quoted=False):
        """
        Add a triple and start indexing.

        :param tuple(rdflib.Identifier) triple: Tuple of three identifiers.
        :param context: Context identifier. ``None`` inserts in the default
            graph.
        :type context: rdflib.Identifier or None
        :param bool quoted: Not used.
        """
        context = self._normalize_context(context)
        if context is None:
            context = RDFLIB_DEFAULT_GRAPH_URI

        Store.add(self, triple, context)

        #logger.info('Adding triple: {}'.format(triple))
        pk_trp = self._pickle(triple)

        pk_s, pk_p, pk_o = [self._pickle(t) for t in triple]
        #logger.debug('Adding quad: {} {}'.format(triple, context))
        pk_c = self._pickle(context)

        # Add new individual terms or gather keys for existing ones.
        keys = [None, None, None, None]
        with self.cur('th:t') as icur:
            for i, pk_t in enumerate((pk_s, pk_p, pk_o, pk_c)):
                thash = self._hash(pk_t)
                if icur.set_key(thash):
                    keys[i] = bytes(icur.value())
                else:
                    # Put new term.
                    with self.cur('t:st') as dcur:
                        keys[i] = self._append(dcur, (pk_t, ))[0]
                    # Index.
                    icur.put(thash, keys[i])

        # Add context in context DB.
        ck = keys[3]
        with self.cur('c:') as cur:
            if not cur.set_key(ck):
                cur.put(ck, b'')

        # Add triple:context association.
        spok = b''.join(keys[:3])
        with self.cur('spo:c') as dcur:
            if not dcur.set_key_dup(spok, ck):
                dcur.put(spok, ck)
        # Index spo:c association.
        with self.cur('c:spo') as icur:
            icur.put(ck, spok)

        self._index_triple('add', spok)
Example #14
0
 def add(self, triple, context, quoted=False):
     assert self.__open, "The Store must be open."
     assert context != self, "Can not add triple directly to store"
     Store.add(self, triple, context, quoted)
     if self.inbatch:
         self.tripleBuffer.append(self.__serialise(triple))
         if len(self.tripleBuffer) >= self.bufferMaxSize:
             self.__flushBuffer()
     else:
         result = self.session.run(
             "CALL n10s.rdf.import.inline($rdf,'N-Triples')",
             rdf=self.__serialise(triple)).single()
         if (result["terminationStatus"]) == "KO":
             raise Exception("Could not persist triple in Neo4j: ",
                             result["extraInfo"])
Example #15
0
 def add(self, xxx_todo_changeme, context, quoted=False):
     if quoted or self.locked:
         raise RuntimeError('nope')
     Store.add(self, xxx_todo_changeme, context, quoted)
     s1, p1, o1 = xxx_todo_changeme
     incoming = s1, p1, o1, context
     for pat in patterns:
         i = self.indexes[pat]
         for incoming_idx, l in enumerate(pat):
             if l == 'f':
                 i = i[None]
             else:
                 i = i[incoming[incoming_idx]]
         i.append((xxx_todo_changeme, context))
     self.quads.append((xxx_todo_changeme, context))
Example #16
0
 def test_namespaces_via_manager(self):
     """
     This tests that NamespaceManager.namespaces works correctly with an
     abstract Store.
     """
     namespace_manager = NamespaceManager(Graph(store=Store()))
     self.assertEqual(list(namespace_manager.namespaces()), [])
Example #17
0
    def add(self, triple, context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        # add dictionary entries for spo[s][p][p] = 1 and pos[p][o][s]
        # = 1, creating the nested dictionaries where they do not yet
        # exits.
        Store.add(self, triple, context, quoted=quoted)
        if context is not None:
            self.__all_contexts.add(context)
        subject, predicate, object_ = triple
        self.__add_triple_context(triple, context, quoted)

        spo = self.__spo
        try:
            po = spo[subject]
        except LookupError:
            po = spo[subject] = {}
        try:
            o = po[predicate]
        except LookupError:
            o = po[predicate] = {}
        o[object_] = 1

        pos = self.__pos
        try:
            os = pos[predicate]
        except LookupError:
            os = pos[predicate] = {}
        try:
            s = os[object_]
        except LookupError:
            s = os[object_] = {}
        s[subject] = 1

        osp = self.__osp
        try:
            sp = osp[object_]
        except LookupError:
            sp = osp[object_] = {}
        try:
            p = sp[subject]
        except LookupError:
            p = sp[subject] = {}
        p[predicate] = 1
Example #18
0
    def __init__(self, configuration=None, identifier=None):
        """ ProxyStore initialization.

            Creates an empty Graph, intializes the HTTP client.
            Use the defaut for internal graph storage, i.e IOMemory.
            The URIref of the graph must be supplied either in identifier or
            in configuration parameter. It will be checked by open().
            The cache file path could be given in the configuration dictionary
            (__init__ only). We have to search about the memory cache.
        """

        LOG.debug("-- ProxyStore.init(configuration=%s, identifer=%s) --\n",
                  configuration, identifier)


        self._identifier = identifier
        self._format = None
        self._etags = None
        self._req_headers = {}

        self.configuration = None
        configuration = self._configuration_extraction(configuration)

        self._graph = Graph()

        # Most important parameter : identifier and graph address
        # If not given, we can not go further
        if (identifier is not None) and len(identifier) > 0:
            if len(configuration) == 0:
                configuration = {PS_CONFIG_URI: identifier}

        # Show the network activity
        if PS_CONFIG_DEBUG_HTTP in configuration.keys():
            httplib2.debuglevel = 1

        # File path for HTTPLIB2 cache
        # As it is a file cache, it is conserved between two executions
        # Should we delete the directory on application end (i.e close()) ?
        if PS_CONFIG_HTTP_CACHE in configuration.keys():
            self.httpserver = httplib2.Http(configuration[PS_CONFIG_HTTP_CACHE])
        else:
            self.httpserver = httplib2.Http(CACHE_DIR)

        # Store will call open() if configuration is not None
        Store.__init__(self, configuration)
Example #19
0
    def triples(self, triple, context=None):
        """ Returns an iterator over all the triples (within the conjunctive
        graph or just the given context) matching the given pattern.

        :param triple: Triple (subject, predicate, object) to remove.
        :param context: ProxyStore is not context aware but it's internal
            cache IOMemory store is. Avoid context parameter.

        :returns: An iterator over the triples.
        """
        LOG.debug("-- ProxyStore.triples(triple=%s, context=%s) --", triple,
                  context)

        Store.triples(self, triple)  #, context=None)

        self._pull()

        return self._graph.store.triples(triple)  #, context=None)
Example #20
0
    def triples(self, triple, context=None):
        """ Returns an iterator over all the triples (within the conjunctive
        graph or just the given context) matching the given pattern.

        :param triple: Triple (subject, predicate, object) to remove.
        :param context: ProxyStore is not context aware but it's internal
            cache IOMemory store is. Avoid context parameter.

        :returns: An iterator over the triples.
        """
        LOG.debug("-- ProxyStore.triples(triple=%s, context=%s) --", 
                  triple, context)

        Store.triples(self, triple) #, context=None)

        self._pull()

        return self._graph.store.triples(triple) #, context=None)
Example #21
0
    def __init__(self, configuration=None, identifier=None):
        """ ProxyStore initialization.

            Creates an empty Graph, intializes the HTTP client.
            Use the defaut for internal graph storage, i.e IOMemory.
            The URIref of the graph must be supplied either in identifier or
            in configuration parameter. It will be checked by open().
            The cache file path could be given in the configuration dictionary
            (__init__ only). We have to search about the memory cache.
        """

        LOG.debug("-- ProxyStore.init(configuration=%s, identifer=%s) --\n",
                  configuration, identifier)

        self._identifier = identifier
        self._format = None
        self._etags = None
        self._req_headers = {}

        self.configuration = None
        configuration = self._configuration_extraction(configuration)

        self._graph = Graph()

        # Most important parameter : identifier and graph address
        # If not given, we can not go further
        if (identifier is not None) and len(identifier) > 0:
            if len(configuration) == 0:
                configuration = {PS_CONFIG_URI: identifier}

        # Show the network activity
        if PS_CONFIG_DEBUG_HTTP in configuration.keys():
            httplib2.debuglevel = 1

        # Use provided Http connection if any
        http_cx = configuration.get(PS_CONFIG_HTTP_CX)
        if http_cx is None:
            http_cx = httplib2.Http()
        else:
            assert isinstance(http_cx, httplib2.Http)
        self.httpserver = http_cx

        # Store will call open() if configuration is not None
        Store.__init__(self, configuration)
Example #22
0
    def add(self, triple, context, quoted=False, txn=None):
        """\
        Add a triple to the store of triples.
        """
        (subject, predicate, object) = triple
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        Store.add(self, (subject, predicate, object), context, quoted)

        _to_string = self._to_string

        s = _to_string(subject, txn=txn)
        p = _to_string(predicate, txn=txn)
        o = _to_string(object, txn=txn)
        c = _to_string(context, txn=txn)

        cspo, cpos, cosp = self.__indicies

        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
        if value is None:
            self.__contexts.put(bb(c), "", txn=txn)

            contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o)),
                                      txn=txn) or "".encode("latin-1")
            contexts = set(contexts_value.split("^".encode("latin-1")))
            contexts.add(bb(c))
            contexts_value = "^".encode("latin-1").join(contexts)
            assert contexts_value is not None

            cspo.put(bb("%s^%s^%s^%s^" % (c, s, p, o)), "", txn=txn)
            cpos.put(bb("%s^%s^%s^%s^" % (c, p, o, s)), "", txn=txn)
            cosp.put(bb("%s^%s^%s^%s^" % (c, o, s, p)), "", txn=txn)
            if not quoted:
                cspo.put(bb("%s^%s^%s^%s^" % ("", s, p, o)),
                         contexts_value,
                         txn=txn)
                cpos.put(bb("%s^%s^%s^%s^" % ("", p, o, s)),
                         contexts_value,
                         txn=txn)
                cosp.put(bb("%s^%s^%s^%s^" % ("", o, s, p)),
                         contexts_value,
                         txn=txn)

            self.__needs_sync = True
Example #23
0
    def remove(self, triplepat, context=None):
        Store.remove(self, triplepat, context)
        context = getattr(context, 'identifier', context)
        if context is None:
            context = DEFAULT
        defid = self._obj2id(DEFAULT)
        req_cid = self._obj2id(context)

        was_in_all_triples = False

        for triple, contexts in self.triples(triplepat, context):
            was_in_all_triples = self._is_in_all_triples
            enctriple = self._encodeTriple(triple)
            for cid in self._getTripleContexts(enctriple):
                if context is not DEFAULT and req_cid != cid:
                    continue
                self._removeTripleContext(enctriple, cid)

            ctxs = self._getTripleContexts(enctriple, skipQuoted=True)
            if defid in ctxs and (context is DEFAULT or len(ctxs) == 1):
                self._removeTripleContext(enctriple, defid)
            if len(self._getTripleContexts(enctriple)) == 0:
                # triple has been removed from all contexts
                sid, pid, oid = enctriple
                self._subjectIndex[sid].remove(enctriple)
                self._predicateIndex[pid].remove(enctriple)
                self._objectIndex[oid].remove(enctriple)

                del self._tripleContexts[enctriple]

        if triplepat == (None, None, None) and \
                context in self._all_contexts and \
                not self.graph_aware:
            # remove the whole context but not empty graphs
            self._all_contexts.remove(context)
            self._contextIndex.pop(req_cid)
        if was_in_all_triples:
            trips = self._contextIndex.get(req_cid, None)
            if trips is not None:
                if isinstance(triplepat[2], tuple):
                    raise NotImplementedError('Cannot remove ranges')
                else:
                    self._contextIndex.pop(req_cid)
Example #24
0
    def addN(self, quads):
        c1, c2, c3 = 0, 0, 0
        for qgroup in grouper(quads, 10000):
            encquads = list()
            for q in qgroup:
                ctx = _fix_ctx(q[3])
                Store.add(self, q[:3], q[3], False)
                encquads.append(((self._obj2id(q[0]), self._obj2id(q[1]),
                                  self._obj2id(q[2])), self._obj2id(ctx), ctx))

            for enctriple, cid, context in encquads:
                if context is not DEFAULT:
                    self._all_contexts.add(context)
                self._addTripleContext(enctriple, cid, False)
            c1 += self._addN_helper(encquads, self._subjectIndex, 0)
            c2 += self._addN_helper(encquads, self._predicateIndex, 1)
            c3 += self._addN_helper(encquads, self._objectIndex, 2)
        assert c1 == c2
        assert c2 == c3
        return c1
Example #25
0
    def add(self, triple, context, quoted=False, txn=None):
        """\
        Add a triple to the store of triples.
        """
        (subject, predicate, object) = triple
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        Store.add(self, (subject, predicate, object), context, quoted)

        _to_string = self._to_string

        s = _to_string(subject, txn=txn)
        p = _to_string(predicate, txn=txn)
        o = _to_string(object, txn=txn)
        c = _to_string(context, txn=txn)

        cspo, cpos, cosp = self.__indicies

        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
        if value is None:
            self.__contexts.put(bb(c), "", txn=txn)

            contexts_value = cspo.get(
                bb("%s^%s^%s^%s^" % ("", s, p, o)), txn=txn) or b("")
            contexts = set(contexts_value.split(b("^")))
            contexts.add(bb(c))
            contexts_value = b("^").join(contexts)
            assert contexts_value is not None

            cspo.put(bb("%s^%s^%s^%s^" % (c, s, p, o)), "", txn=txn)
            cpos.put(bb("%s^%s^%s^%s^" % (c, p, o, s)), "", txn=txn)
            cosp.put(bb("%s^%s^%s^%s^" % (c, o, s, p)), "", txn=txn)
            if not quoted:
                cspo.put(bb(
                    "%s^%s^%s^%s^" % ("", s, p, o)), contexts_value, txn=txn)
                cpos.put(bb(
                    "%s^%s^%s^%s^" % ("", p, o, s)), contexts_value, txn=txn)
                cosp.put(bb(
                    "%s^%s^%s^%s^" % ("", o, s, p)), contexts_value, txn=txn)

            self.__needs_sync = True
Example #26
0
    def remove(self, triple, context):
        """Remove the set of triples matching the pattern from the store

        :param triple: Triple (subject, predicate, object) to remove.
        :param context: 

        :returns: 
        """
        # pylint: disable-msg=W0222
        # Signature differs from overriden method
        LOG.debug("-- ProxyStore.remove(triple=%s, context=%s) --", triple,
                  context)

        Store.remove(self, triple, context)

        if triple == (None, None, None):
            self._graph = Graph()
            # the default implementation of Graph is not efficient in doing
            # this, so better create a new empty one
        else:
            self._graph.store.remove(triple)
Example #27
0
    def remove(self, triple, context):
        """Remove the set of triples matching the pattern from the store

        :param triple: Triple (subject, predicate, object) to remove.
        :param context: 

        :returns: 
        """
        # pylint: disable-msg=W0222
        # Signature differs from overriden method
        LOG.debug("-- ProxyStore.remove(triple=%s, context=%s) --", 
                  triple, context)

        Store.remove(self, triple, context)

        if triple == (None, None, None):
            self._graph = Graph()
            # the default implementation of Graph is not efficient in doing
            # this, so better create a new empty one
        else:
            self._graph.store.remove(triple)
    def add(self, xxx_todo_changeme, context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        (subject, predicate, object) = xxx_todo_changeme
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        # Add the triple to the Store, triggering TripleAdded events
        Store.add(self, (subject, predicate, object), context, quoted)

        _to_string = self._to_string

        s = _to_string(subject)
        p = _to_string(predicate)
        o = _to_string(object)
        c = _to_string(context)

        cspo, cpos, cosp = self.__indices

        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
        if value is None:
            self.__contexts.set(bb(c), "")

            contexts_value = cspo.get(bb(
                "%s^%s^%s^%s^" % ("", s, p, o))) or b("")
            contexts = set(contexts_value.split(b("^")))
            contexts.add(bb(c))
            contexts_value = b("^").join(contexts)
            assert contexts_value != None

            cspo.set(bb("%s^%s^%s^%s^" % (c, s, p, o)), "")
            cpos.set(bb("%s^%s^%s^%s^" % (c, p, o, s)), "")
            cosp.set(bb("%s^%s^%s^%s^" % (c, o, s, p)), "")
            if not quoted:
                cspo.set(bb("%s^%s^%s^%s^" % ("", s, p, o)), contexts_value)
                cpos.set(bb("%s^%s^%s^%s^" % ("", p, o, s)), contexts_value)
                cosp.set(bb("%s^%s^%s^%s^" % ("", o, s, p)), contexts_value)
            self.__needs_sync = True
Example #29
0
    def add(self, triple, context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        (subject, predicate, object) = triple
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        # Add the triple to the Store, triggering TripleAdded events
        Store.add(self, (subject, predicate, object), context, quoted)

        _to_string = self._to_string

        s = _to_string(subject)
        p = _to_string(predicate)
        o = _to_string(object)
        c = _to_string(context)

        cspo, cpos, cosp = self.__indices

        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
        if value is None:
            self.__contexts.set(bb(c), "")

            contexts_value = cspo.get(bb(
                "%s^%s^%s^%s^" % ("", s, p, o))) or b("")
            contexts = set(contexts_value.split(b("^")))
            contexts.add(bb(c))
            contexts_value = b("^").join(contexts)
            assert contexts_value != None

            cspo.set(bb("%s^%s^%s^%s^" % (c, s, p, o)), "")
            cpos.set(bb("%s^%s^%s^%s^" % (c, p, o, s)), "")
            cosp.set(bb("%s^%s^%s^%s^" % (c, o, s, p)), "")
            if not quoted:
                cspo.set(bb("%s^%s^%s^%s^" % ("", s, p, o)), contexts_value)
                cpos.set(bb("%s^%s^%s^%s^" % ("", p, o, s)), contexts_value)
                cosp.set(bb("%s^%s^%s^%s^" % ("", o, s, p)), contexts_value)
            self.__needs_sync = True
Example #30
0
        self.__prefix.close()
        self.__i2k.close()
        self.__k2i.close()
        self.db_env.close()

    def add(self,
            (subject, predicate, object),
            context,
            quoted=False,
            txn=None):
        """\
        Add a triple to the store of triples.
        """
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        Store.add(self, (subject, predicate, object), context, quoted)

        _to_string = self._to_string

        s = _to_string(subject, txn=txn)
        p = _to_string(predicate, txn=txn)
        o = _to_string(object, txn=txn)
        c = _to_string(context, txn=txn)

        cspo, cpos, cosp = self.__indicies

        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
        if value is None:
            self.__contexts.put(bb(c), "", txn=txn)

            contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o)),
Example #31
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
         self.update("CREATE GRAPH <%s>" % graph.identifier)
Example #32
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
         self.update("CREATE GRAPH <%s>" % graph.identifier)
Example #33
0
 def addN(self, quads):
     Store.addN(self, quads)
Example #34
0
        for i, _to_key, _from_key in self.__indicies_info:
            i.delete(_to_key((s, p, o), c), txn=txn)
        if not quoted:
            if contexts_value:
                for i, _to_key, _from_key in self.__indicies_info:
                    i.put(_to_key((s, p, o), b("")), contexts_value, txn=txn)
            else:
                for i, _to_key, _from_key in self.__indicies_info:
                    try:
                        i.delete(_to_key((s, p, o), b("")), txn=txn)
                    except db.DBNotFoundError:
                        pass  # TODO: is it okay to ignore these?

    def remove(self, (subject, predicate, object), context, txn=None):
        assert self.__open, "The Store must be open."
        Store.remove(self, (subject, predicate, object), context)
        _to_string = self._to_string

        if context is not None:
            if context == self:
                context = None

        if subject is not None \
                and predicate is not None \
                and object is not None \
                and context is not None:
            s = _to_string(subject, txn=txn)
            p = _to_string(predicate, txn=txn)
            o = _to_string(object, txn=txn)
            c = _to_string(context, txn=txn)
            value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
Example #35
0
    def remove(self, spo, context, txn=None):
        subject, predicate, object = spo
        assert self.__open, "The Store must be open."
        Store.remove(self, (subject, predicate, object), context)
        _to_string = self._to_string

        if context is not None:
            if context == self:
                context = None

        if subject is not None \
                and predicate is not None \
                and object is not None \
                and context is not None:
            s = _to_string(subject, txn=txn)
            p = _to_string(predicate, txn=txn)
            o = _to_string(object, txn=txn)
            c = _to_string(context, txn=txn)
            value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
                                              (c, s, p, o)), txn=txn)
            if value is not None:
                self.__remove((bb(s), bb(p), bb(o)), bb(c), txn=txn)
                self.__needs_sync = True
        else:
            cspo, cpos, cosp = self.__indicies
            index, prefix, from_key, results_from_key = self.__lookup(
                (subject, predicate, object), context, txn=txn)

            cursor = index.cursor(txn=txn)
            try:
                current = cursor.set_range(prefix)
                needs_sync = True
            except db.DBNotFoundError:
                current = None
                needs_sync = False
            cursor.close()
            while current:
                key, value = current
                cursor = index.cursor(txn=txn)
                try:
                    cursor.set_range(key)
                    # Hack to stop 2to3 converting this to next(cursor)
                    current = getattr(cursor, 'next')()
                except db.DBNotFoundError:
                    current = None
                cursor.close()
                if key.startswith(prefix):
                    c, s, p, o = from_key(key)
                    if context is None:
                        contexts_value = index.get(key, txn=txn) or b("")
                        # remove triple from all non quoted contexts
                        contexts = set(contexts_value.split(b("^")))
                        # and from the conjunctive index
                        contexts.add(b(""))
                        for c in contexts:
                            for i, _to_key, _ in self.__indicies_info:
                                i.delete(_to_key((s, p, o), c), txn=txn)
                    else:
                        self.__remove((s, p, o), c, txn=txn)
                else:
                    break

            if context is not None:
                if subject is None and predicate is None and object is None:
                    # TODO: also if context becomes empty and not just on
                    # remove((None, None, None), c)
                    try:
                        self.__contexts.delete(
                            bb(_to_string(context, txn=txn)), txn=txn)
                    except db.DBNotFoundError:
                        pass

            self.__needs_sync = needs_sync
    def destroy(self, configuration=''):
        import os
        path = configuration or self.homeDir
        if os.path.exists(path):
            for f in os.listdir(path): 
                os.unlink(path+'/'+f)
            os.rmdir(path)

    def add(self, (subject, predicate, object), context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        assert self.__open, "The Store must be open."
        assert context != self, "Can not add triple directly to store"
        # Add the triple to the Store, triggering TripleAdded events
        Store.add(self, (subject, predicate, object), context, quoted)
        
        _to_string = self._to_string
        
        s = _to_string(subject)
        p = _to_string(predicate)
        o = _to_string(object)
        c = _to_string(context)
        
        cspo, cpos, cosp = self.__indices
        
        value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)))
        if value is None:
            self.__contexts.set(bb(c), "")
            
            contexts_value = cspo.get(bb("%s^%s^%s^%s^" % ("", s, p, o))) or b("")
Example #37
0
 def add(self, xxx_todo_changeme, context, quoted=False):
     Store.add(self, xxx_todo_changeme, context, quoted)
     s1, p1, o1 = xxx_todo_changeme
     self.quads.append(((s1, p1, o1), context))
     self.quoted_info.append(quoted)
Example #38
0
 def test_namespaces(self):
     """
     This tests that Store.namespaces is an empty generator.
     """
     store = Store()
     self.assertEqual(list(store.namespaces()), [])
Example #39
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     else:
         self._all_contexts.add(getattr(graph, 'identifier', graph))
Example #40
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     else:
         self.__all_contexts.add(graph)
Example #41
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
         self.update(
             "CREATE GRAPH %s" % self.node_to_sparql(graph.identifier))
Example #42
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
         self.update("CREATE GRAPH %s" %
                     self.node_to_sparql(graph.identifier))
Example #43
0
    def remove(self, spo, context, txn=None):
        subject, predicate, object = spo
        assert self.__open, "The Store must be open."
        Store.remove(self, (subject, predicate, object), context)
        _to_string = self._to_string

        if context is not None:
            if context == self:
                context = None

        if (
            subject is not None
            and predicate is not None
            and object is not None
            and context is not None
        ):
            s = _to_string(subject, txn=txn)
            p = _to_string(predicate, txn=txn)
            o = _to_string(object, txn=txn)
            c = _to_string(context, txn=txn)
            value = self.__indicies[0].get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
            if value is not None:
                self.__remove((bb(s), bb(p), bb(o)), bb(c), txn=txn)
                self.__needs_sync = True
        else:
            cspo, cpos, cosp = self.__indicies
            index, prefix, from_key, results_from_key = self.__lookup(
                (subject, predicate, object), context, txn=txn
            )

            cursor = index.cursor(txn=txn)
            try:
                current = cursor.set_range(prefix)
                needs_sync = True
            except db.DBNotFoundError:
                current = None
                needs_sync = False
            cursor.close()
            while current:
                key, value = current
                cursor = index.cursor(txn=txn)
                try:
                    cursor.set_range(key)
                    # Hack to stop 2to3 converting this to next(cursor)
                    current = getattr(cursor, "next")()
                except db.DBNotFoundError:
                    current = None
                cursor.close()
                if key.startswith(prefix):
                    c, s, p, o = from_key(key)
                    if context is None:
                        contexts_value = index.get(key, txn=txn) or "".encode("latin-1")
                        # remove triple from all non quoted contexts
                        contexts = set(contexts_value.split("^".encode("latin-1")))
                        # and from the conjunctive index
                        contexts.add("".encode("latin-1"))
                        for c in contexts:
                            for i, _to_key, _ in self.__indicies_info:
                                i.delete(_to_key((s, p, o), c), txn=txn)
                    else:
                        self.__remove((s, p, o), c, txn=txn)
                else:
                    break

            if context is not None:
                if subject is None and predicate is None and object is None:
                    # TODO: also if context becomes empty and not just on
                    # remove((None, None, None), c)
                    try:
                        self.__contexts.delete(
                            bb(_to_string(context, txn=txn)), txn=txn
                        )
                    except db.DBNotFoundError:
                        pass

            self.__needs_sync = needs_sync
Example #44
0
        for i, _to_key, _from_key in self.__indicies_info:
            i.delete(_to_key((s, p, o), c), txn=txn)
        if not quoted:
            if contexts_value:
                for i, _to_key, _from_key in self.__indicies_info:
                    i.put(_to_key((s, p, o), b("")), contexts_value, txn=txn)
            else:
                for i, _to_key, _from_key in self.__indicies_info:
                    try:
                        i.delete(_to_key((s, p, o), b("")), txn=txn)
                    except db.DBNotFoundError:
                        pass  # TODO: is it okay to ignore these?

    def remove(self, (subject, predicate, object), context, txn=None):
        assert self.__open, "The Store must be open."
        Store.remove(self, (subject, predicate, object), context)
        _to_string = self._to_string

        if context is not None:
            if context == self:
                context = None

        if subject is not None \
                and predicate is not None \
                and object is not None \
                and context is not None:
            s = _to_string(subject, txn=txn)
            p = _to_string(predicate, txn=txn)
            o = _to_string(object, txn=txn)
            c = _to_string(context, txn=txn)
            value = self.__indicies[0].get(bb("%s^%s^%s^%s^" %
Example #45
0
 def add_graph(self, graph):
     if not self.graph_aware:
         Store.add_graph(self, graph)
     else:
         self.__all_contexts.add(graph)
Example #46
0
    def add(self, triple, context, quoted=False):
        """\
        Add a triple to the store.
        """
        Store.add(self, triple, context, quoted)
        for triple, cg in self.triples(triple, context):
            # triple is already in the store.
            return

        subject, predicate, object = triple

        f = self.forward
        r = self.reverse

        # assign keys for new identifiers

        if subject not in r:
            si = randid()
            while si in f:
                si = randid()
            f[si] = subject
            r[subject] = si
        else:
            si = r[subject]

        if predicate not in r:
            pi = randid()
            while pi in f:
                pi = randid()
            f[pi] = predicate
            r[predicate] = pi
        else:
            pi = r[predicate]

        if object not in r:
            oi = randid()
            while oi in f:
                oi = randid()
            f[oi] = object
            r[object] = oi
        else:
            oi = r[object]

        if context not in r:
            ci = randid()
            while ci in f:
                ci = randid()
            f[ci] = context
            r[context] = ci
        else:
            ci = r[context]

        # add dictionary entries for cspo[c][s][p][o] = 1,
        # cpos[c][p][o][s] = 1, and cosp[c][o][s][p] = 1, creating the
        # nested {} where they do not yet exits.
        self._setNestedIndex(self.cspo, ci, si, pi, oi)
        self._setNestedIndex(self.cpos, ci, pi, oi, si)
        self._setNestedIndex(self.cosp, ci, oi, si, pi)

        if not quoted:
            self._setNestedIndex(self.spo, si, pi, oi, ci)
            self._setNestedIndex(self.pos, pi, oi, si, ci)
            self._setNestedIndex(self.osp, oi, si, pi, ci)
Example #47
0
 def addN(self, quads):
     Store.addN(self,quads)
Example #48
0
    def add(self, triple, context, quoted=False):
        """\
        Add a triple to the store.
        """
        Store.add(self, triple, context, quoted)
        for triple, cg in self.triples(triple, context):
            #triple is already in the store.
            return

        subject, predicate, object = triple

        f = self.forward
        r = self.reverse

        # assign keys for new identifiers

        if not r.has_key(subject):
            si = randid()
            while f.has_key(si):
                si = randid()
            f[si] = subject
            r[subject] = si
        else:
            si = r[subject]

        if not r.has_key(predicate):
            pi = randid()
            while f.has_key(pi):
                pi = randid()
            f[pi] = predicate
            r[predicate] = pi
        else:
            pi = r[predicate]

        if not r.has_key(object):
            oi = randid()
            while f.has_key(oi):
                oi = randid()
            f[oi] = object
            r[object] = oi
        else:
            oi = r[object]

        if not r.has_key(context):
            ci = randid()
            while f.has_key(ci):
                ci = randid()
            f[ci] = context
            r[context] = ci
        else:
            ci = r[context]

        # add dictionary entries for cspo[c][s][p][o] = 1,
        # cpos[c][p][o][s] = 1, and cosp[c][o][s][p] = 1, creating the
        # nested {} where they do not yet exits.
        self._setNestedIndex(self.cspo, ci, si, pi, oi)
        self._setNestedIndex(self.cpos, ci, pi, oi, si)
        self._setNestedIndex(self.cosp, ci, oi, si, pi)

        if not quoted:
            self._setNestedIndex(self.spo, si, pi, oi, ci)
            self._setNestedIndex(self.pos, pi, oi, si, ci)
            self._setNestedIndex(self.osp, oi, si, pi, ci)