Example #1
0
    def _construct_filters(self, domain, filterq):
        if not domain or not filterq:
            return None, None

        subqlist = []
        append = subqlist.append
        included, excluded, opers = parse_filters(filterq)
        args = []

        if included:
            subq = ("exists (select 1 from attributes where serial = v.serial "
                    "and domain = ? and ")
            subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
            subq += ")"
            args += [domain]
            args += included
            append(subq)

        if excluded:
            subq = ("not exists (select 1 from attributes where "
                    "serial = v.serial and domain = ? and ")
            subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
            subq += ")"
            args += [domain]
            args += excluded
            append(subq)

        if opers:
            for k, o, v in opers:
                subq = ("exists (select 1 from attributes where "
                        "serial = v.serial and domain = ? and ")
                subq += "key = ? and value %s ?" % (o,)
                subq += ")"
                args += [domain, k, v]
                append(subq)

        if not subqlist:
            return None, None

        subq = ' and ' + ' and '.join(subqlist)

        return subq, args
Example #2
0
    def _construct_filters(self, domain, filterq):
        if not domain or not filterq:
            return None, None

        subqlist = []
        append = subqlist.append
        included, excluded, opers = parse_filters(filterq)
        args = []

        if included:
            subq = ("exists (select 1 from attributes where serial = v.serial "
                    "and domain = ? and ")
            subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
            subq += ")"
            args += [domain]
            args += included
            append(subq)

        if excluded:
            subq = ("not exists (select 1 from attributes where "
                    "serial = v.serial and domain = ? and ")
            subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
            subq += ")"
            args += [domain]
            args += excluded
            append(subq)

        if opers:
            for k, o, v in opers:
                subq = ("exists (select 1 from attributes where "
                        "serial = v.serial and domain = ? and ")
                subq += "key = ? and value %s ?" % (o, )
                subq += ")"
                args += [domain, k, v]
                append(subq)

        if not subqlist:
            return None, None

        subq = ' and ' + ' and '.join(subqlist)

        return subq, args
Example #3
0
    def latest_version_list(self, parent, prefix='', delimiter=None,
                            start='', limit=10000, before=inf,
                            except_cluster=0, pathq=[], domain=None,
                            filterq=[], sizeq=None, all_props=False):
        """Return a (list of (path, serial) tuples, list of common prefixes)
           for the current versions of the paths with the given parent,
           matching the following criteria.

           The property tuple for a version is returned if all
           of these conditions are true:

                a. parent matches

                b. path > start

                c. path starts with prefix (and paths in pathq)

                d. version is the max up to before

                e. version is not in cluster

                f. the path does not have the delimiter occuring
                   after the prefix, or ends with the delimiter

                g. serial matches the attribute filter query.

                   A filter query is a comma-separated list of
                   terms in one of these three forms:

                   key
                       an attribute with this key must exist

                   !key
                       an attribute with this key must not exist

                   key ?op value
                       the attribute with this key satisfies the value
                       where ?op is one of ==, != <=, >=, <, >.

                h. the size is in the range set by sizeq

           The list of common prefixes includes the prefixes
           matching up to the first delimiter after prefix,
           and are reported only once, as "virtual directories".
           The delimiter is included in the prefixes.

           If arguments are None, then the corresponding matching rule
           will always match.

           Limit applies to the first list of tuples returned.

           If all_props is True, return all properties after path,
           not just serial.
        """

        if not start or start < prefix:
            start = strprevling(prefix)
        nextling = strnextling(prefix)

        v = self.versions.alias('v')
        if before != inf:
            filtered = select([func.max(v.c.serial)],
                              and_(v.c.mtime < before,
                                   v.c.node == self.versions.c.node))
            inner_join = self.nodes.join(
                self.versions,
                onclause=self.versions.c.serial == filtered)
        else:
            filtered = select([self.nodes.c.latest_version])
            filtered = filtered.where(
                self.nodes.c.node == self.versions.c.node).\
                correlate(self.versions)
            inner_join = self.nodes.join(
                self.versions,
                onclause=self.versions.c.serial == filtered)
        if not all_props:
            s = select([self.nodes.c.path, self.versions.c.serial],
                       from_obj=[inner_join]).distinct()
        else:
            s = select([self.nodes.c.path,
                        self.versions.c.serial, self.versions.c.node,
                        self.versions.c.hash,
                        self.versions.c.size, self.versions.c.type,
                        self.versions.c.source,
                        self.versions.c.mtime, self.versions.c.muser,
                        self.versions.c.uuid,
                        self.versions.c.checksum,
                        self.versions.c.cluster],
                       from_obj=[inner_join]).distinct()

        s = s.where(self.versions.c.cluster != except_cluster)
        s = s.where(self.versions.c.node.in_(select([self.nodes.c.node],
                                             self.nodes.c.parent == parent)))

        s = s.where(self.versions.c.node == self.nodes.c.node)
        s = s.where(and_(self.nodes.c.path > bindparam('start'),
                    self.nodes.c.path < nextling))
        conja = []
        conjb = []
        for path, match in pathq:
            if match == MATCH_PREFIX:
                conja.append(
                    self.nodes.c.path.like(self.escape_like(path) + '%',
                                           escape=ESCAPE_CHAR))
            elif match == MATCH_EXACT:
                conjb.append(path)
        if conja or conjb:
            s = s.where(or_(self.nodes.c.path.in_(conjb), *conja))

        if sizeq and len(sizeq) == 2:
            if sizeq[0]:
                s = s.where(self.versions.c.size >= sizeq[0])
            if sizeq[1]:
                s = s.where(self.versions.c.size < sizeq[1])

        if domain and filterq:
            included, excluded, opers = parse_filters(filterq)
            if included:
                subs = select([1])
                subs = subs.where(
                    self.attributes.c.serial ==
                    self.versions.c.serial).correlate(self.versions)
                subs = subs.where(self.attributes.c.domain == domain)
                subs = subs.where(or_(*[self.attributes.c.key.op('=')(x) for
                                  x in included]))
                s = s.where(exists(subs))
            if excluded:
                subs = select([1])
                subs = subs.where(
                    self.attributes.c.serial == self.versions.c.serial).\
                    correlate(self.versions)
                subs = subs.where(self.attributes.c.domain == domain)
                subs = subs.where(or_(*[self.attributes.c.key.op('=')(x) for
                                  x in excluded]))
                s = s.where(not_(exists(subs)))
            if opers:
                for k, o, val in opers:
                    subs = select([1])
                    subs = subs.where(
                        self.attributes.c.serial == self.versions.c.serial).\
                        correlate(self.versions)
                    subs = subs.where(self.attributes.c.domain == domain)
                    subs = subs.where(
                        and_(self.attributes.c.key.op('=')(k),
                             self.attributes.c.value.op(o)(val)))
                    s = s.where(exists(subs))

        s = s.order_by(self.nodes.c.path)

        if not delimiter:
            s = s.limit(limit)
            rp = self.conn.execute(s, start=start)
            r = rp.fetchall()
            rp.close()
            return r, ()

        pfz = len(prefix)
        dz = len(delimiter)
        count = 0
        prefixes = []
        pappend = prefixes.append
        matches = []
        mappend = matches.append

        rp = self.conn.execute(s, start=start)
        while True:
            props = rp.fetchone()
            if props is None:
                break
            path = props[0]
            idx = path.find(delimiter, pfz)

            if idx < 0:
                mappend(props)
                count += 1
                if count >= limit:
                    break
                continue

            if idx + dz == len(path):
                mappend(props)
                count += 1
                continue  # Get one more, in case there is a path.
            pf = path[:idx + dz]
            pappend(pf)
            if count >= limit:
                break

            rp = self.conn.execute(s, start=strnextling(pf))  # New start.
        rp.close()

        return matches, prefixes
Example #4
0
    def latest_version_list(self,
                            parent,
                            prefix='',
                            delimiter=None,
                            start='',
                            limit=10000,
                            before=inf,
                            except_cluster=0,
                            pathq=[],
                            domain=None,
                            filterq=[],
                            sizeq=None,
                            all_props=False):
        """Return a (list of (path, serial) tuples, list of common prefixes)
           for the current versions of the paths with the given parent,
           matching the following criteria.

           The property tuple for a version is returned if all
           of these conditions are true:

                a. parent matches

                b. path > start

                c. path starts with prefix (and paths in pathq)

                d. version is the max up to before

                e. version is not in cluster

                f. the path does not have the delimiter occuring
                   after the prefix, or ends with the delimiter

                g. serial matches the attribute filter query.

                   A filter query is a comma-separated list of
                   terms in one of these three forms:

                   key
                       an attribute with this key must exist

                   !key
                       an attribute with this key must not exist

                   key ?op value
                       the attribute with this key satisfies the value
                       where ?op is one of ==, != <=, >=, <, >.

                h. the size is in the range set by sizeq

           The list of common prefixes includes the prefixes
           matching up to the first delimiter after prefix,
           and are reported only once, as "virtual directories".
           The delimiter is included in the prefixes.

           If arguments are None, then the corresponding matching rule
           will always match.

           Limit applies to the first list of tuples returned.

           If all_props is True, return all properties after path, not just serial.
        """

        if not start or start < prefix:
            start = strprevling(prefix)
        nextling = strnextling(prefix)

        v = self.versions.alias('v')
        n = self.nodes.alias('n')
        if not all_props:
            s = select([n.c.path, v.c.serial]).distinct()
        else:
            s = select([
                n.c.path, v.c.serial, v.c.node, v.c.hash, v.c.size, v.c.type,
                v.c.source, v.c.mtime, v.c.muser, v.c.uuid, v.c.checksum,
                v.c.cluster
            ]).distinct()
        if before != inf:
            filtered = select([func.max(self.versions.c.serial)])
            filtered = filtered.where(self.versions.c.mtime < before)
        else:
            filtered = select([self.nodes.c.latest_version])
        s = s.where(v.c.serial == filtered.where(
            self.nodes.c.node == v.c.node))
        s = s.where(v.c.cluster != except_cluster)
        s = s.where(
            v.c.node.in_(
                select([self.nodes.c.node], self.nodes.c.parent == parent)))

        s = s.where(n.c.node == v.c.node)
        s = s.where(and_(n.c.path > bindparam('start'), n.c.path < nextling))
        conj = []
        for path, match in pathq:
            if match == MATCH_PREFIX:
                conj.append(
                    n.c.path.like(self.escape_like(path) + '%',
                                  escape=ESCAPE_CHAR))
            elif match == MATCH_EXACT:
                conj.append(n.c.path == path)
        if conj:
            s = s.where(or_(*conj))

        if sizeq and len(sizeq) == 2:
            if sizeq[0]:
                s = s.where(v.c.size >= sizeq[0])
            if sizeq[1]:
                s = s.where(v.c.size < sizeq[1])

        if domain and filterq:
            a = self.attributes.alias('a')
            included, excluded, opers = parse_filters(filterq)
            if included:
                subs = select([1])
                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
                subs = subs.where(a.c.domain == domain)
                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in included]))
                s = s.where(exists(subs))
            if excluded:
                subs = select([1])
                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
                subs = subs.where(a.c.domain == domain)
                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in excluded]))
                s = s.where(not_(exists(subs)))
            if opers:
                for k, o, val in opers:
                    subs = select([1])
                    subs = subs.where(a.c.serial == v.c.serial).correlate(v)
                    subs = subs.where(a.c.domain == domain)
                    subs = subs.where(
                        and_(a.c.key.op('=')(k),
                             a.c.value.op(o)(val)))
                    s = s.where(exists(subs))

        s = s.order_by(n.c.path)

        if not delimiter:
            s = s.limit(limit)
            rp = self.conn.execute(s, start=start)
            r = rp.fetchall()
            rp.close()
            return r, ()

        pfz = len(prefix)
        dz = len(delimiter)
        count = 0
        prefixes = []
        pappend = prefixes.append
        matches = []
        mappend = matches.append

        rp = self.conn.execute(s, start=start)
        while True:
            props = rp.fetchone()
            if props is None:
                break
            path = props[0]
            serial = props[1]
            idx = path.find(delimiter, pfz)

            if idx < 0:
                mappend(props)
                count += 1
                if count >= limit:
                    break
                continue

            if idx + dz == len(path):
                mappend(props)
                count += 1
                continue  # Get one more, in case there is a path.
            pf = path[:idx + dz]
            pappend(pf)
            if count >= limit:
                break

            rp = self.conn.execute(s, start=strnextling(pf))  # New start.
        rp.close()

        return matches, prefixes