def _construct_filters(self, domain, filterq):
if not domain or not filterq:
return None, None
subqlist = []
append = subqlist.append
included, excluded, opers = parse_filters(filterq)
args = []
if included:
subq = ("exists (select 1 from attributes where serial = v.serial "
"and domain = ? and ")
subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
subq += ")"
args += [domain]
args += included
append(subq)
if excluded:
subq = ("not exists (select 1 from attributes where "
"serial = v.serial and domain = ? and ")
subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
subq += ")"
args += [domain]
args += excluded
append(subq)
if opers:
for k, o, v in opers:
subq = ("exists (select 1 from attributes where "
"serial = v.serial and domain = ? and ")
subq += "key = ? and value %s ?" % (o,)
subq += ")"
args += [domain, k, v]
append(subq)
if not subqlist:
return None, None
subq = ' and ' + ' and '.join(subqlist)
return subq, args
def _construct_filters(self, domain, filterq):
if not domain or not filterq:
return None, None
subqlist = []
append = subqlist.append
included, excluded, opers = parse_filters(filterq)
args = []
if included:
subq = ("exists (select 1 from attributes where serial = v.serial "
"and domain = ? and ")
subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
subq += ")"
args += [domain]
args += included
append(subq)
if excluded:
subq = ("not exists (select 1 from attributes where "
"serial = v.serial and domain = ? and ")
subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
subq += ")"
args += [domain]
args += excluded
append(subq)
if opers:
for k, o, v in opers:
subq = ("exists (select 1 from attributes where "
"serial = v.serial and domain = ? and ")
subq += "key = ? and value %s ?" % (o, )
subq += ")"
args += [domain, k, v]
append(subq)
if not subqlist:
return None, None
subq = ' and ' + ' and '.join(subqlist)
return subq, args
def latest_version_list(self, parent, prefix='', delimiter=None,
start='', limit=10000, before=inf,
except_cluster=0, pathq=[], domain=None,
filterq=[], sizeq=None, all_props=False):
"""Return a (list of (path, serial) tuples, list of common prefixes)
for the current versions of the paths with the given parent,
matching the following criteria.
The property tuple for a version is returned if all
of these conditions are true:
a. parent matches
b. path > start
c. path starts with prefix (and paths in pathq)
d. version is the max up to before
e. version is not in cluster
f. the path does not have the delimiter occuring
after the prefix, or ends with the delimiter
g. serial matches the attribute filter query.
A filter query is a comma-separated list of
terms in one of these three forms:
key
an attribute with this key must exist
!key
an attribute with this key must not exist
key ?op value
the attribute with this key satisfies the value
where ?op is one of ==, != <=, >=, <, >.
h. the size is in the range set by sizeq
The list of common prefixes includes the prefixes
matching up to the first delimiter after prefix,
and are reported only once, as "virtual directories".
The delimiter is included in the prefixes.
If arguments are None, then the corresponding matching rule
will always match.
Limit applies to the first list of tuples returned.
If all_props is True, return all properties after path,
not just serial.
"""
if not start or start < prefix:
start = strprevling(prefix)
nextling = strnextling(prefix)
v = self.versions.alias('v')
if before != inf:
filtered = select([func.max(v.c.serial)],
and_(v.c.mtime < before,
v.c.node == self.versions.c.node))
inner_join = self.nodes.join(
self.versions,
onclause=self.versions.c.serial == filtered)
else:
filtered = select([self.nodes.c.latest_version])
filtered = filtered.where(
self.nodes.c.node == self.versions.c.node).\
correlate(self.versions)
inner_join = self.nodes.join(
self.versions,
onclause=self.versions.c.serial == filtered)
if not all_props:
s = select([self.nodes.c.path, self.versions.c.serial],
from_obj=[inner_join]).distinct()
else:
s = select([self.nodes.c.path,
self.versions.c.serial, self.versions.c.node,
self.versions.c.hash,
self.versions.c.size, self.versions.c.type,
self.versions.c.source,
self.versions.c.mtime, self.versions.c.muser,
self.versions.c.uuid,
self.versions.c.checksum,
self.versions.c.cluster],
from_obj=[inner_join]).distinct()
s = s.where(self.versions.c.cluster != except_cluster)
s = s.where(self.versions.c.node.in_(select([self.nodes.c.node],
self.nodes.c.parent == parent)))
s = s.where(self.versions.c.node == self.nodes.c.node)
s = s.where(and_(self.nodes.c.path > bindparam('start'),
self.nodes.c.path < nextling))
conja = []
conjb = []
for path, match in pathq:
if match == MATCH_PREFIX:
conja.append(
self.nodes.c.path.like(self.escape_like(path) + '%',
escape=ESCAPE_CHAR))
elif match == MATCH_EXACT:
conjb.append(path)
if conja or conjb:
s = s.where(or_(self.nodes.c.path.in_(conjb), *conja))
if sizeq and len(sizeq) == 2:
if sizeq[0]:
s = s.where(self.versions.c.size >= sizeq[0])
if sizeq[1]:
s = s.where(self.versions.c.size < sizeq[1])
if domain and filterq:
included, excluded, opers = parse_filters(filterq)
if included:
subs = select([1])
subs = subs.where(
self.attributes.c.serial ==
self.versions.c.serial).correlate(self.versions)
subs = subs.where(self.attributes.c.domain == domain)
subs = subs.where(or_(*[self.attributes.c.key.op('=')(x) for
x in included]))
s = s.where(exists(subs))
if excluded:
subs = select([1])
subs = subs.where(
self.attributes.c.serial == self.versions.c.serial).\
correlate(self.versions)
subs = subs.where(self.attributes.c.domain == domain)
subs = subs.where(or_(*[self.attributes.c.key.op('=')(x) for
x in excluded]))
s = s.where(not_(exists(subs)))
if opers:
for k, o, val in opers:
subs = select([1])
subs = subs.where(
self.attributes.c.serial == self.versions.c.serial).\
correlate(self.versions)
subs = subs.where(self.attributes.c.domain == domain)
subs = subs.where(
and_(self.attributes.c.key.op('=')(k),
self.attributes.c.value.op(o)(val)))
s = s.where(exists(subs))
s = s.order_by(self.nodes.c.path)
if not delimiter:
s = s.limit(limit)
rp = self.conn.execute(s, start=start)
r = rp.fetchall()
rp.close()
return r, ()
pfz = len(prefix)
dz = len(delimiter)
count = 0
prefixes = []
pappend = prefixes.append
matches = []
mappend = matches.append
rp = self.conn.execute(s, start=start)
while True:
props = rp.fetchone()
if props is None:
break
path = props[0]
idx = path.find(delimiter, pfz)
if idx < 0:
mappend(props)
count += 1
if count >= limit:
break
continue
if idx + dz == len(path):
mappend(props)
count += 1
continue # Get one more, in case there is a path.
pf = path[:idx + dz]
pappend(pf)
if count >= limit:
break
rp = self.conn.execute(s, start=strnextling(pf)) # New start.
rp.close()
return matches, prefixes
def latest_version_list(self,
parent,
prefix='',
delimiter=None,
start='',
limit=10000,
before=inf,
except_cluster=0,
pathq=[],
domain=None,
filterq=[],
sizeq=None,
all_props=False):
"""Return a (list of (path, serial) tuples, list of common prefixes)
for the current versions of the paths with the given parent,
matching the following criteria.
The property tuple for a version is returned if all
of these conditions are true:
a. parent matches
b. path > start
c. path starts with prefix (and paths in pathq)
d. version is the max up to before
e. version is not in cluster
f. the path does not have the delimiter occuring
after the prefix, or ends with the delimiter
g. serial matches the attribute filter query.
A filter query is a comma-separated list of
terms in one of these three forms:
key
an attribute with this key must exist
!key
an attribute with this key must not exist
key ?op value
the attribute with this key satisfies the value
where ?op is one of ==, != <=, >=, <, >.
h. the size is in the range set by sizeq
The list of common prefixes includes the prefixes
matching up to the first delimiter after prefix,
and are reported only once, as "virtual directories".
The delimiter is included in the prefixes.
If arguments are None, then the corresponding matching rule
will always match.
Limit applies to the first list of tuples returned.
If all_props is True, return all properties after path, not just serial.
"""
if not start or start < prefix:
start = strprevling(prefix)
nextling = strnextling(prefix)
v = self.versions.alias('v')
n = self.nodes.alias('n')
if not all_props:
s = select([n.c.path, v.c.serial]).distinct()
else:
s = select([
n.c.path, v.c.serial, v.c.node, v.c.hash, v.c.size, v.c.type,
v.c.source, v.c.mtime, v.c.muser, v.c.uuid, v.c.checksum,
v.c.cluster
]).distinct()
if before != inf:
filtered = select([func.max(self.versions.c.serial)])
filtered = filtered.where(self.versions.c.mtime < before)
else:
filtered = select([self.nodes.c.latest_version])
s = s.where(v.c.serial == filtered.where(
self.nodes.c.node == v.c.node))
s = s.where(v.c.cluster != except_cluster)
s = s.where(
v.c.node.in_(
select([self.nodes.c.node], self.nodes.c.parent == parent)))
s = s.where(n.c.node == v.c.node)
s = s.where(and_(n.c.path > bindparam('start'), n.c.path < nextling))
conj = []
for path, match in pathq:
if match == MATCH_PREFIX:
conj.append(
n.c.path.like(self.escape_like(path) + '%',
escape=ESCAPE_CHAR))
elif match == MATCH_EXACT:
conj.append(n.c.path == path)
if conj:
s = s.where(or_(*conj))
if sizeq and len(sizeq) == 2:
if sizeq[0]:
s = s.where(v.c.size >= sizeq[0])
if sizeq[1]:
s = s.where(v.c.size < sizeq[1])
if domain and filterq:
a = self.attributes.alias('a')
included, excluded, opers = parse_filters(filterq)
if included:
subs = select([1])
subs = subs.where(a.c.serial == v.c.serial).correlate(v)
subs = subs.where(a.c.domain == domain)
subs = subs.where(or_(*[a.c.key.op('=')(x) for x in included]))
s = s.where(exists(subs))
if excluded:
subs = select([1])
subs = subs.where(a.c.serial == v.c.serial).correlate(v)
subs = subs.where(a.c.domain == domain)
subs = subs.where(or_(*[a.c.key.op('=')(x) for x in excluded]))
s = s.where(not_(exists(subs)))
if opers:
for k, o, val in opers:
subs = select([1])
subs = subs.where(a.c.serial == v.c.serial).correlate(v)
subs = subs.where(a.c.domain == domain)
subs = subs.where(
and_(a.c.key.op('=')(k),
a.c.value.op(o)(val)))
s = s.where(exists(subs))
s = s.order_by(n.c.path)
if not delimiter:
s = s.limit(limit)
rp = self.conn.execute(s, start=start)
r = rp.fetchall()
rp.close()
return r, ()
pfz = len(prefix)
dz = len(delimiter)
count = 0
prefixes = []
pappend = prefixes.append
matches = []
mappend = matches.append
rp = self.conn.execute(s, start=start)
while True:
props = rp.fetchone()
if props is None:
break
path = props[0]
serial = props[1]
idx = path.find(delimiter, pfz)
if idx < 0:
mappend(props)
count += 1
if count >= limit:
break
continue
if idx + dz == len(path):
mappend(props)
count += 1
continue # Get one more, in case there is a path.
pf = path[:idx + dz]
pappend(pf)
if count >= limit:
break
rp = self.conn.execute(s, start=strnextling(pf)) # New start.
rp.close()
return matches, prefixes
