def retrieve(self, gl_rec, cache, fields=None, asdict=False):
"""
Retrieve a subfile - the entire subfile is returned.
Fields is the list of fields of interest.
"""
rv = []
storage = self.storage
gl_id, value = None, None
gbl, piece = storage.split(';')
sub_file = gl_rec[gbl]
# Ignore the header for now.
# Gets hit with nested subfiles.
if fields == None:
asdict = True
fields = self.fields
fieldnames = [f.label for f in fields]
for (rowid, value) in sub_file.keys_with_decendants():
if not valid_rowid(rowid):
break
row = []
sub_file_row = sub_file[rowid]
for field in fields:
row.append(field.retrieve(sub_file_row, cache))
if asdict:
row = dict(zip(fieldnames, row))
row['_rowid'] = rowid
rv.append(row)
return rv
def retrieve(self, gl_rec, cache, fields=None, asdict=False):
"""
Retrieve a subfile - the entire subfile is returned.
Fields is the list of fields of interest.
"""
rv = []
storage = self.storage
gl_id, value = None, None
gbl, piece = storage.split(";")
sub_file = gl_rec[gbl]
# Ignore the header for now.
# Gets hit with nested subfiles.
if fields == None:
asdict = True
fields = self.fields
fieldnames = [f.label for f in fields]
for (rowid, value) in sub_file.keys_with_decendants():
if not valid_rowid(rowid):
break
row = []
sub_file_row = sub_file[rowid]
for field in fields:
row.append(field.retrieve(sub_file_row, cache))
if asdict:
row = dict(zip(fieldnames, row))
row["_rowid"] = rowid
rv.append(row)
return rv
def subfile_traversal(stream, dd, ranges=None, ascending=True, explain=False):
"""
This is chained to a parent file traverser.
It receives a parent file rowid, and pulls the subfile rowids.
"""
parent_field = dd.parent_dd.fields[dd.parent_fieldid]
gl_subpath = parent_field.storage.split(';', 1)[0]
if explain:
for message in stream:
yield message
yield "subfile_traversal, ascending=%s, dd=%s, ranges=%s" % (
ascending, dd, ranges)
return
for rowid, rec_gl_closed_form, rowid_path in stream:
sf = M.Globals.from_closed_form(rec_gl_closed_form)[gl_subpath]
for sf_rowid, value in sf.keys_with_decendants(
): # TODO: smarter filtering
if sf_rowid == "0":
continue # file header
if not valid_rowid(sf_rowid):
break
yield sf_rowid, "%s%s)" % (
sf.open_form, sf_rowid), rowid_path + [gl_subpath, sf_rowid]
def subfile_traversal(stream, dd, ranges=None, ascending=True, explain=False):
"""
This is chained to a parent file traverser.
It receives a parent file rowid, and pulls the subfile rowids.
"""
parent_field = dd.parent_dd.fields[dd.parent_fieldid]
gl_subpath = parent_field.storage.split(";", 1)[0]
if explain:
for message in stream:
yield message
yield "subfile_traversal, ascending=%s, dd=%s, ranges=%s" % (ascending, dd, ranges)
return
for rowid, rec_gl_closed_form, rowid_path in stream:
sf = M.Globals.from_closed_form(rec_gl_closed_form)[gl_subpath]
for sf_rowid, value in sf.keys_with_decendants(): # TODO: smarter filtering
if sf_rowid == "0":
continue # file header
if not valid_rowid(sf_rowid):
break
yield sf_rowid, "%s%s)" % (sf.open_form, sf_rowid), rowid_path + [gl_subpath, sf_rowid]
def next(self):
# Have we exceeded limit
if self.limit and self.results_returned >= self.limit:
raise StopIteration
lastrowid = self.lastrowid # This value should be a string throughout.
if self.ascending:
asc = 1
else:
asc = -1
while not self.results_complete:
# If this is the first pass, we may have the id of a record, which needs to
# be verified
found = False
if self.first_pass:
self.first_pass = False
if lastrowid is None and asc == -1:
lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % self.gl, M.INOUT('%'))
if valid_rowid(lastrowid):
found = True
elif lastrowid and float(lastrowid) > 0:
row_exists, = M.mexec("""set s0=$data(%ss0))""" % (self.gl), M.INOUT(lastrowid))
if valid_rowid(row_exists):
found = True
if not found:
lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (self.gl, asc), M.INOUT(lastrowid))
if not valid_rowid(lastrowid):
break
# Check boundary values
f_lastrowid = float(lastrowid)
if self.ascending:
if self.from_rowid is not None:
if f_lastrowid == self.from_rowid and self.from_rule == ">":
continue
if self.to_rowid is not None:
if f_lastrowid >= self.to_rowid and self.to_rule == "<":
break
if f_lastrowid > self.to_rowid and self.to_rule == "<=":
break
else: # descending:
if f_lastrowid == 0:
break # header record
if self.from_rowid is not None:
if f_lastrowid == self.from_rowid and self.from_rule == "<":
continue
if self.to_rowid is not None:
if f_lastrowid <= self.to_rowid and self.to_rule == ">":
break
if f_lastrowid < self.to_rowid and self.to_rule == ">=":
break
if self.filters:
# Are filters to be applied?
if not self.filters(lastrowid):
continue
if self.skip_rows > 0:
self.skip_rows -= 1
continue
self.lastrowid = lastrowid
self.results_returned += 1
if self.raw:
return self.lastrowid
return self.getter(self.lastrowid)
self.results_complete = True
raise StopIteration
def next(self):
# Have we exceeded limit
if self.limit and self.results_returned >= self.limit:
raise StopIteration
lastrowid = self.lastrowid # This value should be a string throughout.
if self.ascending:
asc = 1
else:
asc = -1
while not self.results_complete:
# If this is the first pass, we may have the id of a record, which needs to
# be verified
found = False
if self.first_pass:
self.first_pass = False
if lastrowid is None and asc == -1:
lastrowid, = M.mexec(
"""set s0=$order(%ss0),-1)""" % self.gl, M.INOUT('%'))
if valid_rowid(lastrowid):
found = True
elif lastrowid and float(lastrowid) > 0:
row_exists, = M.mexec(
"""set s0=$data(%ss0))""" % (self.gl),
M.INOUT(lastrowid))
if valid_rowid(row_exists):
found = True
if not found:
lastrowid, = M.mexec(
"""set s0=$order(%ss0),%d)""" % (self.gl, asc),
M.INOUT(lastrowid))
if not valid_rowid(lastrowid):
break
# Check boundary values
f_lastrowid = float(lastrowid)
if self.ascending:
if self.from_rowid is not None:
if f_lastrowid == self.from_rowid and self.from_rule == ">":
continue
if self.to_rowid is not None:
if f_lastrowid >= self.to_rowid and self.to_rule == "<":
break
if f_lastrowid > self.to_rowid and self.to_rule == "<=":
break
else: # descending:
if f_lastrowid == 0:
break # header record
if self.from_rowid is not None:
if f_lastrowid == self.from_rowid and self.from_rule == "<":
continue
if self.to_rowid is not None:
if f_lastrowid <= self.to_rowid and self.to_rule == ">":
break
if f_lastrowid < self.to_rowid and self.to_rule == ">=":
break
if self.filters:
# Are filters to be applied?
if not self.filters(lastrowid):
continue
if self.skip_rows > 0:
self.skip_rows -= 1
continue
self.lastrowid = lastrowid
self.results_returned += 1
if self.raw:
return self.lastrowid
return self.getter(self.lastrowid)
self.results_complete = True
raise StopIteration
def file_order_traversal(gl,
ranges=None,
ascending=True,
sf_path=[],
explain=False):
"""
Originate records by traversing the file in file order (i.e. no index)
"""
if ranges:
r = ranges[0]
from_rowid = r['from_value']
to_rowid = r['to_value']
from_rule = r['from_rule']
to_rule = r['to_rule']
else:
from_rowid, to_rowid, from_rule, to_rule = None, None, None, None
if explain:
yield "file_order_traversal, ascending=%s, gl=%s, X %s %s AND X %s %s" % (
ascending, gl, from_rule, from_rowid, to_rule, to_rowid)
return
# the new person file has non-integer user ids
if from_rowid != None:
from_rowid = float(from_rowid)
if to_rowid != None:
to_rowid = float(to_rowid)
if from_rowid != None and to_rowid != None:
if ascending:
assert (from_rowid <= to_rowid)
else:
assert (to_rowid <= from_rowid)
if from_rowid is None:
if ascending:
lastrowid = "0"
else:
lastrowid = None
else:
# TODO: I have this in code in shared
lastrowid = ('%f' % from_rowid).rstrip('0').rstrip('.').lstrip('0')
if from_rowid > 0 and lastrowid[0] == "0":
lastrowid = lastrowid[1:]
if lastrowid.endswith(".0"):
lastrowid = lastrowid[:-2]
first_pass = True
if ascending:
asc = 1
else:
asc = -1
while 1:
# If this is the first pass, we may have the id of a record, which needs to
# be verified
found = False
if first_pass:
first_pass = False
if lastrowid is None and asc == -1:
lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % gl,
M.INOUT('%'))
if valid_rowid(lastrowid):
found = True
elif lastrowid and float(lastrowid) > 0:
row_exists, = M.mexec("""set s0=$data(%ss0))""" % (gl),
M.INOUT(lastrowid))
if valid_rowid(row_exists):
found = True
if not found:
lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (gl, asc),
M.INOUT(lastrowid))
if not valid_rowid(lastrowid):
break
# Check boundary values
f_lastrowid = float(lastrowid)
if ascending:
if from_rowid is not None:
if f_lastrowid == from_rowid and from_rule == ">":
continue
if to_rowid is not None:
if f_lastrowid >= to_rowid and to_rule == "<":
break
if f_lastrowid > to_rowid and to_rule == "<=":
break
else: # descending:
if f_lastrowid == 0:
break # header record
if from_rowid is not None:
if f_lastrowid == from_rowid and from_rule == "<":
continue
if to_rowid is not None:
if f_lastrowid <= to_rowid and to_rule == ">":
break
if f_lastrowid < to_rowid and to_rule == ">=":
break
# If this is a subfile, I need to return the full path.
yield (lastrowid, "%s%s)" % (gl, lastrowid), sf_path + [lastrowid])
def file_order_traversal(gl, ranges=None, ascending=True, sf_path=[], explain=False):
"""
Originate records by traversing the file in file order (i.e. no index)
"""
if ranges:
r = ranges[0]
from_rowid = r["from_value"]
to_rowid = r["to_value"]
from_rule = r["from_rule"]
to_rule = r["to_rule"]
else:
from_rowid, to_rowid, from_rule, to_rule = None, None, None, None
if explain:
yield "file_order_traversal, ascending=%s, gl=%s, X %s %s AND X %s %s" % (
ascending,
gl,
from_rule,
from_rowid,
to_rule,
to_rowid,
)
return
# the new person file has non-integer user ids
if from_rowid != None:
from_rowid = float(from_rowid)
if to_rowid != None:
to_rowid = float(to_rowid)
if from_rowid != None and to_rowid != None:
if ascending:
assert from_rowid <= to_rowid
else:
assert to_rowid <= from_rowid
if from_rowid is None:
if ascending:
lastrowid = "0"
else:
lastrowid = None
else:
# TODO: I have this in code in shared
lastrowid = ("%f" % from_rowid).rstrip("0").rstrip(".").lstrip("0")
if from_rowid > 0 and lastrowid[0] == "0":
lastrowid = lastrowid[1:]
if lastrowid.endswith(".0"):
lastrowid = lastrowid[:-2]
first_pass = True
if ascending:
asc = 1
else:
asc = -1
while 1:
# If this is the first pass, we may have the id of a record, which needs to
# be verified
found = False
if first_pass:
first_pass = False
if lastrowid is None and asc == -1:
lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % gl, M.INOUT("%"))
if valid_rowid(lastrowid):
found = True
elif lastrowid and float(lastrowid) > 0:
row_exists, = M.mexec("""set s0=$data(%ss0))""" % (gl), M.INOUT(lastrowid))
if valid_rowid(row_exists):
found = True
if not found:
lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (gl, asc), M.INOUT(lastrowid))
if not valid_rowid(lastrowid):
break
# Check boundary values
f_lastrowid = float(lastrowid)
if ascending:
if from_rowid is not None:
if f_lastrowid == from_rowid and from_rule == ">":
continue
if to_rowid is not None:
if f_lastrowid >= to_rowid and to_rule == "<":
break
if f_lastrowid > to_rowid and to_rule == "<=":
break
else: # descending:
if f_lastrowid == 0:
break # header record
if from_rowid is not None:
if f_lastrowid == from_rowid and from_rule == "<":
continue
if to_rowid is not None:
if f_lastrowid <= to_rowid and to_rule == ">":
break
if f_lastrowid < to_rowid and to_rule == ">=":
break
# If this is a subfile, I need to return the full path.
yield (lastrowid, "%s%s)" % (gl, lastrowid), sf_path + [lastrowid])
