def retrieve(self, gl_rec, cache, fields=None, asdict=False): """ Retrieve a subfile - the entire subfile is returned. Fields is the list of fields of interest. """ rv = [] storage = self.storage gl_id, value = None, None gbl, piece = storage.split(';') sub_file = gl_rec[gbl] # Ignore the header for now. # Gets hit with nested subfiles. if fields == None: asdict = True fields = self.fields fieldnames = [f.label for f in fields] for (rowid, value) in sub_file.keys_with_decendants(): if not valid_rowid(rowid): break row = [] sub_file_row = sub_file[rowid] for field in fields: row.append(field.retrieve(sub_file_row, cache)) if asdict: row = dict(zip(fieldnames, row)) row['_rowid'] = rowid rv.append(row) return rv
def retrieve(self, gl_rec, cache, fields=None, asdict=False): """ Retrieve a subfile - the entire subfile is returned. Fields is the list of fields of interest. """ rv = [] storage = self.storage gl_id, value = None, None gbl, piece = storage.split(";") sub_file = gl_rec[gbl] # Ignore the header for now. # Gets hit with nested subfiles. if fields == None: asdict = True fields = self.fields fieldnames = [f.label for f in fields] for (rowid, value) in sub_file.keys_with_decendants(): if not valid_rowid(rowid): break row = [] sub_file_row = sub_file[rowid] for field in fields: row.append(field.retrieve(sub_file_row, cache)) if asdict: row = dict(zip(fieldnames, row)) row["_rowid"] = rowid rv.append(row) return rv
def subfile_traversal(stream, dd, ranges=None, ascending=True, explain=False): """ This is chained to a parent file traverser. It receives a parent file rowid, and pulls the subfile rowids. """ parent_field = dd.parent_dd.fields[dd.parent_fieldid] gl_subpath = parent_field.storage.split(';', 1)[0] if explain: for message in stream: yield message yield "subfile_traversal, ascending=%s, dd=%s, ranges=%s" % ( ascending, dd, ranges) return for rowid, rec_gl_closed_form, rowid_path in stream: sf = M.Globals.from_closed_form(rec_gl_closed_form)[gl_subpath] for sf_rowid, value in sf.keys_with_decendants( ): # TODO: smarter filtering if sf_rowid == "0": continue # file header if not valid_rowid(sf_rowid): break yield sf_rowid, "%s%s)" % ( sf.open_form, sf_rowid), rowid_path + [gl_subpath, sf_rowid]
def subfile_traversal(stream, dd, ranges=None, ascending=True, explain=False): """ This is chained to a parent file traverser. It receives a parent file rowid, and pulls the subfile rowids. """ parent_field = dd.parent_dd.fields[dd.parent_fieldid] gl_subpath = parent_field.storage.split(";", 1)[0] if explain: for message in stream: yield message yield "subfile_traversal, ascending=%s, dd=%s, ranges=%s" % (ascending, dd, ranges) return for rowid, rec_gl_closed_form, rowid_path in stream: sf = M.Globals.from_closed_form(rec_gl_closed_form)[gl_subpath] for sf_rowid, value in sf.keys_with_decendants(): # TODO: smarter filtering if sf_rowid == "0": continue # file header if not valid_rowid(sf_rowid): break yield sf_rowid, "%s%s)" % (sf.open_form, sf_rowid), rowid_path + [gl_subpath, sf_rowid]
def next(self): # Have we exceeded limit if self.limit and self.results_returned >= self.limit: raise StopIteration lastrowid = self.lastrowid # This value should be a string throughout. if self.ascending: asc = 1 else: asc = -1 while not self.results_complete: # If this is the first pass, we may have the id of a record, which needs to # be verified found = False if self.first_pass: self.first_pass = False if lastrowid is None and asc == -1: lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % self.gl, M.INOUT('%')) if valid_rowid(lastrowid): found = True elif lastrowid and float(lastrowid) > 0: row_exists, = M.mexec("""set s0=$data(%ss0))""" % (self.gl), M.INOUT(lastrowid)) if valid_rowid(row_exists): found = True if not found: lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (self.gl, asc), M.INOUT(lastrowid)) if not valid_rowid(lastrowid): break # Check boundary values f_lastrowid = float(lastrowid) if self.ascending: if self.from_rowid is not None: if f_lastrowid == self.from_rowid and self.from_rule == ">": continue if self.to_rowid is not None: if f_lastrowid >= self.to_rowid and self.to_rule == "<": break if f_lastrowid > self.to_rowid and self.to_rule == "<=": break else: # descending: if f_lastrowid == 0: break # header record if self.from_rowid is not None: if f_lastrowid == self.from_rowid and self.from_rule == "<": continue if self.to_rowid is not None: if f_lastrowid <= self.to_rowid and self.to_rule == ">": break if f_lastrowid < self.to_rowid and self.to_rule == ">=": break if self.filters: # Are filters to be applied? if not self.filters(lastrowid): continue if self.skip_rows > 0: self.skip_rows -= 1 continue self.lastrowid = lastrowid self.results_returned += 1 if self.raw: return self.lastrowid return self.getter(self.lastrowid) self.results_complete = True raise StopIteration
def next(self): # Have we exceeded limit if self.limit and self.results_returned >= self.limit: raise StopIteration lastrowid = self.lastrowid # This value should be a string throughout. if self.ascending: asc = 1 else: asc = -1 while not self.results_complete: # If this is the first pass, we may have the id of a record, which needs to # be verified found = False if self.first_pass: self.first_pass = False if lastrowid is None and asc == -1: lastrowid, = M.mexec( """set s0=$order(%ss0),-1)""" % self.gl, M.INOUT('%')) if valid_rowid(lastrowid): found = True elif lastrowid and float(lastrowid) > 0: row_exists, = M.mexec( """set s0=$data(%ss0))""" % (self.gl), M.INOUT(lastrowid)) if valid_rowid(row_exists): found = True if not found: lastrowid, = M.mexec( """set s0=$order(%ss0),%d)""" % (self.gl, asc), M.INOUT(lastrowid)) if not valid_rowid(lastrowid): break # Check boundary values f_lastrowid = float(lastrowid) if self.ascending: if self.from_rowid is not None: if f_lastrowid == self.from_rowid and self.from_rule == ">": continue if self.to_rowid is not None: if f_lastrowid >= self.to_rowid and self.to_rule == "<": break if f_lastrowid > self.to_rowid and self.to_rule == "<=": break else: # descending: if f_lastrowid == 0: break # header record if self.from_rowid is not None: if f_lastrowid == self.from_rowid and self.from_rule == "<": continue if self.to_rowid is not None: if f_lastrowid <= self.to_rowid and self.to_rule == ">": break if f_lastrowid < self.to_rowid and self.to_rule == ">=": break if self.filters: # Are filters to be applied? if not self.filters(lastrowid): continue if self.skip_rows > 0: self.skip_rows -= 1 continue self.lastrowid = lastrowid self.results_returned += 1 if self.raw: return self.lastrowid return self.getter(self.lastrowid) self.results_complete = True raise StopIteration
def file_order_traversal(gl, ranges=None, ascending=True, sf_path=[], explain=False): """ Originate records by traversing the file in file order (i.e. no index) """ if ranges: r = ranges[0] from_rowid = r['from_value'] to_rowid = r['to_value'] from_rule = r['from_rule'] to_rule = r['to_rule'] else: from_rowid, to_rowid, from_rule, to_rule = None, None, None, None if explain: yield "file_order_traversal, ascending=%s, gl=%s, X %s %s AND X %s %s" % ( ascending, gl, from_rule, from_rowid, to_rule, to_rowid) return # the new person file has non-integer user ids if from_rowid != None: from_rowid = float(from_rowid) if to_rowid != None: to_rowid = float(to_rowid) if from_rowid != None and to_rowid != None: if ascending: assert (from_rowid <= to_rowid) else: assert (to_rowid <= from_rowid) if from_rowid is None: if ascending: lastrowid = "0" else: lastrowid = None else: # TODO: I have this in code in shared lastrowid = ('%f' % from_rowid).rstrip('0').rstrip('.').lstrip('0') if from_rowid > 0 and lastrowid[0] == "0": lastrowid = lastrowid[1:] if lastrowid.endswith(".0"): lastrowid = lastrowid[:-2] first_pass = True if ascending: asc = 1 else: asc = -1 while 1: # If this is the first pass, we may have the id of a record, which needs to # be verified found = False if first_pass: first_pass = False if lastrowid is None and asc == -1: lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % gl, M.INOUT('%')) if valid_rowid(lastrowid): found = True elif lastrowid and float(lastrowid) > 0: row_exists, = M.mexec("""set s0=$data(%ss0))""" % (gl), M.INOUT(lastrowid)) if valid_rowid(row_exists): found = True if not found: lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (gl, asc), M.INOUT(lastrowid)) if not valid_rowid(lastrowid): break # Check boundary values f_lastrowid = float(lastrowid) if ascending: if from_rowid is not None: if f_lastrowid == from_rowid and from_rule == ">": continue if to_rowid is not None: if f_lastrowid >= to_rowid and to_rule == "<": break if f_lastrowid > to_rowid and to_rule == "<=": break else: # descending: if f_lastrowid == 0: break # header record if from_rowid is not None: if f_lastrowid == from_rowid and from_rule == "<": continue if to_rowid is not None: if f_lastrowid <= to_rowid and to_rule == ">": break if f_lastrowid < to_rowid and to_rule == ">=": break # If this is a subfile, I need to return the full path. yield (lastrowid, "%s%s)" % (gl, lastrowid), sf_path + [lastrowid])
def file_order_traversal(gl, ranges=None, ascending=True, sf_path=[], explain=False): """ Originate records by traversing the file in file order (i.e. no index) """ if ranges: r = ranges[0] from_rowid = r["from_value"] to_rowid = r["to_value"] from_rule = r["from_rule"] to_rule = r["to_rule"] else: from_rowid, to_rowid, from_rule, to_rule = None, None, None, None if explain: yield "file_order_traversal, ascending=%s, gl=%s, X %s %s AND X %s %s" % ( ascending, gl, from_rule, from_rowid, to_rule, to_rowid, ) return # the new person file has non-integer user ids if from_rowid != None: from_rowid = float(from_rowid) if to_rowid != None: to_rowid = float(to_rowid) if from_rowid != None and to_rowid != None: if ascending: assert from_rowid <= to_rowid else: assert to_rowid <= from_rowid if from_rowid is None: if ascending: lastrowid = "0" else: lastrowid = None else: # TODO: I have this in code in shared lastrowid = ("%f" % from_rowid).rstrip("0").rstrip(".").lstrip("0") if from_rowid > 0 and lastrowid[0] == "0": lastrowid = lastrowid[1:] if lastrowid.endswith(".0"): lastrowid = lastrowid[:-2] first_pass = True if ascending: asc = 1 else: asc = -1 while 1: # If this is the first pass, we may have the id of a record, which needs to # be verified found = False if first_pass: first_pass = False if lastrowid is None and asc == -1: lastrowid, = M.mexec("""set s0=$order(%ss0),-1)""" % gl, M.INOUT("%")) if valid_rowid(lastrowid): found = True elif lastrowid and float(lastrowid) > 0: row_exists, = M.mexec("""set s0=$data(%ss0))""" % (gl), M.INOUT(lastrowid)) if valid_rowid(row_exists): found = True if not found: lastrowid, = M.mexec("""set s0=$order(%ss0),%d)""" % (gl, asc), M.INOUT(lastrowid)) if not valid_rowid(lastrowid): break # Check boundary values f_lastrowid = float(lastrowid) if ascending: if from_rowid is not None: if f_lastrowid == from_rowid and from_rule == ">": continue if to_rowid is not None: if f_lastrowid >= to_rowid and to_rule == "<": break if f_lastrowid > to_rowid and to_rule == "<=": break else: # descending: if f_lastrowid == 0: break # header record if from_rowid is not None: if f_lastrowid == from_rowid and from_rule == "<": continue if to_rowid is not None: if f_lastrowid <= to_rowid and to_rule == ">": break if f_lastrowid < to_rowid and to_rule == ">=": break # If this is a subfile, I need to return the full path. yield (lastrowid, "%s%s)" % (gl, lastrowid), sf_path + [lastrowid])