def from_AST(cls, node, factory): """Construct from an Enumerator node of form <vars> in <rel> Alternatively, the rhs may be a setmatch of a rel, where the mask is a lookupmask and the key is a vartuple. """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) rhs = node.iter if L.is_name(rhs): rel = L.get_name(rhs) elif isinstance(rhs, L.SetMatch) and L.is_vartuple(rhs.key): keyvars = L.get_vartuple(rhs.key) # Make sure we're dealing with a lookupmask and that the # key vars agree with the mask. mask = Mask(rhs.mask) assert mask.is_lookupmask assert mask.lookup_arity == len(keyvars) lhs = keyvars + lhs rel = L.get_name(rhs.target) else: raise TypeError return cls(lhs, rel)
def from_node(cls, node): checktype(node, L.Aggregate) if isinstance(node.value, L.DemQuery): assert all(isinstance(a, L.Name) for a in node.value.args) oper_demparams = tuple(a.id for a in node.value.args) oper_demname = node.value.demname oper = node.value.value else: oper_demparams = None oper_demname = None oper = node.value if isinstance(oper, L.Name): rel = oper.id relmask = Mask.U params = () elif (isinstance(oper, L.SetMatch) and isinstance(oper.target, L.Name) and L.is_vartuple(oper.key)): rel = oper.target.id relmask = Mask(oper.mask) params = L.get_vartuple(oper.key) else: raise L.ProgramError('Bad aggregate operand', node=node) return cls(node.op, rel, relmask, params, oper_demname, oper_demparams)
def inc_relcomp_helper(tree, manager, inccomp): """Incrementalize a comprehension based on an IncComp structure. Also return maintenance comprehensions. """ if manager.options.get_opt('verbose'): s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45) s += L.ts(inccomp.comp) print(s) # FIXME: Is the below demand code correct when the inner query's # demand invariant must be reference-counted? Given that change # tracking doesn't handle reference counting? # Create invariants for demand sets for demand-driven subqueries. # This only fires if we have subqueries with demand and this outer # query is being transformed WITHOUT filtering. If we are being # transformed WITH filtering, the inner queries would have been # rewritten without demand first; see demand/demtrans.py. deminvs = get_subquery_demnames(inccomp.spec) # Incrementalize them. Use a delta-set for deferring the propagation # of demand until after the inner query's maintenance code already # runs. (The inner query has already been incrementalized.) for demname, demspec in deminvs: # Hack: OuterDemandMaintainer should be refactored to move # to here, to avoid this import. from incoq.demand.demtrans import OuterDemandMaintainer # Determine dependencies of demand invariant. at_rels = set(e.enumrel for e in demspec.join.clauses) deltaname = L.N.deltaset(demname) demcomp = demspec.to_comp({}) # Add delta maintenance code as per the invariant. tree = inc_changetrack(tree, manager, demcomp, deltaname) # Add code (outside all other maintenance) to propagate # the delta changes to the actual inner query demand function. tree = OuterDemandMaintainer.run( tree, manager, deltaname, demname, at_rels, L.get_vartuple(demcomp.resexp), None) # Unwrap the demand clauses in the comp now that we've handled them. spec = inccomp.spec new_clauses = [] for cl in spec.join.clauses: if cl.has_demand: cl = cl.cl new_clauses.append(cl) new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses)) inccomp.spec = new_spec tree = CompReplacer.run(tree, manager, inccomp) tree, comps = RelcompMaintainer.run(tree, manager, inccomp) # If this was an original query, register it with the manager. if 'in_original' in inccomp.comp.options: manager.original_queryinvs.add(inccomp.name) return tree, comps
def inc_relcomp_helper(tree, manager, inccomp): """Incrementalize a comprehension based on an IncComp structure. Also return maintenance comprehensions. """ if manager.options.get_opt('verbose'): s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45) s += L.ts(inccomp.comp) print(s) # FIXME: Is the below demand code correct when the inner query's # demand invariant must be reference-counted? Given that change # tracking doesn't handle reference counting? # Create invariants for demand sets for demand-driven subqueries. # This only fires if we have subqueries with demand and this outer # query is being transformed WITHOUT filtering. If we are being # transformed WITH filtering, the inner queries would have been # rewritten without demand first; see demand/demtrans.py. deminvs = get_subquery_demnames(inccomp.spec) # Incrementalize them. Use a delta-set for deferring the propagation # of demand until after the inner query's maintenance code already # runs. (The inner query has already been incrementalized.) for demname, demspec in deminvs: # Hack: OuterDemandMaintainer should be refactored to move # to here, to avoid this import. from incoq.demand.demtrans import OuterDemandMaintainer # Determine dependencies of demand invariant. at_rels = set(e.enumrel for e in demspec.join.clauses) deltaname = L.N.deltaset(demname) demcomp = demspec.to_comp({}) # Add delta maintenance code as per the invariant. tree = inc_changetrack(tree, manager, demcomp, deltaname) # Add code (outside all other maintenance) to propagate # the delta changes to the actual inner query demand function. tree = OuterDemandMaintainer.run(tree, manager, deltaname, demname, at_rels, L.get_vartuple(demcomp.resexp), None) # Unwrap the demand clauses in the comp now that we've handled them. spec = inccomp.spec new_clauses = [] for cl in spec.join.clauses: if cl.has_demand: cl = cl.cl new_clauses.append(cl) new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses)) inccomp.spec = new_spec tree = CompReplacer.run(tree, manager, inccomp) tree, comps = RelcompMaintainer.run(tree, manager, inccomp) # If this was an original query, register it with the manager. if 'in_original' in inccomp.comp.options: manager.original_queryinvs.add(inccomp.name) return tree, comps
def visit_Enumerator(self, node): if node.iter == self.comp: if not L.is_vartuple(node.target): raise self.Failure arity = len(L.get_vartuple(node.target)) if self.arity != arity: raise self.Failure return self.generic_visit(node)
def from_AST(cls, node, factory): """Construct from Enumerator node of form <vars> in {<expr>} """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) val = L.get_singletonset(node.iter) return cls(lhs, val)
def from_expr(cls, node): """Construct from a condition expression of form <vars> == <rel> """ checktype(node, L.AST) left, op, val = L.get_cmp(node) checktype(op, L.Eq) lhs = L.get_vartuple(left) return cls(lhs, val)
def visit_For(self, node): # Recurse only after we've handled the potential special case. if node.iter != self.comp: return self.generic_visit(node) spec = self.spec special_case = ( node.orelse == () and spec.is_duplicate_safe and L.is_vartuple(node.target) and L.is_vartuple(spec.resexp) and (L.get_vartuple(node.target) == L.get_vartuple(spec.resexp) or (L.is_name(node.target) and L.get_name(node.target) == '_'))) if special_case: code = () code += (L.Comment('Iterate ' + str(spec)), ) code += spec.join.get_code(spec.params, node.body, augmented=self.augmented) return self.visit(code) else: return self.generic_visit(node)
def from_AST(cls, node, factory): """Construct from Enumerator node of form (<var>, <var>) in _M """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) rel = L.get_name(node.iter) if not len(lhs) == 2: raise TypeError cont, item = lhs if not is_mrel(rel): raise TypeError return cls(cont, item)
def visit_For(self, node): # Recurse only after we've handled the potential special case. if node.iter != self.comp: return self.generic_visit(node) spec = self.spec special_case = ( node.orelse == () and spec.is_duplicate_safe and L.is_vartuple(node.target) and L.is_vartuple(spec.resexp) and (L.get_vartuple(node.target) == L.get_vartuple(spec.resexp) or (L.is_name(node.target) and L.get_name(node.target) == '_')) ) if special_case: code = () code += (L.Comment('Iterate ' + str(spec)),) code += spec.join.get_code(spec.params, node.body, augmented=self.augmented) return self.visit(code) else: return self.generic_visit(node)
def from_options(cls, options): """Construct from comprehension options dict. If delta info isn't provided, return None instead of an instance. """ if options is None or '_deltarel' not in options: return None rel = options['_deltarel'] elem = options['_deltaelem'] elem = L.pe(elem) lhs = options['_deltalhs'] lhs = L.get_vartuple(L.pe(lhs)) op = options['_deltaop'] return cls(rel, elem, lhs, op)
def from_expr(cls, node): """Construct from a membership expression (<var>, <var>) in _M """ checktype(node, L.AST) left, op, right = L.get_cmp(node) checktype(op, L.In) lhs = L.get_vartuple(left) assert len(lhs) == 2 cont, item = lhs rel = L.get_name(right) assert is_mrel(rel) return cls(cont, item)
def from_AST(cls, node, factory): """Construct from Enumerator node of form (<var>, <var>, <var>) in _MAP """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) rel = L.get_name(node.iter) if not len(lhs) == 3: raise TypeError map, key, value = lhs if not is_maprel(rel): raise TypeError return cls(map, key, value)
def expr_tosizecost(self, expr): """Turn an iterated expression into a cost bound for its cardinality. """ if isinstance(expr, L.Name): return NameCost(expr.id) # Catch case of iterating over a delta set. # We'll just say O(delta set), even though it can have # duplicates. elif (isinstance(expr, L.Call) and isinstance(expr.func, L.Attribute) and isinstance(expr.func.value, L.Name) and expr.func.attr == 'elements'): return NameCost(expr.func.value.id) elif isinstance(expr, L.SetMatch): if isinstance(expr.target, (L.Set, L.DeltaMatch)): return UnitCost() elif (isinstance(expr.target, L.Name) and L.is_vartuple(expr.key)): keys = L.get_vartuple(expr.key) if all(k in self.args for k in keys): return DefImgsetCost(expr.target.id, Mask(expr.mask), L.get_vartuple(expr.key)) else: return IndefImgsetCost(expr.target.id, Mask(expr.mask)) else: return self.WarnUnknownCost(expr) elif isinstance(expr, L.DeltaMatch): return UnitCost() elif isinstance(expr, (L.Set, L.List, L.Tuple, L.Dict)): return UnitCost() else: return self.WarnUnknownCost(expr)
def from_expr(cls, node): """Construct from a membership condition expression of form <vars> in <rel> Note that this is syntactically different from the form used in comprehensions, even though their textual representation in source code is the same. """ checktype(node, L.AST) left, op, right = L.get_cmp(node) checktype(op, L.In) lhs = L.get_vartuple(left) rel = L.get_name(right) return cls(lhs, rel)
def from_AST(cls, node, factory): """Construct from enumerator of form (tupvar, elt1, ..., eltn) in _TUPN """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) rel = L.get_name(node.iter) if not is_trel(rel): raise TypeError tup, *elts = lhs arity = get_trel(rel) assert arity == len(elts) return cls(tup, tuple(elts))
def from_AST(cls, node, factory): """Construct from an Enumerator node of form var in {<rel>.smlookup(<mask>, <key vars>)} """ checktype(node, L.Enumerator) var = L.get_name(node.target) sm = L.get_singletonset(node.iter) checktype(sm, L.SMLookup) rel = L.get_name(sm.target) mask = Mask(sm.mask) keyvars = L.get_vartuple(sm.key) # Ensure the mask is consistent with how it's used. if mask != Mask.from_keylen(len(keyvars)): raise TypeError lhs = keyvars + (var,) return cls(lhs, rel)
def from_AST(cls, node, factory): """Construct from Enumerator node of form <vars> in deltamatch(<rel>, <mask>, <val>, <limit>) """ checktype(node, L.Enumerator) lhs = L.get_vartuple(node.target) checktype(node.iter, L.DeltaMatch) rel = L.get_name(node.iter.target) mask = Mask(node.iter.mask) val = node.iter.elem limit = node.iter.limit if limit not in [0, 1]: raise TypeError inferred_mask = Mask.from_vars(lhs, lhs) assert mask == inferred_mask return cls(lhs, rel, val, limit)
def from_AST(cls, node, factory): """Construct from an Enumerator node of form var in {<rel>.smlookup(<mask>, <key vars>)} """ checktype(node, L.Enumerator) var = L.get_name(node.target) sm = L.get_singletonset(node.iter) checktype(sm, L.SMLookup) rel = L.get_name(sm.target) mask = Mask(sm.mask) keyvars = L.get_vartuple(sm.key) # Ensure the mask is consistent with how it's used. if mask != Mask.from_keylen(len(keyvars)): raise TypeError lhs = keyvars + (var, ) return cls(lhs, rel)
def membercond_to_enum(cls, cl): """For a condition clause that expresses a membership, return an equivalent enumerator clause. For other kinds of conditions, return the same clause. For enumerators, raise TypeError. """ if cl.kind is not Clause.KIND_COND: raise TypeError compre_ast = None clast = cl.to_AST() if L.is_cmp(clast): lhs, op, rhs = L.get_cmp(clast) if (L.is_vartuple(lhs) and isinstance(op, L.In)): compre_ast = L.Enumerator( L.tuplify(L.get_vartuple(lhs), lval=True), rhs) if compre_ast is None: return cl else: return cls.from_AST(compre_ast)
def deminc_relcomp(tree, manager, comp, compname): """Incrementalize a relational comprehension, add appropriate incrementalized demand structures, and rewrite the maint comps to use these structures. """ verbose = manager.options.get_opt('verbose') use_tag_checks = manager.options.get_opt('tag_checks') factory = manager.factory force_uset = manager.options.get_queryopt(comp, 'uset_force') if force_uset is None: force_uset = manager.options.get_opt('default_uset_force') subdem_tags = manager.options.get_opt('subdem_tags') reorder = manager.options.get_queryopt(comp, 'demand_reorder') # Get the CompSpec and inc info of the original comprehension. inccomp = make_inccomp(tree, manager, comp, compname, force_uset=force_uset) spec = inccomp.spec augmented = inccomp.selfjoin == 'aug' # Make tags/filters/usets structures. ds = make_structures(spec.join.clauses, compname, singletag=manager.options.get_opt('single_tag'), subdem_tags=subdem_tags, reorder=reorder) if verbose: print(' Tags/filters/usets: ' + ' ' * 31 + ', '.join(s.name for s in ds.structs)) # Eliminate Demand from clauses. new_clauses = [] for cl in spec.join.clauses: if isinstance(cl, DemClause): new_clauses.append(cl.cl) else: new_clauses.append(cl) inccomp.spec = spec = spec._replace(join=spec.join._replace( clauses=new_clauses)) # Incrementalize query comp. # (Since we've unwrapped demand clauses, the logic for defining # inner queries' U-sets for if filtering weren't used won't fire.) tree, maintcomps = inc_relcomp_helper(tree, manager, inccomp) # Rewrite maintcomps to use filters. Prune structures. tree, ds = filter_comps(tree, factory, ds, maintcomps, use_tag_checks, augmented=augmented, subdem_tags=subdem_tags) manager.stats['dem structs'] += len(ds.structs) # Incrementalize tags and filters. demcomps = structures_to_comps(ds, factory) for name, comp in demcomps: # When using augmented maintenance code, since the query # maintenance goes before an addition and after a removal, # make sure the demand invariant maintenance still comes # before and after that query maintenance respectively. outsideinvs = [compname] if augmented else [] tree = inc_relcomp(tree, manager, comp, name, outsideinvs=outsideinvs) # Take care of usets. usets = sorted(ds.usets, key=attrgetter('i')) for uset in usets: at_rels = set(e.enumrel for i, e in enumerate(spec.join.clauses) if i < uset.i) # Maintain U-set change set. # FIXME: The delta set name should probably be freshly generated # for this comprehension, so as to ensure it does not interfere # with another delta set for a different use of the same demand # function (i.e. same nested query) in a different occurrence # (possibly even within this same outer query?). demname = uset.name deltaname = L.N.deltaset(demname) uset_comp = uset_to_comp(ds, uset, factory, spec.join.clauses[0]) tree = inc_changetrack(tree, manager, uset_comp, deltaname) tree = OuterDemandMaintainer.run( tree, manager, deltaname, demname, at_rels, L.get_vartuple(uset_comp.resexp), None) return tree