def from_AST(cls, node, factory):
"""Construct from an Enumerator node of form
<vars> in <rel>
Alternatively, the rhs may be a setmatch of a rel, where
the mask is a lookupmask and the key is a vartuple.
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rhs = node.iter
if L.is_name(rhs):
rel = L.get_name(rhs)
elif isinstance(rhs, L.SetMatch) and L.is_vartuple(rhs.key):
keyvars = L.get_vartuple(rhs.key)
# Make sure we're dealing with a lookupmask and that the
# key vars agree with the mask.
mask = Mask(rhs.mask)
assert mask.is_lookupmask
assert mask.lookup_arity == len(keyvars)
lhs = keyvars + lhs
rel = L.get_name(rhs.target)
else:
raise TypeError
return cls(lhs, rel)
def from_AST(cls, node, factory):
"""Construct from an Enumerator node of form
<vars> in <rel>
Alternatively, the rhs may be a setmatch of a rel, where
the mask is a lookupmask and the key is a vartuple.
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rhs = node.iter
if L.is_name(rhs):
rel = L.get_name(rhs)
elif isinstance(rhs, L.SetMatch) and L.is_vartuple(rhs.key):
keyvars = L.get_vartuple(rhs.key)
# Make sure we're dealing with a lookupmask and that the
# key vars agree with the mask.
mask = Mask(rhs.mask)
assert mask.is_lookupmask
assert mask.lookup_arity == len(keyvars)
lhs = keyvars + lhs
rel = L.get_name(rhs.target)
else:
raise TypeError
return cls(lhs, rel)
def from_node(cls, node):
checktype(node, L.Aggregate)
if isinstance(node.value, L.DemQuery):
assert all(isinstance(a, L.Name) for a in node.value.args)
oper_demparams = tuple(a.id for a in node.value.args)
oper_demname = node.value.demname
oper = node.value.value
else:
oper_demparams = None
oper_demname = None
oper = node.value
if isinstance(oper, L.Name):
rel = oper.id
relmask = Mask.U
params = ()
elif (isinstance(oper, L.SetMatch) and isinstance(oper.target, L.Name)
and L.is_vartuple(oper.key)):
rel = oper.target.id
relmask = Mask(oper.mask)
params = L.get_vartuple(oper.key)
else:
raise L.ProgramError('Bad aggregate operand', node=node)
return cls(node.op, rel, relmask, params, oper_demname, oper_demparams)
def from_node(cls, node):
checktype(node, L.Aggregate)
if isinstance(node.value, L.DemQuery):
assert all(isinstance(a, L.Name) for a in node.value.args)
oper_demparams = tuple(a.id for a in node.value.args)
oper_demname = node.value.demname
oper = node.value.value
else:
oper_demparams = None
oper_demname = None
oper = node.value
if isinstance(oper, L.Name):
rel = oper.id
relmask = Mask.U
params = ()
elif (isinstance(oper, L.SetMatch) and
isinstance(oper.target, L.Name) and
L.is_vartuple(oper.key)):
rel = oper.target.id
relmask = Mask(oper.mask)
params = L.get_vartuple(oper.key)
else:
raise L.ProgramError('Bad aggregate operand', node=node)
return cls(node.op, rel, relmask, params,
oper_demname, oper_demparams)
def inc_relcomp_helper(tree, manager, inccomp):
"""Incrementalize a comprehension based on an IncComp structure.
Also return maintenance comprehensions.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45)
s += L.ts(inccomp.comp)
print(s)
# FIXME: Is the below demand code correct when the inner query's
# demand invariant must be reference-counted? Given that change
# tracking doesn't handle reference counting?
# Create invariants for demand sets for demand-driven subqueries.
# This only fires if we have subqueries with demand and this outer
# query is being transformed WITHOUT filtering. If we are being
# transformed WITH filtering, the inner queries would have been
# rewritten without demand first; see demand/demtrans.py.
deminvs = get_subquery_demnames(inccomp.spec)
# Incrementalize them. Use a delta-set for deferring the propagation
# of demand until after the inner query's maintenance code already
# runs. (The inner query has already been incrementalized.)
for demname, demspec in deminvs:
# Hack: OuterDemandMaintainer should be refactored to move
# to here, to avoid this import.
from incoq.demand.demtrans import OuterDemandMaintainer
# Determine dependencies of demand invariant.
at_rels = set(e.enumrel for e in demspec.join.clauses)
deltaname = L.N.deltaset(demname)
demcomp = demspec.to_comp({})
# Add delta maintenance code as per the invariant.
tree = inc_changetrack(tree, manager, demcomp, deltaname)
# Add code (outside all other maintenance) to propagate
# the delta changes to the actual inner query demand function.
tree = OuterDemandMaintainer.run(
tree, manager, deltaname,
demname, at_rels,
L.get_vartuple(demcomp.resexp),
None)
# Unwrap the demand clauses in the comp now that we've handled them.
spec = inccomp.spec
new_clauses = []
for cl in spec.join.clauses:
if cl.has_demand:
cl = cl.cl
new_clauses.append(cl)
new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses))
inccomp.spec = new_spec
tree = CompReplacer.run(tree, manager, inccomp)
tree, comps = RelcompMaintainer.run(tree, manager, inccomp)
# If this was an original query, register it with the manager.
if 'in_original' in inccomp.comp.options:
manager.original_queryinvs.add(inccomp.name)
return tree, comps
def inc_relcomp_helper(tree, manager, inccomp):
"""Incrementalize a comprehension based on an IncComp structure.
Also return maintenance comprehensions.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45)
s += L.ts(inccomp.comp)
print(s)
# FIXME: Is the below demand code correct when the inner query's
# demand invariant must be reference-counted? Given that change
# tracking doesn't handle reference counting?
# Create invariants for demand sets for demand-driven subqueries.
# This only fires if we have subqueries with demand and this outer
# query is being transformed WITHOUT filtering. If we are being
# transformed WITH filtering, the inner queries would have been
# rewritten without demand first; see demand/demtrans.py.
deminvs = get_subquery_demnames(inccomp.spec)
# Incrementalize them. Use a delta-set for deferring the propagation
# of demand until after the inner query's maintenance code already
# runs. (The inner query has already been incrementalized.)
for demname, demspec in deminvs:
# Hack: OuterDemandMaintainer should be refactored to move
# to here, to avoid this import.
from incoq.demand.demtrans import OuterDemandMaintainer
# Determine dependencies of demand invariant.
at_rels = set(e.enumrel for e in demspec.join.clauses)
deltaname = L.N.deltaset(demname)
demcomp = demspec.to_comp({})
# Add delta maintenance code as per the invariant.
tree = inc_changetrack(tree, manager, demcomp, deltaname)
# Add code (outside all other maintenance) to propagate
# the delta changes to the actual inner query demand function.
tree = OuterDemandMaintainer.run(tree, manager, deltaname,
demname, at_rels,
L.get_vartuple(demcomp.resexp), None)
# Unwrap the demand clauses in the comp now that we've handled them.
spec = inccomp.spec
new_clauses = []
for cl in spec.join.clauses:
if cl.has_demand:
cl = cl.cl
new_clauses.append(cl)
new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses))
inccomp.spec = new_spec
tree = CompReplacer.run(tree, manager, inccomp)
tree, comps = RelcompMaintainer.run(tree, manager, inccomp)
# If this was an original query, register it with the manager.
if 'in_original' in inccomp.comp.options:
manager.original_queryinvs.add(inccomp.name)
return tree, comps
def visit_Enumerator(self, node):
if node.iter == self.comp:
if not L.is_vartuple(node.target):
raise self.Failure
arity = len(L.get_vartuple(node.target))
if self.arity != arity:
raise self.Failure
return
self.generic_visit(node)
def visit_Enumerator(self, node):
if node.iter == self.comp:
if not L.is_vartuple(node.target):
raise self.Failure
arity = len(L.get_vartuple(node.target))
if self.arity != arity:
raise self.Failure
return
self.generic_visit(node)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
<vars> in {<expr>}
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
val = L.get_singletonset(node.iter)
return cls(lhs, val)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
<vars> in {<expr>}
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
val = L.get_singletonset(node.iter)
return cls(lhs, val)
def from_expr(cls, node):
"""Construct from a condition expression of form
<vars> == <rel>
"""
checktype(node, L.AST)
left, op, val = L.get_cmp(node)
checktype(op, L.Eq)
lhs = L.get_vartuple(left)
return cls(lhs, val)
def from_expr(cls, node):
"""Construct from a condition expression of form
<vars> == <rel>
"""
checktype(node, L.AST)
left, op, val = L.get_cmp(node)
checktype(op, L.Eq)
lhs = L.get_vartuple(left)
return cls(lhs, val)
def visit_For(self, node):
# Recurse only after we've handled the potential special case.
if node.iter != self.comp:
return self.generic_visit(node)
spec = self.spec
special_case = (
node.orelse == () and spec.is_duplicate_safe
and L.is_vartuple(node.target) and L.is_vartuple(spec.resexp)
and (L.get_vartuple(node.target) == L.get_vartuple(spec.resexp) or
(L.is_name(node.target) and L.get_name(node.target) == '_')))
if special_case:
code = ()
code += (L.Comment('Iterate ' + str(spec)), )
code += spec.join.get_code(spec.params,
node.body,
augmented=self.augmented)
return self.visit(code)
else:
return self.generic_visit(node)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
(<var>, <var>) in _M
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rel = L.get_name(node.iter)
if not len(lhs) == 2:
raise TypeError
cont, item = lhs
if not is_mrel(rel):
raise TypeError
return cls(cont, item)
def visit_For(self, node):
# Recurse only after we've handled the potential special case.
if node.iter != self.comp:
return self.generic_visit(node)
spec = self.spec
special_case = (
node.orelse == () and
spec.is_duplicate_safe and
L.is_vartuple(node.target) and
L.is_vartuple(spec.resexp) and
(L.get_vartuple(node.target) == L.get_vartuple(spec.resexp) or
(L.is_name(node.target) and L.get_name(node.target) == '_'))
)
if special_case:
code = ()
code += (L.Comment('Iterate ' + str(spec)),)
code += spec.join.get_code(spec.params, node.body,
augmented=self.augmented)
return self.visit(code)
else:
return self.generic_visit(node)
def from_options(cls, options):
"""Construct from comprehension options dict.
If delta info isn't provided, return None instead
of an instance.
"""
if options is None or '_deltarel' not in options:
return None
rel = options['_deltarel']
elem = options['_deltaelem']
elem = L.pe(elem)
lhs = options['_deltalhs']
lhs = L.get_vartuple(L.pe(lhs))
op = options['_deltaop']
return cls(rel, elem, lhs, op)
def from_expr(cls, node):
"""Construct from a membership expression
(<var>, <var>) in _M
"""
checktype(node, L.AST)
left, op, right = L.get_cmp(node)
checktype(op, L.In)
lhs = L.get_vartuple(left)
assert len(lhs) == 2
cont, item = lhs
rel = L.get_name(right)
assert is_mrel(rel)
return cls(cont, item)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
(<var>, <var>, <var>) in _MAP
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rel = L.get_name(node.iter)
if not len(lhs) == 3:
raise TypeError
map, key, value = lhs
if not is_maprel(rel):
raise TypeError
return cls(map, key, value)
def from_expr(cls, node):
"""Construct from a membership expression
(<var>, <var>) in _M
"""
checktype(node, L.AST)
left, op, right = L.get_cmp(node)
checktype(op, L.In)
lhs = L.get_vartuple(left)
assert len(lhs) == 2
cont, item = lhs
rel = L.get_name(right)
assert is_mrel(rel)
return cls(cont, item)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
(<var>, <var>) in _M
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rel = L.get_name(node.iter)
if not len(lhs) == 2:
raise TypeError
cont, item = lhs
if not is_mrel(rel):
raise TypeError
return cls(cont, item)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
(<var>, <var>, <var>) in _MAP
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rel = L.get_name(node.iter)
if not len(lhs) == 3:
raise TypeError
map, key, value = lhs
if not is_maprel(rel):
raise TypeError
return cls(map, key, value)
def from_options(cls, options):
"""Construct from comprehension options dict.
If delta info isn't provided, return None instead
of an instance.
"""
if options is None or '_deltarel' not in options:
return None
rel = options['_deltarel']
elem = options['_deltaelem']
elem = L.pe(elem)
lhs = options['_deltalhs']
lhs = L.get_vartuple(L.pe(lhs))
op = options['_deltaop']
return cls(rel, elem, lhs, op)
def expr_tosizecost(self, expr):
"""Turn an iterated expression into a cost bound for its
cardinality.
"""
if isinstance(expr, L.Name):
return NameCost(expr.id)
# Catch case of iterating over a delta set.
# We'll just say O(delta set), even though it can have
# duplicates.
elif (isinstance(expr, L.Call) and isinstance(expr.func, L.Attribute)
and isinstance(expr.func.value, L.Name)
and expr.func.attr == 'elements'):
return NameCost(expr.func.value.id)
elif isinstance(expr, L.SetMatch):
if isinstance(expr.target, (L.Set, L.DeltaMatch)):
return UnitCost()
elif (isinstance(expr.target, L.Name) and L.is_vartuple(expr.key)):
keys = L.get_vartuple(expr.key)
if all(k in self.args for k in keys):
return DefImgsetCost(expr.target.id, Mask(expr.mask),
L.get_vartuple(expr.key))
else:
return IndefImgsetCost(expr.target.id, Mask(expr.mask))
else:
return self.WarnUnknownCost(expr)
elif isinstance(expr, L.DeltaMatch):
return UnitCost()
elif isinstance(expr, (L.Set, L.List, L.Tuple, L.Dict)):
return UnitCost()
else:
return self.WarnUnknownCost(expr)
def from_expr(cls, node):
"""Construct from a membership condition expression of form
<vars> in <rel>
Note that this is syntactically different from the form used
in comprehensions, even though their textual representation
in source code is the same.
"""
checktype(node, L.AST)
left, op, right = L.get_cmp(node)
checktype(op, L.In)
lhs = L.get_vartuple(left)
rel = L.get_name(right)
return cls(lhs, rel)
def from_expr(cls, node):
"""Construct from a membership condition expression of form
<vars> in <rel>
Note that this is syntactically different from the form used
in comprehensions, even though their textual representation
in source code is the same.
"""
checktype(node, L.AST)
left, op, right = L.get_cmp(node)
checktype(op, L.In)
lhs = L.get_vartuple(left)
rel = L.get_name(right)
return cls(lhs, rel)
def from_AST(cls, node, factory):
"""Construct from enumerator of form
(tupvar, elt1, ..., eltn) in _TUPN
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
rel = L.get_name(node.iter)
if not is_trel(rel):
raise TypeError
tup, *elts = lhs
arity = get_trel(rel)
assert arity == len(elts)
return cls(tup, tuple(elts))
def from_AST(cls, node, factory):
"""Construct from an Enumerator node of form
var in {<rel>.smlookup(<mask>, <key vars>)}
"""
checktype(node, L.Enumerator)
var = L.get_name(node.target)
sm = L.get_singletonset(node.iter)
checktype(sm, L.SMLookup)
rel = L.get_name(sm.target)
mask = Mask(sm.mask)
keyvars = L.get_vartuple(sm.key)
# Ensure the mask is consistent with how it's used.
if mask != Mask.from_keylen(len(keyvars)):
raise TypeError
lhs = keyvars + (var,)
return cls(lhs, rel)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
<vars> in deltamatch(<rel>, <mask>, <val>, <limit>)
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
checktype(node.iter, L.DeltaMatch)
rel = L.get_name(node.iter.target)
mask = Mask(node.iter.mask)
val = node.iter.elem
limit = node.iter.limit
if limit not in [0, 1]:
raise TypeError
inferred_mask = Mask.from_vars(lhs, lhs)
assert mask == inferred_mask
return cls(lhs, rel, val, limit)
def from_AST(cls, node, factory):
"""Construct from an Enumerator node of form
var in {<rel>.smlookup(<mask>, <key vars>)}
"""
checktype(node, L.Enumerator)
var = L.get_name(node.target)
sm = L.get_singletonset(node.iter)
checktype(sm, L.SMLookup)
rel = L.get_name(sm.target)
mask = Mask(sm.mask)
keyvars = L.get_vartuple(sm.key)
# Ensure the mask is consistent with how it's used.
if mask != Mask.from_keylen(len(keyvars)):
raise TypeError
lhs = keyvars + (var, )
return cls(lhs, rel)
def from_AST(cls, node, factory):
"""Construct from Enumerator node of form
<vars> in deltamatch(<rel>, <mask>, <val>, <limit>)
"""
checktype(node, L.Enumerator)
lhs = L.get_vartuple(node.target)
checktype(node.iter, L.DeltaMatch)
rel = L.get_name(node.iter.target)
mask = Mask(node.iter.mask)
val = node.iter.elem
limit = node.iter.limit
if limit not in [0, 1]:
raise TypeError
inferred_mask = Mask.from_vars(lhs, lhs)
assert mask == inferred_mask
return cls(lhs, rel, val, limit)
def membercond_to_enum(cls, cl):
"""For a condition clause that expresses a membership, return
an equivalent enumerator clause. For other kinds of conditions,
return the same clause. For enumerators, raise TypeError.
"""
if cl.kind is not Clause.KIND_COND:
raise TypeError
compre_ast = None
clast = cl.to_AST()
if L.is_cmp(clast):
lhs, op, rhs = L.get_cmp(clast)
if (L.is_vartuple(lhs) and isinstance(op, L.In)):
compre_ast = L.Enumerator(
L.tuplify(L.get_vartuple(lhs), lval=True), rhs)
if compre_ast is None:
return cl
else:
return cls.from_AST(compre_ast)
def membercond_to_enum(cls, cl):
"""For a condition clause that expresses a membership, return
an equivalent enumerator clause. For other kinds of conditions,
return the same clause. For enumerators, raise TypeError.
"""
if cl.kind is not Clause.KIND_COND:
raise TypeError
compre_ast = None
clast = cl.to_AST()
if L.is_cmp(clast):
lhs, op, rhs = L.get_cmp(clast)
if (L.is_vartuple(lhs) and
isinstance(op, L.In)):
compre_ast = L.Enumerator(
L.tuplify(L.get_vartuple(lhs), lval=True),
rhs)
if compre_ast is None:
return cl
else:
return cls.from_AST(compre_ast)
def deminc_relcomp(tree, manager, comp, compname):
"""Incrementalize a relational comprehension, add appropriate
incrementalized demand structures, and rewrite the maint comps
to use these structures.
"""
verbose = manager.options.get_opt('verbose')
use_tag_checks = manager.options.get_opt('tag_checks')
factory = manager.factory
force_uset = manager.options.get_queryopt(comp, 'uset_force')
if force_uset is None:
force_uset = manager.options.get_opt('default_uset_force')
subdem_tags = manager.options.get_opt('subdem_tags')
reorder = manager.options.get_queryopt(comp, 'demand_reorder')
# Get the CompSpec and inc info of the original comprehension.
inccomp = make_inccomp(tree, manager, comp, compname,
force_uset=force_uset)
spec = inccomp.spec
augmented = inccomp.selfjoin == 'aug'
# Make tags/filters/usets structures.
ds = make_structures(spec.join.clauses, compname,
singletag=manager.options.get_opt('single_tag'),
subdem_tags=subdem_tags,
reorder=reorder)
if verbose:
print(' Tags/filters/usets: ' + ' ' * 31 +
', '.join(s.name for s in ds.structs))
# Eliminate Demand from clauses.
new_clauses = []
for cl in spec.join.clauses:
if isinstance(cl, DemClause):
new_clauses.append(cl.cl)
else:
new_clauses.append(cl)
inccomp.spec = spec = spec._replace(join=spec.join._replace(
clauses=new_clauses))
# Incrementalize query comp.
# (Since we've unwrapped demand clauses, the logic for defining
# inner queries' U-sets for if filtering weren't used won't fire.)
tree, maintcomps = inc_relcomp_helper(tree, manager, inccomp)
# Rewrite maintcomps to use filters. Prune structures.
tree, ds = filter_comps(tree, factory, ds, maintcomps,
use_tag_checks,
augmented=augmented, subdem_tags=subdem_tags)
manager.stats['dem structs'] += len(ds.structs)
# Incrementalize tags and filters.
demcomps = structures_to_comps(ds, factory)
for name, comp in demcomps:
# When using augmented maintenance code, since the query
# maintenance goes before an addition and after a removal,
# make sure the demand invariant maintenance still comes
# before and after that query maintenance respectively.
outsideinvs = [compname] if augmented else []
tree = inc_relcomp(tree, manager, comp, name,
outsideinvs=outsideinvs)
# Take care of usets.
usets = sorted(ds.usets, key=attrgetter('i'))
for uset in usets:
at_rels = set(e.enumrel for i, e in enumerate(spec.join.clauses)
if i < uset.i)
# Maintain U-set change set.
# FIXME: The delta set name should probably be freshly generated
# for this comprehension, so as to ensure it does not interfere
# with another delta set for a different use of the same demand
# function (i.e. same nested query) in a different occurrence
# (possibly even within this same outer query?).
demname = uset.name
deltaname = L.N.deltaset(demname)
uset_comp = uset_to_comp(ds, uset, factory, spec.join.clauses[0])
tree = inc_changetrack(tree, manager, uset_comp, deltaname)
tree = OuterDemandMaintainer.run(
tree, manager, deltaname,
demname, at_rels,
L.get_vartuple(uset_comp.resexp),
None)
return tree