def updateopts(self, options):
"""Return a modified options dict with the delta keys set."""
options = dict(options)
options['_deltarel'] = self.rel
options['_deltaelem'] = L.ts(self.elem)
options['_deltalhs'] = L.ts(L.tuplify(self.lhs, lval=True))
options['_deltaop'] = self.op
return options
def updateopts(self, options):
"""Return a modified options dict with the delta keys set."""
options = dict(options)
options['_deltarel'] = self.rel
options['_deltaelem'] = L.ts(self.elem)
options['_deltalhs'] = L.ts(L.tuplify(self.lhs, lval=True))
options['_deltaop'] = self.op
return options
def __str__(self):
clast = self.to_AST()
s = L.ts(clast).strip()
if isinstance(clast, L.Enumerator):
# Get rid of "for" at the beginning.
s = s[s.find(' ') + 1:]
return s
def visit_SetUpdate(self, node):
rel = L.get_name(node.target)
if rel not in self.at_rels:
return
# New code gets inserted after the update.
# This is true even if the update was a removal.
# It shouldn't matter where we do the U-set update,
# so long as the invariants are properly maintained
# at the time.
prefix = self.manager.namegen.next_prefix()
vars = [prefix + v for v in self.projvars]
if node.op == 'add':
funcname = L.N.demfunc(self.demname)
else:
funcname = L.N.undemfunc(self.demname)
call_func = L.Call(L.ln(funcname),
tuple(L.ln(v) for v in vars),
(), None, None)
postcode = L.pc('''
for S_PROJVARS in DELTA.elements():
CALL_FUNC
DELTA.clear()
''', subst={'S_PROJVARS': L.tuplify(vars, lval=True),
'DELTA': self.delta_name,
'CALL_FUNC': call_func})
return self.with_outer_maint(node, funcname, L.ts(node),
(), postcode)
def helper(self, node, var, op, elem):
assert op in ['add', 'remove']
# Maintenance goes after addition updates and before removals,
# except when we're using augmented code, which relies on the
# value of the set *without* the updated element.
after_add = self.inccomp.selfjoin != 'aug'
is_add = op == 'add'
if self.inccomp.change_tracker:
# For change trackers, all removals turn into additions,
# but are still run in the same spot they would have been.
funcdict = self.addfuncs
else:
funcdict = self.addfuncs if is_add else self.removefuncs
func = funcdict[var]
code = L.pc('FUNC(ELEM)',
subst={'FUNC': func,
'ELEM': elem})
if after_add ^ is_add:
precode = code
postcode = ()
else:
precode = ()
postcode = code
# Respect outsideinvs. This ensures that demand invariant
# maintenance is inserted before/after the query maintenance.
return self.with_outer_maint(node, self.inccomp.name, L.ts(node),
precode, postcode)
def inc_aggr(tree, manager, aggr, name, *, demand, half_demand):
"""Incrementalize an aggregate query.
If the aggregate is not of the right form, then, if the global
options permit it, skip transforming this query. In this case,
the query gets marked with 'impl' = 'batch' to prevent handling
it again. If the options don't permit it, raise an exception.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + name + ': ').ljust(45)
s += L.ts(aggr)
print(s)
spec = AggrSpec.from_node(aggr)
uset_lru = manager.options.get_queryopt(aggr, 'uset_lru')
if uset_lru is None:
uset_lru = manager.options.get_opt('default_uset_lru')
demname = name if demand else None
if not demand:
half_demand = False
incaggr = IncAggr(aggr, spec, name, demname, uset_lru, half_demand)
tree = AggrReplacer.run(tree, manager, incaggr)
tree = AggrMaintainer.run(tree, manager, incaggr)
if 'in_original' in aggr.options:
manager.original_queryinvs.add(name)
return tree
def preprocess_tree(manager, tree, opts):
opman = manager.options
tree = import_distalgo(tree)
tree = L.import_incast(tree)
# Remove the runtimelib declaration.
# It will be added back at the end.
tree = L.remove_runtimelib(tree)
# Complain if there are redundant function definitions.
FunctionUniqueChecker.run(tree)
# Grab all options.
tree, opts = L.parse_options(tree, ext_opts=opts)
nopts, qopts = opts
opman.import_opts(nopts, qopts)
# Fill in missing param/option info.
tree, unused = L.attach_qopts_info(tree, opts)
tree = L.infer_params(tree, obj_domain=opman.get_opt('obj_domain'))
# Error if unused comps in options (prevent typos from causing
# much frustration).
if len(unused) > 0:
raise L.ProgramError('Options given for non-existent queries: ' +
quote_items(L.ts(c) for c in unused))
return tree, opman
def WarnUnknownCost(self, node):
"""Return UnknownCost, but also print the node that led to it
if self.warn is True.
"""
if self.warn:
print('---- Unknown cost ---- ' + str(L.ts(node)))
return UnknownCost()
def __str__(self):
clast = self.to_AST()
s = L.ts(clast).strip()
if isinstance(clast, L.Enumerator):
# Get rid of "for" at the beginning.
s = s[s.find(' ') + 1:]
return s
def inc_relcomp_helper(tree, manager, inccomp):
"""Incrementalize a comprehension based on an IncComp structure.
Also return maintenance comprehensions.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45)
s += L.ts(inccomp.comp)
print(s)
# FIXME: Is the below demand code correct when the inner query's
# demand invariant must be reference-counted? Given that change
# tracking doesn't handle reference counting?
# Create invariants for demand sets for demand-driven subqueries.
# This only fires if we have subqueries with demand and this outer
# query is being transformed WITHOUT filtering. If we are being
# transformed WITH filtering, the inner queries would have been
# rewritten without demand first; see demand/demtrans.py.
deminvs = get_subquery_demnames(inccomp.spec)
# Incrementalize them. Use a delta-set for deferring the propagation
# of demand until after the inner query's maintenance code already
# runs. (The inner query has already been incrementalized.)
for demname, demspec in deminvs:
# Hack: OuterDemandMaintainer should be refactored to move
# to here, to avoid this import.
from incoq.demand.demtrans import OuterDemandMaintainer
# Determine dependencies of demand invariant.
at_rels = set(e.enumrel for e in demspec.join.clauses)
deltaname = L.N.deltaset(demname)
demcomp = demspec.to_comp({})
# Add delta maintenance code as per the invariant.
tree = inc_changetrack(tree, manager, demcomp, deltaname)
# Add code (outside all other maintenance) to propagate
# the delta changes to the actual inner query demand function.
tree = OuterDemandMaintainer.run(
tree, manager, deltaname,
demname, at_rels,
L.get_vartuple(demcomp.resexp),
None)
# Unwrap the demand clauses in the comp now that we've handled them.
spec = inccomp.spec
new_clauses = []
for cl in spec.join.clauses:
if cl.has_demand:
cl = cl.cl
new_clauses.append(cl)
new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses))
inccomp.spec = new_spec
tree = CompReplacer.run(tree, manager, inccomp)
tree, comps = RelcompMaintainer.run(tree, manager, inccomp)
# If this was an original query, register it with the manager.
if 'in_original' in inccomp.comp.options:
manager.original_queryinvs.add(inccomp.name)
return tree, comps
def helper(self, node, var, op, elem):
assert op in ['add', 'remove']
# Maintenance goes after addition updates and before removals,
# except when we're using augmented code, which relies on the
# value of the set *without* the updated element.
after_add = self.inccomp.selfjoin != 'aug'
is_add = op == 'add'
if self.inccomp.change_tracker:
# For change trackers, all removals turn into additions,
# but are still run in the same spot they would have been.
funcdict = self.addfuncs
else:
funcdict = self.addfuncs if is_add else self.removefuncs
func = funcdict[var]
code = L.pc('FUNC(ELEM)', subst={'FUNC': func, 'ELEM': elem})
if after_add ^ is_add:
precode = code
postcode = ()
else:
precode = ()
postcode = code
# Respect outsideinvs. This ensures that demand invariant
# maintenance is inserted before/after the query maintenance.
return self.with_outer_maint(node, self.inccomp.name, L.ts(node),
precode, postcode)
def visit_SetUpdate(self, node):
node = self.generic_visit(node)
# No action if
# - this is not an update to a variable
# - this is not the variable you are looking for (jedi hand wave)
if not node.is_varupdate():
return node
var, op, elem = node.get_varupdate()
if var != self.spec.rel:
return node
precode = postcode = ()
if op == 'add':
postcode = L.pc('ADDFUNC(ELEM)',
subst={'ADDFUNC': self.addfunc_name,
'ELEM': elem})
elif op == 'remove':
precode = L.pc('REMOVEFUNC(ELEM)',
subst={'REMOVEFUNC': self.removefunc_name,
'ELEM': elem})
else:
assert()
code = L.Maintenance(self.spec.map_name, L.ts(node),
precode, (node,), postcode)
return code
def inc_aggr(tree, manager, aggr, name,
*, demand, half_demand):
"""Incrementalize an aggregate query.
If the aggregate is not of the right form, then, if the global
options permit it, skip transforming this query. In this case,
the query gets marked with 'impl' = 'batch' to prevent handling
it again. If the options don't permit it, raise an exception.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + name + ': ').ljust(45)
s += L.ts(aggr)
print(s)
spec = AggrSpec.from_node(aggr)
uset_lru = manager.options.get_queryopt(aggr, 'uset_lru')
if uset_lru is None:
uset_lru = manager.options.get_opt('default_uset_lru')
demname = name if demand else None
if not demand:
half_demand = False
incaggr = IncAggr(aggr, spec, name, demname, uset_lru, half_demand)
tree = AggrReplacer.run(tree, manager, incaggr)
tree = AggrMaintainer.run(tree, manager, incaggr)
if 'in_original' in aggr.options:
manager.original_queryinvs.add(name)
return tree
def visit_SetUpdate(self, node):
id = node.target.id
name = 'Q_' + id
desc = ts(node)
precode = _self.pc('print(N)', subst={'N': id + '_pre'})
postcode = _self.pc('print(N)', subst={'N': id + '_post'})
return self.with_outer_maint(node, name, desc,
precode, postcode)
def visit_SetUpdate(self, node):
spec = self.incaggr.spec
node = self.generic_visit(node)
if not node.is_varupdate():
return node
var, op, elem = node.get_varupdate()
if var == spec.rel:
precode = postcode = ()
if op == 'add':
postcode = L.pc('ADDFUNC(ELEM)',
subst={
'ADDFUNC': self.addfunc,
'ELEM': elem
})
elif op == 'remove':
precode = L.pc('REMOVEFUNC(ELEM)',
subst={
'REMOVEFUNC': self.removefunc,
'ELEM': elem
})
else:
assert ()
code = L.Maintenance(self.incaggr.name, L.ts(node), precode,
(node, ), postcode)
elif var == L.N.uset(self.incaggr.name):
prefix = self.manager.namegen.next_prefix()
precode = postcode = ()
if op == 'add':
postcode = self.cg.make_addu_maint(prefix)
elif op == 'remove':
precode = self.cg.make_removeu_maint(prefix)
else:
assert ()
code = L.Maintenance(self.incaggr.name, L.ts(node), precode,
(node, ), postcode)
else:
code = node
return code
def inc_relcomp_helper(tree, manager, inccomp):
"""Incrementalize a comprehension based on an IncComp structure.
Also return maintenance comprehensions.
"""
if manager.options.get_opt('verbose'):
s = ('Incrementalizing ' + inccomp.name + ': ').ljust(45)
s += L.ts(inccomp.comp)
print(s)
# FIXME: Is the below demand code correct when the inner query's
# demand invariant must be reference-counted? Given that change
# tracking doesn't handle reference counting?
# Create invariants for demand sets for demand-driven subqueries.
# This only fires if we have subqueries with demand and this outer
# query is being transformed WITHOUT filtering. If we are being
# transformed WITH filtering, the inner queries would have been
# rewritten without demand first; see demand/demtrans.py.
deminvs = get_subquery_demnames(inccomp.spec)
# Incrementalize them. Use a delta-set for deferring the propagation
# of demand until after the inner query's maintenance code already
# runs. (The inner query has already been incrementalized.)
for demname, demspec in deminvs:
# Hack: OuterDemandMaintainer should be refactored to move
# to here, to avoid this import.
from incoq.demand.demtrans import OuterDemandMaintainer
# Determine dependencies of demand invariant.
at_rels = set(e.enumrel for e in demspec.join.clauses)
deltaname = L.N.deltaset(demname)
demcomp = demspec.to_comp({})
# Add delta maintenance code as per the invariant.
tree = inc_changetrack(tree, manager, demcomp, deltaname)
# Add code (outside all other maintenance) to propagate
# the delta changes to the actual inner query demand function.
tree = OuterDemandMaintainer.run(tree, manager, deltaname,
demname, at_rels,
L.get_vartuple(demcomp.resexp), None)
# Unwrap the demand clauses in the comp now that we've handled them.
spec = inccomp.spec
new_clauses = []
for cl in spec.join.clauses:
if cl.has_demand:
cl = cl.cl
new_clauses.append(cl)
new_spec = spec._replace(join=spec.join._replace(clauses=new_clauses))
inccomp.spec = new_spec
tree = CompReplacer.run(tree, manager, inccomp)
tree, comps = RelcompMaintainer.run(tree, manager, inccomp)
# If this was an original query, register it with the manager.
if 'in_original' in inccomp.comp.options:
manager.original_queryinvs.add(inccomp.name)
return tree, comps
def transform_source(source, *, nopts=None, qopts=None):
"""Like transform_ast, but from source code to source code."""
tree = L.p(source)
tree, manager = transform_ast(tree, nopts=nopts, qopts=qopts)
result = L.ts(tree)
manager.stats['lines'] = get_loc_source(result)
return result, manager
def WarnNameCost(self, node, name):
"""As above, but return a NameCost to act as a placeholder
instead.
"""
unname = 'UNKNOWN_' + name
if self.warn:
print('---- Unknown cost ---- ' + str(L.ts(node)) +
' (using placeholder ' + unname + ')')
return NameCost(unname)
def from_AST(cls, node):
# Try each clause until one doesn't raise TypeError.
for enumcls in cls.get_clause_kinds():
try:
return enumcls.from_AST(node, cls)
except TypeError:
pass
else:
raise TypeError('Cannot construct clause from node: ' + L.ts(node))
def from_AST(cls, node):
# Try each clause until one doesn't raise TypeError.
for enumcls in cls.get_clause_kinds():
try:
return enumcls.from_AST(node, cls)
except TypeError:
pass
else:
raise TypeError('Cannot construct clause from node: ' +
L.ts(node))
def visit_SetUpdate(self, node):
spec = self.incaggr.spec
node = self.generic_visit(node)
if not node.is_varupdate():
return node
var, op, elem = node.get_varupdate()
if var == spec.rel:
precode = postcode = ()
if op == 'add':
postcode = L.pc('ADDFUNC(ELEM)',
subst={'ADDFUNC': self.addfunc,
'ELEM': elem})
elif op == 'remove':
precode = L.pc('REMOVEFUNC(ELEM)',
subst={'REMOVEFUNC': self.removefunc,
'ELEM': elem})
else:
assert()
code = L.Maintenance(self.incaggr.name, L.ts(node),
precode, (node,), postcode)
elif var == L.N.uset(self.incaggr.name):
prefix = self.manager.namegen.next_prefix()
precode = postcode = ()
if op == 'add':
postcode = self.cg.make_addu_maint(prefix)
elif op == 'remove':
precode = self.cg.make_removeu_maint(prefix)
else:
assert()
code = L.Maintenance(self.incaggr.name, L.ts(node),
precode, (node,), postcode)
else:
code = node
return code
def impl_auxonly_relcomp(tree, manager, comp, name):
if manager.options.get_opt('verbose'):
s = ('Auxonly ' + name + ': ').ljust(45)
s += L.ts(comp)
print(s)
augmented = manager.options.get_opt('selfjoin_strat') == 'aug'
tree = AuxonlyTransformer.run(tree, manager, comp, name,
augmented=augmented)
return tree
def impl_auxonly_relcomp(tree, manager, comp, name):
if manager.options.get_opt('verbose'):
s = ('Auxonly ' + name + ': ').ljust(45)
s += L.ts(comp)
print(s)
augmented = manager.options.get_opt('selfjoin_strat') == 'aug'
tree = AuxonlyTransformer.run(tree,
manager,
comp,
name,
augmented=augmented)
return tree
def __str__(self):
param_str = ((', '.join(self.params) + ' -> ')
if len(self.params) > 0 else '')
proj_str = L.ts(self.resexp)
join_str = str(self.join)
return '{}{{{} : {}}}'.format(param_str, proj_str, join_str)
def __str__(self):
return L.ts(self.node)
def __str__(self):
return L.ts(self.node)
def transform_query(tree, manager, query, info):
"""Transform a single query. info is the dictionary returned
by QueryFinder.
"""
opman = manager.options
comp_dem_fallback = opman.get_opt('comp_dem_fallback')
aggr_batch_fallback = opman.get_opt('aggr_batch_fallback')
aggr_dem_fallback = opman.get_opt('aggr_dem_fallback')
impl = info['impl']
in_inccomp = info['in_inccomp']
if isinstance(query, L.Comp):
# If we can't handle this query, flag it and skip it.
if is_original(query) and not comp_isvalid(manager, query):
new_options = dict(query.options)
new_options['_invalid'] = True
rewritten_query = query._replace(options=new_options)
tree = L.QueryReplacer.run(tree, query, rewritten_query)
manager.stats['queries skipped'] += 1
if manager.options.get_opt('verbose'):
print('Skipping query ' + L.ts(query))
return tree
# Flatten lookups (e.g. into aggregate result maps) first,
# then rewrite patterns. (Opposite order fails to rewrite
# all occurrences of vars in the condition, since our
# renamer doesn't catch some cases like demparams of DEMQUERY
# nodes.)
rewritten_query = flatten_smlookups(query)
tree = L.QueryReplacer.run(tree, query, rewritten_query)
query = rewritten_query
if not opman.get_opt('pattern_in'):
rewritten_query = patternize_comp(query, manager.factory)
tree = L.QueryReplacer.run(tree, query, rewritten_query)
query = rewritten_query
# See if fallback applies.
if (impl == 'inc' and comp_inc_needs_dem(manager, query) and
comp_dem_fallback):
impl = 'dem'
name = next(manager.compnamegen)
if impl == 'auxonly':
tree = impl_auxonly_relcomp(tree, manager, query, name)
manager.stats['comps expanded'] += 1
elif impl == 'inc':
tree = inc_relcomp(tree, manager, query, name)
elif impl == 'dem':
tree = deminc_relcomp(tree, manager, query, name)
else:
assert()
if impl in ['inc', 'dem']:
if is_original(query):
manager.stats['orig queries'] += 1
manager.stats['incr queries'] += 1
manager.stats['incr comps'] += 1
elif isinstance(query, L.Aggregate):
# 'auxonly' doesn't apply to aggregates, but may appear here if
# it is selected as the default_impl. In any case, treat it as
# 'batch'.
if impl == 'auxonly':
impl = 'batch'
# See if fallbacks apply.
if (impl in ['inc', 'dem'] and aggr_needs_batch(query) and
aggr_batch_fallback):
new_options = dict(query.options)
new_options['_invalid'] = True
rewritten_query = query._replace(options=new_options)
tree = L.QueryReplacer.run(tree, query, rewritten_query)
manager.stats['queries skipped'] += 1
print('Skipping query ' + L.ts(query))
return tree
if (impl == 'inc' and (in_inccomp or aggr_needs_dem(query)) and
aggr_dem_fallback):
impl = 'dem'
if impl in ['inc', 'dem']:
name = next(manager.aggrnamegen)
half_demand = (info['half_demand'] and
aggr_canuse_halfdemand(query))
tree = inc_aggr(tree, manager, query, name,
demand=(impl=='dem'),
half_demand=half_demand)
if is_original(query):
manager.stats['orig queries'] += 1
manager.stats['incr queries'] += 1
manager.stats['incr aggrs'] += 1
else:
assert()
# Helpful for those long-running transformations.
manager.stats['queries processed'] += 1
processed = manager.stats['queries processed']
if processed % 100 == 0:
print('---- Transformed {} queries so far ----'.format(processed))
return tree
