def test_replacer(self):
look = L.pe('R.smlookup("bu", x)')
dem1 = L.pe('DEMQUERY(foo, [y], R.smlookup("bu", y))')
dem2 = L.pe('DEMQUERY(bar, [z], R.smlookup("bu", z))')
tree = L.pe('x + LOOK + DEM1 + DEM1 + DEM2',
subst={
'LOOK': look,
'DEM1': dem1,
'DEM2': dem2
})
namer = L.NameGenerator()
replacer = LookupReplacer(namer)
tree, clauses = replacer.process(tree)
repls = replacer.repls
exp_tree = L.pe('x + v1 + v2 + v2 + v3')
exp_clauses = [
L.Enumerator(L.sn('v1'), L.pe('{R.smlookup("bu", x)}')),
L.Enumerator(L.sn('v2'),
L.pe('DEMQUERY(foo, [y], {R.smlookup("bu", y)})')),
L.Enumerator(L.sn('v3'),
L.pe('DEMQUERY(bar, [z], {R.smlookup("bu", z)})')),
]
exp_repls = {
look: 'v1',
dem1: 'v2',
dem2: 'v3',
}
self.assertEqual(tree, exp_tree)
self.assertEqual(clauses, exp_clauses)
self.assertEqual(repls, exp_repls)
def test_replacer(self):
look = L.pe('R.smlookup("bu", x)')
dem1 = L.pe('DEMQUERY(foo, [y], R.smlookup("bu", y))')
dem2 = L.pe('DEMQUERY(bar, [z], R.smlookup("bu", z))')
tree = L.pe('x + LOOK + DEM1 + DEM1 + DEM2',
subst={'LOOK': look, 'DEM1': dem1, 'DEM2': dem2})
namer = L.NameGenerator()
replacer = LookupReplacer(namer)
tree, clauses = replacer.process(tree)
repls = replacer.repls
exp_tree = L.pe('x + v1 + v2 + v2 + v3')
exp_clauses = [
L.Enumerator(L.sn('v1'),
L.pe('{R.smlookup("bu", x)}')),
L.Enumerator(L.sn('v2'),
L.pe('DEMQUERY(foo, [y], {R.smlookup("bu", y)})')),
L.Enumerator(L.sn('v3'),
L.pe('DEMQUERY(bar, [z], {R.smlookup("bu", z)})')),
]
exp_repls = {
look: 'v1',
dem1: 'v2',
dem2: 'v3',
}
self.assertEqual(tree, exp_tree)
self.assertEqual(clauses, exp_clauses)
self.assertEqual(repls, exp_repls)
def make_addu_maint(self, prefix):
"""Generate code for after an addition to U."""
incaggr = self.incaggr
assert incaggr.has_demand
spec = incaggr.spec
mv_var = prefix + 'val'
elemvar = prefix + 'elem'
# If we're using half-demand, there's no demand to propagate
# to the operand. All we need to do is add an entry with count
# 0 if one is not already there.
if incaggr.half_demand:
return L.pc('''
S_MV = A.smdeflookup(MASK, KEY, None)
if MV is None:
A.smassignkey(MASK, KEY, ZERO, PREFIX)
''', subst={'A': L.ln(incaggr.name),
'S_MV': L.sn(mv_var),
'MV': L.ln(mv_var),
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'ZERO': self.make_zero_mapval_expr(),
'PREFIX': L.Str(prefix)})
update_code = self.make_update_state_code(
L.sn(mv_var), L.ln(mv_var), 'add',
L.ln(elemvar), prefix)
# Make operand demand function call, if operand uses demand.
if spec.has_oper_demand:
demfunc = L.N.demfunc(spec.oper_demname)
call_demfunc = L.Call(L.ln(demfunc),
tuple(L.ln(v) for v in spec.oper_demparams),
(), None, None)
propagate_code = (L.Expr(call_demfunc),)
else:
propagate_code = ()
code = L.pc('''
S_MV = ZERO
for S_ELEM in setmatch(R, RELMASK, PARAMS):
UPDATE_MAPVAL
A.smassignkey(AGGRMASK, KEY, MV, PREFIX)
PROPAGATE_DEMAND
''', subst={'S_MV': L.sn(mv_var),
'ZERO': self.make_zero_mapval_expr(),
'S_ELEM': L.sn(elemvar),
'R': spec.rel,
'RELMASK': spec.relmask.make_node(),
'PARAMS': L.tuplify(spec.params),
'<c>UPDATE_MAPVAL': update_code,
'A': L.ln(incaggr.name),
'AGGRMASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'MV': L.ln(mv_var),
'PREFIX': L.Str(prefix),
'<c>PROPAGATE_DEMAND': propagate_code})
return code
def visit_Tuple(self, node):
# No need to recurse, that's taken care of by the caller
# of this visitor.
tupvar = self.tupvar_namer.next()
arity = len(node.elts)
trel = make_trel(arity)
elts = (L.sn(tupvar), ) + node.elts
new_cl = L.Enumerator(L.tuplify(elts, lval=True), L.ln(trel))
self.new_clauses.append(new_cl)
self.trels.add(trel)
return L.sn(tupvar)
def visit_Tuple(self, node):
# No need to recurse, that's taken care of by the caller
# of this visitor.
tupvar = self.tupvar_namer.next()
arity = len(node.elts)
trel = make_trel(arity)
elts = (L.sn(tupvar),) + node.elts
new_cl = L.Enumerator(L.tuplify(elts, lval=True),
L.ln(trel))
self.new_clauses.append(new_cl)
self.trels.add(trel)
return L.sn(tupvar)
def visit_SetUpdate(self, node):
target_ok = LegalUpdateValidator.run(node.target)
elem_ok = LegalUpdateValidator.run(node.elem)
if target_ok and elem_ok:
return node
code = ()
if not target_ok:
targetvar = next(self.namegen)
code += (L.Assign((L.sn(targetvar),), node.target),)
node = node._replace(target=L.ln(targetvar))
if not elem_ok:
elemvar = next(self.namegen)
code += (L.Assign((L.sn(elemvar),), node.elem),)
node = node._replace(elem=L.ln(elemvar))
return code + (node,)
def to_AST(self):
mask = Mask.from_keylen(len(self.lhs) - 1)
keyvars = self.lhs[:-1]
var = self.lhs[-1]
sm = L.SMLookup(L.ln(self.rel),
mask.make_node().s, L.tuplify(keyvars), None)
return L.Enumerator(L.sn(var), L.Set((sm, )))
def visit_SetUpdate(self, node):
target_ok = LegalUpdateValidator.run(node.target)
elem_ok = LegalUpdateValidator.run(node.elem)
if target_ok and elem_ok:
return node
code = ()
if not target_ok:
targetvar = next(self.namegen)
code += (L.Assign((L.sn(targetvar), ), node.target), )
node = node._replace(target=L.ln(targetvar))
if not elem_ok:
elemvar = next(self.namegen)
code += (L.Assign((L.sn(elemvar), ), node.elem), )
node = node._replace(elem=L.ln(elemvar))
return code + (node, )
def make_update_state_code(self, state_snode, state_lnode, op, val_node,
prefix):
add_template = L.trim('''
S_TREE, _ = STATE
TREE[VAL] = None
S_STATE = (TREE, TREE.MINMAX())
''')
remove_template = L.trim('''
S_TREE, _ = STATE
del TREE[VAL]
S_STATE = (TREE, TREE.MINMAX())
''')
template = {'add': add_template, 'remove': remove_template}[op]
treevar = prefix + 'tree'
minmax = {'min': '__min__', 'max': '__max__'}[self.kind]
code = L.pc(template,
subst={
'S_TREE': L.sn(treevar),
'TREE': L.ln(treevar),
'@MINMAX': minmax,
'STATE': state_lnode,
'S_STATE': state_snode,
'VAL': val_node
})
return code
def to_AST(self):
mask = Mask.from_keylen(len(self.lhs) - 1)
keyvars = self.lhs[:-1]
var = self.lhs[-1]
sm = L.SMLookup(L.ln(self.rel), mask.make_node().s,
L.tuplify(keyvars), None)
return L.Enumerator(L.sn(var), L.Set((sm,)))
def make_update_state_code(self, state_snode, state_lnode,
op, val_node, prefix):
add_template = L.trim('''
S_TREE, _ = STATE
TREE[VAL] = None
S_STATE = (TREE, TREE.MINMAX())
''')
remove_template = L.trim('''
S_TREE, _ = STATE
del TREE[VAL]
S_STATE = (TREE, TREE.MINMAX())
''')
template = {'add': add_template, 'remove': remove_template}[op]
treevar = prefix + 'tree'
minmax = {'min': '__min__', 'max': '__max__'}[self.kind]
code = L.pc(template,
subst={'S_TREE': L.sn(treevar),
'TREE': L.ln(treevar),
'@MINMAX': minmax,
'STATE': state_lnode,
'S_STATE': state_snode,
'VAL': val_node})
return code
def visit_DemQuery(self, node):
self.demwrapped_nodes.add(node.value)
node = self.generic_visit(node)
if not isinstance(node.value, L.SMLookup):
return node
sm = node.value
assert sm.default is None
v = self.repls.get(node, None)
if v is not None:
# Reuse existing entry.
var = v
else:
# Create new entry.
self.repls[node] = var = next(self.namer)
# Create accompanying clause. Has form
# var in DEMQUERY(..., {smlookup})
# The clause constructor logic will later rewrite that,
# or else fail if there's a syntax problem.
cl_target = L.sn(var)
cl_iter = node._replace(value=L.Set((sm,)))
new_cl = L.Enumerator(cl_target, cl_iter)
self.new_clauses.append(new_cl)
return L.ln(var)
def visit_DelKey(self, node):
target_ok = LegalUpdateValidator.run(node.target)
key_ok = LegalUpdateValidator.run(node.key)
if target_ok and key_ok:
return node
code = ()
if not target_ok:
targetvar = next(self.namegen)
code += (L.Assign((L.sn(targetvar),), node.target),)
node = node._replace(target=L.ln(targetvar))
if not key_ok:
keyvar = next(self.namegen)
code += (L.Assign((L.sn(keyvar),), node.key),)
node = node._replace(key=L.ln(keyvar))
return code + (node,)
def visit_DelKey(self, node):
target_ok = LegalUpdateValidator.run(node.target)
key_ok = LegalUpdateValidator.run(node.key)
if target_ok and key_ok:
return node
code = ()
if not target_ok:
targetvar = next(self.namegen)
code += (L.Assign((L.sn(targetvar), ), node.target), )
node = node._replace(target=L.ln(targetvar))
if not key_ok:
keyvar = next(self.namegen)
code += (L.Assign((L.sn(keyvar), ), node.key), )
node = node._replace(key=L.ln(keyvar))
return code + (node, )
def make_update_mapval_code(self, mv_snode, mv_lnode, op, val_node,
prefix):
"""Produce code to make a new mapval, given an update to
the corresponding operand. The mapval is read from mv_lnode
and written to mv_snode.
"""
# If we don't track counts, the mapvals are the same as
# the states.
if not self.incaggr.tracks_counts:
return self.make_update_state_code(mv_snode, mv_lnode, op,
val_node, prefix)
statevar = prefix + 'state'
state_lnode = L.ln(statevar)
state_snode = L.sn(statevar)
countvar = prefix + 'count'
updatestate_code = self.make_update_state_code(state_snode,
state_lnode, op,
val_node, prefix)
if op == 'add':
template = 'COUNTVAR + 1'
elif op == 'remove':
template = 'COUNTVAR - 1'
else:
assert ()
new_count_node = L.pe(template, subst={'COUNTVAR': L.ln(countvar)})
return L.pc('''
S_STATE, S_COUNTVAR = MV
UPDATE_STATE
S_MV = STATE, NEW_COUNT
''',
subst={
'S_STATE': state_snode,
'S_COUNTVAR': L.sn(countvar),
'MV': mv_lnode,
'<c>UPDATE_STATE': updatestate_code,
'STATE': state_lnode,
'NEW_COUNT': new_count_node,
'S_MV': mv_snode
})
def visit_Module(self, node):
node = self.generic_visit(node)
decls = ()
if self.use_mset:
decls += L.pc('''
M = MSet()
''', subst={'M': L.sn(make_mrel())})
for field in self.fields:
decls += L.pc('''
F = FSet()
''', subst={'F': L.sn(make_frel(field))})
if self.use_mapset:
decls += L.pc('''
MAP = MAPSet()
''', subst={'MAP': L.sn(make_maprel())})
node = node._replace(body=decls + node.body)
return node
def visit_SetUpdate(self, node):
if not (isinstance(node.target, L.Name)
and node.target.id == self.rel):
return
fresh = next(self.namegen)
tvar = '_t' + fresh
ftvar = '_ft' + fresh
code = make_flattup_code(self.tuptype, node.elem, L.sn(ftvar), tvar)
update = node._replace(elem=L.ln(ftvar))
return code + (update, )
def visit_SetUpdate(self, node):
if not (isinstance(node.target, L.Name) and
node.target.id == self.rel):
return
fresh = next(self.namegen)
tvar = '_t' + fresh
ftvar = '_ft' + fresh
code = make_flattup_code(self.tuptype, node.elem,
L.sn(ftvar), tvar)
update = node._replace(elem=L.ln(ftvar))
return code + (update,)
def make_update_mapval_code(self, mv_snode, mv_lnode,
op, val_node, prefix):
"""Produce code to make a new mapval, given an update to
the corresponding operand. The mapval is read from mv_lnode
and written to mv_snode.
"""
# If we don't track counts, the mapvals are the same as
# the states.
if not self.incaggr.tracks_counts:
return self.make_update_state_code(mv_snode, mv_lnode,
op, val_node, prefix)
statevar = prefix + 'state'
state_lnode = L.ln(statevar)
state_snode = L.sn(statevar)
countvar = prefix + 'count'
updatestate_code = self.make_update_state_code(
state_snode, state_lnode,
op, val_node, prefix)
if op == 'add':
template = 'COUNTVAR + 1'
elif op == 'remove':
template = 'COUNTVAR - 1'
else:
assert()
new_count_node = L.pe(template, subst={'COUNTVAR': L.ln(countvar)})
return L.pc('''
S_STATE, S_COUNTVAR = MV
UPDATE_STATE
S_MV = STATE, NEW_COUNT
''', subst={'S_STATE': state_snode,
'S_COUNTVAR': L.sn(countvar),
'MV': mv_lnode,
'<c>UPDATE_STATE': updatestate_code,
'STATE': state_lnode,
'NEW_COUNT': new_count_node,
'S_MV': mv_snode})
def visit_Module(self, node):
node = self.generic_visit(node)
decls = ()
if self.use_mset:
decls += L.pc('''
M = MSet()
''',
subst={'M': L.sn(make_mrel())})
for field in self.fields:
decls += L.pc('''
F = FSet()
''',
subst={'F': L.sn(make_frel(field))})
if self.use_mapset:
decls += L.pc('''
MAP = MAPSet()
''',
subst={'MAP': L.sn(make_maprel())})
node = node._replace(body=decls + node.body)
return node
def make_removeu_maint(self, prefix):
"""Generate code for before a removal from U."""
incaggr = self.incaggr
assert incaggr.has_demand
spec = incaggr.spec
mv_var = prefix + 'val'
# If we're using half-demand, there's no demand to propagate
# to the operand. All we need to do is determine whether to
# do the removal by checking whether the count is 0.
if incaggr.half_demand:
return L.pc('''
S_MV = A.smlookup(MASK, KEY)
if COUNT == 0:
A.smdelkey(MASK, KEY, PREFIX)
''',
subst={
'S_MV': L.sn(mv_var),
'COUNT': self.mapval_proj_count(L.ln(mv_var)),
'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'PREFIX': L.Str(prefix)
})
# Generate operand undemand function call, if operand
# uses demand.
if spec.has_oper_demand:
undemfunc = L.N.undemfunc(spec.oper_demname)
call_undemfunc = L.Call(
L.ln(undemfunc), tuple(L.ln(v) for v in spec.oper_demparams),
(), None, None)
propagate_code = (L.Expr(call_undemfunc), )
else:
propagate_code = ()
code = L.pc('''
PROPAGATE_DEMAND
A.smdelkey(MASK, KEY, PREFIX)
''',
subst={
'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'PREFIX': L.Str(prefix),
'<c>PROPAGATE_DEMAND': propagate_code
})
return code
def test_flattup_code(self):
in_node = L.ln('x')
out_node = L.sn('y')
code = make_flattup_code(self.tuptype, in_node, out_node, 't')
exp_code = L.pc('''
y = (x[0], x[1][0], x[1][1])
''')
self.assertEqual(code, exp_code)
in_node = L.pe('x.a')
code = make_flattup_code(self.tuptype, in_node, out_node, 't')
exp_code = L.pc('''
t = x.a
y = (t[0], t[1][0], t[1][1])
''')
self.assertEqual(code, exp_code)
def test_flattup_code(self):
in_node = L.ln('x')
out_node = L.sn('y')
code = make_flattup_code(self.tuptype, in_node, out_node, 't')
exp_code = L.pc('''
y = (x[0], x[1][0], x[1][1])
''')
self.assertEqual(code, exp_code)
in_node = L.pe('x.a')
code = make_flattup_code(self.tuptype, in_node, out_node, 't')
exp_code = L.pc('''
t = x.a
y = (t[0], t[1][0], t[1][1])
''')
self.assertEqual(code, exp_code)
def test_parseclause(self):
comp = L.pe('''
COMP({... for (a, b) in _M for (c, d) in _F_e
for (f, g, h) in _MAP if x > 5}, [], {})
''')
res1 = get_menum(comp.clauses[0])
exp_res1 = (L.sn('a'), L.sn('b'))
res2 = get_fenum(comp.clauses[1])
exp_res2 = (L.sn('c'), L.sn('d'), 'e')
res3 = get_mapenum(comp.clauses[2])
exp_res3 = (L.sn('f'), L.sn('g'), L.sn('h'))
self.assertEqual(res1, exp_res1)
self.assertEqual(res2, exp_res2)
self.assertEqual(res3, exp_res3)
def test_parseclause(self):
comp = L.pe('''
COMP({... for (a, b) in _M for (c, d) in _F_e
for (f, g, h) in _MAP if x > 5}, [], {})
''')
res1 = get_menum(comp.clauses[0])
exp_res1 = (L.sn('a'), L.sn('b'))
res2 = get_fenum(comp.clauses[1])
exp_res2 = (L.sn('c'), L.sn('d'), 'e')
res3 = get_mapenum(comp.clauses[2])
exp_res3 = (L.sn('f'), L.sn('g'), L.sn('h'))
self.assertEqual(res1, exp_res1)
self.assertEqual(res2, exp_res2)
self.assertEqual(res3, exp_res3)
def helper(self, node, no_update=False):
fresh = next(self.namegen)
if no_update:
template = L.trim('''
S_VAR = Set()
''')
else:
template = L.trim('''
S_VAR = Set()
L_VAR.update(EXPR)
''')
new_code = L.pc(template, subst={'L_VAR': L.ln(fresh),
'S_VAR': L.sn(fresh),
'EXPR': node})
self.pre_stmts.extend(new_code)
return L.ln(fresh)
def make_flattup_code(tuptype, in_node, out_node, tempvar):
"""Given a tuple tree type, make code to take the tuple given by the
expression in_node and store the flattened form in the tuple given
by out_node.
"""
def leaf_to_expr(root, path):
"""Turn a leaf path into a series of subscript expressions
that obtain the leaf value from the root value.
"""
node = root
for i in path:
node = L.Subscript(node, L.Index(L.Num(i)), L.Load())
return node
leaves = tuptype_leaves(tuptype)
code = ()
# If in_expr is just a variable name, use it as is.
# Otherwise store it in a temporary variable to avoid redundant
# evaluation of in_expr.
if isinstance(in_node, L.Name):
root_node = in_node
else:
rootname = tempvar
code += L.pc('''
ROOT = IN_NODE
''',
subst={
'IN_NODE': in_node,
'ROOT': L.sn(rootname)
})
root_node = L.ln(rootname)
flattuple_expr = L.Tuple(
tuple(leaf_to_expr(root_node, leaf) for leaf in leaves), L.Load())
code += L.pc('''
OUT_NODE = FLATTUP
''',
subst={
'FLATTUP': flattuple_expr,
'OUT_NODE': out_node
})
return code
def make_removeu_maint(self, prefix):
"""Generate code for before a removal from U."""
incaggr = self.incaggr
assert incaggr.has_demand
spec = incaggr.spec
mv_var = prefix + 'val'
# If we're using half-demand, there's no demand to propagate
# to the operand. All we need to do is determine whether to
# do the removal by checking whether the count is 0.
if incaggr.half_demand:
return L.pc('''
S_MV = A.smlookup(MASK, KEY)
if COUNT == 0:
A.smdelkey(MASK, KEY, PREFIX)
''', subst={'S_MV': L.sn(mv_var),
'COUNT': self.mapval_proj_count(L.ln(mv_var)),
'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'PREFIX': L.Str(prefix)})
# Generate operand undemand function call, if operand
# uses demand.
if spec.has_oper_demand:
undemfunc = L.N.undemfunc(spec.oper_demname)
call_undemfunc = L.Call(L.ln(undemfunc),
tuple(L.ln(v) for v in spec.oper_demparams),
(), None, None)
propagate_code = (L.Expr(call_undemfunc),)
else:
propagate_code = ()
code = L.pc('''
PROPAGATE_DEMAND
A.smdelkey(MASK, KEY, PREFIX)
''', subst={'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'PREFIX': L.Str(prefix),
'<c>PROPAGATE_DEMAND': propagate_code})
return code
def visit_SMLookup(self, node):
node = self.generic_visit(node)
if node in self.demwrapped_nodes:
return node
sm = node
assert sm.default is None
v = self.repls.get(node, None)
if v is not None:
var = v
else:
self.repls[node] = var = next(self.namer)
cl_target = L.sn(var)
cl_iter = L.Set((sm,))
new_cl = L.Enumerator(cl_target, cl_iter)
self.new_clauses.append(new_cl)
return L.ln(var)
def test_lookupclause(self):
cl = LookupClause(('x', 'y', 'z'), 'R')
# AST round-trip.
clast = cl.to_AST()
sm = L.SMLookup(L.ln('R'), 'bbu', L.tuplify(['x', 'y']), None)
exp_clast = L.Enumerator(L.sn('z'), L.Set((sm, )))
self.assertEqual(clast, exp_clast)
cl2 = LookupClause.from_AST(exp_clast, DummyFactory)
self.assertEqual(cl2, cl)
# Attributes.
self.assertEqual(cl.enumvars, ('x', 'y', 'z'))
# Rewriting.
cl2 = cl.rewrite_subst({'x': 'xx', 'z': 'zz'}, DummyFactory)
self.assertEqual(cl2, LookupClause(('xx', 'y', 'zz'), 'R'))
# Rating.
self.assertEqual(cl.rate(['x']), Rate.NORMAL)
self.assertEqual(cl.rate(['x', 'y']), Rate.CONSTANT)
def test_lookupclause(self):
cl = LookupClause(('x', 'y', 'z'), 'R')
# AST round-trip.
clast = cl.to_AST()
sm = L.SMLookup(L.ln('R'), 'bbu', L.tuplify(['x', 'y']), None)
exp_clast = L.Enumerator(L.sn('z'), L.Set((sm,)))
self.assertEqual(clast, exp_clast)
cl2 = LookupClause.from_AST(exp_clast, DummyFactory)
self.assertEqual(cl2, cl)
# Attributes.
self.assertEqual(cl.enumvars, ('x', 'y', 'z'))
# Rewriting.
cl2 = cl.rewrite_subst({'x': 'xx', 'z': 'zz'}, DummyFactory)
self.assertEqual(cl2, LookupClause(('xx', 'y', 'zz'), 'R'))
# Rating.
self.assertEqual(cl.rate(['x']), Rate.NORMAL)
self.assertEqual(cl.rate(['x', 'y']), Rate.CONSTANT)
def helper(self, node, no_update=False):
fresh = next(self.namegen)
if no_update:
template = L.trim('''
S_VAR = Set()
''')
else:
template = L.trim('''
S_VAR = Set()
L_VAR.update(EXPR)
''')
new_code = L.pc(template,
subst={
'L_VAR': L.ln(fresh),
'S_VAR': L.sn(fresh),
'EXPR': node
})
self.pre_stmts.extend(new_code)
return L.ln(fresh)
def make_flattup_code(tuptype, in_node, out_node, tempvar):
"""Given a tuple tree type, make code to take the tuple given by the
expression in_node and store the flattened form in the tuple given
by out_node.
"""
def leaf_to_expr(root, path):
"""Turn a leaf path into a series of subscript expressions
that obtain the leaf value from the root value.
"""
node = root
for i in path:
node = L.Subscript(node, L.Index(L.Num(i)), L.Load())
return node
leaves = tuptype_leaves(tuptype)
code = ()
# If in_expr is just a variable name, use it as is.
# Otherwise store it in a temporary variable to avoid redundant
# evaluation of in_expr.
if isinstance(in_node, L.Name):
root_node = in_node
else:
rootname = tempvar
code += L.pc('''
ROOT = IN_NODE
''', subst={'IN_NODE': in_node,
'ROOT': L.sn(rootname)})
root_node = L.ln(rootname)
flattuple_expr = L.Tuple(tuple(leaf_to_expr(root_node, leaf)
for leaf in leaves),
L.Load())
code += L.pc('''
OUT_NODE = FLATTUP
''', subst={'FLATTUP': flattuple_expr,
'OUT_NODE': out_node})
return code
def visit_Aggregate(self, node):
node = self.generic_visit(node)
operand = node.value
if isinstance(operand, L.Comp):
return node
if not is_retrievalchain(operand):
# Bailout, looks like we won't be able to incrementalize
# this later anyway.
return node
# Replace with {_e for _e in OPERAND}.
# This case is for both single vars and retrieval chains.
# The comp's options are inherited from the aggregate.
params = get_retrieval_params(operand)
elem = '_e'
clause = L.Enumerator(target=L.sn(elem), iter=operand)
node = node._replace(value=L.Comp(resexp=L.ln(elem),
clauses=(clause, ),
params=params,
options=node.options))
return node
def visit_Aggregate(self, node):
node = self.generic_visit(node)
operand = node.value
if isinstance(operand, L.Comp):
return node
if not is_retrievalchain(operand):
# Bailout, looks like we won't be able to incrementalize
# this later anyway.
return node
# Replace with {_e for _e in OPERAND}.
# This case is for both single vars and retrieval chains.
# The comp's options are inherited from the aggregate.
params = get_retrieval_params(operand)
elem = '_e'
clause = L.Enumerator(target=L.sn(elem),
iter=operand)
node = node._replace(value=L.Comp(resexp=L.ln(elem),
clauses=(clause,),
params=params,
options=node.options))
return node
def visit_Module(self, node):
mapname = self.spec.map_name
addcode = make_auxmap_maint_code(self.manager, self.spec,
L.ln('_e'), 'add')
removecode = make_auxmap_maint_code(self.manager, self.spec,
L.ln('_e'), 'remove')
code = L.pc('''
MAP = Map()
def ADDFUNC(_e):
ADDCODE
def REMOVEFUNC(_e):
REMOVECODE
''', subst={'MAP': L.sn(mapname),
'<def>ADDFUNC': self.addfunc_name,
'<c>ADDCODE': addcode,
'<def>REMOVEFUNC': self.removefunc_name,
'<c>REMOVECODE': removecode})
node = node._replace(body=code + node.body)
node = self.generic_visit(node)
return node
def make_addu_maint(self, prefix):
"""Generate code for after an addition to U."""
incaggr = self.incaggr
assert incaggr.has_demand
spec = incaggr.spec
mv_var = prefix + 'val'
elemvar = prefix + 'elem'
# If we're using half-demand, there's no demand to propagate
# to the operand. All we need to do is add an entry with count
# 0 if one is not already there.
if incaggr.half_demand:
return L.pc('''
S_MV = A.smdeflookup(MASK, KEY, None)
if MV is None:
A.smassignkey(MASK, KEY, ZERO, PREFIX)
''',
subst={
'A': L.ln(incaggr.name),
'S_MV': L.sn(mv_var),
'MV': L.ln(mv_var),
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'ZERO': self.make_zero_mapval_expr(),
'PREFIX': L.Str(prefix)
})
update_code = self.make_update_state_code(L.sn(mv_var), L.ln(mv_var),
'add', L.ln(elemvar), prefix)
# Make operand demand function call, if operand uses demand.
if spec.has_oper_demand:
demfunc = L.N.demfunc(spec.oper_demname)
call_demfunc = L.Call(L.ln(demfunc),
tuple(L.ln(v) for v in spec.oper_demparams),
(), None, None)
propagate_code = (L.Expr(call_demfunc), )
else:
propagate_code = ()
code = L.pc('''
S_MV = ZERO
for S_ELEM in setmatch(R, RELMASK, PARAMS):
UPDATE_MAPVAL
A.smassignkey(AGGRMASK, KEY, MV, PREFIX)
PROPAGATE_DEMAND
''',
subst={
'S_MV': L.sn(mv_var),
'ZERO': self.make_zero_mapval_expr(),
'S_ELEM': L.sn(elemvar),
'R': spec.rel,
'RELMASK': spec.relmask.make_node(),
'PARAMS': L.tuplify(spec.params),
'<c>UPDATE_MAPVAL': update_code,
'A': L.ln(incaggr.name),
'AGGRMASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(incaggr.params),
'MV': L.ln(mv_var),
'PREFIX': L.Str(prefix),
'<c>PROPAGATE_DEMAND': propagate_code
})
return code
def fset_bindmatch(field, mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over an F-set."""
# If we had such a thing as a negated field membership clause,
# then the negative test for hasattr() would actually conflict
# with the type check.
tc_applicable = False
if mask == Mask.BB:
tc_applicable = True
cont, item = bvars
code = L.pc('''
if CONT.FIELD == ITEM:
BODY
''', subst={'CONT': L.ln(cont),
'@FIELD': field,
'ITEM': L.ln(item),
'<c>BODY': body})
elif mask == Mask.OUT:
tc_applicable = True
(cont,) = bvars
(item,) = uvars
code = L.pc('''
ITEM = CONT.FIELD
BODY
''', subst={'CONT': L.ln(cont),
'@FIELD': field,
'ITEM': L.sn(item),
'<c>BODY': body})
elif mask == Mask.B1:
tc_applicable = True
(cont,) = (item,) = bvars
code = L.pc('''
if CONT == CONT.FIELD:
BODY
''', subst={'CONT': L.ln(cont),
'@FIELD': field,
'<c>BODY': body})
elif mask == Mask.BW:
# Not applicable because the code and the type check
# are the same thing.
tc_applicable = False
(cont,) = bvars
code = L.pc('''
if hasattr(CONT, FIELD):
BODY
''', subst={'CONT': L.ln(cont),
'FIELD': L.Str(field),
'<c>BODY': body})
elif mask == Mask.UU:
raise AssertionError('No object-domain equivalent for iterating over '
'the M-set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''', subst={'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln('_F_' + field),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body})
if typecheck and tc_applicable:
code = L.pc('''
if hasattr(CONT, FIELD):
CODE
''', subst={'CONT': cont,
'FIELD': L.Str(field),
'<c>CODE': code})
return code
def fset_bindmatch(field, mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over an F-set."""
# If we had such a thing as a negated field membership clause,
# then the negative test for hasattr() would actually conflict
# with the type check.
tc_applicable = False
if mask == Mask.BB:
tc_applicable = True
cont, item = bvars
code = L.pc('''
if CONT.FIELD == ITEM:
BODY
''',
subst={
'CONT': L.ln(cont),
'@FIELD': field,
'ITEM': L.ln(item),
'<c>BODY': body
})
elif mask == Mask.OUT:
tc_applicable = True
(cont, ) = bvars
(item, ) = uvars
code = L.pc('''
ITEM = CONT.FIELD
BODY
''',
subst={
'CONT': L.ln(cont),
'@FIELD': field,
'ITEM': L.sn(item),
'<c>BODY': body
})
elif mask == Mask.B1:
tc_applicable = True
(cont, ) = (item, ) = bvars
code = L.pc('''
if CONT == CONT.FIELD:
BODY
''',
subst={
'CONT': L.ln(cont),
'@FIELD': field,
'<c>BODY': body
})
elif mask == Mask.BW:
# Not applicable because the code and the type check
# are the same thing.
tc_applicable = False
(cont, ) = bvars
code = L.pc('''
if hasattr(CONT, FIELD):
BODY
''',
subst={
'CONT': L.ln(cont),
'FIELD': L.Str(field),
'<c>BODY': body
})
elif mask == Mask.UU:
raise AssertionError('No object-domain equivalent for iterating over '
'the M-set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''',
subst={
'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln('_F_' + field),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body
})
if typecheck and tc_applicable:
code = L.pc('''
if hasattr(CONT, FIELD):
CODE
''',
subst={
'CONT': cont,
'FIELD': L.Str(field),
'<c>CODE': code
})
return code
def test_getclausevars(self):
lhs = L.Tuple((L.sn('x'), L.Tuple((L.sn('y'), L.sn('z')), L.Store())),
L.Store())
vars = get_clause_vars(L.Enumerator(lhs, L.ln('R')), self.tuptype)
exp_vars = ['x', 'y', 'z']
self.assertEqual(vars, exp_vars)
def test_getclausevars(self):
lhs = L.Tuple((L.sn('x'), L.Tuple((L.sn('y'), L.sn('z')), L.Store())),
L.Store())
vars = get_clause_vars(L.Enumerator(lhs, L.ln('R')), self.tuptype)
exp_vars = ['x', 'y', 'z']
self.assertEqual(vars, exp_vars)
def mapset_bindmatch(mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over the MAP set."""
tc_applicable = False
if mask == Mask('bbb'):
tc_applicable = True
map, key, value = bvars
code = L.pc('''
if KEY in MAP and MAP[KEY] == VALUE:
BODY
''',
subst={
'MAP': L.ln(map),
'KEY': L.ln(key),
'VALUE': L.ln(value),
'<c>BODY': body
})
elif mask == Mask('bbu'):
tc_applicable = True
map, key = bvars
(value, ) = uvars
code = L.pc('''
if KEY in MAP:
VALUE = MAP[KEY]
BODY
''',
subst={
'MAP': L.ln(map),
'KEY': L.ln(key),
'VALUE': L.sn(value),
'<c>BODY': body
})
elif mask == Mask('buu'):
tc_applicable = True
(map, ) = bvars
key, value = uvars
code = L.pc('''
for KEY, VALUE in MAP.items():
BODY
''',
subst={
'MAP': L.ln(map),
'KEY': L.sn(key),
'VALUE': L.sn(value),
'<c>BODY': body
})
# Other variations involving wildcards and equalities are possible,
# but for now they're handled by the general auxmap case.
elif mask == Mask('uuu'):
raise AssertionError('No object-domain equivalent for iterating over '
'the MAP set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''',
subst={
'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln(make_maprel()),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body
})
if typecheck and tc_applicable:
code = L.pc('''
if isinstance(MAP, Map):
CODE
''',
subst={
'MAP': map,
'<c>CODE': code
})
return code
def make_oper_maint(self, prefix, op, elem):
"""Generate code for an addition or removal update to the operand."""
incaggr = self.incaggr
spec = incaggr.spec
relmask = spec.relmask
mv_var = prefix + 'val'
uset = L.N.uset(incaggr.name)
vars = tuple(prefix + 'v' + str(i) for i in range(1, len(relmask) + 1))
bvars, uvars, _ = relmask.split_vars(vars)
nextmapval_code = self.make_update_mapval_code(L.sn(mv_var),
L.ln(mv_var), op,
L.tuplify(uvars),
prefix)
subst = {
'S_VARS': L.tuplify(vars, lval=True),
'ELEM': elem,
'KEY': L.tuplify(bvars),
'ZERO': self.make_zero_mapval_expr(),
'U': L.ln(uset),
'S_MV': L.sn(mv_var),
'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(bvars),
'<c>NEXT_MAPVAL': nextmapval_code,
'MV': L.ln(mv_var),
'PREFIX': L.Str(prefix)
}
# We break into different cases based on whether we're using
# demand or not, because the invariants are different in terms
# of what keys are in the map.
if incaggr.has_demand and not incaggr.half_demand:
# If the U-set check passes, the key is definitely in
# the map, so use strict lookups and updates.
code = L.pc('''
S_VARS = ELEM
if KEY in U:
S_MV = A.smlookup(MASK, KEY)
NEXT_MAPVAL
A.smreassignkey(MASK, KEY, MV, PREFIX)
''',
subst=subst)
else:
# The keys in the map should exist iff the corresponding
# operand set is non-empty. For addition, use non-strict
# lookups and updates, since we don't know whether it was
# empty before. For removal, use strict lookup since we
# know it's non-empty, but check the count to tell whether
# to delete it or strictly reassign it. When using half-
# demand, only delete it if it's not demanded.
subst['COUNT'] = self.mapval_proj_count(L.ln(mv_var))
if incaggr.half_demand:
# Check for a count of 1, not 0, because it's the value
# of count before the update.
delete_cond = L.pe('COUNT == 1 and KEY not in U', subst=subst)
else:
delete_cond = L.pe('COUNT == 1', subst=subst)
subst['DELETE_COND'] = delete_cond
if op == 'add':
code = L.pc('''
S_VARS = ELEM
S_MV = A.smdeflookup(MASK, KEY, ZERO)
NEXT_MAPVAL
A.smnsassignkey(MASK, KEY, MV, PREFIX)
''',
subst=subst)
elif op == 'remove':
code = L.pc('''
S_VARS = ELEM
S_MV = A.smlookup(MASK, KEY)
if DELETE_COND:
A.smdelkey(MASK, KEY, PREFIX)
else:
NEXT_MAPVAL
A.smreassignkey(MASK, KEY, MV, PREFIX)
''',
subst=subst)
else:
assert ()
# Guard code in a delta check if necessary.
if relmask.has_wildcards or relmask.has_equalities:
code = L.pc('''
if deltamatch(R, MASK, ELEM, 1):
CODE
''',
subst={
'R': spec.rel,
'MASK': incaggr.oper_deltamask,
'ELEM': elem,
'<c>CODE': code
})
return code
def mapset_bindmatch(mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over the MAP set."""
tc_applicable = False
if mask == Mask('bbb'):
tc_applicable = True
map, key, value = bvars
code = L.pc('''
if KEY in MAP and MAP[KEY] == VALUE:
BODY
''', subst={'MAP': L.ln(map),
'KEY': L.ln(key),
'VALUE': L.ln(value),
'<c>BODY': body})
elif mask == Mask('bbu'):
tc_applicable = True
map, key = bvars
(value,) = uvars
code = L.pc('''
if KEY in MAP:
VALUE = MAP[KEY]
BODY
''', subst={'MAP': L.ln(map),
'KEY': L.ln(key),
'VALUE': L.sn(value),
'<c>BODY': body})
elif mask == Mask('buu'):
tc_applicable = True
(map,) = bvars
key, value = uvars
code = L.pc('''
for KEY, VALUE in MAP.items():
BODY
''', subst={'MAP': L.ln(map),
'KEY': L.sn(key),
'VALUE': L.sn(value),
'<c>BODY': body})
# Other variations involving wildcards and equalities are possible,
# but for now they're handled by the general auxmap case.
elif mask == Mask('uuu'):
raise AssertionError('No object-domain equivalent for iterating over '
'the MAP set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''', subst={'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln(make_maprel()),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body})
if typecheck and tc_applicable:
code = L.pc('''
if isinstance(MAP, Map):
CODE
''', subst={'MAP': map,
'<c>CODE': code})
return code
def make_oper_maint(self, prefix, op, elem):
"""Generate code for an addition or removal update to the operand."""
incaggr = self.incaggr
spec = incaggr.spec
relmask = spec.relmask
mv_var = prefix + 'val'
uset = L.N.uset(incaggr.name)
vars = tuple(prefix + 'v' + str(i)
for i in range(1, len(relmask) + 1))
bvars, uvars, _ = relmask.split_vars(vars)
nextmapval_code = self.make_update_mapval_code(
L.sn(mv_var), L.ln(mv_var),
op, L.tuplify(uvars), prefix)
subst = {'S_VARS': L.tuplify(vars, lval=True),
'ELEM': elem,
'KEY': L.tuplify(bvars),
'ZERO': self.make_zero_mapval_expr(),
'U': L.ln(uset),
'S_MV': L.sn(mv_var),
'A': incaggr.name,
'MASK': incaggr.aggrmask.make_node(),
'KEY': L.tuplify(bvars),
'<c>NEXT_MAPVAL': nextmapval_code,
'MV': L.ln(mv_var),
'PREFIX': L.Str(prefix)}
# We break into different cases based on whether we're using
# demand or not, because the invariants are different in terms
# of what keys are in the map.
if incaggr.has_demand and not incaggr.half_demand:
# If the U-set check passes, the key is definitely in
# the map, so use strict lookups and updates.
code = L.pc('''
S_VARS = ELEM
if KEY in U:
S_MV = A.smlookup(MASK, KEY)
NEXT_MAPVAL
A.smreassignkey(MASK, KEY, MV, PREFIX)
''', subst=subst)
else:
# The keys in the map should exist iff the corresponding
# operand set is non-empty. For addition, use non-strict
# lookups and updates, since we don't know whether it was
# empty before. For removal, use strict lookup since we
# know it's non-empty, but check the count to tell whether
# to delete it or strictly reassign it. When using half-
# demand, only delete it if it's not demanded.
subst['COUNT'] = self.mapval_proj_count(L.ln(mv_var))
if incaggr.half_demand:
# Check for a count of 1, not 0, because it's the value
# of count before the update.
delete_cond = L.pe('COUNT == 1 and KEY not in U', subst=subst)
else:
delete_cond = L.pe('COUNT == 1', subst=subst)
subst['DELETE_COND'] = delete_cond
if op == 'add':
code = L.pc('''
S_VARS = ELEM
S_MV = A.smdeflookup(MASK, KEY, ZERO)
NEXT_MAPVAL
A.smnsassignkey(MASK, KEY, MV, PREFIX)
''', subst=subst)
elif op == 'remove':
code = L.pc('''
S_VARS = ELEM
S_MV = A.smlookup(MASK, KEY)
if DELETE_COND:
A.smdelkey(MASK, KEY, PREFIX)
else:
NEXT_MAPVAL
A.smreassignkey(MASK, KEY, MV, PREFIX)
''', subst=subst)
else:
assert()
# Guard code in a delta check if necessary.
if relmask.has_wildcards or relmask.has_equalities:
code = L.pc('''
if deltamatch(R, MASK, ELEM, 1):
CODE
''', subst={'R': spec.rel,
'MASK': incaggr.oper_deltamask,
'ELEM': elem,
'<c>CODE': code})
return code
def mset_bindmatch(mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over the M-set."""
tc_applicable = False
if mask == Mask.BB:
tc_applicable = True
cont, item = bvars
code = L.pc('''
if ITEM in CONT:
BODY
''', subst={'CONT': L.ln(cont),
'ITEM': L.ln(item),
'<c>BODY': body})
elif mask == Mask.OUT:
tc_applicable = True
(cont,) = bvars
(item,) = uvars
code = L.pc('''
for ITEM in CONT:
BODY
''', subst={'CONT': L.ln(cont),
'ITEM': L.sn(item),
'<c>BODY': body})
elif mask == Mask.B1:
tc_applicable = True
(cont,) = (item,) = bvars
code = L.pc('''
if CONT in CONT:
BODY
''', subst={'CONT': L.ln(cont),
'<c>BODY': body})
elif mask == Mask.BW:
tc_applicable = True
(cont,) = bvars
code = L.pc('''
if not CONT.isempty():
BODY
''', subst={'CONT': L.ln(cont),
'<c>BODY': body})
elif mask == Mask.UU:
raise AssertionError('No object-domain equivalent for iterating over '
'the M-set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''', subst={'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln('_M'),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body})
if typecheck and tc_applicable:
code = L.pc('''
if isinstance(CONT, Set):
CODE
''', subst={'CONT': cont,
'<c>CODE': code})
return code
def mset_bindmatch(mask, bvars, uvars, body, *, typecheck):
"""Form code for bindmatch over the M-set."""
tc_applicable = False
if mask == Mask.BB:
tc_applicable = True
cont, item = bvars
code = L.pc('''
if ITEM in CONT:
BODY
''',
subst={
'CONT': L.ln(cont),
'ITEM': L.ln(item),
'<c>BODY': body
})
elif mask == Mask.OUT:
tc_applicable = True
(cont, ) = bvars
(item, ) = uvars
code = L.pc('''
for ITEM in CONT:
BODY
''',
subst={
'CONT': L.ln(cont),
'ITEM': L.sn(item),
'<c>BODY': body
})
elif mask == Mask.B1:
tc_applicable = True
(cont, ) = (item, ) = bvars
code = L.pc('''
if CONT in CONT:
BODY
''',
subst={
'CONT': L.ln(cont),
'<c>BODY': body
})
elif mask == Mask.BW:
tc_applicable = True
(cont, ) = bvars
code = L.pc('''
if not CONT.isempty():
BODY
''',
subst={
'CONT': L.ln(cont),
'<c>BODY': body
})
elif mask == Mask.UU:
raise AssertionError('No object-domain equivalent for iterating over '
'the M-set')
else:
code = L.pc('''
for UVARS in setmatch(SET, MASK, BVARS):
BODY
''',
subst={
'UVARS': L.tuplify(uvars, lval=True),
'SET': L.ln('_M'),
'MASK': mask.make_node(),
'BVARS': L.tuplify(bvars),
'<c>BODY': body
})
if typecheck and tc_applicable:
code = L.pc('''
if isinstance(CONT, Set):
CODE
''',
subst={
'CONT': cont,
'<c>CODE': code
})
return code