def visit_AugAssign(self, node):
# Aug assign to a name
if isinstance(node.target, ast.Name):
target = Var(node.target.id)
self.addtype(node.target.id, '*')
value = self.visit_expr(node.value)
rhs = Op(node.op.__class__.__name__,
target,
value,
line=node.lineno)
self.addexpr(target.name, rhs)
# Aug assign to a index
elif isinstance(node.target, ast.Subscript):
if isinstance(node.target.slice, ast.Index):
var = Var(node.target.value.id)
right = self.visit_expr(node.value)
index = self.visit_expr(node.target.slice.value)
rhs = Op(node.op.__class__.__name__,
self.visit(node.target),
right,
line=node.lineno)
self.addexpr(
var.name,
Op('AssignElement', var, index, rhs, line=node.lineno))
else:
raise NotSupported(
'Subscript assignments only allowed to Indices',
line=node.lineno)
else:
raise NotSupported('Assignments to {} not supported'.format(
node.target.__class__.__name__),
line=node.lineno)
def visit_Print(self, node):
'''
Only used in Python 2.x, ignores destination and newline
'''
values_model = map(self.visit_expr, node.values)
expr = Op('StrAppend', Var(VAR_OUT), *values_model, line=node.lineno)
self.addexpr(VAR_OUT, expr)
def visit_printf(self, node, args):
'''
printf function call
'''
# Extract format and args
if len(args) == 0:
self.addwarn("'printf' with zero args at line %s" % (
node.coord.line,))
fmt = Const('?', line=node.coord.line)
else:
if isinstance(args[0], Const):
fmt = args[0]
args = args[1:]
else:
self.addwarn("First argument of 'printf' at lines %s should \
be a format" % (node.coord.line,))
fmt = Const('?', line=node.coord.line)
fmt.value = fmt.value.replace('%lf', '%f')
fmt.value = fmt.value.replace('%ld', '%d')
fmt.value = fmt.value.replace('%lld', '%d')
expr = Op('StrAppend', Var(VAR_OUT),
Op('StrFormat', fmt, *args, line=node.coord.line),
line=node.coord.line)
self.addexpr(VAR_OUT, expr)
def ssa(self, fnc):
'''
Converts exprs of each loc to SSA form
'''
for loc in fnc.locs():
# Find last appearance of each var
last = {}
for i, (var, _) in enumerate(fnc.exprs(loc)):
last[var] = i
# Replace non-last appearance by a fresh var
m = {}
exprs = []
for i, (var, expr) in enumerate(fnc.exprs(loc)):
for v1, v2 in m.items():
expr = expr.replace(v1, Var(v2))
if var == VAR_RET:
newvar = var
else:
if last[var] > i:
newvar = m[var] = self.ssavar(var)
else:
m.pop(var, None)
newvar = var
exprs.append((newvar, expr))
fnc.replaceexprs(loc, exprs)
def optimizeif(self, preloc, condexpr, trueloc, falseloc):
'''
Optimized "simple" or "loop-less" if statement
'''
# Remove unneded part of the graph
self.fnc.rmtrans(preloc, True)
self.loc = preloc
# Keep track of assigned vars
varss = set()
varsl = []
mt = {}
mf = {}
# Add exprs from branches
def addvars(loc, m):
for (var, expr) in self.fnc.exprs(loc):
newvar = self.ssavar(var)
if var not in varss:
varss.add(var)
varsl.append(var)
# Replace vars mapped so far
for (v1, v2) in m.items():
expr = expr.replace(v1, Var(v2))
expr.original = (var, self.cnt)
self.addexpr(newvar, expr)
# Remember replacement
m[var] = newvar
addvars(trueloc, mt)
if falseloc is not None:
addvars(falseloc, mf)
# Add condition
condvar = self.ssavar('$cond')
self.addexpr(condvar, condexpr.copy())
# Merge branches
for var in varsl:
self.addexpr(
var,
Op('ite', Var(condvar), Var(mt.get(var, var)),
Var(mf.get(var, var))))
def rmtmp(self, fnc):
'''
Removes (merges) "tmp" or SSA-generated assignments
'''
for loc in fnc.locs():
m = {}
exprs = []
primed = set([])
lastret = None
# Remember "real" vars and replace temps
for var, expr in fnc.exprs(loc):
expr.prime(primed)
for v, e in m.items():
expr = expr.replace(v, e)
if var.endswith('&'):
m[var] = expr
else:
if var == VAR_RET:
lastret = len(exprs)
exprs.append((var, expr))
primed.add(var)
# "Merge" return stmts
nexprs = []
retexpr = None
retcond = None
for i, (var, expr) in enumerate(exprs):
if var == VAR_RET:
tmpretcond = self.getretcond(expr)
if tmpretcond is True or retcond is None:
retcond = tmpretcond
elif tmpretcond is not None and retcond is not True:
retcond = Op(self.OROP, retcond, tmpretcond)
if retexpr:
retexpr = retexpr.replace(VAR_RET, expr)
else:
retexpr = expr
if i == lastret:
nexprs.append((var, retexpr))
else:
if retcond is True:
continue
elif retcond:
expr = Op('ite', Op(self.NOTOP, retcond), expr,
Var(var))
nexprs.append((var, expr))
fnc.replaceexprs(loc, nexprs)
def visit_Call(self, node):
if len(node.keywords) > 0 or node.starargs or node.kwargs:
raise NotSupported(
'starargs, kwargs and keyword arguments not supported',
line=node.lineno)
if isinstance(node.func, ast.Name):
if node.func.id in self.BUILTIN_FNCS:
fncname = node.func.id
args = list(map(self.visit_expr, node.args))
return Op(fncname, *args, line=node.lineno)
else:
fnc = Var(node.func.id)
args = list(map(self.visit_expr, node.args))
return Op('FuncCall', fnc, *args, line=node.lineno)
elif isinstance(node.func, ast.Num):
num = self.visit_expr(node.func)
self.addwarn("Call to a number '%s' on line %s ignored", num,
num.line)
return Const('?')
elif isinstance(node.func, ast.Attribute):
attr = node.func.attr
val = self.visit_expr(node.func.value)
args = list(map(self.visit_expr, node.args))
if isinstance(val, Var) and val.name in self.MODULE_NAMES:
return Op('%s_%s' % (val, attr), *args, line=node.lineno)
if attr == 'pop':
if isinstance(val, Var):
popvar = self.ssavar('pop#')
self.addexpr(popvar, Op(attr, val, *args,
line=node.lineno))
self.addexpr(val.name,
Op('GetElement', Var(popvar), Const('0')))
return Op('GetElement', Var(popvar), Const('1'))
else:
raise NotSupported('Pop to a non-name list')
return Op(attr, val, *args, line=node.lineno)
else:
raise NotSupported('Call of {} not supported'.format(
node.func.__class__.__name__),
line=node.lineno)
def visit_ID(self, node):
'''
ID
Attrs: name
'''
if node.name in self.CONSTS:
return Const(node.name, line=node.coord.line)
return Var(node.name, line=node.coord.line)
def addvars(loc, m):
for (var, expr) in self.fnc.exprs(loc):
newvar = self.ssavar(var)
if var not in varss:
varss.add(var)
varsl.append(var)
# Replace vars mapped so far
for (v1, v2) in m.items():
expr = expr.replace(v1, Var(v2))
self.addexpr(newvar, expr)
# Remember replacement
m[var] = newvar
def visit_Assign(self, node):
if len(node.targets) != 1:
raise NotSupported('Only single assignments allowed')
target = node.targets[0]
# Assignment to a variable
if isinstance(target, ast.Name):
right = self.visit_expr(node.value)
self.addtype(target.id, '*')
self.addexpr(target.id, right)
# Assignment to indexed element
elif isinstance(target, ast.Subscript):
if isinstance(target.slice, ast.Index):
var = Var(target.value.id)
self.addtype(target.value.id, '*')
right = self.visit_expr(node.value)
index = self.visit_expr(target.slice.value)
self.addexpr(
target.value.id,
Op('AssignElement', var, index, right, line=right.line))
else:
raise NotSupported(
'Subscript assignments only allowed to Indices',
line=node.lineno)
# Assignment to a tuple
elif isinstance(target, ast.Tuple):
targets = map(self.visit_expr, target.elts)
right = self.visit_expr(node.value)
for i, target in enumerate(targets):
expr = Op('GetElement',
right.copy(),
Const(str(i)),
line=right.line)
if isinstance(target, Var):
self.addexpr(target.name, expr)
else:
raise NotSupported("Tuple non-var assignment",
line=target.line)
else:
raise NotSupported('Assignments to {} not supported'.format(
target.__class__.__name__),
line=node.lineno)
def visit_Name(self, node):
if node.id in self.CONSTS:
return Const(node.id)
self.addtype(node.id, '*')
return Var(node.id)
def visit_For(self, node):
if node.orelse:
raise NotSupported("For-Else not supported",
line=self.getline(node.orelse))
# Iterated expression
it = self.visit_expr(node.iter)
# Targets of iteration
if isinstance(node.target, ast.Name):
self.addtype(node.target.id, '*')
targets = [self.visit_expr(node.target)]
elif isinstance(node.target, ast.Tuple):
for el in node.target.elts:
if isinstance(el, ast.Name):
self.addtype(el.id, '*')
targets = map(self.visit_expr, node.target.elts)
else:
raise NotSupported(
'For loop with {} as target'.format(
node.target.__class__.__name__),
line=node.lineno)
hiddenvar = self.hiddenvarcnt
self.hiddenvarcnt += 1
# Set up the iterated variable
iter_name = 'iter#{}'.format(hiddenvar)
it_var = Var(iter_name)
self.addtype(iter_name, '*')
# Set up the iteration index
ind_name = 'ind#{}'.format(hiddenvar)
ind_var = Var(ind_name)
self.addtype(ind_name, 'int')
# Add assignments to iterators
self.addexpr(it_var.name, it)
self.addexpr(ind_var.name, Const(str(0), line=node.lineno))
# Condition is ind_var < len(iter_var)
cond = Op('Lt', ind_var.copy(), Op('len', it_var.copy()),
line=node.iter.lineno)
# Assignments to iterated variable(s)
prebody = []
el = Op('GetElement', it_var.copy(), ind_var.copy(),
line=node.target.lineno)
if len(targets) == 1:
prebody.append((targets[0].name, el.copy()))
else:
for i, t in enumerate(targets):
eli = Op('GetElement', el.copy(), Const(str(i)),
line=node.target.lineno)
prebody.append((t.name, eli))
# Add index variable increment
prebody.append((ind_var.name,
Op('Add', ind_var.copy(), Const(str(1)),
line=node.iter.lineno)))
self.visit_loop(node, None, cond, None, node.body, False, 'for',
prebody=prebody)
def visit_scanf(self, node, args):
'''
scanf function call
'''
# Check format
if len(args) == 0:
self.warn("'scanf' without arguments at line %s (ignored)",
node.coord.line)
else:
fmt = args[0]
if isinstance(fmt, Const) and fmt.value[0] == '"' \
and fmt.value[-1] == '"':
fmt = fmt.value[1:-1]
args = args[1:]
else:
self.warn("First argument of 'scanf' at line %s should be a \
(string) format (ignored)",
node.coord.line)
fmt = ''
args = []
# Extract format arguments
fs = list(re.findall(
r'(%((d)|(i)|(li)|(lli)|(ld)|(lld)|(lf)|(f)|(s)|(c)))', fmt))
# Check argument number
if len(fs) != len(args):
self.addwarn("Mismatch between format and number of argument(s)\
of 'scanf' at line %s.",
node.coord.line)
if len(args) > len(fs):
fs += ['*' for _ in xrange(len(args) - len(fs))]
# Iterate formats and arguments
for f, a in zip(fs, args):
if f:
f = f[0]
# Get type from an argument
if f in ['%d', '%ld', '%i', '%li', '%lli', '%lld']:
t = 'int'
elif f in ['%c']:
t = 'char'
elif f in ['%s']:
t = 'string'
elif f in ['%f', '%lf']:
t = 'float'
elif f == '*':
t = '*'
else:
self.addwarn("Invalid 'scanf' format at line %s.",
node.coord.line)
t = '*'
# Check argument type
if isinstance(a, Op) and a.name == '&' and len(a.args) == 1:
a = a.args[0]
elif isinstance(a, Var) or (isinstance(a, Op) and a.name == '[]'):
self.addwarn("Forgoten '&' in 'scanf' at line %s?",
node.coord.line)
else:
raise NotSupported("Argument to scanf: '%s'" % (a,),
line=node.coord.line)
# Add operations
rexpr = Op('ListHead', Const(t), Var(VAR_IN), line=node.coord.line)
if isinstance(a, Var):
self.addexpr(a.name, rexpr)
elif isinstance(a, Op) and a.name == '[]' and isinstance(a.args[0],
Var):
self.addexpr(a.args[0].name,
Op('ArrayAssign', a.args[0], a.args[1], rexpr,
line=node.coord.line))
else:
raise NotSupported("Argument to scanf: '%s'" % (a,),
line=node.coord.line)
self.addexpr(VAR_IN,
Op('ListTail', Var(VAR_IN), line=node.coord.line))
def potential(self, f1, f2, loc1, var1, loc2):
varp1 = prime(var1)
expr1 = self.E1[loc1][var1]
isid = (isinstance(expr1, Var) and expr1.name == var1
and expr1.primed == False)
tree1 = self.T1[loc1][var1]
vars1 = list(set(map(unprimes, expr1.vars())) | set([var1]))
vars1.sort()
V1 = list(self.V1 - set(['-']))
V1.sort()
V2 = list(self.V2)
V2.sort()
for var2 in V2:
sofar = set()
# Special vars can be only mapped to special vars
if (var1 in SPECIAL_VARS or var2 in SPECIAL_VARS) and var1 != var2:
continue
# other special variables
if var2 != '*':
if var1.startswith('ind#') != var2.startswith('ind#'):
continue
if var1.startswith('iter#') != var2.startswith('iter#'):
continue
# Params can only be mapped to params
if ((var1 in self.pmap or var2 in self.pmap.keys())
and var2 != self.pmap.get(var1)):
continue
# Cannot delete new variable
if var1 == '-' and var2 == '*':
continue
expr2 = self.E2[loc2][var2]
tree2 = self.T2[loc2][var2]
vars2 = list(set(map(unprimes, expr2.vars())) | set([var2]))
vars2.sort()
# (0) Deletes are special
if var1 == '-':
delexpr = Var(var2)
deltree = self.totree(delexpr)
delcost = self.distance(tree2, deltree, {var2: var2})
if delcost:
yield ([(var1, var2)], delcost, (), None)
continue
# (1) Generate corrects (if not new variable)
if var2 != '*':
for m in self.one_to_ones(vars2, V1, var2, var1):
m = [(s2, s1) for (s1, s2) in m]
ok = True
for mem1 in self.trace.get(f1.name, {}).get(loc1, []):
val1 = mem1.get(varp1)
if isundef(val1) and (var1 != VAR_RET):
continue
if isinstance(val1, str) and self.cleanstrings:
val1 = val1.strip()
mem2 = {v2: mem1.get(v1) for (v1, v2) in m}
mem2.update(
{prime(v2): mem1.get(prime(v1))
for (v1, v2) in m})
try:
val2 = self.inter.execute(expr2, mem2)
if isinstance(val2, str) and self.cleanstrings:
val2 = val2.strip()
if not equals(val2, val1):
ok = False
break
except RuntimeErr:
ok = False
break
if ok:
order = self.getorder(var2, expr2,
{v: v
for v in vars2})
if order is None:
assert False, 'order error %s %s' % (var2, expr2)
ms = list(m)
ms.sort()
sofar.add((tuple(ms), tuple(order)))
yield (m, 0, set(order), None)
# (2) Generate repairs
tmprepairs = {}
#for rexpr, rtree in zip([self.E1[loc1][var1]], [self.T1[loc1][var1]]):
for idx, ((rexpr, _), rtree) in enumerate(
zip(self.ER[loc1][var1], self.TR[loc1][var1])):
risid = (isinstance(rexpr, Var) and rexpr.name == var1
and rexpr.primed == False)
rvars = list(set(map(unprimes, rexpr.vars())) | set([var1]))
for m in self.one_to_ones(rvars, V2, var1, var2):
order = self.getorder(var2, rexpr, dict(m))
if order is None:
self.debug(
'skipping repair %s := %s (%s) because impossible order',
var1, expr1, m)
continue
if risid and var2 == '*':
cost = 0
else:
cost = self.distance(tree2, rtree, dict(m))
# Account for *declaring* a new variable
if var2 == '*' and loc1 == 1:
cost += 1
ms = list(m)
ms.sort()
tms = tuple(ms)
torder = tuple(order)
if (tms, torder) in sofar:
continue
# From each 'm'-'order' pair we first remember all pairs
# and later yield only the one with the smallest cost
# since other ones have no sense
tmp = (tms, torder)
if tmp not in tmprepairs:
tmprepairs[tmp] = []
tmprepairs[tmp].append((cost, (m, cost, set(order), idx)))
#yield (m, cost, set(order))
for treps in tmprepairs.values():
treps.sort()
#print treps[0][1]
yield treps[0][1]
def query_transformation_initiated_at(query):
args0, args1 = query.args
args0args0 = args0.args[0]
return query(args0(args0args0, Var('X')), args1)
def query_transformation_happens_at(query):
args0, _, args2 = query.args
return query(args0, Var('X'), args2)
def visit_For(self, node):
if node.orelse:
raise NotSupported("For-Else not supported",
line=self.getline(node.orelse))
# Iterated expression
it = self.visit_expr(node.iter)
# Targets of iteration
if isinstance(node.target, ast.Name):
targets = [self.visit_expr(node.target)]
elif isinstance(node.target, ast.Tuple):
targets = map(self.visit_expr, node.target.elts)
else:
raise NotSupported('For loop with {} as target'.format(
node.target.__class__.__name__),
line=node.lineno)
hiddenvar = self.hiddenvarcnt
self.hiddenvarcnt += 1
# Set up the iterated variable
iter_name = 'iter#{}'.format(hiddenvar)
it_var = Var(iter_name)
self.addtype(iter_name, '*')
# Set up the iteration index
ind_name = 'ind#{}'.format(hiddenvar)
ind_var = Var(ind_name)
self.addtype(ind_name, 'int')
# Add assignments to iterators
self.addexpr(it_var.name, it)
self.addexpr(ind_var.name, Const(str(0), line=node.lineno))
# Condition is ind_var < len(iter_var)
cond = Op('Lt',
ind_var.copy(),
Op('len', it_var.copy()),
line=node.iter.lineno)
# Assignments to iterated variable(s)
prebody = []
el = Op('GetElement',
it_var.copy(),
ind_var.copy(),
line=node.target.lineno)
if len(targets) == 1:
prebody.append((targets[0].name, el.copy()))
else:
for i, t in enumerate(targets):
eli = Op('GetElement',
el.copy(),
Const(str(i)),
line=node.target.lineno)
prebody.append((t.name, eli))
# Add index variable increment
prebody.append((ind_var.name,
Op('Add',
ind_var.copy(),
Const(str(1)),
line=node.iter.lineno)))
self.visit_loop(node,
None,
cond,
None,
node.body,
False,
'for',
prebody=prebody)
def potential(self, f1, f2, loc1, var1, loc2):
varp1 = prime(var1)
expr1 = self.E1[loc1][var1]
isid = isinstance(expr1, Var) and expr1.name == var1
tree1 = self.T1[loc1][var1]
vars1 = list(set(map(unprimes, expr1.vars())) | set([var1]))
vars1.sort()
V1 = list(self.V1 - set(['-']))
V1.sort()
V2 = list(self.V2)
V2.sort()
for var2 in V2:
sofar = set()
# Special vars can be only mapped to special vars
if (var1 in SPECIAL_VARS or var2 in SPECIAL_VARS) and var1 != var2:
continue
# Params can only be mapped to params
if var1 in self.pmap and var2 != self.pmap[var1]:
continue
# Cannot delete new variable
if var1 == '-' and var2 == '*':
continue
expr2 = self.E2[loc2][var2]
tree2 = self.T2[loc2][var2]
vars2 = list(set(map(unprimes, expr2.vars())) | set([var2]))
vars2.sort()
# (0) Deletes are special
if var1 == '-':
delexpr = Var(var2)
deltree = self.totree(delexpr)
delcost = self.distance(tree2, deltree, {var2: var2})
if delcost:
yield ([(var1, var2)], delcost, ())
continue
# (1) Generate corrects (if not new variable)
if var2 != '*':
for m in self.one_to_ones(vars2, V1, var2, var1):
m = [(s2, s1) for (s1, s2) in m]
ok = True
for mem1 in self.trace.get(f1.name, {}).get(loc1, []):
val1 = mem1.get(varp1)
if isundef(val1):
continue
if isinstance(val1, str) and self.cleanstrings:
val1 = val1.strip()
mem2 = {v2: mem1.get(v1) for (v1, v2) in m}
mem2.update(
{prime(v2): mem1.get(prime(v1))
for (v1, v2) in m})
try:
val2 = self.inter.execute(expr2, mem2)
if isinstance(val2, str) and self.cleanstrings:
val2 = val2.strip()
if val2 != val1:
ok = False
break
except RuntimeErr:
ok = False
break
if ok:
order = self.getorder(var2, expr2,
{v: v
for v in vars2})
if order is None:
assert False, 'order error %s %s' % (var2, expr2)
ms = list(m)
ms.sort()
sofar.add((tuple(ms), tuple(order)))
yield (m, 0, set(order))
# (2) Generate repairs
for m in self.one_to_ones(vars1, V2, var1, var2):
order = self.getorder(var2, expr1, dict(m))
if order is None:
self.debug(
'skipping repair %s := %s (%s) because \
impossible order', var1, expr1, m)
continue
if isid and var2 == '*':
cost = 0
else:
cost = self.distance(tree2, tree1, dict(m))
# Account for *declaring* a new variable
if var2 == '*' and loc1 == 1:
cost += 1
ms = list(m)
ms.sort()
if (tuple(ms), tuple(order)) in sofar:
continue
yield (m, cost, set(order))