def generic_ConstFolding(self, origstring, refstring, nbits, lvl=False):
'Generic test for ConstFolding transformer'
orig = ast.parse(origstring)
ref = ast.parse(refstring)
if lvl:
orig = asttools.LevelOperators().visit(orig)
ref = asttools.LevelOperators().visit(orig)
orig = asttools.ConstFolding(orig, nbits).visit(orig)
self.assertTrue(asttools.Comparator().visit(orig, ref))
def match(target_str, pattern_str):
'Apply all pre-processing, then pattern matcher'
target_ast = ast.parse(target_str, mode="eval").body
target_ast = pre_processing.all_preprocessings(target_ast)
target_ast = asttools.LevelOperators(ast.Add).visit(target_ast)
pattern_ast = ast.parse(pattern_str, mode="eval").body
pattern_ast = pre_processing.all_preprocessings(pattern_ast)
pattern_ast = asttools.LevelOperators(ast.Add).visit(pattern_ast)
return PatternMatcher(target_ast).visit(target_ast, pattern_ast)
def replace(target_str, pattern_str, replacement_str):
'Apply pre-processing and replace'
target_ast = ast.parse(target_str, mode="eval").body
target_ast = pre_processing.all_preprocessings(target_ast)
target_ast = asttools.LevelOperators(ast.Add).visit(target_ast)
patt_ast = ast.parse(pattern_str, mode="eval").body
patt_ast = pre_processing.all_preprocessings(patt_ast)
patt_ast = asttools.LevelOperators(ast.Add).visit(patt_ast)
rep_ast = ast.parse(replacement_str)
rep = PatternReplacement(patt_ast, target_ast, rep_ast)
return rep.visit(target_ast)
def visit_Expr(self, node):
'Simplify expression and replace it'
old_value = deepcopy(node.value)
old_value = asttools.LevelOperators().visit(old_value)
node.value = self.simplify(node.value, self.nbits)
copyvalue = deepcopy(node.value)
copyvalue = asttools.LevelOperators().visit(copyvalue)
# simplify until fixpoint is reached
while not asttools.Comparator().visit(old_value, copyvalue):
old_value = deepcopy(node.value)
old_value = asttools.LevelOperators().visit(old_value)
node.value = self.simplify(node.value, self.nbits)
copyvalue = deepcopy(node.value)
copyvalue = asttools.LevelOperators().visit(copyvalue)
return node
def test_with_funcs(self):
'Tests with functions'
tests = [
("f(1 + 1 + 1)",
ast.Call(ast.Name('f', ast.Load()),
[ast.BoolOp(ast.Add(),
[ast.Num(n) for n in [1, 1, 1]])],
[],
None,
None)),
("f(1 + 1 + g(2 + 2 + 2))",
ast.Call(ast.Name('f', ast.Load()),
[ast.BoolOp(ast.Add(),
[ast.Num(1),
ast.Num(1),
ast.Call(ast.Name('g', ast.Load()),
[ast.BoolOp(ast.Add(),
[ast.Num(2),
ast.Num(2),
ast.Num(2)])],
[],
None,
None)])],
[],
None,
None))]
for teststring, ref_ast in tests:
test_ast = ast.parse(teststring, mode="eval").body
test_ast = asttools.LevelOperators(ast.Add).visit(test_ast)
self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def test_differentops(self):
'Test with other types of operators'
tests = [("(3 & 5 & 6)",
ast.BoolOp(ast.BitAnd(),
[ast.Num(3), ast.Num(5), ast.Num(6)])),
("(1 ^ 2 ^ 3) - 4",
ast.BinOp(ast.BoolOp(ast.BitXor(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.Add(),
ast.BinOp(ast.Num(-1), ast.Mult(), ast.Num(4)))),
("((1 + 2 + 3) & (4 + 5))",
ast.BinOp(ast.BoolOp(ast.Add(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.BitAnd(),
ast.BinOp(ast.Num(4), ast.Add(), ast.Num(5)))),
("(1 & 2 & 3) - (4 & 5)",
ast.BinOp(ast.BoolOp(ast.BitAnd(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.Add(),
ast.BinOp(ast.Num(-1), ast.Mult(),
ast.BinOp(ast.Num(4), ast.BitAnd(),
ast.Num(5))))),
("(1 & 2 & 3) << (4 & 5)",
ast.BinOp(ast.BoolOp(ast.BitAnd(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.LShift(),
ast.BinOp(ast.Num(4), ast.BitAnd(), ast.Num(5))))]
for teststring, ref_ast in tests:
test_ast = ast.parse(teststring, mode="eval").body
test_ast = pre_processing.all_preprocessings(test_ast)
test_ast = asttools.LevelOperators().visit(test_ast)
self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def test_afterSubMult(self):
'Tests after SubToMult pre-processing'
tests = [("1 + 2 - 3", ast.BoolOp(ast.Add(), [ast.Num(1), ast.Num(2),
ast.BinOp(ast.Num(-1),
ast.Mult(),
ast.Num(3))])),
("1 + 2 - 3 + 4", ast.BoolOp(ast.Add(),
[ast.Num(1),
ast.Num(2),
ast.BinOp(ast.Num(-1),
ast.Mult(),
ast.Num(3)),
ast.Num(4)])),
("(1 + 2) - (3 + 4)",
ast.BoolOp(ast.Add(),
[ast.Num(1), ast.Num(2),
ast.BinOp(ast.Num(-1), ast.Mult(),
ast.BinOp(ast.Num(3), ast.Add(),
ast.Num(4)))]))]
for teststring, ref_ast in tests:
test_ast = ast.parse(teststring, mode="eval").body
test_ast = pre_processing.all_preprocessings(test_ast)
test_ast = asttools.LevelOperators(ast.Add).visit(test_ast)
self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def test_leveled(self):
'Test positive matchings for leveled ast'
pattern_string = "A + 2*B + 3*C"
test_pos = ["x + 2*y + 3*z", "3*z + 2*y + x", "2*y + 3*z + x"]
for input_string in test_pos:
self.generic_test_positive(input_string, pattern_string, True)
# actual pre-processing only level ADD nodes, but this test is
# for code coverage
test_neg = ast.parse("x ^ 2*y ^ 2*z")
test_neg = pre_processing.all_preprocessings(ast.parse(test_neg))
test_neg = asttools.LevelOperators().visit(test_neg)
patt_ast = ast.parse(pattern_string)
patt_ast = pre_processing.all_preprocessings(patt_ast)
patt_ast = asttools.LevelOperators(ast.Add).visit(patt_ast)
pat = pattern_matcher.PatternMatcher(test_neg)
self.assertFalse(pat.visit(test_neg, patt_ast))
def simplify(self, expr_ast, nbits):
'Apply pattern matching and arithmetic simplification'
if DEBUG:
print "before: "
print unparse(expr_ast)
print ""
expr_ast = all_target_preprocessings(expr_ast, self.nbits)
expr_ast = asttools.LevelOperators(ast.Add).visit(expr_ast)
for pattern, repl in self.patterns:
rep = pattern_matcher.PatternReplacement(pattern, expr_ast, repl)
new_ast = rep.visit(deepcopy(expr_ast))
if DEBUG:
if not asttools.Comparator().visit(new_ast, expr_ast):
print "replaced! "
expr_debug = deepcopy(expr_ast)
expr_debug = asttools.Unleveling().visit(expr_debug)
print unparse(expr_debug)
new_debug = deepcopy(new_ast)
new_debug = asttools.Unleveling().visit(new_debug)
print unparse(new_debug)
print "before: ", ast.dump(expr_ast)
print "pattern: ", ast.dump(pattern)
patt_debug = asttools.Unleveling().visit(deepcopy(pattern))
print unparse(patt_debug)
print ""
print ""
print "after: ", ast.dump(new_ast)
print ""
expr_ast = new_ast
# bitwise simplification: this is a ugly hack, should be
# "generalized"
expr_ast = asttools.LevelOperators(ast.BitXor).visit(expr_ast)
expr_ast = asttools.ConstFolding(expr_ast, self.nbits).visit(expr_ast)
expr_ast = asttools.Unleveling().visit(expr_ast)
if DEBUG:
print "after PM: "
print unparse(expr_ast)
print ""
expr_ast = arithm_simpl.run(expr_ast, nbits)
expr_ast = asttools.GetConstMod(self.nbits).visit(expr_ast)
if DEBUG:
print "arithm simpl: "
print unparse(expr_ast)
print ""
print "-" * 80
return expr_ast
def test_astform(self):
'Tests with different types of ast'
t1 = ast.parse("1 + 2 + 3", mode="eval").body
t1_ref = ast.BoolOp(ast.Add(), [ast.Num(1), ast.Num(2), ast.Num(3)])
t2 = ast.parse("1 + 2 + 3", mode="eval")
t3 = ast.parse("1 + 2 + 3").body[0]
tests = [(t1, t1_ref), (t2, ast.Expression(t1_ref)),
(t3, ast.Expr(t1_ref))]
for test, ref in tests:
ltest = asttools.LevelOperators().visit(test)
self.assertTrue(asttools.Comparator().visit(ltest, ref))
def test_leveled(self):
'Test on leveled ast'
patt_string = "A + 2*B + 3*C"
rep_string = "A"
test_pos = "3*z + x + 2*y"
ref_ast = ast.parse("x", mode='eval').body
output_ast = pattern_matcher.replace(test_pos, patt_string, rep_string)
self.assertTrue(asttools.Comparator().visit(output_ast, ref_ast))
# only ADD nodes are leveled right now, this is for code
# coverage
test_neg = ast.parse("3*z ^ x ^ 2*y")
test_neg = asttools.LevelOperators().visit(test_neg)
patt_ast = ast.parse("A + 3*z")
patt_ast = asttools.LevelOperators().visit(patt_ast)
rep_ast = ast.parse(rep_string)
ref_ast = ast.parse("3*z ^ x ^ 2*y")
ref_ast = asttools.LevelOperators().visit(ref_ast)
rep = pattern_matcher.PatternReplacement(patt_ast, test_neg, rep_ast)
output_ast = rep.visit(test_neg)
self.assertTrue(asttools.Comparator().visit(output_ast, ref_ast))
def test_withUnaryOp(self):
'Test with UnaryOp involved'
tests = [("5 + (-(6 + 2)) + 3",
ast.BoolOp(ast.Add(),
[ast.Num(5),
ast.UnaryOp(ast.USub(), ast.BinOp(ast.Num(6),
ast.Add(),
ast.Num(2))),
ast.Num(3)]))]
for teststring, ref_ast in tests:
test_ast = ast.parse(teststring, mode="eval").body
test_ast = asttools.LevelOperators(ast.Add).visit(test_ast)
self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def __init__(self, nbits, rules_list=DEFAULT_RULES):
'Init context : correspondance between variables and values'
# pylint: disable=dangerous-default-value
self.context = {}
self.nbits = nbits
self.patterns = []
for pattern, replace in rules_list:
patt_ast = ast.parse(pattern, mode="eval").body
patt_ast = all_preprocessings(patt_ast, self.nbits)
patt_ast = asttools.LevelOperators(ast.Add).visit(patt_ast)
rep_ast = ast.parse(replace, mode="eval").body
self.patterns.append((patt_ast, rep_ast))
def test_unleveling(self):
'Tests to see if unleveling is correct'
tests = [("x + (3 + y)", "3 + (y + x)"),
("x*(2*z)", "2*(z*x)"),
("x + (y + (z*(5*var)))", "y + (5*(var*z) + x)")]
for test, ref in tests:
ref_ast = ast.parse(ref)
ast_test = ast.parse(test)
asttools.LevelOperators().visit(ast_test)
asttools.Unleveling().visit(ast_test)
self.assertTrue(asttools.Comparator().visit(ast_test, ref_ast))
self.assertFalse('BoolOp' in astunparse.unparse(ast_test))
def visit_Assign(self, node):
'Simplify value of assignment and update context'
# use EvalPattern to replace known variables
node.value = pattern_matcher.EvalPattern(self.context).visit(
node.value)
old_value = deepcopy(node.value)
old_value = asttools.LevelOperators().visit(old_value)
node.value = self.simplify(node.value, self.nbits)
copyvalue = deepcopy(node.value)
copyvalue = asttools.LevelOperators().visit(copyvalue)
# simplify until fixpoint is reached
while not asttools.Comparator().visit(old_value, copyvalue):
old_value = deepcopy(node.value)
node.value = self.simplify(node.value, self.nbits)
copyvalue = deepcopy(node.value)
if len(unparse(copyvalue)) > len(unparse(old_value)):
node.value = deepcopy(old_value)
break
copyvalue = asttools.LevelOperators().visit(copyvalue)
old_value = asttools.LevelOperators().visit(old_value)
for target in node.targets:
self.context[target.id] = node.value
return node
def generic_leveling(self, refstring_list, result):
'Test matching of leveled AST and ref AST'
for refstring in refstring_list:
ref = ast.parse(refstring, mode="eval").body
ref = asttools.LevelOperators().visit(ref)
self.assertTrue(asttools.Comparator().visit(ref, result))
