def check(expr1=None, expr2=None):
ok=1
expr1=expr1 or sys.argv[1]
expr2=expr2 or sys.argv[2]
l1=munge(astl(expr1))
l2=astl(expr2)
try: c1=compileast(sequence2ast(l1))
except:
traceback.print_exc
c1=None
c2=compileast(sequence2ast(l2))
if c1 !=c2:
ok=0
print 'test failed', expr1, expr2
print
print l1
print
print l2
print
ast=parser.sequence2ast(l1)
c=parser.compileast(ast)
pretty(expr1)
pret(l1)
pret(l2)
return ok
def check(expr1=None, expr2=None):
ok = 1
expr1 = expr1 or sys.argv[1]
expr2 = expr2 or sys.argv[2]
l1 = munge(astl(expr1))
l2 = astl(expr2)
try:
c1 = compileast(sequence2ast(l1))
except:
traceback.print_exc
c1 = None
c2 = compileast(sequence2ast(l2))
if c1 != c2:
ok = 0
print 'test failed', expr1, expr2
print
print l1
print
print l2
print
ast = parser.sequence2ast(l1)
c = parser.compileast(ast)
pretty(expr1)
pret(l1)
pret(l2)
return ok
def test_funccall(args):
t = create_file_input(_create_funccall_expr_stmt('set_test_args',
map(lambda c: create_constant_test(c), args)))
test_args = [ 'UNSET' ]
def set_test_args(*args): test_args[:] = args
scope = { 'set_test_args': set_test_args }
exec parser.sequence2ast(t).compile() in scope
assert tuple(test_args) == args
def check_bad_tree(tree, label):
print
print label
try:
sequence2ast(tree)
except parser.ParserError:
print "caught expected exception for invalid tree"
pass
else:
print "test failed: did not properly detect invalid tree:"
pprint.pprint(tree)
def check_bad_tree(tree, label):
print
print label
try:
sequence2ast(tree)
except parser.ParserError:
print "caught expected exception for invalid tree"
pass
else:
print "test failed: did not properly detect invalid tree:"
pprint.pprint(tree)
def test_funccall(args):
t = create_file_input(
_create_funccall_expr_stmt(
'set_test_args', map(lambda c: create_constant_test(c), args)))
test_args = ['UNSET']
def set_test_args(*args):
test_args[:] = args
scope = {'set_test_args': set_test_args}
exec parser.sequence2ast(t).compile() in scope
assert tuple(test_args) == args
def compile(src, file_name, ctype):
if ctype=='eval': ast=parser.expr(src)
elif ctype=='exec': ast=parser.suite(src)
l=ast2list(ast)
l=munge(l)
ast=sequence2ast(l)
return parser.compileast(ast, file_name)
def _standardize_st(self, st, format='tuple'):
"""Given a syntax tree and a desired format, return the tree in that
format.
"""
# convert the incoming ast/cst into an AST
if type(st) is type(parser.suite('')):
ast = st
else:
if type(st) in (type(()), type([])):
ast = parser.sequence2ast(st)
else:
raise ASTutilsException, "incoming type unrecognized: " +\
repr(type(st))
# return the tree in the desired format
formats = {
'tuple': ast.totuple,
'list': ast.tolist,
'ast': lambda: ast
}
outgoing = formats.get(format.lower())
if outgoing is None:
raise ASTutilsException, "requested format unrecognized: " + format
return outgoing()
def rewrite_and_compile(self, output_func_name=None, print_func_name=None):
"""
Compiles the parse tree into code, while rewriting the parse tree according to the
output_func_name and print_func_name arguments.
At the same time, the code is scanned for possible mutations, and a list is returned.
In the list:
- A string indicates the mutation of a variable by assignment to a slice of it,
or to an attribute.
- A tuple of (variable_name, method_name) indicates the invocation of a method
on the variable; this will sometimes be a mutation (e.g., list.append(value)),
and sometimes not.
@param output_func_name: the name of function used to wrap statements that are simply expressions.
(More than one argument will be passed if the statement is in the form of a list.)
Can be None.
@param print_func_name: the name of a function used to replace print statements without a destination
file. Can be None.
@returns: a tuple of the compiled code followed by a list of mutations
"""
state = _RewriteState(output_func_name=output_func_name,
print_func_name=print_func_name,
future_features=self.future_features)
rewritten = _rewrite_file_input(self.original, state)
encoded = (symbol.encoding_decl, rewritten, self.encoding)
compiled = parser.sequence2ast(encoded).compile()
return (compiled, state.mutated)
def compile_unicode(source, filename, method):
'''Compile Python source represented as unicode object. All string
litterals containing non-ASCII character will be unicode objects.'''
import parser
from token import ISNONTERMINAL, STRING
source = source.encode('utf-8') # parser complains about unicode source
if method=='exec':
ast = parser.suite(source)
elif method=='eval':
ast = parser.expr(source)
else:
raise ValueError('Unsupported compilation method: %r' % (method,))
ast_seq = ast.tolist(True)
# non-recursive method to walk through tree
stack = [iter([ast_seq]).next]
while stack:
try:
node = stack[-1]()
except StopIteration:
stack.pop()
continue
if ISNONTERMINAL(node[0]):
stack.append(iter(node[1:]).next)
elif node[0]==STRING:
s = eval(node[1])
try:
s.decode('ascii')
except UnicodeDecodeError:
s = s.decode('utf-8')
node[1] = repr(s)
return parser.sequence2ast(ast_seq).compile(filename)
def rewrite_and_compile(code, output_func_name=None, output_func_self=None, print_func_name=None, encoding="utf8"):
"""
Compiles the supplied text into code, while rewriting the parse tree so:
* Print statements without a destination file are transformed into calls to
<print_func_name>(*args), if print_func_name is not None
* Statements which are simply expressions are transformed into calls to
<output_func_name>(*args), if output_fnuc_name is not None
(More than one argument is passed if the statement is in the form of a list; for example '1,2'.)
At the same time, the code is scanned for possible mutations, and a list is returned.
In the list:
* A string indicates the mutation of a variable by assignment to a slice of it,
or to an attribute.
* A tuple of (variable_name, method_name) indicates the invocation of a method
on the variable; this will sometimes be a mutation (e.g., list.append(value)),
and sometimes not.
"""
state = _RewriteState(output_func_name=output_func_name, output_func_self=output_func_self, print_func_name=print_func_name)
if (isinstance(code, str)):
code = code.encode("utf8")
encoding = "utf8"
original = parser.suite(code)
rewritten = _rewrite_file_input(original.totuple(), state)
encoded = (symbol.encoding_decl, rewritten, encoding)
compiled = parser.sequence2ast(encoded).compile()
return (compiled, state.mutated)
def interpolate(self, block):
"""Given a block of code, return the same code + our testing framework.
Our testing framework consists of calls to the Observer.intercept method
wrapped around all tests and print statements. Tests are any explicit
comparison statement, i.e., those with a comparison operator. The
Observer.run method adds the Observer instance to the test script's
namespace as __pytest__.
Example:
>>> block = "1 + 1 == 2"
>>> Interpolator.interpolate(block)
'__pytest__ . intercept ( "1 + 1 == 2" , 1 , globals ( ) , locals ( ) , COMPARING = True , PRINTING = False )'
>>> block = "print 'hello world'"
>>> Interpolator.interpolate(block)
'__pytest__ . intercept ( "print \\\\\\'hello world\\\\\\'" , 1 , globals ( ) , locals ( ) , COMPARING = False , PRINTING = True )'
"""
self.cst = parser.suite(block).tolist(line_info=True)
self._walk(self.cst)
ast = parser.sequence2ast(self.cst)
return ASTutils.ast2text(ast)
def compile(src, file_name, ctype):
if ctype == 'eval': ast = parser.expr(src)
elif ctype == 'exec': ast = parser.suite(src)
l = ast2list(ast)
l = munge(l)
ast = sequence2ast(l)
return parser.compileast(ast, file_name)
def _walk(self, cst):
"""walk an AST list (a cst?) and do our interpolation
"""
i = 0
for node in cst:
i += 1
if type(node) is type([]):
# we have a list of subnodes; recurse
self._walk(node)
else:
# we have an actual node; interpret it and act accordingly
# if the node is a simple comparison, wrap it
# TODO account for multiple small_stmts
if (node in (symbol.stmt, symbol.suite)) and (i == 1):
# the i flag is to guard against the rare case where the
# node constant would be stmt or suite, and the line number
# would too
if cst[1][0] == symbol.simple_stmt:
# convert the cst stmt fragment to an AST
ast = parser.sequence2ast(self._stmt2file_input(cst))
if self._is_test(ast):
cst[1] = self._wrap(cst, COMPARING=True)[1]
elif ASTutils.hasnode(ast, symbol.print_stmt):
cst[1] = self._wrap(cst, PRINTING=True)[1]
def eval_seq(t, d=None):
if d is None:
d = {}
ast = parser.sequence2ast((257, t, (0, '')))
co = ast.compile()
eval(co, d, d)
del d["__builtins__"]
return d
def eval_seq(t, d=None):
if d is None:
d = {}
ast = parser.sequence2ast( (257, t, (0, '')) )
co = ast.compile()
eval(co, d, d)
del d["__builtins__"]
return d
def eval_simple(t, d=None):
if d is None:
d = {}
# put a valid top-level wrapper around the simple statement fragment.
ast = parser.sequence2ast((257, (264, (265, t, (4, ''))), (0, '')))
co = ast.compile()
eval(co, d, d)
del d["__builtins__"] # somehow this gets in here.
return d
def eval_simple(t, d=None):
if d is None:
d = {}
# put a valid top-level wrapper around the simple statement fragment.
ast = parser.sequence2ast( (257, (264, (265, t, (4, '') )), (0, '')) )
co = ast.compile()
eval(co, d, d)
del d["__builtins__"] # somehow this gets in here.
return d
def test_funccall(args):
t = create_file_input(_create_funccall_expr_stmt('set_test_args',
[create_constant_test(c) for c in args]))
test_args = [ 'UNSET' ]
def set_test_args(*args): test_args[:] = args
scope = { 'set_test_args': set_test_args }
exec(parser.sequence2ast(t).compile(), scope)
assert tuple(test_args) == args
def spam():
# Regression test
import traceback
ok=1
for expr1, expr2 in (
("a*b", "__guarded_mul__(_vars, a, b)"),
("a*b*c",
"__guarded_mul__(_vars, __guarded_mul__(_vars, a, b), c)"
),
("a.b", "__guarded_getattr__(_vars, a, 'b')"),
("a[b]", "__guarded_getitem__(_vars, a, b)"),
("a[b,c]", "__guarded_getitem__(_vars, a, b, c)"),
("a[b:c]", "__guarded_getslice__(_vars, a, b, c)"),
("a[:c]", "__guarded_getslice__(_vars, a, 0, c)"),
("a[b:]", "__guarded_getslice__(_vars, a, b)"),
("a[:]", "__guarded_getslice__(_vars, a)"),
("_vars['sequence-index'] % 2",
"__guarded_getitem__(_vars, _vars, 'sequence-index') % 2"
),
):
l1=munge(astl(expr1))
l2=astl(expr2)
try: c1=compileast(sequence2ast(l1))
except:
traceback.print_exc
c1=None
c2=compileast(sequence2ast(l2))
if c1 !=c2:
ok=0
print 'test failed', expr1, expr2
print
print l1
print
print l2
print
ast=parser.sequence2ast(l1)
c=parser.compileast(ast)
if ok: print 'all tests succeeded'
def modify_AST(myAST, assign_name, assign_val):
myAST_Visitor = AST_Visitor(1)
old_AST_list = myAST.tolist(1)
assign_val_type = 0
if isinstance(assign_val, int):
assign_val_type = 2
elif isinstance(assign_val, float):
assign_val_type = 1
elif type(assign_val) == str:
assign_val_type = 3
new_AST_list = myAST_Visitor.traverse(old_AST_list, assign_name, gen_assign_val_subtree(assign_val, assign_val_type))
myNewAST = parser.sequence2ast(new_AST_list)
return myNewAST
def spam():
# Regression test
import traceback
ok = 1
for expr1, expr2 in (
("a*b", "__guarded_mul__(_vars, a, b)"),
("a*b*c", "__guarded_mul__(_vars, __guarded_mul__(_vars, a, b), c)"),
("a.b", "__guarded_getattr__(_vars, a, 'b')"),
("a[b]", "__guarded_getitem__(_vars, a, b)"),
("a[b,c]", "__guarded_getitem__(_vars, a, b, c)"),
("a[b:c]", "__guarded_getslice__(_vars, a, b, c)"),
("a[:c]", "__guarded_getslice__(_vars, a, 0, c)"),
("a[b:]", "__guarded_getslice__(_vars, a, b)"),
("a[:]", "__guarded_getslice__(_vars, a)"),
("_vars['sequence-index'] % 2",
"__guarded_getitem__(_vars, _vars, 'sequence-index') % 2"),
):
l1 = munge(astl(expr1))
l2 = astl(expr2)
try:
c1 = compileast(sequence2ast(l1))
except:
traceback.print_exc
c1 = None
c2 = compileast(sequence2ast(l2))
if c1 != c2:
ok = 0
print 'test failed', expr1, expr2
print
print l1
print
print l2
print
ast = parser.sequence2ast(l1)
c = parser.compileast(ast)
if ok: print 'all tests succeeded'
def test_funccall(args):
t = create_file_input(
_create_funccall_expr_stmt('set_test_args',
[create_constant_test(c)
for c in args]))
test_args = ['UNSET']
def set_test_args(*args):
test_args[:] = args
scope = {'set_test_args': set_test_args}
exec(parser.sequence2ast(t).compile(), scope)
assert tuple(test_args) == args
def modify_AST(myAST, assign_name, assign_val):
myAST_Visitor = AST_Visitor(1)
old_AST_list = myAST.tolist(1)
assign_val_type = 0
if isinstance(assign_val, int):
assign_val_type = 2
elif isinstance(assign_val, float):
assign_val_type = 1
elif type(assign_val) == str:
assign_val_type = 3
new_AST_list = myAST_Visitor.traverse(
old_AST_list, assign_name,
gen_assign_val_subtree(assign_val, assign_val_type))
myNewAST = parser.sequence2ast(new_AST_list)
return myNewAST
def rewrite_and_compile(self, output_func_name=None, print_func_name=None, copy_func_name="__copy"):
"""
Compiles the parse tree into code, while rewriting the parse tree according to the
output_func_name and print_func_name arguments.
At the same time, the code is scanned for possible mutations, and a list is returned.
Each item in the list is a tuple of:
- The name of the variable at the root of the path to the object
(e.g., for a.b.c, "a")
- A string describing what should be copied. The string may include ellipses (...)
for complex areas - it's meant as a human description
- Code that can be evaluated to copy the object.
@param output_func_name: the name of function used to wrap statements that are simply expressions.
(More than one argument will be passed if the statement is in the form of a list.)
Can be None.
@param print_func_name: the name of a function used to replace print statements without a destination
file. Can be None.
@param copy_func_name: the name of a function used to make shallow copies of objects.
Should have the same semantics as copy.copy (will normally be an import of copy.copy)
Defaults to __copy.
@returns: a tuple of the compiled code followed by a list of mutations
"""
state = _RewriteState(output_func_name=output_func_name,
print_func_name=print_func_name,
future_features=self.future_features)
rewritten = _rewrite_file_input(self.original, state)
encoded = (symbol.encoding_decl, rewritten, self.encoding)
try:
compiled = parser.sequence2ast(encoded).compile()
except parser.ParserError, e:
if "Illegal number of children for try/finally node" in e.message:
raise UnsupportedSyntaxError("try/except/finally not supported due to Python issue 4529")
else:
raise UnsupportedSyntaxError("Unexpected parser error: " + e.message);
def _parseconf(confstr):
"""
Parse the configuration *confstr* string and remove anything else
than the supported constructs, which are:
Assignments, bool, dict list, string, float, bool, and, or, xor,
arithmetics, string expressions and if..then..else.
The *entire* statement containing the unsupported statement is removed
from the parser; the effect is that the whole expression is ignored
from the 'root' down.
The modified AST object is returned to the Python parser for evaluation.
"""
# Initialise the parse tree, convert to list format and get a list of
# the symbol ID's for the unwanted statements. Might raise SyntaxError.
ast = parser.suite(confstr)
#ast = parser.expr(confstr)
stmts = parser.ast2list(ast)
rmsym = _get_forbidden_symbols()
result = list()
# copy 256: 'single_input', 257: 'file_input' or 258: 'eval_input'. The
# parse tree must begin with one of these to compile back to an AST obj.
result.append(stmts[0])
# NOTE: This might be improved with reduce(...) builtin!? How can we get
# line number for better warnings?
for i in range(1, len(stmts)):
# censor the parse tree produced from parsing the configuration.
if _check_ast(stmts[i], rmsym):
result.append(stmts[i])
else:
pass
return parser.sequence2ast(result)
def rewrite_and_compile(code,
output_func_name=None,
output_func_self=None,
print_func_name=None,
encoding="utf8"):
"""
Compiles the supplied text into code, while rewriting the parse tree so:
* Print statements without a destination file are transformed into calls to
<print_func_name>(*args), if print_func_name is not None
* Statements which are simply expressions are transformed into calls to
<output_func_name>(*args), if output_fnuc_name is not None
(More than one argument is passed if the statement is in the form of a list; for example '1,2'.)
At the same time, the code is scanned for possible mutations, and a list is returned.
In the list:
* A string indicates the mutation of a variable by assignment to a slice of it,
or to an attribute.
* A tuple of (variable_name, method_name) indicates the invocation of a method
on the variable; this will sometimes be a mutation (e.g., list.append(value)),
and sometimes not.
"""
state = _RewriteState(output_func_name=output_func_name,
output_func_self=output_func_self,
print_func_name=print_func_name)
if (isinstance(code, unicode)):
code = code.encode("utf8")
encoding = "utf8"
original = parser.suite(code)
rewritten = _rewrite_file_input(original.totuple(), state)
encoded = (symbol.encoding_decl, rewritten, encoding)
compiled = parser.sequence2ast(encoded).compile()
return (compiled, state.mutated)
def _wrap(self, stmt, COMPARING=False, PRINTING=False):
"""given a single simple comparison statement as a cst, return that
statement wrapped with our testing function, also as a cst
"""
# convert statement to source code
cst = self._stmt2file_input(stmt)
old_source = ASTutils.ast2text(parser.sequence2ast(cst))
old_source = self._escape_source(old_source)
# escape string delimiters in old source code; convert to single line
template = """\
__pytest__.intercept("%s", %s, globals(), locals(), COMPARING=%s,PRINTING=%s)"""
new_source = template % ( old_source
, self._line_number(cst)
, COMPARING
, PRINTING
)
# convert back to a cst, extract our statement, and return
cst = parser.suite(new_source).tolist()
return cst[1]
name = token.tok_name.get(node[0])
print(name, end=' ')
for i in range(1, len(node)):
item = node[i]
if type(item) is type([]):
dump_and_modify(item)
else:
print(repr(item))
if name == "NUMBER":
# increment all numbers!
node[i] = repr(int(item)+1)
ast = parser.expr("1 + 3")
list = ast.tolist()
dump_and_modify(list)
ast = parser.sequence2ast(list)
print(eval(parser.compileast(ast)))
## eval_input testlist test and_test not_test comparison
## expr xor_expr and_expr shift_expr arith_expr term factor
## power atom NUMBER '1'
## PLUS '+'
## term factor power atom NUMBER '3'
## NEWLINE ''
## ENDMARKER ''
## 6
def roundtrip(f, s):
st1 = f(s)
t = st1.totuple()
st2 = parser.sequence2ast(t)
def _compile_copy_code(path, copy_func_name):
copy_code = _create_copy_code(path, copy_func_name)
return parser.sequence2ast(copy_code).compile()
def roundtrip(f, s):
st1 = f(s)
t = st1.totuple()
st2 = parser.sequence2ast(t)
name = token.tok_name.get(node[0])
print name,
for i in range(1, len(node)):
item = node[i]
if type(item) is type([]):
dump_and_modify(item)
else:
print repr(item)
if name == "NUMBER":
# increment all numbers!
node[i] = repr(int(item)+1)
ast = parser.expr("1 + 3")
list = ast.tolist()
dump_and_modify(list)
ast = parser.sequence2ast(list)
print eval(parser.compileast(ast))
## eval_input testlist test and_test not_test comparison
## expr xor_expr and_expr shift_expr arith_expr term factor
## power atom NUMBER '1'
## PLUS '+'
## term factor power atom NUMBER '3'
## NEWLINE ''
## ENDMARKER ''
## 6