def visit_Return(self, node):
print ast.dump(node)
print '(' + str(codegen.to_source(node.value)) + ')'
ast.dump(node)
a = HoareStmt("ASSIGNMENT",['ret' + self.function_name, codegen.to_source(node.value)])# [node.targets[0].id, codegen.to_source(node.value)])
self.paths[-1].append(a)
self.paths[-1].append(self.postcondition)
def perform_replace(args, file, node, base, child, field, index,
newnode=AST_NONE):
"Replaces a node with another"
print "\tReplacing %s (Line %d)" % (str(child), child.lineno)
oldnodes = None
if index is not None:
oldnodes = getattr(node, field)
newnodes = oldnodes[:]
newnodes[index] = newnode
setattr(node, field, newnodes)
else:
setattr(node, field, newnode)
new_source = codegen.to_source(base)
open(file, mode="w").write(new_source)
# It's a failure when the tests pass
if run_tests(args):
print "\t\tFound uncovered code"
log = open("%s.log" % file, mode="a")
log.write("Failure:\n")
log.write("Line %d:%d\n" % (child.lineno, child.col_offset))
log.write("Code:\n")
log.write(codegen.to_source(child))
log.write("\n\n\n")
log.close()
if index is not None:
setattr(node, field, oldnodes)
else:
setattr(node, field, child)
def _create_line_variable(self, node):
if not self._node_is_variable(node):
return (None, None)
attributes = []
is_subscript = isinstance(node.parent, ast.Subscript)
is_write = False
if isinstance(node.parent, ast.Assign) and (node in node.parent.targets):
is_write = True
elif is_subscript and isinstance(node.parent.ctx, ast.Store):
is_write = True
key = node.lineno + (1 if is_write else 0)
attributes.append('node = {0}'.format(repr('variable')))
attributes.append('name = {0}'.format(repr(node.id)))
attributes.append('vid = id({0})'.format(node.id))
attributes.append('type = str(type({0}))'.format(node.id))
attributes.append('access = {0}'.format(repr('write' if is_write else 'read')))
if is_subscript:
slice = codegen.to_source(node.parent.slice.value)
element = codegen.to_source(node.parent)
attributes.append('slice = ({0})'.format(slice))
attributes.append('slice_type = str(type({0}))'.format(element))
attributes.append('line_offset = {0}'.format(node.line_offset))
value = ('print repr(dict({0}))'.format(', '.join(attributes)), node.line_offset)
return (key, value)
def visit_Assign(self, node):
print codegen.to_source(node.value)
if isinstance(node.targets[0], ast.Name):
assign = HoareStmt("ASSIGNMENT", [node.targets[0].id, codegen.to_source(node.value)])
if isinstance(node.value, ast.Call):
assign = HoareStmt("ASSIGNMENT", [node.targets[0].id, "ret" + str(codegen.to_source(node.value.func))])
self.generic_visit(node)
self.paths[-1].append(assign)
def print_possible(k):
for i in k:
print '#' * 10
try:
d = deserialize(i)
print codegen.to_source(d)
except Exception as e:
print e
def find_conds(self, node, fname):
pre = [d for d in node.decorator_list if d.func.id == 'precondition']
post = [d for d in node.decorator_list if d.func.id == 'postcondition']
temp = [codegen.to_source(a) for a in post[0].args]
temp = [str(j).replace("ret", "ret" + fname) for j in temp]
precond = HoareStmt("STMT", [codegen.to_source(p) for p in pre[0].args][0])
postcond = HoareStmt("STMT", temp[0])
return precond, postcond
def patterns(func):
empty_argspec = inspect.ArgSpec(args=[], varargs=None, keywords=None, defaults=None)
assert inspect.getargspec(func) == empty_argspec, 'Pattern function should not have arguments'
# TODO: make it not as weird and dirty
func.__globals__['Mismatch'] = Mismatch
tree = get_ast(func)
print("\n"+codegen.to_source(tree)+"\n")
transform_function(tree.body[0])
print("\n"+codegen.to_source(tree)+"\n")
return compile_func(func, tree)
def warn(self, func, arg):
"""Warn about calls of bitbake APIs which pass a non-literal
argument for the variable name, as we're not able to track such
a reference.
"""
try:
funcstr = codegen.to_source(func)
argstr = codegen.to_source(arg)
except TypeError:
self.log.debug(2, 'Failed to convert function and argument to source form')
else:
self.log.debug(1, self.unhandled_message % (funcstr, argstr))
def warn(self, func, arg):
"""Warn about calls of bitbake APIs which pass a non-literal
argument for the variable name, as we're not able to track such
a reference.
"""
try:
funcstr = codegen.to_source(func)
argstr = codegen.to_source(arg)
except TypeError:
self.log.debug(2, 'Failed to convert function and argument to source form')
else:
self.log.debug(1, self.unhandled_message % (funcstr, argstr))
def explore(self):
copy_ast = self.program_ast#copy.deepcopy(self.program_ast)
copy_fnc = find_function(copy_ast, 'main')
general_dict = {}
RewriteTest(copy_fnc, general_dict , copy_ast).generic_visit(copy_fnc)
print codegen.to_source(copy_ast)
# RewriteAssigns(general_dict, copy_ast).generic_visit(copy_fnc)
#for i in self.program_ast.body:
# print self.fnc.body
# stmts = []
#for stmt in self.fnc.body:
# stmts.append(stmt)
# print self.fnc.body.index(stmts[2])
# print stmts[:2]
# print i
codegen.to_source(self.program_ast)
# print (codegen.to_source(self.program_ast))
# RewriteReturn(self).visit(self.program_ast)
# print (codegen.to_source(self.program_ast))
#Dictionary
store = {}
# List of Constraints
constr = []
s = (store, constr)
#List for inputs
input_to_ret = []
#List of statements
stmts = []
# Calculate Inputs
for stmt in self.fnc.body:
stmts.append(stmt)
eval_body(s, stmts, self)
#print self.inputs_to_ret
# Calculate return values to inputs
for e in self.inputs_to_ret:
input = generate_inputs(self.fnc,e )
f = FunctionEvaluator(self.fnc, self.program_ast, input)
ret = f.eval()
input_to_ret.append((input, ret))
return (input_to_ret, self.assetion_violations_to_input)
def warn(cls, func, arg):
"""Warn about calls of bitbake APIs which pass a non-literal
argument for the variable name, as we're not able to track such
a reference.
"""
try:
funcstr = codegen.to_source(func)
argstr = codegen.to_source(arg)
except TypeError:
logger.debug(2, 'Failed to convert function and argument to source form')
else:
logger.debug(1, "Warning: in call to '%s', argument '%s' is "
"not a literal", funcstr, argstr)
def print_screen(f, stmt, indent_level):
screen = stmt.screen
# The creator of the python ast module also created a script to revert it.
sourcecode = codegen.to_source(screen.code.source, u" " * 4)
# why suddenly ast in the source code field
f.write(u"screen %s" % screen.name)
if hasattr(screen, 'parameters') and screen.parameters:
print_params(f, screen.parameters)
f.write(u":\n")
if screen.tag:
indent(f, indent_level + 1)
f.write(u"tag %s\n" % screen.tag)
if screen.zorder and screen.zorder != '0':
indent(f, indent_level + 1)
f.write(u"zorder %s\n" % screen.zorder)
if screen.modal:
indent(f, indent_level + 1)
f.write(u"modal %s\n" % screen.modal)
if screen.variant != "None":
indent(f, indent_level + 1)
f.write(u"variant %s\n" % screen.variant)
# actual screen decompilation is HARD
if DECOMPILE_SCREENS:
screendecompiler.print_screen(f, sourcecode, indent_level + 1)
else:
indent(f, indent_level + 1)
f.write('python:\n')
f.write('\n'.join([
" " * (indent_level + 2) + line
for line in sourcecode.splitlines()
]))
f.write(u"\n")
def transform_function(func_tree):
assert all(isinstance(t, If) for t in func_tree.body), \
'Patterns function should only have if statements'
# Adjust arglist and decorators
func_tree.args.args.append(A('value'))
func_tree.decorator_list = []
# Transform tests to pattern matching
for test in func_tree.body:
cond = test.test
if isinstance(cond, (Num, Str, List, Tuple, Dict)) and not has_vars(cond):
test.test = make_eq(N('value'), cond)
elif isinstance(cond, (Num, Str, Name, NameConstant, Compare, List, Tuple, Dict, BinOp)):
tests, assigns = destruct_to_tests_and_assigns(N('value'), cond)
test.test = BoolOp(op=And(), values=tests) if len(tests) > 1 else \
tests[0] if tests else V(1)
test.body = assigns + test.body
else:
raise TypeError("Don't know how to match %s"
% (codegen.to_source(cond).strip() or cond))
func_tree.body = lmap(wrap_tail_expr, func_tree.body)
func_tree.body.append(make_raise(N('Mismatch')))
# Set raise Mismatch lineno just after function end
func_tree.body[-1].lineno = last_lineno(func_tree) + 1
def update_funs(t0, t1):
d = OrderedDict()
used = set()
class FunDefScanner(ast.NodeVisitor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.stack = []
def visit_FunctionDef(self, node):
if not self.stack:
# collecting top level function only
d[node.name] = node
self.stack.append(True)
super().generic_visit(node)
self.stack.pop()
visitor = FunDefScanner()
visitor.visit(t1)
class FunDefMerger(ast.NodeVisitor):
def visit_FunctionDef(self, node):
if node.name in d:
node.args = d[node.name].args
ast.fix_missing_locations(node.args)
used.add(node.name)
visitor = FunDefMerger()
visitor.visit(t0)
for name, node in d.items():
if name not in used:
t0.body.append(node)
ast.fix_missing_locations(t0.body[-1])
return codegen.to_source(t0)
def print_screen(f, stmt, indent_level):
screen = stmt.screen
sourcecode = unicode(codegen.to_source(screen.code.source,
u" " * 4)).splitlines()
if sourcecode[0] == u'_1 = (_name, 0)':
sourcecode.remove(u'_1 = (_name, 0)')
sourcecode = u"\n".join(
[u" " * 4 * (indent_level + 2) + i for i in sourcecode])
#why suddenly ast in the source code field
f.write(u"screen %s" % screen.name)
if screen.parameters:
print_params(f, screen.parameters)
f.write(u":\n")
if screen.tag:
indent(f, indent_level + 1)
f.write(u"tag %s\n" % screen.tag)
if screen.zorder:
indent(f, indent_level + 1)
f.write(u"zorder %s\n" % screen.zorder)
if screen.modal:
indent(f, indent_level + 1)
f.write(u"modal %s\n" % screen.modal)
if screen.variant != "None":
indent(f, indent_level + 1)
f.write(u"variant %s\n" % screen.variant)
indent(f, indent_level + 1)
f.write(u"python:\n")
f.write(sourcecode)
f.write(u"\n")
def transform_function(func_tree):
assert all(isinstance(t, If) for t in func_tree.body), \
'Patterns function should only have if statements'
# Adjust arglist and decorators
func_tree.args.args.append(A('value'))
func_tree.decorator_list = []
# Transform tests to pattern matching
for test in func_tree.body:
cond = test.test
if isinstance(cond, (Num, Str, List, Tuple, Dict)) and not has_vars(cond):
test.test = make_eq(N('value'), cond)
elif isinstance(cond, (Num, Str, Name, NameConstant, Compare, List, Tuple, Dict, BinOp)):
tests, assigns = destruct_to_tests_and_assigns(N('value'), cond)
test.test = BoolOp(op=And(), values=tests) if len(tests) > 1 else \
tests[0] if tests else V(1)
test.body = assigns + test.body
else:
raise TypeError("Don't know how to match %s"
% (codegen.to_source(cond).strip() or cond))
func_tree.body = lmap(wrap_tail_expr, func_tree.body)
func_tree.body.append(make_raise(N('Mismatch')))
# Set raise Mismatch lineno just after function end
func_tree.body[-1].lineno = last_lineno(func_tree) + 1
def visit_While(self, node):
inv_node = [n for n in node.body if (isinstance(n, ast.Expr) and isinstance(n.value, ast.Call) and (n.value.func.id == 'invariant'))]
inv = HoareStmt('STMT', [codegen.to_source(a) for a in inv_node[0].value.args][0])
post_inv = HoareStmt("STMT", str("Not" + codegen.to_source(node.test)))
self.paths[-1].append(inv)
self.paths.append([])
self.paths[-1].append(inv)
self.generic_visit(node)
self.paths[-1].append(inv)
self.paths.append([])
self.paths[-1].append(inv)
self.paths[-1].append(post_inv)
def hoareify(func, toverify):
hoareify = []
postconditions = []
post = [d for d in func.decorator_list if d.func.id == 'postcondition']
temp = [codegen.to_source(a) for a in post[0].args]
temp = [str(j).replace("ret", "ret" + func.name) for j in temp]
postcond = HoareStmt("STMT", temp[0])
postconditions.append(postcond)
def walk(node):
if not hasattr(node, 'body'):
return
for p in node.body:
walk(p)
p = Paths(toverify).visit(func)
print "\nPATHS:\n", p
hoareify = p
for i in range(len(hoareify)):
hoareify[i:i] = if_situation([hoareify[i]])
del hoareify[i]
for i in range(len(hoareify)):
if hoareify[i][-1] == hoareify[i][-2]:
del hoareify[i][-1]
#for i in p:
# print i
return hoareify
def visit_Name(self, node):
if node.id in arg_names:
if node.id.find("seq") >= 0:
return node
# create z3 node
if node.id.find("int") >= 0:
z3_node = ast.parse("z3.Int(x)")
elif node.id.find("real") >= 0:
z3_node = ast.parse("z3.Real(x)")
else:
return node
# fix the symbolic names
new_node_str = codegen.to_source(node) + "#"
# replace x by this new_node
s_new_node = ast.Str(new_node_str)
ReplaceByNode('x', s_new_node).visit(z3_node)
return z3_node
elif node.id in symbolic_state:
# print node.id, symbolic_state[node.id]
return ast.parse(symbolic_state[node.id])
elif node.id in local_vars:
return ast.copy_location(ast.Num(local_vars[node.id]), node)
else:
return node
def visit_Call(self, node):
if isinstance(node.func, ast.Name): # Could be Attribute node
n_id = node.func.id
if n_id == 'requires' or n_id == 'assures':
# Uncompile 'bodyx' in 'requires/assures(body1, body2, ...) for all x
# Since codegen fails to convert all bodyx's, we must do this:
args = codegen.to_source(node)
args = re.sub(n_id, '', args) # Remove function call
args = args.strip(
' ()') # Remove surrounding spaces and parenthesis
args = args.split(",") # Convert to list
for body in args:
body = body.strip(' ()')
if n_id == 'assures':
global post_conditions
post_conditions.append(
body
) # To be executed at end of parent visit_FunctionDef
else:
# Add assersion to the global solver
eval("global_solver.add(" + body + ")")
return
else: # Some other function call
if self.Z3_INFO:
print("call to:", n_id, "passing...")
# call/apply z3 representation of this function
# self.visit(node.value)
else:
if self.Z3_INFO:
print("call function is of type:", node.func)
print("passing...")
def print_screen(f, stmt, indent_level):
screen = stmt.screen
sourcecode = unicode(codegen.to_source(screen.code.source, u" "*4)).splitlines()
if sourcecode[0] == u'_1 = (_name, 0)':
sourcecode.remove(u'_1 = (_name, 0)')
sourcecode = u"\n".join([u" "*4*(indent_level+2)+i for i in sourcecode])
#why suddenly ast in the source code field
f.write(u"screen %s" % screen.name)
if screen.parameters:
print_params(f, screen.parameters)
f.write(u":\n")
if screen.tag:
indent(f, indent_level+1)
f.write(u"tag %s\n" % screen.tag)
if screen.zorder:
indent(f, indent_level+1)
f.write(u"zorder %s\n" % screen.zorder)
if screen.modal:
indent(f, indent_level+1)
f.write(u"modal %s\n" % screen.modal)
if screen.variant != "None":
indent(f, indent_level+1)
f.write(u"variant %s\n" % screen.variant)
indent(f, indent_level+1)
f.write(u"python:\n")
f.write(sourcecode)
f.write(u"\n")
def test_codegen():
"""This function will rewrite the init in class Foo, substituting
the last parameter for two other parameters and constructing a
dictionary with them.
"""
return codegen.to_source(rewrite_tree())
def print_object(self, ast):
# handles the printing of anything unknown which inherts from object.
# prints the values of relevant attributes in a dictionary-like way
# it will not print anything which is a bound method or starts with a _
self.p('<')
self.p(str(ast.__class__)[8:-2] if hasattr(ast, '__class__') else str(ast))
if isinstance(ast, py_ast.Module) and self.decompile_python:
self.p('.code = ')
self.print_ast(codegen.to_source(ast, unicode(self.indentation)))
self.p('>')
return
keys = list(i for i in dir(ast) if self.should_print_key(ast, i))
if keys:
self.p(' ')
self.ind(1, keys)
for i, key in enumerate(keys):
self.p('.')
self.p(str(key))
self.p(' = ')
self.print_ast(getattr(ast, key))
if i+1 != len(keys):
self.p(',')
self.ind()
self.ind(-1, keys)
self.p('>')
def print_object(self, ast):
# handles the printing of anything unknown which inherts from object.
# prints the values of relevant attributes in a dictionary-like way
# it will not print anything which is a bound method or starts with a _
self.p('<')
self.p(str(ast.__class__)[8:-2] if hasattr(ast, '__class__') else str(ast))
if isinstance(ast, py_ast.Module):
self.p('.code = ')
if self.decompile_python:
self.print_ast(codegen.to_source(ast, unicode(self.indentation)))
self.p('>')
return
keys = list(i for i in dir(ast) if not i.startswith('_') and hasattr(ast, i) and not inspect.isroutine(getattr(ast, i)))
if keys:
self.p(' ')
self.ind(1, keys)
for i, key in enumerate(keys):
self.p('.')
self.p(str(key))
self.p(' = ')
self.print_ast(getattr(ast, key))
if i+1 != len(keys):
self.p(',')
self.ind()
self.ind(-1, keys)
self.p('>')
def update_funs(t0, t1):
d = OrderedDict()
used = set()
class FunDefScanner(ast.NodeVisitor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.stack = []
def visit_FunctionDef(self, node):
if not self.stack:
# collecting top level function only
d[node.name] = node
self.stack.append(True)
super().generic_visit(node)
self.stack.pop()
visitor = FunDefScanner()
visitor.visit(t1)
class FunDefMerger(ast.NodeVisitor):
def visit_FunctionDef(self, node):
if node.name in d:
node.args = d[node.name].args
ast.fix_missing_locations(node.args)
used.add(node.name)
visitor = FunDefMerger()
visitor.visit(t0)
for name, node in d.items():
if name not in used:
t0.body.append(node)
ast.fix_missing_locations(t0.body[-1])
return codegen.to_source(t0)
def test_codegen():
"""This function will rewrite the init in class Foo, substituting
the last parameter for two other parameters and constructing a
dictionary with them.
"""
return codegen.to_source(rewrite_tree())
def print_screen(f, stmt, indent_level):
screen = stmt.screen
# The creator of the python ast module also created a script to revert it.
sourcecode = codegen.to_source(screen.code.source, u" "*4)
# why suddenly ast in the source code field
f.write(u"screen %s" % screen.name)
if hasattr(screen, 'parameters') and screen.parameters:
print_params(f, screen.parameters)
f.write(u":\n")
if screen.tag:
indent(f, indent_level+1)
f.write(u"tag %s\n" % screen.tag)
if screen.zorder and screen.zorder != '0':
indent(f, indent_level+1)
f.write(u"zorder %s\n" % screen.zorder)
if screen.modal:
indent(f, indent_level+1)
f.write(u"modal %s\n" % screen.modal)
if screen.variant != "None":
indent(f, indent_level+1)
f.write(u"variant %s\n" % screen.variant)
# actual screen decompilation is HARD
if DECOMPILE_SCREENS:
screendecompiler.print_screen(f, sourcecode, indent_level+1)
else:
indent(f, indent_level+1)
f.write('python:\n')
f.write('\n'.join([" "*(indent_level+2)+line for line in sourcecode.splitlines()]))
f.write(u"\n")
def format_class(node, exclude=['_*']):
template = textwrap.dedent('''
.. py:class:: {name}{args}
{documentation}
''')
methods = []
for attr_node in node.body:
if isinstance(attr_node, ast.FunctionDef):
if any(fnmatch.fnmatch(attr_node.name, ex) for ex in exclude):
continue
# adding the method to the rendered part
methods.append(indent(format_method(attr_node.name, attr_node)))
# Refer to the __init__ method for class documentation
# and signature
init = [
attr_node for attr_node in node.body if
isinstance(attr_node, ast.FunctionDef) and attr_node.name == '__init__'
]
if init:
doc = ast.get_docstring(init[0])
args = '(' + codegen.to_source(init[0].args) + ')'
else:
doc = ast.get_docstring(node)
args = ''
doc = indent(doc) if doc else ''
ret = template.format(name=node.name, documentation=doc, args=args)
ret += '\n'.join(methods)
return ret
def visit_UnaryOp(self, node):
# print "XX", codegen.to_source(node), node.op, node.op == ast.NotEq, node.op == ast.Not, node.op == ast.IsNot, codegen.to_source(node.op)
old_source = codegen.to_source(node)
if old_source.find("not") >= 0:
new_source = old_source.replace("not", "z3.Not")
return ast.parse(new_source)
else:
return node
def print_ast(cls, tree):
"""
Print out a specified abstract syntax tree.
:param tree: abstract syntax tree to print
"""
# print out the mutated tree
code = codegen.to_source(tree)
print code
def evaluate(self, expression):
assert expression is not None
if isinstance(expression, AST):
expression_code = to_source(expression)
try:
eval(expression_code, self._globals, self._locals)
except Exception, e:
print 'Expression raised error:', str(e), 'in', expression_code
def readFunction(astree):
functions={}
for i in range(len(astree.body)):
line = codegen.to_source(astree.body[i]).splitlines()[0]
if line.startswith("def"):
functions[line.split(' ')[1].split('(')[0].strip()]=i
return functions
def write_clean_strings():
import ast
import codegen
try:
with open('name.py', 'rw') as file1:
p = ast.parse(file1)
print(codegen.to_source(p))
except SyntaxError as e:
print e
def grep_visit(self, node):
"""
Transform node into source, and append a hit to results if the
regexp matches the source.
"""
source = codegen.to_source(node)
if self.finder.search(source):
self.results.append(_fmt_res(self.where,
source))
def test_line_mutator(self):
code = "x = 1\ny = -1 + 10\nz = 'skip this'\nw = 200.0"
number_mutator = mutator.NumberMutator()
mutations = list(mutator.LineMutator(number_mutator, code))
self.assertEquals([codegen.to_source(m) for line_no, pos, m in mutations],
["x = 2\ny = -1 + 10\nz = 'skip this'\nw = 200.0",
"x = 1\ny = 0 + 10\nz = 'skip this'\nw = 200.0",
"x = 1\ny = -1 + 11\nz = 'skip this'\nw = 200.0",
"x = 1\ny = -1 + 10\nz = 'skip this'\nw = 201.0"])
def visit_If(self, node):
pattern = re.compile(r"and|or")
temp = pattern.split(codegen.to_source(node.test))
ifs = HoareStmt("IFSTART", self.apply_and(temp))
print "IFF", ifs
ife = HoareStmt("IFEND", 7)
self.paths[-1].append(ifs)
self.generic_visit(node)
self.paths[-1].append(ife)
def display_file(a_filename, is_main=True):
try:
# READ FILE
l_f = file(a_filename, 'r')
# TRANSFORM group x -> def pycado_object_x
l_src = ""
l = l_f.readline()
while l:
if l.strip().startswith("group "):
#todo: to improve. Here fails when several spaces between group and group name
l = l.replace("group ", "def pycado_object_")
l_src += l
l = l_f.readline()
l_f.close()
# PERFORM PREPROCESSING:
# - transform groups in classes with some initialisation
# - add cs0 in primitives calling
# - make a function grouping lonely instructions
l_ast = ast.parse(l_src, a_filename)
#print dump(l_ast)
l_pre_compiler = pre_compiler(a_filename)
l_pre_compiler.visit(l_ast)
fixer = fix_tree()
fixer.visit(l_ast)
# EXECUTE!
print("# CODE GENERATED")
print(codegen.to_source(l_ast))
#print dump(l_ast)
exec compile(l_ast, a_filename, 'exec') in globals()
if is_main:
pycado_main()
print("\n# PYCADO_OBJ LIST")
for o in nspace.get_objs():
o.build()
o.display()
#print o.name, o
nspace.display_edges()
nspace.fitAll()
except:
print sys.exc_info()
nspace.log(traceback.format_exc())
def destruct_to_tests_and_assigns(topic, pattern, names=None):
if names is None:
names = {}
if isinstance(pattern, (Num, Str)):
return [make_eq(topic, pattern)], []
elif isinstance(pattern, Name):
if pattern.id in NAMED_CONSTS:
return [make_op(Is, topic, pattern)], []
elif pattern.id == '_':
return [], []
else:
if pattern.id in names:
return [make_eq(topic, names[pattern.id])], []
else:
names[pattern.id] = topic
return [], [make_assign(pattern.id, topic)]
elif isinstance(pattern, NameConstant):
return [make_op(Is, topic, pattern)], []
elif isinstance(pattern, Compare) and len(pattern.ops) == 1 and isinstance(
pattern.ops[0], Is):
left_tests, left_assigns = destruct_to_tests_and_assigns(
topic, pattern.left, names)
test = make_call('isinstance', topic, pattern.comparators[0])
return left_tests + [test], left_assigns
elif isinstance(pattern, (List, Tuple, Dict)):
elts = getattr(pattern, 'elts', []) or getattr(pattern, 'values', [])
coll_tests = [
make_call('isinstance', topic,
N(pattern.__class__.__name__.lower())),
make_eq(make_call('len', topic), len(elts))
]
tests, assigns = subscript_tests_and_assigns(topic, pattern, names)
return coll_tests + tests, assigns
elif isinstance(pattern, BinOp) and isinstance(pattern.op, Add) \
and isinstance(pattern.left, (List, Tuple)) and isinstance(pattern.right, Name):
coll_tests = [
make_call('isinstance', topic,
N(pattern.left.__class__.__name__.lower())),
make_op(GtE, make_call('len', topic), len(pattern.left.elts)),
]
coll_assigns = [
make_assign(
pattern.right.id,
Subscript(ctx=Load(),
value=topic,
slice=Slice(lower=V(len(pattern.left.elts)),
upper=None,
step=None)))
]
tests, assigns = subscript_tests_and_assigns(topic, pattern.left,
names)
return coll_tests + tests, assigns + coll_assigns
else:
raise TypeError("Don't know how to match %s" %
(codegen.to_source(pattern).strip() or pattern))
def test_code_mutator_with_filter(self):
def line_filter(line, line_no):
return line_no % 2 == 1
code = "x = 1\ny = 2\nz = 3\nw = 4"
mutators = [mutator.NumberMutator]
mutations = list(mutator.code_mutator(mutators, code, line_filter))
self.assertEquals(["x = 2\ny = 2\nz = 3\nw = 4",
"x = 1\ny = 2\nz = 4\nw = 4"],
[codegen.to_source(m) for line_no, pos, m, class_name in mutations])
def apply_transformations(self, code):
"""
Applies the created transformers to the given code segment
:param code: Python code to transform
:return: Transformed Python code with mutations applied
"""
tree = ast.parse(code)
for transformation in self.transformers:
transformation().visit(tree)
return codegen.to_source(tree)
def visit_If(self, node):
"""Subclassed ast module method."""
if isinstance(node.body[0], ast.If):
print("[> Nested 'if' error number {}\t<]".format(self.count))
print("[> Error node starts on line {}\t<]".format(node.lineno))
print("-" * 55)
print(codegen.to_source(node))
print("-" * 55)
input("Hit Enter to continue\n")
ReadIHNIL.count += 1
def test_code_mutator_multiple_classes(self):
def line_filter(line, line_no):
return True
code = "x = 1\ny = True"
mutators = [mutator.NumberMutator, mutator.BooleanMutator]
mutations = list(mutator.code_mutator(mutators, code, line_filter))
self.assertEquals(["x = 2\ny = True",
"x = 1\ny = False"],
[codegen.to_source(m) for line_no, pos, m, class_name in mutations])
def visit_For(self, node):
inv_node = [n for n in node.body if (isinstance(n, ast.Expr) and isinstance(n.value, ast.Call) and (n.value.func.id == 'invariant'))]
pre_inv = HoareStmt("STMT", str(node.target.id + ' == ' + str(eval(codegen.to_source(node.iter))[0])))
post_inv = HoareStmt("STMT", str(node.target.id + '==' + str(codegen.to_source(node.iter.args[0]))))
inv = HoareStmt('STMT', [codegen.to_source(a) for a in inv_node[0].value.args][0])
invplus = HoareStmt("ASSIGNMENT", [node.target.id, node.target.id + '+ 1'])
self.paths[-1].append(pre_inv)
self.paths[-1].append(inv)
self.paths.append([])
self.paths[-1].append(inv)
self.generic_visit(node)
self.paths[-1].append(invplus)
self.paths[-1].append(inv)
self.paths.append([])
self.paths[-1].append(inv)
self.paths[-1].append(post_inv)
def display_file(a_filename, is_main=True):
try:
# READ FILE
l_f = file(a_filename, 'r')
# TRANSFORM group x -> def pycado_object_x
l_src = ""
l = l_f.readline()
while l:
if l.strip().startswith("group "):
#todo: to improve. Here fails when several spaces between group and group name
l = l.replace("group ", "def pycado_object_")
l_src += l
l = l_f.readline()
l_f.close()
# PERFORM PREPROCESSING:
# - transform groups in classes with some initialisation
# - add cs0 in primitives calling
# - make a function grouping lonely instructions
l_ast = ast.parse(l_src, a_filename)
#print dump(l_ast)
l_pre_compiler = pre_compiler(a_filename)
l_pre_compiler.visit(l_ast)
fixer = fix_tree()
fixer.visit(l_ast)
# EXECUTE!
print("# CODE GENERATED")
print(codegen.to_source(l_ast))
#print dump(l_ast)
exec compile(l_ast, a_filename, 'exec') in globals()
if is_main:
pycado_main()
print("\n# PYCADO_OBJ LIST")
for o in nspace.get_objs():
o.build()
o.display()
#print o.name, o
nspace.display_edges()
nspace.fitAll()
except:
print sys.exc_info()
nspace.log(traceback.format_exc())
def get_template(filename):
cached = cache.get(filename)
if not cached:
with open(filename) as fp:
source = fp.read()
if sys.version_info[0] < 3:
source = source.decode("utf-8")
tree = parse(source)
compiled = compile_tree(tree)
module = ast.Module(compiled)
optimized = optimize(module)
template_source = ""
try:
from astmonkey import visitors
template_source = visitors.to_source(optimized)
except ImportError:
try:
import codegen
template_source = codegen.to_source(optimized)
except ImportError:
template_source = ""
code = compile(ast.fix_missing_locations(optimized), filename, "exec")
globs = {
ESCAPE: escape,
QUOTEATTR: quoteattr,
TO_STRING: soft_unicode,
WRITE_ATTRS: write_attrs
}
scope = {}
exec_(code, globs, scope)
main_fun = scope[MAIN]
concat = "".join
def render(**kwargs):
output = []
context = {WRITE: output.append, WRITE_MULTI: output.extend}
context.update(kwargs)
main_fun(**context)
return concat(output)
setattr(render, "template_source", template_source)
cached = render
cache[filename] = cached
return cached
def format_method(name, node):
template = textwrap.dedent('''
.. py:method:: {name}({arguments})
{documentation}
''')
doc = ast.get_docstring(node)
doc = indent(doc) if doc else ''
return template.format(name=name,
arguments=codegen.to_source(node.args),
documentation=doc)
def visit_FunctionDef(self, node):
# 拿到运行时环境的全局命名空间
code = "__globals = globals()\n"
# 把需要降级的全局变量“实例化”出来。 比如 INCR = __globals['INCR']
code += "\n".join("{0} = __globals['{0}']".format(name)
for name in self.lowered_name)
code_ast = ast.parse(code, mode="exec")
# 把前面降级的代码插入到方法体，插入到原有代码的前面
# 这里使用node.body[:0]的原因是body也是一个list，但是我们需要插入到最前面 list[:0]得到空列表[]
node.body[:0] = code_ast.body
# 查看修改后的源码
print(codegen.to_source(node))
self.func = node
def run(self, statement):
"""
Run the given statement inside this context. Return the statement if the
execution did not raise any error, ``None`` otherwise.
"""
assert statement is not None
if isinstance(statement, AST):
statement_code = to_source(statement)
try:
exec statement_code in self._globals, self._locals
except Exception, e:
print 'Statement raised error:', str(
e), 'in', statement_code, 'in context', id(self)
return
def Exec(self, source):
self.execution_count += 1
try:
nodes = ast.parse(source, self.filename)
except IndentationError as e:
raise ParseError(e)
except (OverflowError, SyntaxError, ValueError,
TypeError, MemoryError) as e:
raise ParseError(e)
stdout = StringIO.StringIO()
stderr = StringIO.StringIO()
prev_stdout = sys.stdout
prev_stderr = sys.stderr
sys.stdout = stdout
sys.stderr = stderr
try:
if isinstance(nodes.body[-1], ast.Expr):
exec_nodes = nodes.body[:-1]
interactive_nodes = nodes.body[-1:]
else:
exec_nodes, interactive_nodes = nodes.body, []
for node in exec_nodes:
mod = ast.Module([node])
code = compile(mod, self.filename, "exec")
exec(code, self.global_context, self.local_context)
result = None
for node in interactive_nodes:
source = codegen.to_source(node)
new_node = ast.parse(source, self.filename, mode="eval")
mod = ast.Expression(new_node.body)
code = compile(mod, self.filename, "eval")
result = eval(code, self.global_context, self.local_context)
sys.stdout = prev_stdout
sys.stderr = prev_stderr
return stdout.getvalue(), stderr.getvalue(), result
except Exception as e:
raise ExecError(stdout.getvalue(), stderr.getvalue(), e)
finally:
sys.stdout = prev_stdout
sys.stderr = prev_stderr
def tf_keywords(self, ast_tree):
def nuke_all_strings(mod):
if isinstance(mod, ast.Str):
mod.s = ''
for child in children(mod):
nuke_all_strings(child)
ast_tree1 = copy.deepcopy(ast_tree)
nuke_all_strings(ast_tree1)
codestr = cg.to_source(ast_tree1)
ws = re.compile(("\_*[a-zA-Z]+[a-zA-Z0-9\_]*"))
out = defaultdict(int)
for x in ws.findall(codestr):
idx = self.keywords.try_index(x)
if idx != -1:
out[idx] += 1
return out
def dt(self, lang, inps, resp, args=None):
if isinstance(inps, basestring):
inps = [inps]
# transform response(s) to a python code snipped, has it + put in db
if isinstance(resp, list):
code_src = "def _resp(c):\n"
for r in resp:
code_src += " c.resp(u\"%s\", 0.0, [])\n" % r
code_fn = '_resp'
elif isinstance(resp, basestring):
code_src = "def _resp(c):\n"
code_src += " c.resp(u\"%s\", 0.0, [])\n" % resp
code_fn = '_resp'
else:
code_src = inspect.getsource(resp)
code_src = self._unindent(code_src)
logging.debug('dte: code_src=%s' % code_src)
code_ast = ast.parse(code_src)
for node in ast.walk(code_ast):
if isinstance(node, ast.FunctionDef):
code_fn = node.name
code_src = codegen.to_source(node)
break
md5s = self.store_code(code_src, code_fn)
# caller's source location:
curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe, 2)
self.src_location = (calframe[1][1], calframe[1][2])
# use macro engine to generate input strings
self.generate_training_data(lang, inps, md5s, code_fn, args)
