def visit_FunctionDef(self, node, parent):
if node in self.seen:
return node
self.seen.add(node)
oldNode = dict(ast.iter_fields(node))
finalBlock = []
newNode = ast.FunctionDef(name=oldNode['name'], args=oldNode['args'], body=[ast.TryFinally(body=dict(ast.iter_fields(node))['body'], finalbody=finalBlock)], decorator_list=oldNode['decorator_list'])
self.funcName = dict(ast.iter_fields(node))['name']
if self.funcName in self.localFuncs:
args = dict(ast.iter_fields(dict(ast.iter_fields(node))['args']))
args['args'].append(ast.Name(id=self.calleeInfo, ctx=ast.Param()))
args['defaults'].append(ast.Str(s=''))
self.addChild(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.locStack, ctx=ast.Load()), attr='append', ctx=ast.Load()), args=[ast.Name(id=self.calleeInfo, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), newNode)
#self.addChild(ast.Print(dest=None, values=[ast.Name(id=self.funcStack, ctx=ast.Load())], nl=True), node)
self.addChild(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.funcStack, ctx=ast.Load()), attr='append', ctx=ast.Load()), args=[ast.Str(s=self.funcName)], keywords=[], starargs=None, kwargs=None)), newNode)
self.addChild(ast.Assign(targets=[ast.Name(id=self.loopSize, ctx=ast.Store())], value=ast.Call(func=ast.Name(id='len', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), newNode)
finalBlock.append(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.funcStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None)))
if self.funcName in self.localFuncs:
finalBlock.append(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.locStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None)))
loopCorr = ast.While(test=ast.Compare(left=ast.Call(func=ast.Name(id='len', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), ops=[ast.Gt()], comparators=[ast.Name(id=self.loopSize, ctx=ast.Load())]), body=[ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.loopStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None))], orelse=[])
self.seen.add(loopCorr)
finalBlock.append(loopCorr)
#self.addChild(ast.Print(dest=None, values=[ast.Str(s=self.funcName + '_afterPop')], nl=True), node, loc=len(dict(ast.iter_fields(node))['body']))
ast.fix_missing_locations(newNode)
#print ast.dump(newNode)
self.funcs.append(newNode)
self.generic_visit(newNode, parent)
return newNode
def parse_imports(s):
module = ast.parse(s)
imports = []
for c in ast.iter_child_nodes(module):
if isinstance(c, ast.ImportFrom):
fields = dict(ast.iter_fields(c))
from_module = fields['module']
if from_module != '__future__':
raise ProcessingException(
'non-future ImportFrom on line ' + str(c.lineno))
names = [dict(ast.iter_fields(i))['name'] for i in fields['names']]
if len(names) != 1:
raise ProcessingException(
'multiple imports on line ' + str(c.lineno))
imports.append(ParsedImport(c.lineno, names[0], True))
if isinstance(c, ast.Import):
fields = dict(ast.iter_fields(c))
names = [dict(ast.iter_fields(i))['name'] for i in fields['names']]
if len(names) != 1:
raise ProcessingException(
'multiple imports on line ' + str(c.lineno))
imports.append(ParsedImport(c.lineno, names[0], False))
return imports
def attach_data(self, node):
"""Generic method called for visit_XXXX() with XXXX in
GatherOMPData.statements list
"""
if self.current:
for curr in self.current:
md = OMPDirective(curr)
metadata.add(node, md)
self.current = list()
# add a Pass to hold some directives
for field_name, field in ast.iter_fields(node):
if field_name in GatherOMPData.statement_lists:
if field and isinstance(field[-1], ast.Expr) and self.isompdirective(field[-1].value):
field.append(ast.Pass())
self.generic_visit(node)
# add an If to hold scoping OpenMP directives
directives = metadata.get(node, OMPDirective)
field_names = {n for n, _ in ast.iter_fields(node)}
has_no_scope = field_names.isdisjoint(GatherOMPData.statement_lists)
if directives and has_no_scope:
# some directives create a scope, but the holding stmt may not
# artificially create one here if needed
sdirective = "".join(d.s for d in directives)
scoping = ("parallel", "task", "section")
if any(s in sdirective for s in scoping):
node = ast.If(ast.Num(1), [node], [])
return node
def visit_Call(self, node):
super(srcpuller, self).generic_visit(node)
state = 0
for n in ast.iter_child_nodes(node):
if state == 0:
if type(n) == ast.Name:
for field, value in ast.iter_fields(n):
if (field == 'id' and value == 'source'):
state = 1
break
continue
elif state == 1:
if type(n) == ast.Str:
for field, value in ast.iter_fields(n):
if (field == 's'):
print 'sourc name:', value
state = 2
break
continue
elif state == 2:
if type(n) == ast.Str:
for field, value in ast.iter_fields(n):
if (field == 's'):
print 'pat name:', value
state = 3
break
continue
elif state == 3:
if type(n) == ast.Str:
for field, value in ast.iter_fields(n):
if (field == 's'):
print 'inst name:', value
break
break
def nodes_equal(x, y):
__tracebackhide__ = True
assert type(x) == type(y), "Ast nodes do not have the same type: '%s' != '%s' " % (
type(x),
type(y),
)
if isinstance(x, (ast.Expr, ast.FunctionDef, ast.ClassDef)):
assert (
x.lineno == y.lineno
), "Ast nodes do not have the same line number : %s != %s" % (
x.lineno,
y.lineno,
)
assert x.col_offset == y.col_offset, (
"Ast nodes do not have the same column offset number : %s != %s"
% (x.col_offset, y.col_offset)
)
for (xname, xval), (yname, yval) in zip(ast.iter_fields(x), ast.iter_fields(y)):
assert xname == yname, (
"Ast nodes fields differ : %s (of type %s) != %s (of type %s)"
% (xname, type(xval), yname, type(yval))
)
assert type(xval) == type(yval), (
"Ast nodes fields differ : %s (of type %s) != %s (of type %s)"
% (xname, type(xval), yname, type(yval))
)
for xchild, ychild in zip(ast.iter_child_nodes(x), ast.iter_child_nodes(y)):
assert nodes_equal(xchild, ychild), "Ast node children differs"
return True
def visit_arguments(self, node):
args = None
vararg = None
kwarg = None
defaults = None
for field, value in ast.iter_fields(node):
if field == "args":
args = value
for arg in args:
#print "Arg: ", arg
for field, value in ast.iter_fields(arg):
if field == "id":
#print "Define arg: ", value
self.env[value] = "arg"
else:
JSONVisitorException("Unexpected error: argument's field is not id.")
elif field == "vararg":
vararg = value
elif field == "kwarg":
kwarg = value
elif field == "defaults":
defaults = value
if vararg or kwarg or defaults:
raise JSONVisitorException("Unexpected error: Missed case: vararg, kwarg or defaults is not empty.")
def match_recurse(pattern_node, actual_node, collected):
supertypes = [actual_node.__class__]
while supertypes[-1] != object:
supertypes.append(supertypes[-1].__base__)
if is_placeholder(pattern_node, ["__" + t.__name__ for t in supertypes]):
collected.append(actual_node)
return True
elif type(pattern_node) != type(actual_node):
return False
elif isinstance(pattern_node, ast.AST):
return all(
match_recurse(left, right, collected)
for ((left_name, left), (right_name, right))
in zip(ast.iter_fields(pattern_node), ast.iter_fields(actual_node))
)
elif isinstance(pattern_node, list):
if len(pattern_node) != len(actual_node):
return False
else:
return all(
match_recurse(left, right, collected)
for (left, right) in zip(pattern_node, actual_node)
)
else:
if pattern_node == "__" and type(actual_node) is str:
collected.append(actual_node)
return True
else:
return pattern_node == actual_node
def match(node, pat):
"""Return `True` if AST tree `node` matches AST pattern `pat`.
"""
if isinstance(pat, ast.Name) and pat.id == '_':
return True
elif isinstance(pat, ast.AST):
if not isinstance(node, ast.AST):
return False
if not (issubclass(node.__class__, pat.__class__) or
issubclass(pat.__class__, node.__class__)):
return False
assert _check_fields(node, pat)
for (field1, val1), (field2, val2) in \
zip(ast.iter_fields(node),
ast.iter_fields(pat)):
assert(field1 == field2)
if not match(val1, val2):
return False
elif isinstance(pat, list):
if not isinstance(node, list):
return False
if len(node) != len(pat):
return False
for val1, val2 in zip(node, pat):
if not match(val1, val2):
return False
else:
# Primitive comparison.
if node != pat:
return False
return True
def editfunctiondef(self, node):
for fname,fnode in ast.iter_fields(node):
if fname == 'args':
for argname, argnode in ast.iter_fields(fnode):
if argname == 'kwarg' and argnode != None:
del self.functionstofix[node.lineno]
if node.lineno in self.functionstofix.keys():
print "%d | Fixing function definition." % (node.lineno)
def visit_Call(self, node, parent):
if node in self.seen:
return node
self.seen.add(node)
callName = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('id', None)
callType = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('attr',None)
#print ast.dump(dict(ast.iter_fields(node))['func']), callType, node.lineno, node.col_offset
#print callName, self.localFuncs
if callName in self.localFuncs:
#print ast.dump(node)
#print callName, node.lineno, node.col_offset
dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg=self.calleeInfo, value=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset))))
if callType in erps.keys() or callType == "stocPrim":
if callType not in self.primsNumArgs:
self.primsNumArgs[callType] = len(inspect.getargspec(globals()[callType]).args)
namedArgs = map(lambda x: dict(ast.iter_fields(x))['arg'], dict(ast.iter_fields(node))['keywords'])
numArgs = len(namedArgs) + len(dict(ast.iter_fields(node))['args'])
#print callType, node.lineno, node.col_offset
#print ast.dump(parent)
if not ('name' in namedArgs or numArgs == self.primsNumArgs[callType]): #check if name already supplied
dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg='name', value=ast.BinOp(left=ast.BinOp(left=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.funcStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.locStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), op=ast.Add(), right=ast.BinOp(left=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset)), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)))))
ast.fix_missing_locations(node)
#print map(ast.dump, dict(ast.iter_fields(node))['keywords'])
self.generic_visit(node)
return node
def shallow_match_main(self, ins_node, std_node, check_meta=True, ignores=[]):
"""
Checks that all non astNode attributes are equal between ins_node and std_node
:param ins_node: Instructor ast root node
:param std_node: Student AST root node
:param check_meta: flag to check whether the fields of the instructor node and the student node should match
:return: a mapping between the isntructor and student root nodes, or False if such a mapping doesn't exist
"""
ins = ins_node.astNode
std = std_node.astNode
ins_field_list = list(ast.iter_fields(ins))
std_field_list = list(ast.iter_fields(std))
meta_matched = self.metas_match(ins_node, std_node, check_meta)
is_match = len(ins_field_list) == len(std_field_list) and type(ins).__name__ == type(
std).__name__ and meta_matched
for insTup, stdTup in zip(ins_field_list, std_field_list):
if not is_match:
break
ins_field = insTup[0]
ins_value = insTup[1]
std_field = stdTup[0]
std_value = stdTup[1]
if ins_value is None:
continue
ignore_field = ins_field in ignores
is_match = (ins_field == std_field) or ignore_field
if not isinstance(ins_value, list):
ins_value = [ins_value]
if not isinstance(std_value, list):
std_value = [std_value]
# is_match = len(ins_value) == len(std_value)# for stretchy matching this isn't True
# Reference ast_node_visitor.js for the original behavior and keep note of it for the purposes of handling
# the children noting the special case when the nodes of the array are actually parameters of the node
# (e.g. a load function) instead of a child node
if not ignore_field:
for inssub_value, stdsub_value in zip(ins_value, std_value):
if not is_match:
break
# TODO: make this a smarter comparison, maybe handle dictionaries, f-strings, tuples, etc.
if is_primitive(inssub_value):
is_match = inssub_value == stdsub_value
mapping = False
if is_match:
mapping = AstMap() # return MAPPING
mapping.add_node_pairing(ins_node, std_node)
mapping = [mapping]
return mapping
def generic_visit(self, node):
for key, value in ast.iter_fields(node):
if isinstance(value, AST):
for k, v in ast.iter_fields(value):
if isinstance(v, AST):
return True
elif isinstance(value, list):
for node in value:
if isinstance(node, AST):
return True
return False
def visit_ImportFrom(self, node):
fields = dict(ast.iter_fields(node))
alias_nodes = list(ast.iter_child_nodes(node))
for alias_node in alias_nodes:
alias_fields = dict(ast.iter_fields(alias_node))
if fields['level'] == 0:
# from a import b
# it might mean a.b is a module
# or b might be a function of a module
self.absolute_imports.append(fields['module'])
self.absolute_imports.append('{}.{}'.format(fields['module'], alias_fields['name']))
else:
self.relative_imports.append((fields['level'], fields['module'], alias_fields['name']))
def nodes_equal(x, y):
__tracebackhide__ = True
assert type(x) == type(y)
if isinstance(x, (ast.Expr, ast.FunctionDef, ast.ClassDef)):
assert x.lineno == y.lineno
assert x.col_offset == y.col_offset
for (xname, xval), (yname, yval) in zip(ast.iter_fields(x),
ast.iter_fields(y)):
assert xname == yname
assert type(xval) == type(yval)
for xchild, ychild in zip(ast.iter_child_nodes(x),
ast.iter_child_nodes(y)):
assert nodes_equal(xchild, ychild)
return True
def generic_visit(self, node):
if not (isinstance(node, ast.AST)):
print node
raise JSONVisitorException("Unexpected error: Non-ast passed to visit. Please report to the TAs.")
for field, value in ast.iter_fields(node):
# print "field: ", field, " value: ", value
self.indent_print(field + ":")
self.indent = self.indent + 1
if isinstance(value, list):
for item in value:
if isinstance(item, ast.AST):
self.indent_print(item.__class__.__name__ + ":")
self.indent = self.indent + 1
self.visit(item)
self.indent = self.indent - 1
else:
raise JSONVisitorException(
"Unexpected error: Missed case: %s. Please report to the TAs." % item
)
elif isinstance(value, ast.AST):
self.visit(value)
else:
raise JSONVisitorException("Unexpected error: Missed case: %s. Please report to the TAs." % value)
self.indent = self.indent - 1
def visit_AllNumVar(self, node):
self.indent_print(node.__class__.__name__ + ":")
for field, value in ast.iter_fields(node):
if field == "id":
self.indent = self.indent + 1
self.indent_print(field + ":" + str(value))
self.indent = self.indent - 1
def _fixup_global_g_inner(self, a): def maybe_replace(value): # search for attrib access like "g.something" if isinstance(value, ast.Attribute) \ and isinstance(value.value, ast.Name) \ and value.value.id == "g": # found one assert value.attr in self._py_globals if isinstance(self._py_globals[value.attr], cparser.CFunc): # overwrite this with "something.__func__" to resolve the staticmethod return ast.Attribute( value=ast.Name(id=value.attr, ctx=ast.Load()), attr="__func__", ctx=ast.Load() ) # overwrite this with just "something" return ast.Name(id=value.attr, ctx=ast.Load()) return value for node in ast.walk(a): for fieldname, value in ast.iter_fields(node): if isinstance(value, ast.AST): setattr(node, fieldname, maybe_replace(value)) elif isinstance(value, list): for i in range(len(value)): value[i] = maybe_replace(value[i]) elif isinstance(value, tuple): setattr(node, fieldname, tuple(map(maybe_replace, value)))
def generic_visit(self, node):
for field, old_value in ast.iter_fields(node):
old_value = getattr(node, field, None)
if isinstance(old_value, list):
new_values = []
for value in old_value:
if isinstance(value, ast.AST):
value = self.visit(value)
if value is None:
continue
elif not isinstance(value, ast.AST):
new_values.extend(value)
continue
new_values.append(value)
old_value[:] = new_values
elif isinstance(old_value, ast.AST):
new_node = self.visit(old_value)
if new_node is None:
delattr(node, field)
elif not isinstance(new_node, ast.AST):
new_node_tuple = ast.copy_location(ast.Tuple(elts=list(new_node), ctx=ast.load()), new_node[0])
setattr(node, field, new_node_tuple)
else:
setattr(node, field, new_node)
return node
def _format(node, field='', level=0):
if isinstance(node, ast.AST):
head_string = get_indent( level ) + node.__class__.__name__
head_string += " ({})".format(field) if field else ""
if show_lineno and hasattr( node, 'lineno' ):
head_string += ' [{}]'.format( node.lineno )
body_string = ""
for label, value in ast.iter_fields(node):
if label == "ctx":
continue
if isinstance( value, (str, int, float, long) ):
head_string += ' {} = {}'.format( label, value )
elif not value:
body_string += get_indent( level+1, pipe=True) + '{} ({})\n'.format( value, label )
else:
body_string += _format( value, label, level+1 )
return head_string + "\n" + body_string
elif isinstance(node, list):
string = ''
for x in node:
string += _format( x, field, level )
return string
return repr(node)
def visit_FunctionDef(self, node):
name = None
args = None
body = None
decorator_list = None
for field, value in ast.iter_fields(node):
if field == "name":
name = value
self.print_field_value(field, value)
elif field == "args":
args = value
self.indent_print(field + ":")
self.indent = self.indent + 1
self.visit(value)
self.indent = self.indent - 1
elif field == "body":
body = value
self.indent_print(field + ":")
self.indent = self.indent + 1
for stmt in body:
self.indent_print(stmt.__class__.__name__ + ":")
self.indent = self.indent + 1
self.visit(stmt)
self.indent = self.indent - 1
self.indent = self.indent - 1
elif field == "decorator_list":
decorator_list = value
self.print_field_value(field, value)
if decorator_list:
raise JSONVisitorException("Unexpected error: Missed case: decorator_list is not empty.")
def visit_Assign(self, node):
lhs = None
rhs = None
for field, value in ast.iter_fields(node):
if field == "targets":
lhs = value[0]
elif field == "value":
rhs = value
else:
raise JSONVisitorException("Unexpected error: Missed case: %s." % value)
if isinstance(lhs, ast.Name):
node.targets = [self.visit(lhs)]
node.value = self.visit(rhs)
elif isinstance(lhs, ast.Tuple):
new_lhs = []
new_rhs = []
for l,r in zip(lhs.elts, rhs.elts):
new_lhs.append(self.visit(l))
new_rhs.append(self.visit(r))
lhs.elts = new_lhs
rhs.elts = new_rhs
if (node.lineno, node.col_offset) in self.locs:
self.fixes.append(node)
return node
def code_generic_visit(self, node):
for field, old_value in ast.iter_fields(node):
old_value = getattr(node, field, None)
if isinstance(old_value, list):
prev_insertion_point = self._insertion_point
new_values = []
if field in ("body", "orelse", "finalbody"):
self._insertion_point = new_values
for value in old_value:
if isinstance(value, ast.AST):
value = self.code_visit(value)
if value is None:
continue
elif not isinstance(value, ast.AST):
new_values.extend(value)
continue
new_values.append(value)
old_value[:] = new_values
self._insertion_point = prev_insertion_point
elif isinstance(old_value, ast.AST):
new_node = self.code_visit(old_value)
if new_node is None:
delattr(node, field)
else:
setattr(node, field, new_node)
return node
def str_node(node):
if isinstance(node, ast.AST):
fields = [(name, str_node(val)) for name, val in ast.iter_fields(node) if name not in ('left', 'right')]
rv = '%s(%s' % (node.__class__.__name__, ', '.join('%s=%s' % field for field in fields))
return rv + ')'
else:
return repr(node)
def generic_visit(self, node):
if isinstance(node, list):
nodestart = "["
nodeend = "]"
children = [("", child) for child in node]
else:
nodestart = type(node).__name__ + "("
nodeend = ")"
children = [(name + "=", value) for name, value in ast.iter_fields(node)]
if len(children) > 1:
self.indentation += 1
self.write(nodestart)
for i, pair in enumerate(children):
attr, child = pair
if len(children) > 1:
self.write("\n" + self.indent_with * self.indentation)
if isinstance(child, (ast.AST, list)):
self.write(attr)
self.visit(child)
else:
self.write(attr + repr(child))
if i != len(children) - 1:
self.write(",")
self.write(nodeend)
if len(children) > 1:
self.indentation -= 1
def visit_arguments(self, node):
args = None
vararg = None
kwarg = None
defaults = None
for field, value in ast.iter_fields(node):
if field == "args":
args = value
self.indent_print(field + ":")
self.indent = self.indent + 1
for arg in args:
self.indent_print(arg.__class__.__name__ + ":")
self.visit(arg)
self.indent = self.indent - 1
elif field == "vararg":
vararg = value
self.print_field_value(field, value)
elif field == "kwarg":
kwarg = value
self.print_field_value(field, value)
elif field == "defaults":
defaults = value
self.print_field_value(field, value)
if vararg or kwarg or defaults:
raise JSONVisitorException(
"Unexpected error: Missed case: vararg, kwarg or defaults is not empty. Please report to the TAs."
)
def find_peers(node):
node_for_line = node._parent
for _field, value in ast.iter_fields(node._parent._parent):
if isinstance(value, list) and node_for_line in value:
return value[value.index(node_for_line) + 1:]
continue
return []
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node.
Overridden from the ast.NodeVisitor base class so that I can add some
local state to keep track of whether or not my node visitor is inside
a try/except block. When a try block is encountered, the node is added
to the `trys` instance attribute and then the try block is recursed in
to. Once the recursion has exited, the node is removed from the `trys`
instance attribute
"""
for field, value in ast.iter_fields(node):
if isinstance(value, list):
for item in value:
# add the node to the try/except block to signify that
# something potentially odd is going on in this import
if isinstance(item, ast.Try):
self.trys[item] = item
if isinstance(item, ast.AST):
self.visit(item)
# after the node has been recursed in to, remove the try node
if isinstance(item, ast.Try):
del self.trys[item]
elif isinstance(value, ast.AST):
# add the node to the try/except block to signify that
# something potentially odd is going on in this import
if isinstance(value, ast.Try):
self.trys[value] = item
self.visit(value)
# after the node has been recursed in to, remove the try node
if isinstance(value, ast.Try):
del self.trys[value]
def visit_If(self, node):
for field, value in ast.iter_fields(node):
if field == "test":
self.indent_print(field + ":")
self.indent = self.indent + 1
self.indent_print(value.__class__.__name__ + ":")
self.indent = self.indent + 1
self.visit(value)
self.indent = self.indent - 1
self.indent = self.indent - 1
elif field == "body":
self.indent_print(field + ":")
self.indent = self.indent + 1
for stmt in value:
self.indent_print(stmt.__class__.__name__ + ":")
self.indent = self.indent + 1
self.visit(stmt)
self.indent = self.indent - 1
self.indent = self.indent - 1
elif field == "orelse":
self.indent_print(field + ":")
self.indent = self.indent + 1
for stmt in value:
self.indent_print(stmt.__class__.__name__ + ":")
self.indent = self.indent + 1
self.visit(stmt)
self.indent = self.indent - 1
self.indent = self.indent - 1
else:
self.print_field_value(field, value)
def visit_FunctionDef(self, node):
is_main = False
for field, value in ast.iter_fields(node):
if field == "name" and value == self.main:
is_main = True
if is_main and field == "args":
return self.visit(value)
def fill_line_numbers(tree, lineno, col_offset, **kw):
"""Fill in line numbers somewhat more cleverly than the
ast.fix_missing_locations method, which doesn't take into account the
fact that line numbers are monotonically increasing down lists of AST
nodes."""
if type(tree) is list:
for sub in tree:
if isinstance(sub, ast.AST) \
and hasattr(sub, "lineno") \
and hasattr(sub, "col_offset") \
and (sub.lineno, sub.col_offset) > (lineno, col_offset):
lineno = sub.lineno
col_offset = sub.col_offset
fill_line_numbers(sub, lineno, col_offset)
elif isinstance(tree, ast.AST):
if not (hasattr(tree, "lineno") and hasattr(tree, "col_offset")):
tree.lineno = lineno
tree.col_offset = col_offset
for name, sub in ast.iter_fields(tree):
fill_line_numbers(sub, tree.lineno, tree.col_offset)
elif isinstance(tree, (str, int, float)) or tree is None:
pass
else:
raise TypeError("Invalid AST node '{!r}', type: '{!r}' "
"after expansion".format(tree, type(tree)))
return tree
def get_last_lineno(node):
"""Recursively find the last line number of the ast node."""
max_lineno = 0
if hasattr(node, "lineno"):
max_lineno = node.lineno
for _, field in ast.iter_fields(node):
if isinstance(field, list):
for value in field:
if isinstance(value, ast.AST):
max_lineno = max(max_lineno, get_last_lineno(value))
elif isinstance(field, ast.AST):
max_lineno = max(max_lineno, get_last_lineno(field))
return max_lineno
def to_rule(node: ast.AST) -> Rule:
parent = typename(node)
children = []
for chname, chval in ast.iter_fields(node):
if chname == 'ctx':
continue
if chval is None or (isinstance(chval, list) and len(chval) == 0):
continue
is_list = isinstance(chval, list)
if is_list:
base_type = base_ast_type(chval[0])
else:
base_type = base_ast_type(chval)
children.append(Node(chname, NodeType(base_type.__name__, is_list)))
return Rule(NodeType(parent, False), tuple(children))
def wasModified(node,var):
if isinstance(node,ast.Assign):
for target in node.targets:
if target.id == var:
return True
else:
for name,value in ast.iter_fields(node):
if isinstance(value,list):
for n in value:
ret = wasModified(n,var)
if ret:
return ret
elif isinstance(value,ast.AST):
ret = wasModified(value,var)
if ret:
return ret
return False
def test_tree_job(self):
def addition(x, y):
z = x + numpy.abs(y)
return x * z
code = inspect.getsource(addition)
node = ast.parse(dedent(code))
stack = [(0, node)]
while len(stack) > 0:
ind, n = stack[-1]
del stack[-1]
att = {name: ch for name, ch in ast.iter_fields(n)}
self.assertTrue(att is not None)
for ch in ast.iter_child_nodes(n):
stack.append((ind + 1, ch))
self.assertEmpty(stack)
def generic_visit(self, node: TAst, *args) -> TAst:
ret_node = node
for field, old_value in ast.iter_fields(node):
if not isinstance(old_value, (AST, list)):
continue
new_node = self.visit(old_value)
assert ( # noqa: IG01
new_node is not None
), f"can't remove AST nodes that aren't part of a list {old_value!r}"
if new_node is not old_value:
if ret_node is node:
ret_node = self.clone_node(node)
setattr(ret_node, field, new_node)
return ret_node
def generic_visit(self, node, **kwargs):
"""
Extract useful information from relevant nodes including docstrings.
This is virtually identical to the standard version contained in
NodeVisitor. It is only overridden because we're tracking extra
information (the hierarchy of containing nodes) not preserved in
the original.
"""
for field, value in iter_fields(node):
if isinstance(value, list):
for item in value:
if isinstance(item, AST):
self.visit(item,
containingNodes=kwargs['containingNodes'])
elif isinstance(value, AST):
self.visit(value, containingNodes=kwargs['containingNodes'])
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node.
Overridden from the ast.NodeVisitor base class so that I can add some
local state to keep track of whether or not my node visitor is inside
a try/except block. When a try block is encountered, the node is added
to the `trys` instance attribute and then the try block is recursed in
to. Once the recursion has exited, the node is removed from the `trys`
instance attribute
"""
for field, value in ast.iter_fields(node):
if isinstance(value, list):
for item in value:
if isinstance(item, ast.AST):
self.visit(item)
elif isinstance(value, ast.AST):
self.visit(value)
def visit_Assign(self, node):
for key, val in ast.iter_fields(node):
if key == 'targets':
for subnode in val:
if type(subnode) is ast.Tuple:
for subsubnode in subnode.elts:
crawler = myextract()
crawler.visit(subsubnode)
self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1
else:
crawler = myextract()
crawler.visit(subnode)
self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1
self.depth += 1
#ast.NodeVisitor.generic_visit(self, node)
self.generic_visit(node)
self.depth -= 1
def _format(node, indent=4):
if isinstance(node, ast.AST):
fields = [(a, _format(b, indent + 4))
for a, b in ast.iter_fields(node)]
rv = node.__class__.__name__ + '(\n'
for field in fields:
rv += ' ' * indent + '%s=%s,\n' % field
if include_attributes and node._attributes:
rv += fields and ', ' or ' '
rv += ', '.join('%s=%s' %
(a, _format(getattr(node, a), indent + 4))
for a in node._attributes)
return rv + ' ' * indent + ')'
elif isinstance(node, list):
return '[\n%s%s\n%s]' % (' ' * indent, (',\n' + ' ' * indent).join(
_format(x, indent + 4) for x in node), ' ' * indent)
return repr(node)
def assert_ast_like(sample, template, _path=None):
"""Check that the sample AST matches the template.
Raises a suitable subclass of :exc:`ASTMismatch` if a difference is detected.
The ``_path`` parameter is used for recursion; you shouldn't normally pass it.
"""
if _path is None:
_path = ["tree"]
if callable(template):
# Checker function at the top level
return template(sample, _path)
if not isinstance(sample, type(template)):
raise ASTNodeTypeMismatch(_path, sample, template)
for name, template_field in ast.iter_fields(template):
# Bug fix - The type of the nodes could be different
if not hasattr(sample, name):
continue
sample_field = getattr(sample, name)
field_path = _path + [name]
if isinstance(template_field, list):
if template_field and (
isinstance(template_field[0], ast.AST) or callable(template_field[0])
):
_check_node_list(field_path, sample_field, template_field)
else:
# List of plain values, e.g. 'global' statement names
if sample_field != template_field:
raise ASTPlainListMismatch(field_path, sample_field, template_field)
elif isinstance(template_field, ast.AST):
assert_ast_like(sample_field, template_field, field_path)
elif callable(template_field):
# Checker function
template_field(sample_field, field_path)
else:
# Single value, e.g. Name.id
if sample_field != template_field:
raise ASTPlainObjMismatch(field_path, sample_field, template_field)
def _format(node, _depth, _max_depth=None):
if (_max_depth is not None) and (_depth > _max_depth):
return '<...>'
_next_depth = _depth + 1
_indent = indent * (_depth - 1)
_subindent = indent * _depth
if isinstance(node, ast.AST):
formatted_fields = [(name,
_format(value,
_depth=_next_depth,
_max_depth=_max_depth))
for name, value in ast.iter_fields(node)]
if include_attributes and node._attributes:
formatted_fields.extend([(attributename,
_format(getattr(node, attributename),
_depth=_next_depth,
_max_depth=_max_depth))
for attributename in node._attributes
])
if annotate_fields:
return f'{node.__class__.__name__}(' + ', '.join(
f'{name}={value}'
for name, value in formatted_fields) + ')'
else:
return f'{node.__class__.__name__}({", ".join( fieldname for fieldname, _ in formatted_fields )})'
elif isinstance(node, (list, set, tuple)):
nodes = node
formatted_nodes = ['[']
formatted_nodes.extend(
f'{_subindent}{_format(subnode, _depth=_next_depth, _max_depth=_max_depth)},'
for subnode in nodes)
if len(formatted_nodes) > 1:
formatted_nodes.append(f'{_indent}]')
else:
formatted_nodes[-1] += ']'
return '\n'.join(formatted_nodes)
else:
return repr(node)
def get_deps(code):
body = ast.parse(code)
_, statements = next(ast.iter_fields(body))
# Line no. at which each identifier was first seen
declaration_line_num_map = {}
ddg = {}
def update_decls(lhs_vars_input, num):
lhs_var_nodes = []
for var_node in lhs_vars_input:
lhs_var_nodes.append(var_node)
if var_node.id not in declaration_line_num_map:
declaration_line_num_map[var_node.id] = num
ddg[var_node.id] = set()
return lhs_var_nodes
# x1, x2, x3, ..., xN = 1, 2, 3, 4, 5, ..., N
# is represented in the AST as:
# - R = ast.Assign is root
# - R.targets gives the LHS
# - R.values
for seq_no, node in enumerate(statements):
if isinstance(node, ast.Assign):
identifier_names = node.targets
lhs_vars = update_decls(identifier_names, seq_no)
self_edge_occurrences_to_ignore = {x: 1 for x in identifier_names}
# DFS in RHS
depends_on = []
for descendant in ast.walk(node):
if descendant in self_edge_occurrences_to_ignore and self_edge_occurrences_to_ignore[
descendant] > 0:
self_edge_occurrences_to_ignore[descendant] -= 1
continue
if isinstance(descendant, ast.Name):
depends_on.append(descendant)
for var in lhs_vars:
for dependency in depends_on:
ddg[var.id].add(dependency.id)
return declaration_line_num_map, ddg
def exec_py(
code: str,
src_name: str = "<py-code>",
globals_: Optional[dict] = None,
locals_: Optional[dict] = None,
) -> object:
"""
Run arbitrary Python code in supplied globals, return evaluated value of last statement.
"""
if globals_ is None:
globals_ = {}
if locals_ is None:
locals_ = globals_
try:
ast_ = ast.parse(code, src_name, "exec")
last_expr = None
last_def_name = None
for field_ in ast.iter_fields(ast_):
if "body" != field_[0]:
continue
if len(field_[1]) > 0:
le = field_[1][-1]
if isinstance(le, ast.Expr):
last_expr = ast.Expression()
last_expr.body = field_[1].pop().value
elif isinstance(le, (ast.FunctionDef, ast.ClassDef)):
last_def_name = le.name
exec(compile(ast_, src_name, "exec"), globals_, locals_)
if last_expr is not None:
return eval(compile(last_expr, src_name, "eval"), globals_, locals_)
elif last_def_name is not None:
# godforbid the code to declare global/nonlocal for the last defined artifact
return locals_[last_def_name]
return None
except Exception:
logger.error(
f"""Error running Python code:
-=-{src_name}-=-
{code!s}
-=*{src_name}*=-
""",
exc_info=True,
)
raise
def ast_visit(node, parentNode, level=0):
#with open("out.txt", 'a') as f:
# f.write(' ' * level + str_node(node) + '\n')
# print(' ' * level + str_node(node))
str_node(node, parentNode)
for field, value in ast.iter_fields(node):
if (type(value) == ast.Call):
print("CALL FOUND")
#print("F: {}\nV:{}\n".format(field, value))
if isinstance(value, list):
for item in value:
if (type(item) == ast.Assign):
print("ASSIGN FOUND")
if isinstance(item, ast.AST):
ast_visit(item, node, level=level + 1)
elif isinstance(value, ast.AST):
ast_visit(value, node, level=level + 1)
def translate_facts(node: ast.AST, names: dict, module: ModuleType) -> ast.AST:
"""Translate facts referred within a node with __Fact_<Class Name>."""
if isinstance(node, ast.Name):
return translate_fact(node, node.id, names, module)
elif isinstance(node, ast.Attribute):
node.value = translate_facts(node.value, names, module)
return translate_fact(node, node_names(node), names, module)
else:
for field, value in ast.iter_fields(node):
if isinstance(value, ast.AST):
setattr(node, field, translate_facts(value, names, module))
elif isinstance(value, list):
value[:] = [
translate_facts(v, names, module) for v in value
if isinstance(v, ast.AST)
]
return node
def generic_visit(self, node):
"""
We use the visit method - it's called in every step of the visit
function that we call (please don't overload that!), so we will see
the entire graph
"""
for attr, vs in ast.iter_fields(node):
if isinstance(vs, ast.AST):
self.graph[self.make_label(node)].append(self.make_label(vs))
self.visit(vs)
elif isinstance(vs, list):
for v in vs:
if isinstance(v, ast.AST):
self.graph[self.make_label(node)].append(
self.make_label(v))
self.visit(v)
else:
pass
def disp(self, node):
if isinstance(node, ast.AST):
string = getClname(node) + '('
cnt = 0
for name, value in ast.iter_fields(node):
val = self.disp(value)
if cnt > 0:
string += ', '
cnt += 1
string += name + '=' + val
return string + ')'
elif isinstance(node, list):
string = ''
for x in node:
string += self.disp(x)
return string
else:
return repr(node)
def generic_visit(self, node):
# label this node
out_string = 'n%s [label="%s"];\n' % (id(node), self.label(node))
# edge to parent
if hasattr(node, 'parent') and node.parent is not None:
out_string += 'n%s -> n%s [label="parent",style=dotted];\n' % (
id(node), id(node.parent))
# edges to children
for fieldname, fieldvalue in ast.iter_fields(node):
for index, child in enumerate_flatten(fieldvalue):
if isinstance(child, ast.AST):
suffix = "".join(["[%d]" % i for i in index])
out_string += 'n%d -> n%d [label="%s%s"];\n' % (
id(node), id(child), fieldname, suffix)
out_string += _to_dot(child)
return out_string
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node."""
if node == self.node_to_check:
self.final_indent = self.current_indent
for field, value in ast.iter_fields(node):
if (isinstance(node, tuple(TYPE_TO_INDENT_FIELD.keys())) and
field in TYPE_TO_INDENT_FIELD[node.__class__]):
self.current_indent += 1
if isinstance(value, list):
for item in value:
if isinstance(item, _ast.AST):
self.visit(item)
elif isinstance(value, _ast.AST):
self.visit(value)
if (isinstance(node, tuple(TYPE_TO_INDENT_FIELD.keys())) and
field in TYPE_TO_INDENT_FIELD[node.__class__]):
self.current_indent -= 1
return node
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
if self._name == node.name:
self._param_type_matches.clear()
for name, field in ast.iter_fields(node):
if name == "args":
for arg in field.args:
if arg.annotation and hasattr(arg.annotation, "id"):
self._param_type_matches[arg.arg] = self._locate(
arg.annotation.id
)
elif arg.annotation and hasattr(arg.annotation, "attr"):
self._param_type_matches[arg.arg] = self._locate(
f"{arg.annotation.value.id}.{arg.annotation.attr}"
)
else:
self._param_type_matches[arg.arg] = None
if node.returns and hasattr(node.returns, "id"):
self._return_type = self._locate(node.returns.id) # type: ignore
def generic_visit_level(self):
stack = [self._root]
self._levelTran = []
while len(stack) != 0:
tmp = []
res_each = []
for i in stack:
res_each.append(i)
self.d[i] = [0]
for field, value in ast.iter_fields(i):
if isinstance(value, list):
for item in value:
if isinstance(item, ast.AST):
tmp.append(item)
elif isinstance(value, ast.AST):
tmp.append(value)
stack = tmp
self._levelTran.insert(0, res_each)
def generic_visit(self, node):
for field, old_value in ast.iter_fields(node):
if isinstance(old_value, list):
for value in old_value:
if isinstance(value, ast.AST):
if (field == 'body' or field == 'orelse'):
clsname = type(value).__name__
if getattr(self, "visit_TopLevel" + clsname,
False):
getattr(self,
"visit_TopLevel" + clsname)(value)
else:
self.visit(value)
else:
self.visit(value)
elif isinstance(old_value, ast.AST):
self.visit(old_value)
return node
def generic_visit(self, node):
"""Drive the visitor."""
for _, value in ast.iter_fields(node):
if isinstance(value, list):
max_idx = len(value) - 1
for idx, item in enumerate(value):
if isinstance(item, ast.AST):
if idx < max_idx:
setattr(item, 'sibling', value[idx + 1])
else:
setattr(item, 'sibling', None)
setattr(item, 'parent', node)
self.visit(node=item)
elif isinstance(value, ast.AST):
setattr(value, 'sibling', None)
setattr(value, 'parent', node)
self.visit(node=value)
def ExpandTree(node):
node_fields = []
to_expand = [node]
while to_expand:
current = to_expand.pop()
if isinstance(current, ast.AST):
node_fields.append(current.__class__)
for field_name, child in ast.iter_fields(current):
node_fields.append(field_name)
if isinstance(child, (list, tuple)):
for item in child:
to_expand.append(item)
else:
to_expand.append(child)
else:
node_fields.append(current)
print node_fields
return node_fields
def __init__(self, ast_node, my_field='', tid=0, lin_tree=None, ancestor=None):
"""
:param ast_node: The AST node to be wrapped
:param my_field: the field of the parent node that produced this child.
"""
self.children = []
self.astNode = ast_node
self.field = my_field
self.tree_id = tid
self.parent = ancestor
if lin_tree is None:
self.linear_tree = [self]
else:
lin_tree.append(self)
self.linear_tree = lin_tree
# reference to the easy node wrapping the ast_node
setattr(ast_node, 'easy_node', self)
tid_count = tid
my_field_generator = ast.iter_fields(self.astNode)
for item in my_field_generator:
field, value = item
# if the field doesn't have a value, no child exists
if value is None:
continue
# If the children are not in an array, wrap it in an array for
# consistency in the code the follows
if not isinstance(value, list):
value = [value]
# Reference ast_node_visitor.js for the original behavior and keep note of it for the purposes of handling
# the children noting the special case when the nodes of the array are actually parameters of the node
# (e.g. a load function) instead of a child node
for sub_value in value:
if isinstance(sub_value, ast.AST):
new_child = EasyNode(sub_value, my_field=field,
tid=tid_count + 1,
lin_tree=self.linear_tree,
ancestor=self)
self.children.append(new_child)
tid_count = len(self.linear_tree) - 1
def pretty(node, key="/", level=0):
"""Used for testing and new test generation via AstView.
Don't change the format without updating tests"""
if isinstance(node, ast.AST):
fields = [(key, val) for key, val in ast.iter_fields(node)]
value_label = node.__class__.__name__
if isinstance(node, ast.Call):
# Try to make 3.4 AST-s more similar to 3.5
if sys.version_info[:2] == (3, 4):
if ("kwargs", None) in fields:
fields.remove(("kwargs", None))
if ("starargs", None) in fields:
fields.remove(("starargs", None))
# TODO: translate also non-None kwargs and starargs
elif isinstance(node, list):
fields = list(enumerate(node))
if len(node) == 0:
value_label = "[]"
else:
value_label = "[...]"
else:
fields = []
value_label = repr(node)
item_text = level * " " + str(key) + "=" + value_label
if hasattr(node, "lineno"):
item_text += " @ " + str(getattr(node, "lineno"))
if hasattr(node, "col_offset"):
item_text += "." + str(getattr(node, "col_offset"))
if hasattr(node, "end_lineno"):
item_text += " - " + str(getattr(node, "end_lineno"))
if hasattr(node, "end_col_offset"):
item_text += "." + str(getattr(node, "end_col_offset"))
lines = [item_text] + [
pretty(field_value, field_key, level + 1)
for field_key, field_value in fields
]
return "\n".join(lines)
def generic_visit(self, node):
try:
return ast.literal_eval(node)
except ValueError:
for field, value in ast.iter_fields(node):
if field in self.excluded_fields:
delattr(node, field)
if value is None:
continue
if isinstance(value, list):
if field in ('keywords', 'kwargs'):
new_value = dict((kw.arg, self.visit(kw.value)) for kw in value)
else:
new_value = [self.visit(i) for i in value]
else:
new_value = self.visit(value)
setattr(node, field, new_value)
return node
def generate_dot_code(python_file):
"""
generate_dot_code
:param py_code:
:return:
"""
with open(python_file) as source:
ast_module_obj = parser.suite(source.read())
ast_obj_list = parser.st2list(ast_module_obj)
# ast_module_obj = ast.parse(source.read(), python_file, mode='exec')
# ast_module_obj_body = ast_module_obj.body
ast_obj = ast_obj_list
print('ast_list\n', repr(ast_module_obj))
print('ast_iter_tuple\n', ast.iter_fields(ast_module_obj))
# print('ast_body\n', ast_module_obj_body)
print('ast_obj\n\n', repr(ast_obj))
# print('ast.iter_child_nodes\n', ast.iter_child_nodes(ast_obj))
# for b in ast.walk(ast_obj):
# print('ast_obj\n', repr(b))
call_graph = {}
construct_call_graph(ast_obj, call_graph)
# pprint.pprint(call_graph)
dot = []
dot.append("digraph G {")
dot.append("rankdir=LR")
for from_fn, to_fns in call_graph.items():
if not to_fns:
dot.append('%s;' % from_fn)
for to_fn in to_fns:
if to_fn not in call_graph: continue
dot.append('%s -> %s;' % (from_fn, to_fn))
dot.append("}")
return '\n'.join(dot)
def _find_line_numbers(self, node, line_numbers):
""" Populates a set containing all line numbers used by the node and its
descendants.
line_numbers is a set that all the line numbers will be added to."""
if FormattedValue is not None and isinstance(node, FormattedValue):
# FormattedValue is a separate code block with its own line nums.
return
line_number = getattr(node, 'lineno', None)
if line_number is not None:
line_numbers.add(line_number)
for _, value in iter_fields(node):
if isinstance(value, list):
for item in value:
if isinstance(item, AST):
self._find_line_numbers(item, line_numbers)
elif isinstance(value, AST):
self._find_line_numbers(value, line_numbers)
def generic_visit(self, node):
# Get new and old values
r = [(field, old_value, self._get_new_value(old_value))
for field, old_value in ast.iter_fields(node)]
if len(r) == 0:
return node
if all(not i[2][1] for i in r):
return node
# Ok - there was a modification. We need to clone the class and pass that
# back up.
new_node = node.__class__()
for f in r:
if f[1] is not None:
setattr(new_node, f[0], f[2][0])
return new_node