def visit_FunctionDef(self, node, parent): if node in self.seen: return node self.seen.add(node) oldNode = dict(ast.iter_fields(node)) finalBlock = [] newNode = ast.FunctionDef(name=oldNode['name'], args=oldNode['args'], body=[ast.TryFinally(body=dict(ast.iter_fields(node))['body'], finalbody=finalBlock)], decorator_list=oldNode['decorator_list']) self.funcName = dict(ast.iter_fields(node))['name'] if self.funcName in self.localFuncs: args = dict(ast.iter_fields(dict(ast.iter_fields(node))['args'])) args['args'].append(ast.Name(id=self.calleeInfo, ctx=ast.Param())) args['defaults'].append(ast.Str(s='')) self.addChild(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.locStack, ctx=ast.Load()), attr='append', ctx=ast.Load()), args=[ast.Name(id=self.calleeInfo, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), newNode) #self.addChild(ast.Print(dest=None, values=[ast.Name(id=self.funcStack, ctx=ast.Load())], nl=True), node) self.addChild(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.funcStack, ctx=ast.Load()), attr='append', ctx=ast.Load()), args=[ast.Str(s=self.funcName)], keywords=[], starargs=None, kwargs=None)), newNode) self.addChild(ast.Assign(targets=[ast.Name(id=self.loopSize, ctx=ast.Store())], value=ast.Call(func=ast.Name(id='len', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), newNode) finalBlock.append(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.funcStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None))) if self.funcName in self.localFuncs: finalBlock.append(ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.locStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None))) loopCorr = ast.While(test=ast.Compare(left=ast.Call(func=ast.Name(id='len', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), ops=[ast.Gt()], comparators=[ast.Name(id=self.loopSize, ctx=ast.Load())]), body=[ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id=self.loopStack, ctx=ast.Load()), attr='pop', ctx=ast.Load()), args=[], keywords=[], starargs=None, kwargs=None))], orelse=[]) self.seen.add(loopCorr) finalBlock.append(loopCorr) #self.addChild(ast.Print(dest=None, values=[ast.Str(s=self.funcName + '_afterPop')], nl=True), node, loc=len(dict(ast.iter_fields(node))['body'])) ast.fix_missing_locations(newNode) #print ast.dump(newNode) self.funcs.append(newNode) self.generic_visit(newNode, parent) return newNode
def parse_imports(s): module = ast.parse(s) imports = [] for c in ast.iter_child_nodes(module): if isinstance(c, ast.ImportFrom): fields = dict(ast.iter_fields(c)) from_module = fields['module'] if from_module != '__future__': raise ProcessingException( 'non-future ImportFrom on line ' + str(c.lineno)) names = [dict(ast.iter_fields(i))['name'] for i in fields['names']] if len(names) != 1: raise ProcessingException( 'multiple imports on line ' + str(c.lineno)) imports.append(ParsedImport(c.lineno, names[0], True)) if isinstance(c, ast.Import): fields = dict(ast.iter_fields(c)) names = [dict(ast.iter_fields(i))['name'] for i in fields['names']] if len(names) != 1: raise ProcessingException( 'multiple imports on line ' + str(c.lineno)) imports.append(ParsedImport(c.lineno, names[0], False)) return imports
def attach_data(self, node): """Generic method called for visit_XXXX() with XXXX in GatherOMPData.statements list """ if self.current: for curr in self.current: md = OMPDirective(curr) metadata.add(node, md) self.current = list() # add a Pass to hold some directives for field_name, field in ast.iter_fields(node): if field_name in GatherOMPData.statement_lists: if field and isinstance(field[-1], ast.Expr) and self.isompdirective(field[-1].value): field.append(ast.Pass()) self.generic_visit(node) # add an If to hold scoping OpenMP directives directives = metadata.get(node, OMPDirective) field_names = {n for n, _ in ast.iter_fields(node)} has_no_scope = field_names.isdisjoint(GatherOMPData.statement_lists) if directives and has_no_scope: # some directives create a scope, but the holding stmt may not # artificially create one here if needed sdirective = "".join(d.s for d in directives) scoping = ("parallel", "task", "section") if any(s in sdirective for s in scoping): node = ast.If(ast.Num(1), [node], []) return node
def visit_Call(self, node): super(srcpuller, self).generic_visit(node) state = 0 for n in ast.iter_child_nodes(node): if state == 0: if type(n) == ast.Name: for field, value in ast.iter_fields(n): if (field == 'id' and value == 'source'): state = 1 break continue elif state == 1: if type(n) == ast.Str: for field, value in ast.iter_fields(n): if (field == 's'): print 'sourc name:', value state = 2 break continue elif state == 2: if type(n) == ast.Str: for field, value in ast.iter_fields(n): if (field == 's'): print 'pat name:', value state = 3 break continue elif state == 3: if type(n) == ast.Str: for field, value in ast.iter_fields(n): if (field == 's'): print 'inst name:', value break break
def nodes_equal(x, y): __tracebackhide__ = True assert type(x) == type(y), "Ast nodes do not have the same type: '%s' != '%s' " % ( type(x), type(y), ) if isinstance(x, (ast.Expr, ast.FunctionDef, ast.ClassDef)): assert ( x.lineno == y.lineno ), "Ast nodes do not have the same line number : %s != %s" % ( x.lineno, y.lineno, ) assert x.col_offset == y.col_offset, ( "Ast nodes do not have the same column offset number : %s != %s" % (x.col_offset, y.col_offset) ) for (xname, xval), (yname, yval) in zip(ast.iter_fields(x), ast.iter_fields(y)): assert xname == yname, ( "Ast nodes fields differ : %s (of type %s) != %s (of type %s)" % (xname, type(xval), yname, type(yval)) ) assert type(xval) == type(yval), ( "Ast nodes fields differ : %s (of type %s) != %s (of type %s)" % (xname, type(xval), yname, type(yval)) ) for xchild, ychild in zip(ast.iter_child_nodes(x), ast.iter_child_nodes(y)): assert nodes_equal(xchild, ychild), "Ast node children differs" return True
def visit_arguments(self, node): args = None vararg = None kwarg = None defaults = None for field, value in ast.iter_fields(node): if field == "args": args = value for arg in args: #print "Arg: ", arg for field, value in ast.iter_fields(arg): if field == "id": #print "Define arg: ", value self.env[value] = "arg" else: JSONVisitorException("Unexpected error: argument's field is not id.") elif field == "vararg": vararg = value elif field == "kwarg": kwarg = value elif field == "defaults": defaults = value if vararg or kwarg or defaults: raise JSONVisitorException("Unexpected error: Missed case: vararg, kwarg or defaults is not empty.")
def match_recurse(pattern_node, actual_node, collected): supertypes = [actual_node.__class__] while supertypes[-1] != object: supertypes.append(supertypes[-1].__base__) if is_placeholder(pattern_node, ["__" + t.__name__ for t in supertypes]): collected.append(actual_node) return True elif type(pattern_node) != type(actual_node): return False elif isinstance(pattern_node, ast.AST): return all( match_recurse(left, right, collected) for ((left_name, left), (right_name, right)) in zip(ast.iter_fields(pattern_node), ast.iter_fields(actual_node)) ) elif isinstance(pattern_node, list): if len(pattern_node) != len(actual_node): return False else: return all( match_recurse(left, right, collected) for (left, right) in zip(pattern_node, actual_node) ) else: if pattern_node == "__" and type(actual_node) is str: collected.append(actual_node) return True else: return pattern_node == actual_node
def match(node, pat): """Return `True` if AST tree `node` matches AST pattern `pat`. """ if isinstance(pat, ast.Name) and pat.id == '_': return True elif isinstance(pat, ast.AST): if not isinstance(node, ast.AST): return False if not (issubclass(node.__class__, pat.__class__) or issubclass(pat.__class__, node.__class__)): return False assert _check_fields(node, pat) for (field1, val1), (field2, val2) in \ zip(ast.iter_fields(node), ast.iter_fields(pat)): assert(field1 == field2) if not match(val1, val2): return False elif isinstance(pat, list): if not isinstance(node, list): return False if len(node) != len(pat): return False for val1, val2 in zip(node, pat): if not match(val1, val2): return False else: # Primitive comparison. if node != pat: return False return True
def editfunctiondef(self, node): for fname,fnode in ast.iter_fields(node): if fname == 'args': for argname, argnode in ast.iter_fields(fnode): if argname == 'kwarg' and argnode != None: del self.functionstofix[node.lineno] if node.lineno in self.functionstofix.keys(): print "%d | Fixing function definition." % (node.lineno)
def visit_Call(self, node, parent): if node in self.seen: return node self.seen.add(node) callName = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('id', None) callType = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('attr',None) #print ast.dump(dict(ast.iter_fields(node))['func']), callType, node.lineno, node.col_offset #print callName, self.localFuncs if callName in self.localFuncs: #print ast.dump(node) #print callName, node.lineno, node.col_offset dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg=self.calleeInfo, value=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset)))) if callType in erps.keys() or callType == "stocPrim": if callType not in self.primsNumArgs: self.primsNumArgs[callType] = len(inspect.getargspec(globals()[callType]).args) namedArgs = map(lambda x: dict(ast.iter_fields(x))['arg'], dict(ast.iter_fields(node))['keywords']) numArgs = len(namedArgs) + len(dict(ast.iter_fields(node))['args']) #print callType, node.lineno, node.col_offset #print ast.dump(parent) if not ('name' in namedArgs or numArgs == self.primsNumArgs[callType]): #check if name already supplied dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg='name', value=ast.BinOp(left=ast.BinOp(left=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.funcStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.locStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), op=ast.Add(), right=ast.BinOp(left=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset)), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None))))) ast.fix_missing_locations(node) #print map(ast.dump, dict(ast.iter_fields(node))['keywords']) self.generic_visit(node) return node
def shallow_match_main(self, ins_node, std_node, check_meta=True, ignores=[]): """ Checks that all non astNode attributes are equal between ins_node and std_node :param ins_node: Instructor ast root node :param std_node: Student AST root node :param check_meta: flag to check whether the fields of the instructor node and the student node should match :return: a mapping between the isntructor and student root nodes, or False if such a mapping doesn't exist """ ins = ins_node.astNode std = std_node.astNode ins_field_list = list(ast.iter_fields(ins)) std_field_list = list(ast.iter_fields(std)) meta_matched = self.metas_match(ins_node, std_node, check_meta) is_match = len(ins_field_list) == len(std_field_list) and type(ins).__name__ == type( std).__name__ and meta_matched for insTup, stdTup in zip(ins_field_list, std_field_list): if not is_match: break ins_field = insTup[0] ins_value = insTup[1] std_field = stdTup[0] std_value = stdTup[1] if ins_value is None: continue ignore_field = ins_field in ignores is_match = (ins_field == std_field) or ignore_field if not isinstance(ins_value, list): ins_value = [ins_value] if not isinstance(std_value, list): std_value = [std_value] # is_match = len(ins_value) == len(std_value)# for stretchy matching this isn't True # Reference ast_node_visitor.js for the original behavior and keep note of it for the purposes of handling # the children noting the special case when the nodes of the array are actually parameters of the node # (e.g. a load function) instead of a child node if not ignore_field: for inssub_value, stdsub_value in zip(ins_value, std_value): if not is_match: break # TODO: make this a smarter comparison, maybe handle dictionaries, f-strings, tuples, etc. if is_primitive(inssub_value): is_match = inssub_value == stdsub_value mapping = False if is_match: mapping = AstMap() # return MAPPING mapping.add_node_pairing(ins_node, std_node) mapping = [mapping] return mapping
def generic_visit(self, node): for key, value in ast.iter_fields(node): if isinstance(value, AST): for k, v in ast.iter_fields(value): if isinstance(v, AST): return True elif isinstance(value, list): for node in value: if isinstance(node, AST): return True return False
def visit_ImportFrom(self, node): fields = dict(ast.iter_fields(node)) alias_nodes = list(ast.iter_child_nodes(node)) for alias_node in alias_nodes: alias_fields = dict(ast.iter_fields(alias_node)) if fields['level'] == 0: # from a import b # it might mean a.b is a module # or b might be a function of a module self.absolute_imports.append(fields['module']) self.absolute_imports.append('{}.{}'.format(fields['module'], alias_fields['name'])) else: self.relative_imports.append((fields['level'], fields['module'], alias_fields['name']))
def nodes_equal(x, y): __tracebackhide__ = True assert type(x) == type(y) if isinstance(x, (ast.Expr, ast.FunctionDef, ast.ClassDef)): assert x.lineno == y.lineno assert x.col_offset == y.col_offset for (xname, xval), (yname, yval) in zip(ast.iter_fields(x), ast.iter_fields(y)): assert xname == yname assert type(xval) == type(yval) for xchild, ychild in zip(ast.iter_child_nodes(x), ast.iter_child_nodes(y)): assert nodes_equal(xchild, ychild) return True
def generic_visit(self, node): if not (isinstance(node, ast.AST)): print node raise JSONVisitorException("Unexpected error: Non-ast passed to visit. Please report to the TAs.") for field, value in ast.iter_fields(node): # print "field: ", field, " value: ", value self.indent_print(field + ":") self.indent = self.indent + 1 if isinstance(value, list): for item in value: if isinstance(item, ast.AST): self.indent_print(item.__class__.__name__ + ":") self.indent = self.indent + 1 self.visit(item) self.indent = self.indent - 1 else: raise JSONVisitorException( "Unexpected error: Missed case: %s. Please report to the TAs." % item ) elif isinstance(value, ast.AST): self.visit(value) else: raise JSONVisitorException("Unexpected error: Missed case: %s. Please report to the TAs." % value) self.indent = self.indent - 1
def visit_AllNumVar(self, node): self.indent_print(node.__class__.__name__ + ":") for field, value in ast.iter_fields(node): if field == "id": self.indent = self.indent + 1 self.indent_print(field + ":" + str(value)) self.indent = self.indent - 1
def _fixup_global_g_inner(self, a): def maybe_replace(value): # search for attrib access like "g.something" if isinstance(value, ast.Attribute) \ and isinstance(value.value, ast.Name) \ and value.value.id == "g": # found one assert value.attr in self._py_globals if isinstance(self._py_globals[value.attr], cparser.CFunc): # overwrite this with "something.__func__" to resolve the staticmethod return ast.Attribute( value=ast.Name(id=value.attr, ctx=ast.Load()), attr="__func__", ctx=ast.Load() ) # overwrite this with just "something" return ast.Name(id=value.attr, ctx=ast.Load()) return value for node in ast.walk(a): for fieldname, value in ast.iter_fields(node): if isinstance(value, ast.AST): setattr(node, fieldname, maybe_replace(value)) elif isinstance(value, list): for i in range(len(value)): value[i] = maybe_replace(value[i]) elif isinstance(value, tuple): setattr(node, fieldname, tuple(map(maybe_replace, value)))
def generic_visit(self, node): for field, old_value in ast.iter_fields(node): old_value = getattr(node, field, None) if isinstance(old_value, list): new_values = [] for value in old_value: if isinstance(value, ast.AST): value = self.visit(value) if value is None: continue elif not isinstance(value, ast.AST): new_values.extend(value) continue new_values.append(value) old_value[:] = new_values elif isinstance(old_value, ast.AST): new_node = self.visit(old_value) if new_node is None: delattr(node, field) elif not isinstance(new_node, ast.AST): new_node_tuple = ast.copy_location(ast.Tuple(elts=list(new_node), ctx=ast.load()), new_node[0]) setattr(node, field, new_node_tuple) else: setattr(node, field, new_node) return node
def _format(node, field='', level=0): if isinstance(node, ast.AST): head_string = get_indent( level ) + node.__class__.__name__ head_string += " ({})".format(field) if field else "" if show_lineno and hasattr( node, 'lineno' ): head_string += ' [{}]'.format( node.lineno ) body_string = "" for label, value in ast.iter_fields(node): if label == "ctx": continue if isinstance( value, (str, int, float, long) ): head_string += ' {} = {}'.format( label, value ) elif not value: body_string += get_indent( level+1, pipe=True) + '{} ({})\n'.format( value, label ) else: body_string += _format( value, label, level+1 ) return head_string + "\n" + body_string elif isinstance(node, list): string = '' for x in node: string += _format( x, field, level ) return string return repr(node)
def visit_FunctionDef(self, node): name = None args = None body = None decorator_list = None for field, value in ast.iter_fields(node): if field == "name": name = value self.print_field_value(field, value) elif field == "args": args = value self.indent_print(field + ":") self.indent = self.indent + 1 self.visit(value) self.indent = self.indent - 1 elif field == "body": body = value self.indent_print(field + ":") self.indent = self.indent + 1 for stmt in body: self.indent_print(stmt.__class__.__name__ + ":") self.indent = self.indent + 1 self.visit(stmt) self.indent = self.indent - 1 self.indent = self.indent - 1 elif field == "decorator_list": decorator_list = value self.print_field_value(field, value) if decorator_list: raise JSONVisitorException("Unexpected error: Missed case: decorator_list is not empty.")
def visit_Assign(self, node): lhs = None rhs = None for field, value in ast.iter_fields(node): if field == "targets": lhs = value[0] elif field == "value": rhs = value else: raise JSONVisitorException("Unexpected error: Missed case: %s." % value) if isinstance(lhs, ast.Name): node.targets = [self.visit(lhs)] node.value = self.visit(rhs) elif isinstance(lhs, ast.Tuple): new_lhs = [] new_rhs = [] for l,r in zip(lhs.elts, rhs.elts): new_lhs.append(self.visit(l)) new_rhs.append(self.visit(r)) lhs.elts = new_lhs rhs.elts = new_rhs if (node.lineno, node.col_offset) in self.locs: self.fixes.append(node) return node
def code_generic_visit(self, node): for field, old_value in ast.iter_fields(node): old_value = getattr(node, field, None) if isinstance(old_value, list): prev_insertion_point = self._insertion_point new_values = [] if field in ("body", "orelse", "finalbody"): self._insertion_point = new_values for value in old_value: if isinstance(value, ast.AST): value = self.code_visit(value) if value is None: continue elif not isinstance(value, ast.AST): new_values.extend(value) continue new_values.append(value) old_value[:] = new_values self._insertion_point = prev_insertion_point elif isinstance(old_value, ast.AST): new_node = self.code_visit(old_value) if new_node is None: delattr(node, field) else: setattr(node, field, new_node) return node
def str_node(node): if isinstance(node, ast.AST): fields = [(name, str_node(val)) for name, val in ast.iter_fields(node) if name not in ('left', 'right')] rv = '%s(%s' % (node.__class__.__name__, ', '.join('%s=%s' % field for field in fields)) return rv + ')' else: return repr(node)
def generic_visit(self, node): if isinstance(node, list): nodestart = "[" nodeend = "]" children = [("", child) for child in node] else: nodestart = type(node).__name__ + "(" nodeend = ")" children = [(name + "=", value) for name, value in ast.iter_fields(node)] if len(children) > 1: self.indentation += 1 self.write(nodestart) for i, pair in enumerate(children): attr, child = pair if len(children) > 1: self.write("\n" + self.indent_with * self.indentation) if isinstance(child, (ast.AST, list)): self.write(attr) self.visit(child) else: self.write(attr + repr(child)) if i != len(children) - 1: self.write(",") self.write(nodeend) if len(children) > 1: self.indentation -= 1
def visit_arguments(self, node): args = None vararg = None kwarg = None defaults = None for field, value in ast.iter_fields(node): if field == "args": args = value self.indent_print(field + ":") self.indent = self.indent + 1 for arg in args: self.indent_print(arg.__class__.__name__ + ":") self.visit(arg) self.indent = self.indent - 1 elif field == "vararg": vararg = value self.print_field_value(field, value) elif field == "kwarg": kwarg = value self.print_field_value(field, value) elif field == "defaults": defaults = value self.print_field_value(field, value) if vararg or kwarg or defaults: raise JSONVisitorException( "Unexpected error: Missed case: vararg, kwarg or defaults is not empty. Please report to the TAs." )
def find_peers(node): node_for_line = node._parent for _field, value in ast.iter_fields(node._parent._parent): if isinstance(value, list) and node_for_line in value: return value[value.index(node_for_line) + 1:] continue return []
def generic_visit(self, node): """Called if no explicit visitor function exists for a node. Overridden from the ast.NodeVisitor base class so that I can add some local state to keep track of whether or not my node visitor is inside a try/except block. When a try block is encountered, the node is added to the `trys` instance attribute and then the try block is recursed in to. Once the recursion has exited, the node is removed from the `trys` instance attribute """ for field, value in ast.iter_fields(node): if isinstance(value, list): for item in value: # add the node to the try/except block to signify that # something potentially odd is going on in this import if isinstance(item, ast.Try): self.trys[item] = item if isinstance(item, ast.AST): self.visit(item) # after the node has been recursed in to, remove the try node if isinstance(item, ast.Try): del self.trys[item] elif isinstance(value, ast.AST): # add the node to the try/except block to signify that # something potentially odd is going on in this import if isinstance(value, ast.Try): self.trys[value] = item self.visit(value) # after the node has been recursed in to, remove the try node if isinstance(value, ast.Try): del self.trys[value]
def visit_If(self, node): for field, value in ast.iter_fields(node): if field == "test": self.indent_print(field + ":") self.indent = self.indent + 1 self.indent_print(value.__class__.__name__ + ":") self.indent = self.indent + 1 self.visit(value) self.indent = self.indent - 1 self.indent = self.indent - 1 elif field == "body": self.indent_print(field + ":") self.indent = self.indent + 1 for stmt in value: self.indent_print(stmt.__class__.__name__ + ":") self.indent = self.indent + 1 self.visit(stmt) self.indent = self.indent - 1 self.indent = self.indent - 1 elif field == "orelse": self.indent_print(field + ":") self.indent = self.indent + 1 for stmt in value: self.indent_print(stmt.__class__.__name__ + ":") self.indent = self.indent + 1 self.visit(stmt) self.indent = self.indent - 1 self.indent = self.indent - 1 else: self.print_field_value(field, value)
def visit_FunctionDef(self, node): is_main = False for field, value in ast.iter_fields(node): if field == "name" and value == self.main: is_main = True if is_main and field == "args": return self.visit(value)
def fill_line_numbers(tree, lineno, col_offset, **kw): """Fill in line numbers somewhat more cleverly than the ast.fix_missing_locations method, which doesn't take into account the fact that line numbers are monotonically increasing down lists of AST nodes.""" if type(tree) is list: for sub in tree: if isinstance(sub, ast.AST) \ and hasattr(sub, "lineno") \ and hasattr(sub, "col_offset") \ and (sub.lineno, sub.col_offset) > (lineno, col_offset): lineno = sub.lineno col_offset = sub.col_offset fill_line_numbers(sub, lineno, col_offset) elif isinstance(tree, ast.AST): if not (hasattr(tree, "lineno") and hasattr(tree, "col_offset")): tree.lineno = lineno tree.col_offset = col_offset for name, sub in ast.iter_fields(tree): fill_line_numbers(sub, tree.lineno, tree.col_offset) elif isinstance(tree, (str, int, float)) or tree is None: pass else: raise TypeError("Invalid AST node '{!r}', type: '{!r}' " "after expansion".format(tree, type(tree))) return tree
def get_last_lineno(node): """Recursively find the last line number of the ast node.""" max_lineno = 0 if hasattr(node, "lineno"): max_lineno = node.lineno for _, field in ast.iter_fields(node): if isinstance(field, list): for value in field: if isinstance(value, ast.AST): max_lineno = max(max_lineno, get_last_lineno(value)) elif isinstance(field, ast.AST): max_lineno = max(max_lineno, get_last_lineno(field)) return max_lineno
def to_rule(node: ast.AST) -> Rule: parent = typename(node) children = [] for chname, chval in ast.iter_fields(node): if chname == 'ctx': continue if chval is None or (isinstance(chval, list) and len(chval) == 0): continue is_list = isinstance(chval, list) if is_list: base_type = base_ast_type(chval[0]) else: base_type = base_ast_type(chval) children.append(Node(chname, NodeType(base_type.__name__, is_list))) return Rule(NodeType(parent, False), tuple(children))
def wasModified(node,var): if isinstance(node,ast.Assign): for target in node.targets: if target.id == var: return True else: for name,value in ast.iter_fields(node): if isinstance(value,list): for n in value: ret = wasModified(n,var) if ret: return ret elif isinstance(value,ast.AST): ret = wasModified(value,var) if ret: return ret return False
def test_tree_job(self): def addition(x, y): z = x + numpy.abs(y) return x * z code = inspect.getsource(addition) node = ast.parse(dedent(code)) stack = [(0, node)] while len(stack) > 0: ind, n = stack[-1] del stack[-1] att = {name: ch for name, ch in ast.iter_fields(n)} self.assertTrue(att is not None) for ch in ast.iter_child_nodes(n): stack.append((ind + 1, ch)) self.assertEmpty(stack)
def generic_visit(self, node: TAst, *args) -> TAst: ret_node = node for field, old_value in ast.iter_fields(node): if not isinstance(old_value, (AST, list)): continue new_node = self.visit(old_value) assert ( # noqa: IG01 new_node is not None ), f"can't remove AST nodes that aren't part of a list {old_value!r}" if new_node is not old_value: if ret_node is node: ret_node = self.clone_node(node) setattr(ret_node, field, new_node) return ret_node
def generic_visit(self, node, **kwargs): """ Extract useful information from relevant nodes including docstrings. This is virtually identical to the standard version contained in NodeVisitor. It is only overridden because we're tracking extra information (the hierarchy of containing nodes) not preserved in the original. """ for field, value in iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, AST): self.visit(item, containingNodes=kwargs['containingNodes']) elif isinstance(value, AST): self.visit(value, containingNodes=kwargs['containingNodes'])
def generic_visit(self, node): """Called if no explicit visitor function exists for a node. Overridden from the ast.NodeVisitor base class so that I can add some local state to keep track of whether or not my node visitor is inside a try/except block. When a try block is encountered, the node is added to the `trys` instance attribute and then the try block is recursed in to. Once the recursion has exited, the node is removed from the `trys` instance attribute """ for field, value in ast.iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, ast.AST): self.visit(item) elif isinstance(value, ast.AST): self.visit(value)
def visit_Assign(self, node): for key, val in ast.iter_fields(node): if key == 'targets': for subnode in val: if type(subnode) is ast.Tuple: for subsubnode in subnode.elts: crawler = myextract() crawler.visit(subsubnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 else: crawler = myextract() crawler.visit(subnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 #ast.NodeVisitor.generic_visit(self, node) self.generic_visit(node) self.depth -= 1
def _format(node, indent=4): if isinstance(node, ast.AST): fields = [(a, _format(b, indent + 4)) for a, b in ast.iter_fields(node)] rv = node.__class__.__name__ + '(\n' for field in fields: rv += ' ' * indent + '%s=%s,\n' % field if include_attributes and node._attributes: rv += fields and ', ' or ' ' rv += ', '.join('%s=%s' % (a, _format(getattr(node, a), indent + 4)) for a in node._attributes) return rv + ' ' * indent + ')' elif isinstance(node, list): return '[\n%s%s\n%s]' % (' ' * indent, (',\n' + ' ' * indent).join( _format(x, indent + 4) for x in node), ' ' * indent) return repr(node)
def assert_ast_like(sample, template, _path=None): """Check that the sample AST matches the template. Raises a suitable subclass of :exc:`ASTMismatch` if a difference is detected. The ``_path`` parameter is used for recursion; you shouldn't normally pass it. """ if _path is None: _path = ["tree"] if callable(template): # Checker function at the top level return template(sample, _path) if not isinstance(sample, type(template)): raise ASTNodeTypeMismatch(_path, sample, template) for name, template_field in ast.iter_fields(template): # Bug fix - The type of the nodes could be different if not hasattr(sample, name): continue sample_field = getattr(sample, name) field_path = _path + [name] if isinstance(template_field, list): if template_field and ( isinstance(template_field[0], ast.AST) or callable(template_field[0]) ): _check_node_list(field_path, sample_field, template_field) else: # List of plain values, e.g. 'global' statement names if sample_field != template_field: raise ASTPlainListMismatch(field_path, sample_field, template_field) elif isinstance(template_field, ast.AST): assert_ast_like(sample_field, template_field, field_path) elif callable(template_field): # Checker function template_field(sample_field, field_path) else: # Single value, e.g. Name.id if sample_field != template_field: raise ASTPlainObjMismatch(field_path, sample_field, template_field)
def _format(node, _depth, _max_depth=None): if (_max_depth is not None) and (_depth > _max_depth): return '<...>' _next_depth = _depth + 1 _indent = indent * (_depth - 1) _subindent = indent * _depth if isinstance(node, ast.AST): formatted_fields = [(name, _format(value, _depth=_next_depth, _max_depth=_max_depth)) for name, value in ast.iter_fields(node)] if include_attributes and node._attributes: formatted_fields.extend([(attributename, _format(getattr(node, attributename), _depth=_next_depth, _max_depth=_max_depth)) for attributename in node._attributes ]) if annotate_fields: return f'{node.__class__.__name__}(' + ', '.join( f'{name}={value}' for name, value in formatted_fields) + ')' else: return f'{node.__class__.__name__}({", ".join( fieldname for fieldname, _ in formatted_fields )})' elif isinstance(node, (list, set, tuple)): nodes = node formatted_nodes = ['['] formatted_nodes.extend( f'{_subindent}{_format(subnode, _depth=_next_depth, _max_depth=_max_depth)},' for subnode in nodes) if len(formatted_nodes) > 1: formatted_nodes.append(f'{_indent}]') else: formatted_nodes[-1] += ']' return '\n'.join(formatted_nodes) else: return repr(node)
def get_deps(code): body = ast.parse(code) _, statements = next(ast.iter_fields(body)) # Line no. at which each identifier was first seen declaration_line_num_map = {} ddg = {} def update_decls(lhs_vars_input, num): lhs_var_nodes = [] for var_node in lhs_vars_input: lhs_var_nodes.append(var_node) if var_node.id not in declaration_line_num_map: declaration_line_num_map[var_node.id] = num ddg[var_node.id] = set() return lhs_var_nodes # x1, x2, x3, ..., xN = 1, 2, 3, 4, 5, ..., N # is represented in the AST as: # - R = ast.Assign is root # - R.targets gives the LHS # - R.values for seq_no, node in enumerate(statements): if isinstance(node, ast.Assign): identifier_names = node.targets lhs_vars = update_decls(identifier_names, seq_no) self_edge_occurrences_to_ignore = {x: 1 for x in identifier_names} # DFS in RHS depends_on = [] for descendant in ast.walk(node): if descendant in self_edge_occurrences_to_ignore and self_edge_occurrences_to_ignore[ descendant] > 0: self_edge_occurrences_to_ignore[descendant] -= 1 continue if isinstance(descendant, ast.Name): depends_on.append(descendant) for var in lhs_vars: for dependency in depends_on: ddg[var.id].add(dependency.id) return declaration_line_num_map, ddg
def exec_py( code: str, src_name: str = "<py-code>", globals_: Optional[dict] = None, locals_: Optional[dict] = None, ) -> object: """ Run arbitrary Python code in supplied globals, return evaluated value of last statement. """ if globals_ is None: globals_ = {} if locals_ is None: locals_ = globals_ try: ast_ = ast.parse(code, src_name, "exec") last_expr = None last_def_name = None for field_ in ast.iter_fields(ast_): if "body" != field_[0]: continue if len(field_[1]) > 0: le = field_[1][-1] if isinstance(le, ast.Expr): last_expr = ast.Expression() last_expr.body = field_[1].pop().value elif isinstance(le, (ast.FunctionDef, ast.ClassDef)): last_def_name = le.name exec(compile(ast_, src_name, "exec"), globals_, locals_) if last_expr is not None: return eval(compile(last_expr, src_name, "eval"), globals_, locals_) elif last_def_name is not None: # godforbid the code to declare global/nonlocal for the last defined artifact return locals_[last_def_name] return None except Exception: logger.error( f"""Error running Python code: -=-{src_name}-=- {code!s} -=*{src_name}*=- """, exc_info=True, ) raise
def ast_visit(node, parentNode, level=0): #with open("out.txt", 'a') as f: # f.write(' ' * level + str_node(node) + '\n') # print(' ' * level + str_node(node)) str_node(node, parentNode) for field, value in ast.iter_fields(node): if (type(value) == ast.Call): print("CALL FOUND") #print("F: {}\nV:{}\n".format(field, value)) if isinstance(value, list): for item in value: if (type(item) == ast.Assign): print("ASSIGN FOUND") if isinstance(item, ast.AST): ast_visit(item, node, level=level + 1) elif isinstance(value, ast.AST): ast_visit(value, node, level=level + 1)
def translate_facts(node: ast.AST, names: dict, module: ModuleType) -> ast.AST: """Translate facts referred within a node with __Fact_<Class Name>.""" if isinstance(node, ast.Name): return translate_fact(node, node.id, names, module) elif isinstance(node, ast.Attribute): node.value = translate_facts(node.value, names, module) return translate_fact(node, node_names(node), names, module) else: for field, value in ast.iter_fields(node): if isinstance(value, ast.AST): setattr(node, field, translate_facts(value, names, module)) elif isinstance(value, list): value[:] = [ translate_facts(v, names, module) for v in value if isinstance(v, ast.AST) ] return node
def generic_visit(self, node): """ We use the visit method - it's called in every step of the visit function that we call (please don't overload that!), so we will see the entire graph """ for attr, vs in ast.iter_fields(node): if isinstance(vs, ast.AST): self.graph[self.make_label(node)].append(self.make_label(vs)) self.visit(vs) elif isinstance(vs, list): for v in vs: if isinstance(v, ast.AST): self.graph[self.make_label(node)].append( self.make_label(v)) self.visit(v) else: pass
def disp(self, node): if isinstance(node, ast.AST): string = getClname(node) + '(' cnt = 0 for name, value in ast.iter_fields(node): val = self.disp(value) if cnt > 0: string += ', ' cnt += 1 string += name + '=' + val return string + ')' elif isinstance(node, list): string = '' for x in node: string += self.disp(x) return string else: return repr(node)
def generic_visit(self, node): # label this node out_string = 'n%s [label="%s"];\n' % (id(node), self.label(node)) # edge to parent if hasattr(node, 'parent') and node.parent is not None: out_string += 'n%s -> n%s [label="parent",style=dotted];\n' % ( id(node), id(node.parent)) # edges to children for fieldname, fieldvalue in ast.iter_fields(node): for index, child in enumerate_flatten(fieldvalue): if isinstance(child, ast.AST): suffix = "".join(["[%d]" % i for i in index]) out_string += 'n%d -> n%d [label="%s%s"];\n' % ( id(node), id(child), fieldname, suffix) out_string += _to_dot(child) return out_string
def generic_visit(self, node): """Called if no explicit visitor function exists for a node.""" if node == self.node_to_check: self.final_indent = self.current_indent for field, value in ast.iter_fields(node): if (isinstance(node, tuple(TYPE_TO_INDENT_FIELD.keys())) and field in TYPE_TO_INDENT_FIELD[node.__class__]): self.current_indent += 1 if isinstance(value, list): for item in value: if isinstance(item, _ast.AST): self.visit(item) elif isinstance(value, _ast.AST): self.visit(value) if (isinstance(node, tuple(TYPE_TO_INDENT_FIELD.keys())) and field in TYPE_TO_INDENT_FIELD[node.__class__]): self.current_indent -= 1 return node
def visit_FunctionDef(self, node: ast.FunctionDef) -> None: if self._name == node.name: self._param_type_matches.clear() for name, field in ast.iter_fields(node): if name == "args": for arg in field.args: if arg.annotation and hasattr(arg.annotation, "id"): self._param_type_matches[arg.arg] = self._locate( arg.annotation.id ) elif arg.annotation and hasattr(arg.annotation, "attr"): self._param_type_matches[arg.arg] = self._locate( f"{arg.annotation.value.id}.{arg.annotation.attr}" ) else: self._param_type_matches[arg.arg] = None if node.returns and hasattr(node.returns, "id"): self._return_type = self._locate(node.returns.id) # type: ignore
def generic_visit_level(self): stack = [self._root] self._levelTran = [] while len(stack) != 0: tmp = [] res_each = [] for i in stack: res_each.append(i) self.d[i] = [0] for field, value in ast.iter_fields(i): if isinstance(value, list): for item in value: if isinstance(item, ast.AST): tmp.append(item) elif isinstance(value, ast.AST): tmp.append(value) stack = tmp self._levelTran.insert(0, res_each)
def generic_visit(self, node): for field, old_value in ast.iter_fields(node): if isinstance(old_value, list): for value in old_value: if isinstance(value, ast.AST): if (field == 'body' or field == 'orelse'): clsname = type(value).__name__ if getattr(self, "visit_TopLevel" + clsname, False): getattr(self, "visit_TopLevel" + clsname)(value) else: self.visit(value) else: self.visit(value) elif isinstance(old_value, ast.AST): self.visit(old_value) return node
def generic_visit(self, node): """Drive the visitor.""" for _, value in ast.iter_fields(node): if isinstance(value, list): max_idx = len(value) - 1 for idx, item in enumerate(value): if isinstance(item, ast.AST): if idx < max_idx: setattr(item, 'sibling', value[idx + 1]) else: setattr(item, 'sibling', None) setattr(item, 'parent', node) self.visit(node=item) elif isinstance(value, ast.AST): setattr(value, 'sibling', None) setattr(value, 'parent', node) self.visit(node=value)
def ExpandTree(node): node_fields = [] to_expand = [node] while to_expand: current = to_expand.pop() if isinstance(current, ast.AST): node_fields.append(current.__class__) for field_name, child in ast.iter_fields(current): node_fields.append(field_name) if isinstance(child, (list, tuple)): for item in child: to_expand.append(item) else: to_expand.append(child) else: node_fields.append(current) print node_fields return node_fields
def __init__(self, ast_node, my_field='', tid=0, lin_tree=None, ancestor=None): """ :param ast_node: The AST node to be wrapped :param my_field: the field of the parent node that produced this child. """ self.children = [] self.astNode = ast_node self.field = my_field self.tree_id = tid self.parent = ancestor if lin_tree is None: self.linear_tree = [self] else: lin_tree.append(self) self.linear_tree = lin_tree # reference to the easy node wrapping the ast_node setattr(ast_node, 'easy_node', self) tid_count = tid my_field_generator = ast.iter_fields(self.astNode) for item in my_field_generator: field, value = item # if the field doesn't have a value, no child exists if value is None: continue # If the children are not in an array, wrap it in an array for # consistency in the code the follows if not isinstance(value, list): value = [value] # Reference ast_node_visitor.js for the original behavior and keep note of it for the purposes of handling # the children noting the special case when the nodes of the array are actually parameters of the node # (e.g. a load function) instead of a child node for sub_value in value: if isinstance(sub_value, ast.AST): new_child = EasyNode(sub_value, my_field=field, tid=tid_count + 1, lin_tree=self.linear_tree, ancestor=self) self.children.append(new_child) tid_count = len(self.linear_tree) - 1
def pretty(node, key="/", level=0): """Used for testing and new test generation via AstView. Don't change the format without updating tests""" if isinstance(node, ast.AST): fields = [(key, val) for key, val in ast.iter_fields(node)] value_label = node.__class__.__name__ if isinstance(node, ast.Call): # Try to make 3.4 AST-s more similar to 3.5 if sys.version_info[:2] == (3, 4): if ("kwargs", None) in fields: fields.remove(("kwargs", None)) if ("starargs", None) in fields: fields.remove(("starargs", None)) # TODO: translate also non-None kwargs and starargs elif isinstance(node, list): fields = list(enumerate(node)) if len(node) == 0: value_label = "[]" else: value_label = "[...]" else: fields = [] value_label = repr(node) item_text = level * " " + str(key) + "=" + value_label if hasattr(node, "lineno"): item_text += " @ " + str(getattr(node, "lineno")) if hasattr(node, "col_offset"): item_text += "." + str(getattr(node, "col_offset")) if hasattr(node, "end_lineno"): item_text += " - " + str(getattr(node, "end_lineno")) if hasattr(node, "end_col_offset"): item_text += "." + str(getattr(node, "end_col_offset")) lines = [item_text] + [ pretty(field_value, field_key, level + 1) for field_key, field_value in fields ] return "\n".join(lines)
def generic_visit(self, node): try: return ast.literal_eval(node) except ValueError: for field, value in ast.iter_fields(node): if field in self.excluded_fields: delattr(node, field) if value is None: continue if isinstance(value, list): if field in ('keywords', 'kwargs'): new_value = dict((kw.arg, self.visit(kw.value)) for kw in value) else: new_value = [self.visit(i) for i in value] else: new_value = self.visit(value) setattr(node, field, new_value) return node
def generate_dot_code(python_file): """ generate_dot_code :param py_code: :return: """ with open(python_file) as source: ast_module_obj = parser.suite(source.read()) ast_obj_list = parser.st2list(ast_module_obj) # ast_module_obj = ast.parse(source.read(), python_file, mode='exec') # ast_module_obj_body = ast_module_obj.body ast_obj = ast_obj_list print('ast_list\n', repr(ast_module_obj)) print('ast_iter_tuple\n', ast.iter_fields(ast_module_obj)) # print('ast_body\n', ast_module_obj_body) print('ast_obj\n\n', repr(ast_obj)) # print('ast.iter_child_nodes\n', ast.iter_child_nodes(ast_obj)) # for b in ast.walk(ast_obj): # print('ast_obj\n', repr(b)) call_graph = {} construct_call_graph(ast_obj, call_graph) # pprint.pprint(call_graph) dot = [] dot.append("digraph G {") dot.append("rankdir=LR") for from_fn, to_fns in call_graph.items(): if not to_fns: dot.append('%s;' % from_fn) for to_fn in to_fns: if to_fn not in call_graph: continue dot.append('%s -> %s;' % (from_fn, to_fn)) dot.append("}") return '\n'.join(dot)
def _find_line_numbers(self, node, line_numbers): """ Populates a set containing all line numbers used by the node and its descendants. line_numbers is a set that all the line numbers will be added to.""" if FormattedValue is not None and isinstance(node, FormattedValue): # FormattedValue is a separate code block with its own line nums. return line_number = getattr(node, 'lineno', None) if line_number is not None: line_numbers.add(line_number) for _, value in iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, AST): self._find_line_numbers(item, line_numbers) elif isinstance(value, AST): self._find_line_numbers(value, line_numbers)
def generic_visit(self, node): # Get new and old values r = [(field, old_value, self._get_new_value(old_value)) for field, old_value in ast.iter_fields(node)] if len(r) == 0: return node if all(not i[2][1] for i in r): return node # Ok - there was a modification. We need to clone the class and pass that # back up. new_node = node.__class__() for f in r: if f[1] is not None: setattr(new_node, f[0], f[2][0]) return new_node