def hy_eval(hytree, namespace, module_name): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 if not isinstance(namespace, dict): raise HyTypeError(foo, "Globals must be a dictionary") # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def _log_function(item): """ Handler function for log message types. :param item: ast object being inspected for certain properties :type item: :class:`~ast.AST` :return: Returns the descriptor and arg offset of the item. :rtype: tuple (str, int) """ level_arg = item.args[0] if isinstance(level_arg, ast.Str): level = level_arg.s.lower() else: any_call_or_name_elements = any( isinstance(element, (ast.Call, ast.Name)) for element in ast.walk(level_arg) ) if any_call_or_name_elements: level = "(dynamic)" else: level = ', '.join( element.s.lower() for element in ast.walk(level_arg) if isinstance(element, ast.Str) ) integer_arg_offset = 1 return level, integer_arg_offset
def nestable_lines(self): """ Return the range of lines that are nestable. Parse the tree for all start and end lines of a nestable group (e.g. a function defintion or if statement). For each, return a tuple of the start and end line number. """ nests = [] nodes = [ast.walk(node) for node in self._tree.body] for node in nodes: end = 0 for subnode in node: if isinstance(subnode, (ast.FunctionDef, ast.ClassDef, ast.If, ast.For, ast.TryExcept, ast.TryFinally)): end = 0 for subsubnode in ast.walk(subnode): try: lineno = subsubnode.lineno except AttributeError: pass else: if lineno > end: end = lineno nests.append((subnode.lineno, end)) return nests
def replace_variables(self): """Replace script variables in string values.""" variables = { 'SCRIPT_SOURCE': self.source_script, 'SCRIPT_NAME': self.script_name, 'SCRIPT_PATH': self.script_path, 'SCRIPT_AUTHOR': 'Sebastien Helleu', 'SCRIPT_VERSION': '1.0', 'SCRIPT_LICENSE': 'GPL3', 'SCRIPT_DESCRIPTION': ('%s scripting API test' % self.language.capitalize()), 'SCRIPT_LANGUAGE': self.language, } # count the total number of tests tests_count = 0 for node in ast.walk(self.tree): if isinstance(node, ast.Call) and \ isinstance(node.func, ast.Name) and \ node.func.id == 'check': tests_count += 1 variables['SCRIPT_TESTS'] = str(tests_count) # replace variables for node in ast.walk(self.tree): if isinstance(node, ast.Str) and \ node.s in variables: node.s = variables[node.s]
def ast_equal(code1, code2, check_line_col=False, ignore_var_names=False): """ Checks whether ast nodes are equivalent recursively. By default does not check line number or col offset """ gen1 = ast.walk(code1) gen2 = ast.walk(code2) for node1, node2 in zip_longest(gen1, gen2): # unequal length if node1 is None or node2 is None: return False if type(node1) != type(node2): return False # ignore the names of load name variables. if ignore_var_names and is_load_name(node1) and is_load_name(node2): continue if not ast_field_equal(node1, node2): return False if check_line_col and hasattr(node1, 'lineno'): if node1.lineno != node2.lineno: return False if node1.col_offset != node2.col_offset: return False return True
def ast_transform(root): parent_node = None dt_class_names = set(c.__name__ for c in (DateTime, Date, Duration)) str_nodes = [] for node in ast.walk(root): try: if isinstance(node, ast.Str): if isinstance(parent_node, ast.Name) and parent_node.id in dt_class_names: pass else: str_nodes.append(node) finally: parent_node = node nt = DTNodeTransformer(str_nodes) nt.visit(root) ast.fix_missing_locations(root) prev_lineno, prev_col_offset = 1, 0 for n in ast.walk(root): if not hasattr(n, 'lineno'): n.lineno = prev_lineno else: prev_lineno = n.lineno if not hasattr(n, 'col_offset'): n.col_offset = prev_col_offset else: prev_col_offset = n.col_offset
def check_for_wrong_tuple(tree, code, noqa): errors = [] candidates = [] for assign in ast.walk(tree): if not isinstance(assign, ast.Assign) or assign.lineno in noqa: continue elif isinstance(assign.value, ast.Call): continue for tuple_el in ast.walk(assign): if isinstance(tuple_el, ast.Tuple) and len(tuple_el.elts) == 1: candidates.append((assign.lineno, assign.col_offset)) break if not candidates: return [] for candidate in candidates: tokens = tokenize.generate_tokens(lambda L=iter(code): next(L)) for t in tokens: x = TokenInfo(*t) if x.start[0] != candidate[0]: continue if x.type == token.OP and x.string == "=": x = TokenInfo(*next(tokens)) if x.type != token.OP and x.string != "(": x_next = TokenInfo(*next(tokens)) if x_next.type == token.OP and x_next.string == ",": errors.append(x.start) return errors
def parse(self, source, filename="source"): self.clear() self.filename = filename data = ast.parse(source, filename) for a in ast.walk(data): if isinstance(a, _ast.Assign) and isinstance(a.targets[0], _ast.Name): name = a.targets[0].id if name == self.stored_dict_name: if isinstance(a.value, _ast.Dict): for x in a.value.values: if isinstance(x, _ast.Str): self.stored_raw.append((x.s, x.lineno, x.col_offset)) else: print "Error in %s:%d:%d: nonstring dict value" % ( self.filename, x.lineno, x.col_offset, ) if not self.using_function: for a in ast.walk(data): if isinstance(a, _ast.Subscript) and isinstance(a.value, _ast.Name): name = a.value.id if name == self.retrieved_dict_name: if hasattr(a.slice, "value") and isinstance(a.slice.value, _ast.Str): x = a.slice.value self.retrieved_raw.append((x.s, x.lineno, x.col_offset)) else: for a in ast.walk(data): if isinstance(a, _ast.Call) and isinstance(a.func, _ast.Name): name = a.func.id if name == self.retrieved_dict_name: if len(a.args) == 1 and isinstance(a.args[0], _ast.Str): x = a.args[0] self.retrieved_raw.append((x.s, x.lineno, x.col_offset)) else: print "Suspicious use of '%s' in %s:%d:%d" % ( self.retrieved_dict_name, self.filename, a.lineno, a.col_offset, ) for item in self.stored_raw: if item[0] in self.stored: self.stored[item[0]].append(item) else: self.stored[item[0]] = [item] for item in self.retrieved_raw: if item[0] in self.retrieved: self.retrieved[item[0]].append(item) else: self.retrieved[item[0]] = [item] s_set = set(self.stored.keys()) r_set = set(self.retrieved.keys()) self.missing = r_set - s_set self.redundant = s_set - r_set
def _find_last_line_for_class(source, classname): all_nodes = list(ast.walk(ast.parse(source))) classes = [n for n in all_nodes if isinstance(n, ast.ClassDef)] our_class = next(c for c in classes if c.name == classname) last_line_in_our_class = max( getattr(thing, 'lineno', 0) for thing in ast.walk(our_class) ) return last_line_in_our_class
def translateFunctions(self, tree): for node in ast.walk(tree): if isinstance(node, ast.FunctionDef): self.forwardDeclaration(node) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef): self.visit(node)
def read_setup_py(setup_py): requirements = set() tree = ast.parse(setup_py) for node in ast.walk(tree): arg = getattr(node, "arg", None) if arg in ("requires", "install_requires"): for node in ast.walk(node.value): if isinstance(node, ast.Str): requirements.add(read_package_name(node.s)) return requirements
def has_main(filepath): with open(filepath, "r") as f: source = f.read() tree = ast.parse(source) first_level_ifs = (n for n in ast.walk(tree) if isinstance(n, _ast.If)) strs_in_ifs = [n.s for _if in first_level_ifs for n in ast.walk(_if) if isinstance(n, _ast.Str)] return "__main__" in strs_in_ifs
def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): """``eval`` evaluates a quoted expression and returns the value. The optional second and third arguments specify the dictionary of globals to use and the module name. The globals dictionary defaults to ``(local)`` and the module name defaults to the name of the current module. => (eval '(print "Hello World")) "Hello World" If you want to evaluate a string, use ``read-str`` to convert it to a form first: => (eval (read-str "(+ 1 1)")) 2""" if namespace is None: frame = inspect.stack()[1][0] namespace = inspect.getargvalues(frame).locals if module_name is None: m = inspect.getmodule(inspect.stack()[1][0]) module_name = '__eval__' if m is None else m.__name__ foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 if ast_callback: ast_callback(_ast, expr) if not isinstance(namespace, dict): raise HyTypeError(foo, "Globals must be a dictionary") # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def compare(self): if not self.solution.strip(): return False try: root = ast.parse(self.solution) except: return False soln_nodes = [ type(n).__name__ for n in ast.walk(root) ] key_nodes = [ type(n).__name__ for n in ast.walk(ast.parse(self.answer_key)) ] return soln_nodes == key_nodes
def fix_linenumbers_ast(self, tree): # Hack because of Python bug(?). When creating the tree, some nodes do # not have the _attributes field. for node in ast.walk(tree): if not hasattr(node, '_attributes'): node._attributes = () tree = ast.fix_missing_locations(tree) for node in ast.walk(tree): if isinstance(node, (ast.stmt, ast.expr)): node.lineno -= 1 return tree
def test_roundtrip(cls, source=None): if source is None: # pragma: no cover try: source = inspect.getsource(mnfy) except IOError: pass original_ast = ast.parse(source) minifier = mnfy.SourceCode() minifier.visit(original_ast) minified_source = str(minifier) minified_ast = ast.parse(minified_source) node_pairs = zip(ast.walk(minified_ast), ast.walk(original_ast)) for minified, original in node_pairs: cls.compare_ast_nodes(minified, original) return minified_source
def parse_python(self, node, lineno=0, filename="<string>"): if not node or not node.strip(): return h = bbhash(str(node)) if h in codeparsercache.pythoncache: self.references = set(codeparsercache.pythoncache[h].refs) self.execs = set(codeparsercache.pythoncache[h].execs) self.contains = {} for i in codeparsercache.pythoncache[h].contains: self.contains[i] = set(codeparsercache.pythoncache[h].contains[i]) return if h in codeparsercache.pythoncacheextras: self.references = set(codeparsercache.pythoncacheextras[h].refs) self.execs = set(codeparsercache.pythoncacheextras[h].execs) self.contains = {} for i in codeparsercache.pythoncacheextras[h].contains: self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) return # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though node = "\n" * int(lineno) + node code = compile(check_indent(str(node)), filename, "exec", ast.PyCF_ONLY_AST) for n in ast.walk(code): if n.__class__.__name__ == "Call": self.visit_Call(n) self.execs.update(self.var_execs) codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains)
def parse(codingrule, _seen=()): """ Parse a condition of a codingrule. Returns a dictionary with the first node of the AST generated from `codingrule.condition`. Each node represented as a dict contains a key "type", which indicates which type of node it is. This must be one of OR, AND, NOT, EQUALS and NOT_EQUALS. All nodes (excluding NOT) contain a key 'values' which is a list of operands. Not only has one operand, which is held in 'value'. Each node NOT represented as a dict must be a CodingSchemaField, Code, str or int. A CodingSchemaField is always on the left side of an equation and compared to either a Code, str or int. All CodingRule's will be replaced be their condition and parsed. @raises: SyntaxError, CodingSchemaField.DoesNotExist, Code.DoesNotExist @returns: root of tree, or None if condition is empty """ tree = ast.parse("({})".format(codingrule.condition)).body[0] # This function can be called from parse_node if the condition refers to another # codingrule. Keeping _seen prevents infinite loops when parsing. if codingrule in _seen: raise SyntaxError("Recursion: condition can't contain itself.") # Not all nodes from the Python language are supported for node in ast.walk(tree): if node.__class__ not in KNOWN_NODES or (isinstance(node, ast.Compare) and len(node.ops) > 1): if hasattr(node, "col_offset"): raise SyntaxError("invalid syntax (col {}, line {})".format(node.col_offset, node.lineno)) raise SyntaxError("invalid syntax") return parse_node(tree, _seen=_seen + (codingrule,))
def checkBody(self, funcdef): for bodyNode in funcdef.body: for node in ast.walk(bodyNode): if isinstance(node, ast.Attribute): if isinstance(node.value, ast.Name): if node.value.id != funcdef.args.args[0].id: self.problem("Model variable '{0}' is used in the rule, but this variable is not first argument in the rule argument list".format(node.value.id), lineno=funcdef.lineno)
def _sympify_string(math_string): "Convert math string into SymPy object." # drop pound symbols ('#') since they denote function names # we detect those automatically expr_string = math_string.replace('#', '') # sympify doesn't recognize LN as ln() expr_string = \ re.sub(r'(?<!\w)LN(?!\w)', 'ln', expr_string, flags=re.IGNORECASE) expr_string = \ re.sub(r'(?<!\w)EXP(?!\w)', 'exp', expr_string, flags=re.IGNORECASE) # Convert raw variables into StateVariable objects variable_fixes = { Symbol('T'): v.T, Symbol('P'): v.P, Symbol('R'): v.R } # sympify uses eval, so we need to sanitize the input nodes = ast.parse(expr_string) nodes = ast.Expression(nodes.body[0].value) for node in ast.walk(nodes): if type(node) not in _AST_WHITELIST: #pylint: disable=W1504 raise ValueError('Expression from TDB file not in whitelist: ' '{}'.format(expr_string)) return sympify(expr_string).xreplace(variable_fixes)
def checkBody(self, funcdef): """Look for statements of the format 'return None'""" for node in ast.walk(funcdef): if isinstance(node, ast.Return): if isinstance(node.value, ast.Name): if node.value.id == 'None': self.problem("Cannot return None from model rule {0}".format(funcdef.name), lineno=funcdef.lineno)
def parse_module_imports(root_name, module): """Parse a python module for all 'import' statements and extract the :param root_name: the name of the module being scanned :param module: the source of a python module :returns: all modules imported :rtype: `nx.DiGraph` """ D = Dependencies() D.add_root(root_name) tree = ast.parse(module) for node in ast.walk(tree): if type(node) is ast.Import: for child in node.names: name = child.name D.add_submodule(root_name, name) elif type(node) is ast.ImportFrom: # relative imports (from . import <name1>, <name2>, ...) # the "module" attribute is None if node.module is not None: D.add_submodule(root_name, node.module) for name in node.names: D.add_subattribute(node.module, name.name) else: continue return D
def make_ast_tree(self): """ Returns a modified AST All it does is modify the line numbers so that tracebacks work nicely """ line_numbers = self.line_numbers try: tree = ast.parse(self.data, filename = self.path) except SyntaxError as e: descr, args = e.args args = list(args) args[1] = line_numbers[e.lineno] args[2] = None args[3] = linecache.getline(e.filename, args[1]).strip('\n') e.__init__(descr, args) raise for node in ast.walk(tree): try: node.lineno = line_numbers[node.lineno] except AttributeError: pass return tree
def _find_all_function_calls_by_name(self, name: str) -> List[ast.Call]: """ :param name: name of the function for which all calls should be returned. :return: a list of function calls for function with given name. For example, for foo('bar') bar('foo') foo('baz') _find_all_function_calls_by_name(m, 'foo') will return [ Call( func=Name(id='foo', ctx=Load()), args=[Str(s='bar')], keywords=[])), Call( func=Name(id='foo', ctx=Load()), args=[Str(s='baz')], keywords=[])) ] """ return [ node for node in ast.walk(self.ast_module) if isinstance(node, ast.Call) and node.func.id == name ]
def nits(self): class_methods = set() all_methods = set(function_def for function_def in ast.walk(self.python_file.tree) if isinstance(function_def, ast.FunctionDef)) for class_def in self.iter_ast_types(ast.ClassDef): if not is_upper_camel(class_def.name): yield self.error('T000', 'Classes must be UpperCamelCased', class_def) for class_global in self.iter_class_globals(class_def): if not is_constant(class_global.id) and class_global.id not in self.CLASS_GLOBAL_BUILTINS: yield self.error('T001', 'Class globals must be UPPER_SNAKE_CASED', class_global) if not class_def.bases or all(isinstance(base, ast.Name) and base.id == 'object' for base in class_def.bases): class_methods.update(self.iter_class_methods(class_def)) else: # If the class is inheriting from anything that is potentially a bad actor, rely # upon checking that bad actor out of band. Fixes PANTS-172. for method in self.iter_class_methods(class_def): all_methods.discard(method) for function_def in all_methods - class_methods: if is_reserved_name(function_def.name): yield self.error('T801', 'Method name overrides a builtin.', function_def) # TODO(wickman) Only enforce this for classes that derive from object. If they # don't derive object, it's possible that the superclass naming is out of its # control. for function_def in all_methods: if not any((is_lower_snake(function_def.name), is_builtin_name(function_def.name), is_reserved_with_trailing_underscore(function_def.name))): yield self.error('T002', 'Method names must be lower_snake_cased', function_def)
def count_lines(self,node): childnodes = list(ast.walk(node)) lines = set() for n in childnodes: if hasattr(n,'lineno'): lines.add(n.lineno) return len(lines)
def looks_like_teardown_function(node): returns = [x for x in ast.walk(node) if isinstance(x, ast.Return)] if len(returns) != 1: return return_def = returns[0] resp_name = node.args.args[0] if not isinstance(return_def.value, ast.Name) or return_def.value.id != resp_name.id: return for body_node in node.body: for child in ast.walk(body_node): if isinstance(child, ast.Name) and child.id == resp_name.id: if child is not return_def.value: return return resp_name.id
def extract_strings_from_file(fn): filedata = open(fn, 'r', encoding="utf8") root_node = ast.parse(filedata.read(), fn, 'exec') filedata.close() for node in ast.walk(root_node): if type(node) == ast.Call: # print("found function at") # print("%s:%d" % (fn, node.lineno)) # lambda's if type(node.func) == ast.Name: continue # getattr(self, con.type)(context, box, con) if not hasattr(node.func, "attr"): continue translate_args = func_translate_args.get(node.func.attr, ()) # do nothing if not found for arg_kw, arg_pos in translate_args: if arg_pos < len(node.args): extract_strings(fn, node.args[arg_pos]) else: for kw in node.keywords: if kw.arg == arg_kw: extract_strings(fn, kw.value)
def root_package_name(name): p = ast.parse(name) for n in ast.walk(p): if isinstance(n, ast.Name): return n.id else: return None
def ClassDef_default(t, x): """Converts a class to an ES6 class.""" _class_guards(t, x) name = x.name body = x.body if len(x.bases) > 0: super_name = x.bases[0].id else: super_name = None # strip docs from body body = [e for e in body if isinstance(e, (ast.FunctionDef, ast.AsyncFunctionDef))] # * Each FunctionDef must have self as its first arg # silly check for methods for node in body: arg_names = [arg.arg for arg in node.args.args] t.unsupported(node, len(arg_names) == 0 or arg_names[0] != 'self', "First arg on method must be 'self'") if len(body) > 0 and body[0].name == '__init__': init = body[0] # * __init__ may not contain a return statement # silly check init_args = [arg.arg for arg in init.args.args] init_body = init.body for stmt in ast.walk(init): assert not isinstance(stmt, ast.Return) if super_name: superclass = JSName(super_name) else: superclass = None return JSClass(JSName(name), superclass, body)
def run(self): for node in ast.walk(self.tree): if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): num_positional_args = len(node.args) if (num_positional_args == 1 and isinstance(node.args[0], ast.GeneratorExp) and node.func.id in ("list", "set")): msg_key = {"list": "C400", "set": "C401"}[node.func.id] yield ( node.lineno, node.col_offset, self.messages[msg_key], type(self), ) elif (num_positional_args == 1 and isinstance(node.args[0], (ast.GeneratorExp, ast.ListComp)) and isinstance(node.args[0].elt, ast.Tuple) and len(node.args[0].elt.elts) == 2 and node.func.id == "dict"): if isinstance(node.args[0], ast.GeneratorExp): msg = "C402" else: msg = "C404" yield ( node.lineno, node.col_offset, self.messages[msg], type(self), ) elif (num_positional_args == 1 and isinstance(node.args[0], ast.ListComp) and node.func.id in ("list", "set")): msg_key = {"list": "C411", "set": "C403"}[node.func.id] yield ( node.lineno, node.col_offset, self.messages[msg_key], type(self), ) elif num_positional_args == 1 and ( isinstance(node.args[0], ast.Tuple) and node.func.id == "tuple" or isinstance(node.args[0], ast.List) and node.func.id == "list"): suffix = "remove the outer call to {func}()." msg_key = {"tuple": "C409", "list": "C410"}[node.func.id] msg = self.messages[msg_key] + suffix yield ( node.lineno, node.col_offset, msg.format(type=type(node.args[0]).__name__.lower(), func=node.func.id), type(self), ) elif (num_positional_args == 1 and isinstance(node.args[0], (ast.Tuple, ast.List)) and node.func.id in ("tuple", "list", "set", "dict")): suffix = "rewrite as a {func} literal." msg_key = { "tuple": "C409", "list": "C410", "set": "C405", "dict": "C406", }[node.func.id] msg = self.messages[msg_key] + suffix yield ( node.lineno, node.col_offset, msg.format(type=type(node.args[0]).__name__.lower(), func=node.func.id), type(self), ) elif (num_positional_args == 1 # These take 1 positional argument + some keyword arguments and (node.func.id in ( "all", "any", "frozenset", "tuple", "max", "min", "sorted", )) and isinstance(node.args[0], ast.ListComp)): yield ( node.lineno, node.col_offset, self.messages["C407"].format(func=node.func.id), type(self), ) elif (num_positional_args in (1, 2) # These can take a second positional argument and (node.func.id in ( "enumerate", "sum", )) and isinstance(node.args[0], ast.ListComp)): yield ( node.lineno, node.col_offset, self.messages["C407"].format(func=node.func.id), type(self), ) elif (num_positional_args == 2 and node.func.id == "filter" and isinstance(node.args[1], ast.ListComp)): yield ( node.lineno, node.col_offset, self.messages["C407"].format(func=node.func.id), type(self), ) elif (num_positional_args >= 2 and node.func.id == "map" and any( isinstance(a, ast.ListComp) for a in node.args[1:])): yield ( node.lineno, node.col_offset, self.messages["C407"].format(func=node.func.id), type(self), ) elif (num_positional_args == 0 and not has_star_args(node) and not has_keyword_args(node) and node.func.id in ("tuple", "list", "dict")): yield ( node.lineno, node.col_offset, self.messages["C408"].format(type=node.func.id), type(self), ) elif (node.func.id in {"list", "reversed"} and num_positional_args > 0 and isinstance(node.args[0], ast.Call) and isinstance(node.args[0].func, ast.Name) and node.args[0].func.id == "sorted"): remediation = "" if node.func.id == "reversed": reverse_flag_value = False for keyword in node.args[0].keywords: if keyword.arg != "reverse": continue if isinstance(keyword.value, ast.NameConstant): reverse_flag_value = keyword.value.value elif isinstance(keyword.value, ast.Num): reverse_flag_value = bool(keyword.value.n) else: # Complex value reverse_flag_value = None if reverse_flag_value is None: remediation = " - toggle reverse argument to sorted()" else: remediation = " - use sorted(..., reverse={!r})".format( not reverse_flag_value) msg = self.messages["C413"].format( inner=node.args[0].func.id, outer=node.func.id, remediation=remediation, ) yield ( node.lineno, node.col_offset, msg, type(self), ) elif (num_positional_args > 0 and isinstance(node.args[0], ast.Call) and isinstance(node.args[0].func, ast.Name) and ((node.func.id in {"set", "sorted"} and node.args[0].func.id in {"list", "reversed", "sorted", "tuple"}) or (node.func.id in {"list", "tuple"} and node.args[0].func.id in {"list", "tuple"}) or (node.func.id == "set" and node.args[0].func.id == "set"))): yield ( node.lineno, node.col_offset, self.messages["C414"].format( inner=node.args[0].func.id, outer=node.func.id), type(self), ) elif (node.func.id in {"reversed", "set", "sorted"} and num_positional_args > 0 and isinstance(node.args[0], ast.Subscript) and isinstance(node.args[0].slice, ast.Slice) and node.args[0].slice.lower is None and node.args[0].slice.upper is None and isinstance(node.args[0].slice.step, ast.UnaryOp) and isinstance(node.args[0].slice.step.op, ast.USub) and isinstance(node.args[0].slice.step.operand, ast.Num) and node.args[0].slice.step.operand.n == 1): yield ( node.lineno, node.col_offset, self.messages["C415"].format(func=node.func.id), type(self), ) elif isinstance(node, ast.Compare): if (len(node.ops) == 1 and isinstance(node.ops[0], ast.In) and len(node.comparators) == 1 and isinstance( node.comparators[0], (ast.DictComp, ast.ListComp, ast.SetComp))): yield ( node.lineno, node.col_offset, self.messages["C412"].format( type=comp_type[node.comparators[0].__class__]), type(self), ) elif isinstance(node, (ast.ListComp, ast.SetComp)): if (len(node.generators) == 1 and not node.generators[0].ifs and not is_async_generator(node.generators[0]) and (isinstance(node.elt, ast.Name) and isinstance(node.generators[0].target, ast.Name) and node.elt.id == node.generators[0].target.id)): yield ( node.lineno, node.col_offset, self.messages["C416"].format( type=comp_type[node.__class__]), type(self), )
def getCallSourceLines(callFrame, icNames, icMethod): """Raises NoSourceAvailableError.""" code = callFrame.f_code # inspect.getblock(), which is called internally by inspect.getsource(), # only returns the first line of <code> when <code> represents a top-level # module, not the entire module's source, as needed here. The # # if ismodule(object): # return lines, 0 # # check in inspect.py doesn't account for code objects of modules, only # actual module objects themselves. # # A workaround is to call findsource() directly on code objects of modules, # which bypasses getblock(). # # Also, the errors raised differ between Python2 and Python3 . In Python2, # inspect.findsource() and inspect.getsource() raise IOErrors. In Python3, # inspect.findsource() and inspect.getsource() raise OSErrors. try: if code.co_name == '<module>': # Module -> use workaround above. parentBlockStartLine = 1 lines = inspect.findsource(code)[0] # Raises [IO/OS]Error. parentBlockSource = ''.join(lines) else: # Not a module -> use inspect.getsource() normally. parentBlockStartLine = code.co_firstlineno parentBlockSource = inspect.getsource(code) # Raises [IO/OS]Error. except (IOError, OSError) as err: if 'source code' in err.args[0]: raise NoSourceAvailableError() else: raise lineno = inspect.getframeinfo(callFrame)[1] linenoRelativeToParent = lineno - parentBlockStartLine + 1 # There could be multiple ic() calls on the same line(s), like # # ic(1); ic(2); ic(3, # 4, # 5); ic(6) # # so include all of them. Which invocation is the appropriate one will be # determined later via bytecode offset calculations. # # TODO(grun): Support invocations of ic() where ic() is an attribute chain # in the AST. For example, support # # import icecream # icecream.ic() # # and # # class Foo: # blah = ic # Foo.blah() # parentBlockSource = textwrap.dedent(parentBlockSource) potentialCalls = [ node for node in ast.walk(ast.parse(parentBlockSource)) if isAstNodeIceCreamCall(node, icNames, icMethod) and ( node.lineno == linenoRelativeToParent or any( arg.lineno == linenoRelativeToParent for arg in node.args)) ] if not potentialCalls: # TODO(grun): Add note that to NoSourceAvailableError that this # situation can occur when the underlying source changed during # execution. raise NoSourceAvailableError() endLine = lineno - parentBlockStartLine + 1 startLine = min(call.lineno for call in potentialCalls) lines = parentBlockSource.splitlines()[startLine - 1:endLine] # inspect's lineno attribute doesn't point to the closing right parenthesis # if the closing right parenthesis is on its own line without any # arguments. E.g. # # ic(1, # 2 <--- inspect's reported lineno. # ) <--- Should be the reported lineno. # # Detect this situation and add the missing right parenthesis. if isCallStrMissingClosingRightParenthesis('\n'.join(lines).strip()): lines.append(')') source = stripCommentsAndNewlines('\n'.join(lines)).strip() absoluteStartLineNum = parentBlockStartLine + startLine - 1 startLineOffset = calculateLineOffsets(code)[absoluteStartLineNum] return source, absoluteStartLineNum, startLineOffset
def dump_py_messages_from_files(msgs, reports, files, settings): """ Dump text inlined in the python files given, e.g. 'My Name' in: layout.prop("someprop", text="My Name") """ import ast bpy_struct = bpy.types.ID.__base__ i18n_contexts = bpy.app.translations.contexts root_paths = tuple( bpy.utils.resource_path(t) for t in ('USER', 'LOCAL', 'SYSTEM')) def make_rel(path): for rp in root_paths: if path.startswith(rp): try: # can't always find the relative path (between drive letters on windows) return os.path.relpath(path, rp) except ValueError: return path # Use binary's dir as fallback... try: # can't always find the relative path (between drive letters on windows) return os.path.relpath(path, os.path.dirname(bpy.app.binary_path)) except ValueError: return path # Helper function def extract_strings_ex(node, is_split=False): """ Recursively get strings, needed in case we have "Blah" + "Blah", passed as an argument in that case it won't evaluate to a string. However, break on some kind of stopper nodes, like e.g. Subscript. """ if type(node) == ast.Str: eval_str = ast.literal_eval(node) if eval_str: yield (is_split, eval_str, (node, )) else: is_split = (type(node) in separate_nodes) for nd in ast.iter_child_nodes(node): if type(nd) not in stopper_nodes: yield from extract_strings_ex(nd, is_split=is_split) def _extract_string_merge(estr_ls, nds_ls): return "".join(s for s in estr_ls if s is not None), tuple(n for n in nds_ls if n is not None) def extract_strings(node): estr_ls = [] nds_ls = [] for is_split, estr, nds in extract_strings_ex(node): estr_ls.append(estr) nds_ls.extend(nds) ret = _extract_string_merge(estr_ls, nds_ls) return ret def extract_strings_split(node): """ Returns a list args as returned by 'extract_strings()', but split into groups based on separate_nodes, this way expressions like ("A" if test else "B") won't be merged but "A" + "B" will. """ estr_ls = [] nds_ls = [] bag = [] for is_split, estr, nds in extract_strings_ex(node): if is_split: bag.append((estr_ls, nds_ls)) estr_ls = [] nds_ls = [] estr_ls.append(estr) nds_ls.extend(nds) bag.append((estr_ls, nds_ls)) return [ _extract_string_merge(estr_ls, nds_ls) for estr_ls, nds_ls in bag ] i18n_ctxt_ids = {v for v in bpy.app.translations.contexts_C_to_py.values()} def _ctxt_to_ctxt(node): # We must try, to some extend, to get contexts from vars instead of only literal strings... ctxt = extract_strings(node)[0] if ctxt: return ctxt # Basically, we search for attributes matching py context names, for now. # So non-literal contexts should be used that way: # i18n_ctxt = bpy.app.translations.contexts # foobar(text="Foo", text_ctxt=i18n_ctxt.id_object) if type(node) == ast.Attribute: if node.attr in i18n_ctxt_ids: #print(node, node.attr, getattr(i18n_contexts, node.attr)) return getattr(i18n_contexts, node.attr) return i18n_contexts.default def _op_to_ctxt(node): # Some smart coders like things like: # >>> row.operator("preferences.addon_disable" if is_enabled else "preferences.addon_enable", ...) # We only take first arg into account here! bag = extract_strings_split(node) opname, _ = bag[0] if not opname: return i18n_contexts.default op = bpy.ops for n in opname.split('.'): op = getattr(op, n) try: return op.get_rna_type().translation_context except Exception as e: default_op_context = i18n_contexts.operator_default print("ERROR: ", str(e)) print(" Assuming default operator context '{}'".format( default_op_context)) return default_op_context # Gather function names. # In addition of UI func, also parse pgettext ones... # Tuples of (module name, (short names, ...)). pgettext_variants = ( ("pgettext", ("_", )), ("pgettext_iface", ("iface_", )), ("pgettext_tip", ("tip_", )), ("pgettext_data", ("data_", )), ) pgettext_variants_args = {"msgid": (0, {"msgctxt": 1})} # key: msgid keywords. # val: tuples of ((keywords,), context_getter_func) to get a context for that msgid. # Note: order is important, first one wins! translate_kw = { "text": ( (("text_ctxt", ), _ctxt_to_ctxt), (("operator", ), _op_to_ctxt), ), "msgid": ((("msgctxt", ), _ctxt_to_ctxt), ), "message": (), } context_kw_set = {} for k, ctxts in translate_kw.items(): s = set() for c, _ in ctxts: s |= set(c) context_kw_set[k] = s # {func_id: {msgid: (arg_pos, # {msgctxt: arg_pos, # ... # } # ), # ... # }, # ... # } func_translate_args = {} # First, functions from UILayout # First loop is for msgid args, second one is for msgctxt args. for func_id, func in bpy.types.UILayout.bl_rna.functions.items(): # check it has one or more arguments as defined in translate_kw for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()): if ((arg_kw in translate_kw) and (not arg.is_output) and (arg.type == 'STRING')): func_translate_args.setdefault(func_id, {})[arg_kw] = (arg_pos, {}) for func_id, func in bpy.types.UILayout.bl_rna.functions.items(): if func_id not in func_translate_args: continue for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()): if (not arg.is_output) and (arg.type == 'STRING'): for msgid, msgctxts in context_kw_set.items(): if arg_kw in msgctxts: func_translate_args[func_id][msgid][1][ arg_kw] = arg_pos # The report() func of operators. for func_id, func in bpy.types.Operator.bl_rna.functions.items(): # check it has one or more arguments as defined in translate_kw for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()): if ((arg_kw in translate_kw) and (not arg.is_output) and (arg.type == 'STRING')): func_translate_args.setdefault(func_id, {})[arg_kw] = (arg_pos, {}) # We manually add funcs from bpy.app.translations for func_id, func_ids in pgettext_variants: func_translate_args[func_id] = pgettext_variants_args for func_id in func_ids: func_translate_args[func_id] = pgettext_variants_args # print(func_translate_args) # Break recursive nodes look up on some kind of nodes. # E.g. we don't want to get strings inside subscripts (blah["foo"])! # we don't want to get strings from comparisons (foo.type == 'BAR'). stopper_nodes = {ast.Subscript, ast.Compare} # Consider strings separate: ("a" if test else "b") separate_nodes = {ast.IfExp} check_ctxt_py = None if reports["check_ctxt"]: check_ctxt = reports["check_ctxt"] check_ctxt_py = { "py_in_rna": (check_ctxt.get("py_in_rna"), set(msgs.keys())), "multi_lines": check_ctxt.get("multi_lines"), "not_capitalized": check_ctxt.get("not_capitalized"), "end_point": check_ctxt.get("end_point"), "spell_checker": check_ctxt.get("spell_checker"), "spell_errors": check_ctxt.get("spell_errors"), } for fp in files: with open(fp, 'r', encoding="utf8") as filedata: root_node = ast.parse(filedata.read(), fp, 'exec') fp_rel = make_rel(fp) for node in ast.walk(root_node): if type(node) == ast.Call: # print("found function at") # print("%s:%d" % (fp, node.lineno)) # We can't skip such situations! from blah import foo\nfoo("bar") would also be an ast.Name func! if type(node.func) == ast.Name: func_id = node.func.id elif hasattr(node.func, "attr"): func_id = node.func.attr # Ugly things like getattr(self, con.type)(context, box, con) else: continue func_args = func_translate_args.get(func_id, {}) # First try to get i18n contexts, for every possible msgid id. msgctxts = dict.fromkeys(func_args.keys(), "") for msgid, (_, context_args) in func_args.items(): context_elements = {} for arg_kw, arg_pos in context_args.items(): if arg_pos < len(node.args): context_elements[arg_kw] = node.args[arg_pos] else: for kw in node.keywords: if kw.arg == arg_kw: context_elements[arg_kw] = kw.value break # print(context_elements) for kws, proc in translate_kw[msgid]: if set(kws) <= context_elements.keys(): args = tuple(context_elements[k] for k in kws) #print("running ", proc, " with ", args) ctxt = proc(*args) if ctxt: msgctxts[msgid] = ctxt break # print(translate_args) # do nothing if not found for arg_kw, (arg_pos, _) in func_args.items(): msgctxt = msgctxts[arg_kw] estr_lst = [(None, ())] if arg_pos < len(node.args): estr_lst = extract_strings_split(node.args[arg_pos]) #print(estr, nds) else: for kw in node.keywords: if kw.arg == arg_kw: estr_lst = extract_strings_split(kw.value) break #print(estr, nds) for estr, nds in estr_lst: if estr: if nds: msgsrc = "{}:{}".format( fp_rel, sorted({nd.lineno for nd in nds})[0]) else: msgsrc = "{}:???".format(fp_rel) process_msg(msgs, msgctxt, estr, msgsrc, reports, check_ctxt_py, settings) reports["py_messages"].append( (msgctxt, estr, msgsrc))
def _find_decorators_node_to_validate(self) -> None: for node in ast.walk(self.tree): if isinstance(node, ast.FunctionDef): decorators = node.decorator_list self._validate_decorators(decorators) # type: ignore
def debug(victim=None, ignore_exceptions=(BdbQuit,), catch_exception=None, depth=0): """A decorator function to catch exceptions and enter debug mode. Args: victim (typing.Union(type, function)): either a class or function to wrap and debug. ignore_exceptions (list): list of classes of exceptions not to catch. catch_exception (type): class of exception to catch and debug. default is None, meaning catch all exceptions. depth (number): how many levels of inner function calls to propagate. Returns: object. wrapped class or function. Note: This wrapper avoids recursion by setting a flag to each wrapped item. """ if victim is None: # Debug is used as a decorator so we need to return wrap function to # get the real victim def wrapper(real_victim): return debug(real_victim, ignore_exceptions, catch_exception, depth) return wrapper register_break_signal() if inspect.isfunction(victim): if hasattr(victim, '_ipdebug_wrapped'): # Don't wrap the function more than once return victim _transformer = ErrorsCatchTransformer( ignore_exceptions=ignore_exceptions, catch_exception=catch_exception, depth=depth) try: # Try to get the source code of the wrapped object. sourcelines, start_num = inspect.getsourcelines(victim.__code__) indent = re.match(r'\s*', sourcelines[0]).group() source = ''.join(l.replace(indent, '', 1) for l in sourcelines) except IOError: # Worst-case scenario we can only catch errors at a granularity # of the whole function return victim else: # If we have access to the source, we can silence errors on a # per-expression basis, which is "better" old_code_tree = ast.parse(source) # Reposition the line numbers to their original value ast.increment_lineno(old_code_tree, start_num - 1) tree = _transformer.visit(old_code_tree) import_debug_cmd = ast.ImportFrom( __name__, [ast.alias("start_debugging", None), ast.alias("debug", None)], 0) # Add import to the debugger as first command tree.body[0].body.insert(0, import_debug_cmd) # Add import to the exception classes if catch_exception is not None: import_exception_cmd = ast.ImportFrom( catch_exception.__module__, [ast.alias(catch_exception.__name__, None)], 0) tree.body[0].body.insert(1, import_exception_cmd) if ignore_exceptions is not None: for exception_class in ignore_exceptions: import_exception_cmd = ast.ImportFrom( exception_class.__module__, [ast.alias(exception_class.__name__, None)], 0) tree.body[0].body.insert(1, import_exception_cmd) # Delete the debugger decorator of the function del tree.body[0].decorator_list[:] # Add pass at the end (to enable debugging the last command) pass_cmd = ast.Pass() func_body = tree.body[0].body pass_cmd.lineno = get_last_lineno(func_body[-1]) + 1 pass_cmd.col_offset = func_body[-1].col_offset func_body.append(pass_cmd) # Fix missing line numbers and column offsets before compiling for node in ast.walk(tree): if not hasattr(node, 'lineno'): node.lineno = 0 ast.fix_missing_locations(tree) # Define the wrapping function object function_definition = "def _free_vars_wrapper(): pass" wrapping_function = ast.parse(function_definition).body[0] # Initialize closure's variables to None body_list = [ast.parse("{var} = None".format(var=free_var)).body[0] for free_var in victim.__code__.co_freevars] # Add the original function ("victim") to the wrapping function body_list.append(tree.body[0]) wrapping_function.body = body_list # Replace original function ("victim") with the wrapping function tree.body[0] = wrapping_function # Create a new runnable code object to replace the original code code = compile(tree, victim.__code__.co_filename, 'exec') victim.__code__ = code.co_consts[0].co_consts[1] # Set a flag to indicate that the method was wrapped victim._ipdebug_wrapped = True return victim elif inspect.ismethod(victim): debug(victim.__func__, ignore_exceptions, catch_exception) return victim elif isinstance(victim, type): # Wrap each method of the class with the debugger for name, member in vars(victim).items(): if isinstance(member, (type, types.FunctionType, types.LambdaType, types.MethodType)): setattr(victim, name, debug(member, ignore_exceptions, catch_exception)) return victim else: raise TypeError( "Debugger can only wrap functions and classes. " "Got object {!r} of type {}".format(victim, type(victim).__name__))
async def meval(code, globs, **kwargs): # Note to self: please don't set globals here as they will be lost. # Don't clutter locals locs = {} # Restore globals later globs = globs.copy() # This code saves __name__ and __package into a kwarg passed to the func. # It is set before the users code runs to make sure relative imports work global_args = "_globs" while global_args in globs.keys(): # Make sure there's no name collision, just keep prepending _s global_args = "_" + global_args kwargs[global_args] = {} for glob in ["__name__", "__package__"]: # Copy data to args we are sending kwargs[global_args][glob] = globs[glob] root = ast.parse(code, "exec") code = root.body ret_name = "_ret" ok = False while True: if ret_name in globs.keys(): ret_name = "_" + ret_name continue for node in ast.walk(root): if isinstance(node, ast.Name) and node.id == ret_name: ret_name = "_" + ret_name break ok = True if ok: break if not code: return None if not any(isinstance(node, ast.Return) for node in code): for i in range(len(code)): if isinstance(code[i], ast.Expr): if (i == len(code) - 1 or not isinstance(code[i].value, ast.Call)): code[i] = ast.copy_location( ast.Expr( ast.Call(func=ast.Attribute(value=ast.Name( id=ret_name, ctx=ast.Load()), attr="append", ctx=ast.Load()), args=[code[i].value], keywords=[])), code[-1]) else: for node in code: if isinstance(node, ast.Return): node.value = ast.List(elts=[node.value], ctx=ast.Load()) code.append( ast.copy_location( ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1])) # globals().update(**<global_args>) glob_copy = ast.Expr( ast.Call( func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals", ctx=ast.Load()), args=[], keywords=[]), attr="update", ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg=None, value=ast.Name(id=global_args, ctx=ast.Load())) ])) ast.fix_missing_locations(glob_copy) code.insert(0, glob_copy) ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())], value=ast.List(elts=[], ctx=ast.Load())) ast.fix_missing_locations(ret_decl) code.insert(1, ret_decl) args = [] for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())): ast.fix_missing_locations(a) args += [a] args = ast.arguments(args=[], vararg=None, kwonlyargs=args, kwarg=None, defaults=[], kw_defaults=[None for i in range(len(args))]) args.posonlyargs = [] fun = ast.AsyncFunctionDef(name="tmp", args=args, body=code, decorator_list=[], returns=None) ast.fix_missing_locations(fun) mod = ast.parse("") mod.body = [fun] comp = compile(mod, "<string>", "exec") exec(comp, {}, locs) r = await locs["tmp"](**kwargs) for i in range(len(r)): if hasattr(r[i], "__await__"): r[i] = await r[i] # workaround for 3.5 i = 0 while i < len(r) - 1: if r[i] is None: del r[i] else: i += 1 if len(r) == 1: [r] = r elif not r: r = None return r
one += one one -= one one | one a[0] += 100 5 < 3 not True del apple["Hearted"] import garbage 8 is 7 """) print("iter_fields:", ast.iter_fields(multiline.body[0])) print("iter_child_nodes:", ast.iter_child_nodes(multiline)) print("walk:", ast.walk(multiline)) print(ast.dump(multiline, True, False)) print("*" * 40) class VisitStuff(ast.NodeVisitor): def __init__(self): self.nums = 0 def visit_Num(self, node): print("Found a ", node.n) self.nums += 1 vs = VisitStuff()
def get_all_imports(path, encoding=None, extra_ignore_dirs=None, follow_links=True): imports = set() raw_imports = set() candidates = [] ignore_errors = False ignore_dirs = [".hg", ".svn", ".git", ".tox", "__pycache__", "env", "venv"] if extra_ignore_dirs: ignore_dirs_parsed = [] for e in extra_ignore_dirs: ignore_dirs_parsed.append(os.path.basename(os.path.realpath(e))) ignore_dirs.extend(ignore_dirs_parsed) walk = os.walk(path, followlinks=follow_links) for root, dirs, files in walk: dirs[:] = [d for d in dirs if d not in ignore_dirs] candidates.append(os.path.basename(root)) files = [fn for fn in files if os.path.splitext(fn)[1] == ".py"] candidates += [os.path.splitext(fn)[0] for fn in files] for file_name in files: file_name = os.path.join(root, file_name) with open_func(file_name, "r", encoding=encoding) as f: contents = f.read() try: tree = ast.parse(contents) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: raw_imports.add(subnode.name) elif isinstance(node, ast.ImportFrom): raw_imports.add(node.module) except Exception as exc: if ignore_errors: traceback.print_exc(exc) logging.warn("Failed on file: %s" % file_name) continue else: logging.error("Failed on file: %s" % file_name) raise exc # Clean up imports for name in [n for n in raw_imports if n]: # Sanity check: Name could have been None if the import # statement was as ``from . import X`` # Cleanup: We only want to first part of the import. # Ex: from django.conf --> django.conf. But we only want django # as an import. cleaned_name, _, _ = name.partition('.') imports.add(cleaned_name) packages = imports - (set(candidates) & imports) logging.debug('Found packages: {0}'.format(packages)) with open(join("stdlib"), "r") as f: data = {x.strip() for x in f} data = {x for x in data if x not in py2_exclude} if py2 else data return list(packages - data)
counts = {} for subdir, dirs, files in os.walk('/home/u180133/PROGECT/qualitas/'): for file in files: #print os.path.join(subdir, file) filepath = subdir + os.sep + file if filepath.endswith(".py"): print filepath + '\n' try: tree = ast.parse(open(filepath).read()) except SyntaxError: print 'Syntax Error Keep Her Lit' pass for node in ast.walk(tree): #print type(node).__name__ nt = type(node).__name__ if nt not in counts: counts[nt] = 0 counts[nt] += 1 print '\n' print counts with open('/home/u180133/PROGECT/allNodeCount.txt', 'a') as bean: #bean.write('Hello World' +'\n') #bean.write( filepath+'\n') json.dump(counts, bean)
def find_unprefixed_string_literals(filename): u""" Find unprefixed string literals in a Python source file. Returns a list of ``(line_number, column)`` tuples (both 1-based) of positions where string literals without a ``u`` or ``b`` prefix start. Note: Due to limitations in Python's ``ast`` module this does not check the rear parts of auto-concatenated string literals (``'foo' 'bar'``). """ with io.open(filename, encoding=u"utf-8") as f: lines = f.readlines() # In some versions of Python, the ast module cannot deal with # encoding declarations (http://bugs.python.org/issue22221). We # therefore replace all comment lines at the beginning of the file # with empty lines (to keep the line numbers correct). for i, line in enumerate(lines): line = line.strip() if line.startswith(u"#"): lines[i] = u"\n" elif line: break root = ast.parse(u"".join(lines), filename.encode(FILESYSTEM_ENCODING)) problems = [] for node in ast.walk(root): if isinstance(node, ast.Str): lineno = node.lineno - 1 col_offset = node.col_offset if col_offset == -1: # `lineno` and `col_offset` are broken for literals that span # multiple lines: For these, `lineno` contains the line of the # *closing* quotes, and `col_offset` is always -1, see # https://bugs.python.org/issue16806. We therefore have to # find the start of the literal manually, which is difficult # since '''-literals can contain """ and vice versa. The # following code assumes that no ''' or """ literal begins on # the same line where a multi-line literal ends. last_line = lines[lineno] if last_line.rfind(u'"""') > last_line.rfind(u"'''"): quotes = u'"""' else: quotes = u"'''" for lineno, line in renumerate(lines[:lineno]): try: i = line.rindex(quotes) if (i > 1) and (line[i - 2:i].lower() == u"ur"): col_offset = i - 2 elif (i > 0) and (line[i - 1].lower() in u"rbu"): col_offset = i - 1 else: col_offset = 0 break except ValueError: continue leading = lines[lineno][col_offset - 1:col_offset + 1] if leading[:-1] == u"[": # data['id'] is unambiguous, ignore these continue if leading[-1:] not in u"ub": # Don't allow capital U and B either problems.append((lineno + 1, col_offset + 1)) return sorted(problems)
def find_classes(module): class_nodes = [c for c in ast.walk(module) if isinstance(c, ast.ClassDef)] return class_nodes
def gather_names(node): """Returns the set of all names present in the node's tree.""" rtn = set(map(get_id, walk(node))) rtn.discard(None) return rtn
def get_config_assignments(config: Path) -> Dict[str, LINES_RANGE]: ast_nodes = ast.walk(ast.parse(config.read_text())) assignments = dict( extract_assignment(ast_node) for ast_node in ast_nodes if isinstance(ast_node, ast.Assign)) return assignments
def min_line(node): """Computes the minimum lineno.""" node_line = get_lineno(node) return min(map(get_lineno, walk(node), itertools.repeat(node_line)))
def check_expression( text ): """ >>> check_expression("c1=='chr1' and c3-c2>=2000 and c6=='+'") True >>> check_expression("eval('1+1')") False >>> check_expression("import sys") False >>> check_expression("[].__str__") False >>> check_expression("__builtins__") False >>> check_expression("'x' in globals") False >>> check_expression("'x' in [1,2,3]") True >>> check_expression("c3=='chr1' and c5>5") True >>> check_expression("c3=='chr1' and d5>5") # Invalid d5 reference False >>> check_expression("c3=='chr1' and c5>5 or exec") False >>> check_expression("type(c1) != type(1)") True >>> check_expression("c1.split(',')[1] == '1'") True >>> check_expression("exec 1") False >>> check_expression("str(c2) in [\\\"a\\\",\\\"b\\\"]") True """ try: module = parse( text ) except SyntaxError: return False if not isinstance(module, Module): return False statements = module.body if not len( statements ) == 1: return False expression = statements[0] if expression.__class__.__name__ != 'Expr': return False for ast_node in walk( expression ): ast_node_class = ast_node.__class__.__name__ # Toss out everything that is not a "simple" expression, # imports, error handling, etc... if ast_node_class not in AST_NODE_TYPE_WHITELIST: return False # White-list more potentially dangerous types AST elements. if ast_node_class == 'Name': # In order to prevent loading 'exec', 'eval', etc... # put string restriction on names allowed. if not __check_name( ast_node ): return False # Check only valid, white-listed functions are called. elif ast_node_class == 'Call': if not __check_call( ast_node ): return False # Check only valid, white-listed attributes are accessed elif ast_node_class == 'Attribute': if not __check_attribute( ast_node ): return False return True
def max_line(node): """Computes the maximum lineno.""" return max(map(get_lineno, walk(node)))
def _update(self): """update tkk """ # we don't need to update the base TKK value when it is still valid now = math.floor(int(time.time() * 1000) / 3600000.0) if self.tkk and int(self.tkk.split('.')[0]) == now: return r = self.session.get(self.host) rawtkk = self.RE_RAWTKK.search(r.text) if rawtkk: self.tkk = rawtkk.group(1) return # this will be the same as python code after stripping out a reserved word 'var' code = unicode(self.RE_TKK.search(r.text).group(1)).replace('var ', '') # unescape special ascii characters such like a \x3d(=) if PY3: # pragma: no cover code = code.encode().decode('unicode-escape') else: # pragma: no cover code = code.decode('string_escape') if code: tree = ast.parse(code) visit_return = False operator = '+' n, keys = 0, dict(a=0, b=0) for node in ast.walk(tree): if isinstance(node, ast.Assign): name = node.targets[0].id if name in keys: if isinstance(node.value, ast.Num): keys[name] = node.value.n # the value can sometimes be negative elif isinstance(node.value, ast.UnaryOp) and \ isinstance(node.value.op, ast.USub): # pragma: nocover keys[name] = -node.value.operand.n elif isinstance(node, ast.Return): # parameters should be set after this point visit_return = True elif visit_return and isinstance(node, ast.Num): n = node.n elif visit_return and n > 0: # the default operator is '+' but implement some more for # all possible scenarios if isinstance(node, ast.Add): # pragma: nocover pass elif isinstance(node, ast.Sub): # pragma: nocover operator = '-' elif isinstance(node, ast.Mult): # pragma: nocover operator = '*' elif isinstance(node, ast.Pow): # pragma: nocover operator = '**' elif isinstance(node, ast.BitXor): # pragma: nocover operator = '^' # a safety way to avoid Exceptions clause = compile( '{1}{0}{2}'.format(operator, keys['a'], keys['b']), '', 'eval') value = eval(clause, dict(__builtin__={})) result = '{}.{}'.format(n, value) self.tkk = result
def _test_sort(elements, _type): values = [e.value for e in elements] for wrong, right in zip(values, natsorted(values)): if wrong != right: print( f"{_type} is not sorted, {right!r} should come before {wrong!r}" ) return True return False if len(sys.argv) == 1: print("Usage: sort.py [filename]") sys.exit(1) with open(sys.argv[1]) as f: contents = f.read() fail = False for node in ast.walk(ast.parse(contents)): if type(node) == ast.List: fail = _test_sort(node.elts, "List") or fail if type(node) == ast.Set: fail = _test_sort(node.elts, "Set") or fail if type(node) == ast.Dict: fail = _test_sort(node.keys, "Dict") or fail sys.exit(fail)
def run(self): for node in ast.walk(self.tree): for rule in ('I200', 'I201'): for err in getattr(self, 'rule_{}'.format(rule))(node): yield err
def _maybe_ignore_child(self, node: ast.AST) -> bool: if isinstance(node, ast.AnnAssign): self._to_ignore.extend(ast.walk(node.annotation)) return node in self._to_ignore
def iter_excepts(cls, tree): for ast_node in ast.walk(tree): if isinstance(ast_node, ast.TryExcept): yield ast_node
def read_project(root_path, verbose=False, ignore=None, encoding=None): """ Reads project into an AST and transforms imports into Nodes :param root_path: String :return: Node """ nodes = {} root_node = None errors = False ignore_files = set( [".hg", ".svn", ".git", ".tox", "__pycache__", "env", "venv"]) # python 2.6 comp if ignore: for ignored_file in ignore: ignore_files.add(os.path.basename(os.path.realpath(ignored_file))) # traverse root directory, and list directories as dirs and files as files for root, dirs, files in os.walk(root_path): dirs[:] = [d for d in dirs if d not in ignore_files] files = [ fn for fn in files if os.path.splitext(fn)[1] == ".py" and fn not in ignore_files ] for file_name in files: full_path = os.path.join(root, file_name) with open_func(full_path, "r", encoding=encoding) as f: try: # fails on empty files file_data = f.read() lines = file_data.splitlines() tree = ast.parse(file_data) if verbose: click.echo( crayons.yellow( 'Trying to parse file: {}'.format(full_path))) if full_path in nodes: node = nodes[full_path] else: node = Node(file_name[:-3], full_path=full_path) nodes[full_path] = node if not root_node: root_node = node for ast_node in ast.walk(tree): if isinstance(ast_node, ast.Import) and ast_node.names: for subnode in ast_node.names: if not subnode.name: continue path_to_module = get_path_from_package_name( root_path, subnode.name) if path_to_module in nodes: new_node = nodes[path_to_module] else: new_node = Node(subnode.name, full_path=path_to_module) nodes[path_to_module] = new_node new_node.is_imported_from[full_path].append( ast_node.lineno) node.add(new_node) elif isinstance(ast_node, ast.ImportFrom) and ast_node.module: current_path = root_path if 0 <= ast_node.lineno - 1 < len(lines) and\ REGEX_RELATIVE_PATTERN.findall(lines[ast_node.lineno - 1]): current_path = root path_to_module = get_path_from_package_name( current_path, ast_node.module) if path_to_module in nodes: new_node = nodes[path_to_module] else: new_node = Node(ast_node.module, full_path=path_to_module) nodes[path_to_module] = new_node for obj_import in ast_node.names: if ast_node.lineno not in node.func_imports: node.func_imports[ast_node.lineno] = [ obj_import.name ] else: node.func_imports[ast_node.lineno].append( obj_import.name) new_node.is_imported_from[full_path].append( ast_node.lineno) node.add(new_node) elif isinstance(ast_node, (ast.ClassDef, ast.FunctionDef)): node.func_defs[ast_node.name] = ast_node.lineno except Exception as e: errors = True click.echo( crayons.yellow( 'Parsing of file failed: {}'.format(full_path))) if verbose: click.echo(crayons.red(traceback.format_exc(e))) if errors: click.echo( crayons.red( 'There were errors during the operation, perhaps you are trying to parse python 3 project, ' 'with python 2 version of the script? (or vice versa)')) return root_node
def visit_Call(self, node): for child in ast.walk(node): if isinstance(child, ast.Name): self.cache.append(child.id)
def _check_contains_yield(self, node: AnyComprehension) -> None: for sub_node in ast.walk(node): if isinstance(sub_node, ast.Yield): # pragma: py-gte-38 self.add_violation(YieldInComprehensionViolation(node))
def find_decorated_funcs(tree): for node in ast.walk(tree): if isinstance(node, ast.FunctionDef): if hasattr(node, "decorator_list"): yield node
def getCallSourceLines(funcName, callFrame): code = callFrame.f_code # inspect.getblock(), which is called internally by inspect.getsource(), # only returns the first line of <code> when <code> represents a top-level # module, not the entire module's source, as needed here. The # # if ismodule(object): return lines, 0 # # check in inspect.py doesn't account for code objects that represent # modules, only module objects themselves. # # A workaround is to call findsource() directly on top-level modules, which # bypasses getblock(). if code.co_name == '<module>': # Module -> use workaround explained above. parentBlockStartLine = 1 parentBlockSource = ''.join(inspect.findsource(code)[0]) else: # Not a module -> use inspect.getsource() normally. parentBlockStartLine = code.co_firstlineno parentBlockSource = inspect.getsource(code) lineno = inspect.getframeinfo(callFrame)[1] linenoRelativeToParent = lineno - parentBlockStartLine + 1 # There could be multiple ic() calls on the same line(s), like # # ic(1); ic(2); ic(3, # 4, # 5); ic(6) # # so include all of them. Which invocation is the appropriate one will be # determined later via bytecode offset calculations. # # TODO(grun): Support invocations of ic() where ic() is an attribute chain # in the AST. For example, support # # import icecream # icecream.ic() # # and # # class Foo: # blah = ic # Foo.blah() # parentBlockSource = textwrap.dedent(parentBlockSource) potentialCalls = [ node for node in ast.walk(ast.parse(parentBlockSource)) if nodeIsIcCall(node, funcName) and ( node.lineno == linenoRelativeToParent or any( arg.lineno == linenoRelativeToParent for arg in node.args)) ] endLine = lineno - parentBlockStartLine + 1 startLine = min(call.lineno for call in potentialCalls) lines = parentBlockSource.splitlines()[startLine - 1:endLine] source = ' '.join(line.strip() for line in lines) callOffset = callFrame.f_lasti absoluteStartLineNum = parentBlockStartLine + startLine - 1 startLineOffset = calculateLineOffsets(code)[absoluteStartLineNum] return source, absoluteStartLineNum, startLineOffset
def calls(abstree): return [ node.func.id for node in ast.walk(abstree) if type(node) is ast.Call and type(node.func) is ast.Name ]
def definitions(abstree): return [ node.name for node in ast.walk(abstree) if type(node) is ast.FunctionDef ]
def min_col(node): """Computes the minimum col_offset.""" return min(map(get_col, walk(node), itertools.repeat(node.col_offset)))
def find_loop(f): for n in ast.walk(ast.parse(_func_source(f))): if type(n) == ast.For: return n if type(n) == ast.While: return n