def hy_eval(hytree, namespace, module_name):
foo = HyObject()
foo.start_line = 0
foo.end_line = 0
foo.start_column = 0
foo.end_column = 0
replace_hy_obj(hytree, foo)
if not isinstance(module_name, string_types):
raise HyTypeError(foo, "Module name must be a string")
_ast, expr = hy_compile(hytree, module_name, get_expr=True)
# Spoof the positions in the generated ast...
for node in ast.walk(_ast):
node.lineno = 1
node.col_offset = 1
for node in ast.walk(expr):
node.lineno = 1
node.col_offset = 1
if not isinstance(namespace, dict):
raise HyTypeError(foo, "Globals must be a dictionary")
# Two-step eval: eval() the body of the exec call
eval(ast_compile(_ast, "<eval_body>", "exec"), namespace)
# Then eval the expression context and return that
return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def _log_function(item):
"""
Handler function for log message types.
:param item: ast object being inspected for certain properties
:type item: :class:`~ast.AST`
:return: Returns the descriptor and arg offset of the item.
:rtype: tuple (str, int)
"""
level_arg = item.args[0]
if isinstance(level_arg, ast.Str):
level = level_arg.s.lower()
else:
any_call_or_name_elements = any(
isinstance(element, (ast.Call, ast.Name))
for element in ast.walk(level_arg)
)
if any_call_or_name_elements:
level = "(dynamic)"
else:
level = ', '.join(
element.s.lower()
for element in ast.walk(level_arg)
if isinstance(element, ast.Str)
)
integer_arg_offset = 1
return level, integer_arg_offset
def nestable_lines(self):
""" Return the range of lines that are nestable.
Parse the tree for all start and end lines of a nestable group (e.g. a
function defintion or if statement). For each, return a tuple of the
start and end line number.
"""
nests = []
nodes = [ast.walk(node) for node in self._tree.body]
for node in nodes:
end = 0
for subnode in node:
if isinstance(subnode, (ast.FunctionDef, ast.ClassDef, ast.If,
ast.For, ast.TryExcept,
ast.TryFinally)):
end = 0
for subsubnode in ast.walk(subnode):
try:
lineno = subsubnode.lineno
except AttributeError:
pass
else:
if lineno > end:
end = lineno
nests.append((subnode.lineno, end))
return nests
def replace_variables(self):
"""Replace script variables in string values."""
variables = {
'SCRIPT_SOURCE': self.source_script,
'SCRIPT_NAME': self.script_name,
'SCRIPT_PATH': self.script_path,
'SCRIPT_AUTHOR': 'Sebastien Helleu',
'SCRIPT_VERSION': '1.0',
'SCRIPT_LICENSE': 'GPL3',
'SCRIPT_DESCRIPTION': ('%s scripting API test' %
self.language.capitalize()),
'SCRIPT_LANGUAGE': self.language,
}
# count the total number of tests
tests_count = 0
for node in ast.walk(self.tree):
if isinstance(node, ast.Call) and \
isinstance(node.func, ast.Name) and \
node.func.id == 'check':
tests_count += 1
variables['SCRIPT_TESTS'] = str(tests_count)
# replace variables
for node in ast.walk(self.tree):
if isinstance(node, ast.Str) and \
node.s in variables:
node.s = variables[node.s]
def ast_equal(code1, code2, check_line_col=False, ignore_var_names=False):
"""
Checks whether ast nodes are equivalent recursively.
By default does not check line number or col offset
"""
gen1 = ast.walk(code1)
gen2 = ast.walk(code2)
for node1, node2 in zip_longest(gen1, gen2):
# unequal length
if node1 is None or node2 is None:
return False
if type(node1) != type(node2):
return False
# ignore the names of load name variables.
if ignore_var_names and is_load_name(node1) and is_load_name(node2):
continue
if not ast_field_equal(node1, node2):
return False
if check_line_col and hasattr(node1, 'lineno'):
if node1.lineno != node2.lineno:
return False
if node1.col_offset != node2.col_offset:
return False
return True
def ast_transform(root):
parent_node = None
dt_class_names = set(c.__name__ for c in (DateTime, Date, Duration))
str_nodes = []
for node in ast.walk(root):
try:
if isinstance(node, ast.Str):
if isinstance(parent_node, ast.Name) and parent_node.id in dt_class_names:
pass
else:
str_nodes.append(node)
finally:
parent_node = node
nt = DTNodeTransformer(str_nodes)
nt.visit(root)
ast.fix_missing_locations(root)
prev_lineno, prev_col_offset = 1, 0
for n in ast.walk(root):
if not hasattr(n, 'lineno'):
n.lineno = prev_lineno
else:
prev_lineno = n.lineno
if not hasattr(n, 'col_offset'):
n.col_offset = prev_col_offset
else:
prev_col_offset = n.col_offset
def check_for_wrong_tuple(tree, code, noqa):
errors = []
candidates = []
for assign in ast.walk(tree):
if not isinstance(assign, ast.Assign) or assign.lineno in noqa:
continue
elif isinstance(assign.value, ast.Call):
continue
for tuple_el in ast.walk(assign):
if isinstance(tuple_el, ast.Tuple) and len(tuple_el.elts) == 1:
candidates.append((assign.lineno, assign.col_offset))
break
if not candidates:
return []
for candidate in candidates:
tokens = tokenize.generate_tokens(lambda L=iter(code): next(L))
for t in tokens:
x = TokenInfo(*t)
if x.start[0] != candidate[0]:
continue
if x.type == token.OP and x.string == "=":
x = TokenInfo(*next(tokens))
if x.type != token.OP and x.string != "(":
x_next = TokenInfo(*next(tokens))
if x_next.type == token.OP and x_next.string == ",":
errors.append(x.start)
return errors
def parse(self, source, filename="source"):
self.clear()
self.filename = filename
data = ast.parse(source, filename)
for a in ast.walk(data):
if isinstance(a, _ast.Assign) and isinstance(a.targets[0], _ast.Name):
name = a.targets[0].id
if name == self.stored_dict_name:
if isinstance(a.value, _ast.Dict):
for x in a.value.values:
if isinstance(x, _ast.Str):
self.stored_raw.append((x.s, x.lineno, x.col_offset))
else:
print "Error in %s:%d:%d: nonstring dict value" % (
self.filename,
x.lineno,
x.col_offset,
)
if not self.using_function:
for a in ast.walk(data):
if isinstance(a, _ast.Subscript) and isinstance(a.value, _ast.Name):
name = a.value.id
if name == self.retrieved_dict_name:
if hasattr(a.slice, "value") and isinstance(a.slice.value, _ast.Str):
x = a.slice.value
self.retrieved_raw.append((x.s, x.lineno, x.col_offset))
else:
for a in ast.walk(data):
if isinstance(a, _ast.Call) and isinstance(a.func, _ast.Name):
name = a.func.id
if name == self.retrieved_dict_name:
if len(a.args) == 1 and isinstance(a.args[0], _ast.Str):
x = a.args[0]
self.retrieved_raw.append((x.s, x.lineno, x.col_offset))
else:
print "Suspicious use of '%s' in %s:%d:%d" % (
self.retrieved_dict_name,
self.filename,
a.lineno,
a.col_offset,
)
for item in self.stored_raw:
if item[0] in self.stored:
self.stored[item[0]].append(item)
else:
self.stored[item[0]] = [item]
for item in self.retrieved_raw:
if item[0] in self.retrieved:
self.retrieved[item[0]].append(item)
else:
self.retrieved[item[0]] = [item]
s_set = set(self.stored.keys())
r_set = set(self.retrieved.keys())
self.missing = r_set - s_set
self.redundant = s_set - r_set
def _find_last_line_for_class(source, classname):
all_nodes = list(ast.walk(ast.parse(source)))
classes = [n for n in all_nodes if isinstance(n, ast.ClassDef)]
our_class = next(c for c in classes if c.name == classname)
last_line_in_our_class = max(
getattr(thing, 'lineno', 0) for thing in ast.walk(our_class)
)
return last_line_in_our_class
def translateFunctions(self, tree):
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef):
self.forwardDeclaration(node)
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef):
self.visit(node)
def read_setup_py(setup_py):
requirements = set()
tree = ast.parse(setup_py)
for node in ast.walk(tree):
arg = getattr(node, "arg", None)
if arg in ("requires", "install_requires"):
for node in ast.walk(node.value):
if isinstance(node, ast.Str):
requirements.add(read_package_name(node.s))
return requirements
def has_main(filepath):
with open(filepath, "r") as f:
source = f.read()
tree = ast.parse(source)
first_level_ifs = (n for n in ast.walk(tree) if isinstance(n, _ast.If))
strs_in_ifs = [n.s for _if in first_level_ifs for n in ast.walk(_if) if isinstance(n, _ast.Str)]
return "__main__" in strs_in_ifs
def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
"""``eval`` evaluates a quoted expression and returns the value. The optional
second and third arguments specify the dictionary of globals to use and the
module name. The globals dictionary defaults to ``(local)`` and the module
name defaults to the name of the current module.
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
=> (eval (read-str "(+ 1 1)"))
2"""
if namespace is None:
frame = inspect.stack()[1][0]
namespace = inspect.getargvalues(frame).locals
if module_name is None:
m = inspect.getmodule(inspect.stack()[1][0])
module_name = '__eval__' if m is None else m.__name__
foo = HyObject()
foo.start_line = 0
foo.end_line = 0
foo.start_column = 0
foo.end_column = 0
replace_hy_obj(hytree, foo)
if not isinstance(module_name, string_types):
raise HyTypeError(foo, "Module name must be a string")
_ast, expr = hy_compile(hytree, module_name, get_expr=True)
# Spoof the positions in the generated ast...
for node in ast.walk(_ast):
node.lineno = 1
node.col_offset = 1
for node in ast.walk(expr):
node.lineno = 1
node.col_offset = 1
if ast_callback:
ast_callback(_ast, expr)
if not isinstance(namespace, dict):
raise HyTypeError(foo, "Globals must be a dictionary")
# Two-step eval: eval() the body of the exec call
eval(ast_compile(_ast, "<eval_body>", "exec"), namespace)
# Then eval the expression context and return that
return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def compare(self):
if not self.solution.strip():
return False
try:
root = ast.parse(self.solution)
except:
return False
soln_nodes = [ type(n).__name__ for n in ast.walk(root) ]
key_nodes = [ type(n).__name__ for n in ast.walk(ast.parse(self.answer_key)) ]
return soln_nodes == key_nodes
def fix_linenumbers_ast(self, tree):
# Hack because of Python bug(?). When creating the tree, some nodes do
# not have the _attributes field.
for node in ast.walk(tree):
if not hasattr(node, '_attributes'):
node._attributes = ()
tree = ast.fix_missing_locations(tree)
for node in ast.walk(tree):
if isinstance(node, (ast.stmt, ast.expr)):
node.lineno -= 1
return tree
def test_roundtrip(cls, source=None):
if source is None: # pragma: no cover
try:
source = inspect.getsource(mnfy)
except IOError:
pass
original_ast = ast.parse(source)
minifier = mnfy.SourceCode()
minifier.visit(original_ast)
minified_source = str(minifier)
minified_ast = ast.parse(minified_source)
node_pairs = zip(ast.walk(minified_ast), ast.walk(original_ast))
for minified, original in node_pairs:
cls.compare_ast_nodes(minified, original)
return minified_source
def parse_python(self, node, lineno=0, filename="<string>"):
if not node or not node.strip():
return
h = bbhash(str(node))
if h in codeparsercache.pythoncache:
self.references = set(codeparsercache.pythoncache[h].refs)
self.execs = set(codeparsercache.pythoncache[h].execs)
self.contains = {}
for i in codeparsercache.pythoncache[h].contains:
self.contains[i] = set(codeparsercache.pythoncache[h].contains[i])
return
if h in codeparsercache.pythoncacheextras:
self.references = set(codeparsercache.pythoncacheextras[h].refs)
self.execs = set(codeparsercache.pythoncacheextras[h].execs)
self.contains = {}
for i in codeparsercache.pythoncacheextras[h].contains:
self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
return
# We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
node = "\n" * int(lineno) + node
code = compile(check_indent(str(node)), filename, "exec",
ast.PyCF_ONLY_AST)
for n in ast.walk(code):
if n.__class__.__name__ == "Call":
self.visit_Call(n)
self.execs.update(self.var_execs)
codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains)
def parse(codingrule, _seen=()):
"""
Parse a condition of a codingrule. Returns a dictionary with the first node of the
AST generated from `codingrule.condition`.
Each node represented as a dict contains a key "type", which indicates which type
of node it is. This must be one of OR, AND, NOT, EQUALS and NOT_EQUALS. All nodes
(excluding NOT) contain a key 'values' which is a list of operands. Not only has one
operand, which is held in 'value'.
Each node NOT represented as a dict must be a CodingSchemaField, Code, str or int. A
CodingSchemaField is always on the left side of an equation and compared to either a
Code, str or int.
All CodingRule's will be replaced be their condition and parsed.
@raises: SyntaxError, CodingSchemaField.DoesNotExist, Code.DoesNotExist
@returns: root of tree, or None if condition is empty
"""
tree = ast.parse("({})".format(codingrule.condition)).body[0]
# This function can be called from parse_node if the condition refers to another
# codingrule. Keeping _seen prevents infinite loops when parsing.
if codingrule in _seen:
raise SyntaxError("Recursion: condition can't contain itself.")
# Not all nodes from the Python language are supported
for node in ast.walk(tree):
if node.__class__ not in KNOWN_NODES or (isinstance(node, ast.Compare) and len(node.ops) > 1):
if hasattr(node, "col_offset"):
raise SyntaxError("invalid syntax (col {}, line {})".format(node.col_offset, node.lineno))
raise SyntaxError("invalid syntax")
return parse_node(tree, _seen=_seen + (codingrule,))
def checkBody(self, funcdef):
for bodyNode in funcdef.body:
for node in ast.walk(bodyNode):
if isinstance(node, ast.Attribute):
if isinstance(node.value, ast.Name):
if node.value.id != funcdef.args.args[0].id:
self.problem("Model variable '{0}' is used in the rule, but this variable is not first argument in the rule argument list".format(node.value.id), lineno=funcdef.lineno)
def _sympify_string(math_string):
"Convert math string into SymPy object."
# drop pound symbols ('#') since they denote function names
# we detect those automatically
expr_string = math_string.replace('#', '')
# sympify doesn't recognize LN as ln()
expr_string = \
re.sub(r'(?<!\w)LN(?!\w)', 'ln', expr_string, flags=re.IGNORECASE)
expr_string = \
re.sub(r'(?<!\w)EXP(?!\w)', 'exp', expr_string,
flags=re.IGNORECASE)
# Convert raw variables into StateVariable objects
variable_fixes = {
Symbol('T'): v.T,
Symbol('P'): v.P,
Symbol('R'): v.R
}
# sympify uses eval, so we need to sanitize the input
nodes = ast.parse(expr_string)
nodes = ast.Expression(nodes.body[0].value)
for node in ast.walk(nodes):
if type(node) not in _AST_WHITELIST: #pylint: disable=W1504
raise ValueError('Expression from TDB file not in whitelist: '
'{}'.format(expr_string))
return sympify(expr_string).xreplace(variable_fixes)
def checkBody(self, funcdef):
"""Look for statements of the format 'return None'"""
for node in ast.walk(funcdef):
if isinstance(node, ast.Return):
if isinstance(node.value, ast.Name):
if node.value.id == 'None':
self.problem("Cannot return None from model rule {0}".format(funcdef.name), lineno=funcdef.lineno)
def parse_module_imports(root_name, module):
"""Parse a python module for all 'import' statements and extract the
:param root_name: the name of the module being scanned
:param module: the source of a python module
:returns: all modules imported
:rtype: `nx.DiGraph`
"""
D = Dependencies()
D.add_root(root_name)
tree = ast.parse(module)
for node in ast.walk(tree):
if type(node) is ast.Import:
for child in node.names:
name = child.name
D.add_submodule(root_name, name)
elif type(node) is ast.ImportFrom:
# relative imports (from . import <name1>, <name2>, ...)
# the "module" attribute is None
if node.module is not None:
D.add_submodule(root_name, node.module)
for name in node.names:
D.add_subattribute(node.module, name.name)
else:
continue
return D
def make_ast_tree(self):
"""
Returns a modified AST
All it does is modify the line numbers
so that tracebacks work nicely
"""
line_numbers = self.line_numbers
try:
tree = ast.parse(self.data, filename = self.path)
except SyntaxError as e:
descr, args = e.args
args = list(args)
args[1] = line_numbers[e.lineno]
args[2] = None
args[3] = linecache.getline(e.filename, args[1]).strip('\n')
e.__init__(descr, args)
raise
for node in ast.walk(tree):
try:
node.lineno = line_numbers[node.lineno]
except AttributeError:
pass
return tree
def _find_all_function_calls_by_name(self, name: str) -> List[ast.Call]:
"""
:param name: name of the function for which all calls should be
returned.
:return: a list of function calls for function with given name.
For example, for
foo('bar')
bar('foo')
foo('baz')
_find_all_function_calls_by_name(m, 'foo') will return [
Call(
func=Name(id='foo', ctx=Load()),
args=[Str(s='bar')],
keywords=[])),
Call(
func=Name(id='foo', ctx=Load()),
args=[Str(s='baz')],
keywords=[]))
]
"""
return [
node
for node in ast.walk(self.ast_module)
if isinstance(node, ast.Call) and node.func.id == name
]
def nits(self):
class_methods = set()
all_methods = set(function_def for function_def in ast.walk(self.python_file.tree)
if isinstance(function_def, ast.FunctionDef))
for class_def in self.iter_ast_types(ast.ClassDef):
if not is_upper_camel(class_def.name):
yield self.error('T000', 'Classes must be UpperCamelCased', class_def)
for class_global in self.iter_class_globals(class_def):
if not is_constant(class_global.id) and class_global.id not in self.CLASS_GLOBAL_BUILTINS:
yield self.error('T001', 'Class globals must be UPPER_SNAKE_CASED', class_global)
if not class_def.bases or all(isinstance(base, ast.Name) and base.id == 'object'
for base in class_def.bases):
class_methods.update(self.iter_class_methods(class_def))
else:
# If the class is inheriting from anything that is potentially a bad actor, rely
# upon checking that bad actor out of band. Fixes PANTS-172.
for method in self.iter_class_methods(class_def):
all_methods.discard(method)
for function_def in all_methods - class_methods:
if is_reserved_name(function_def.name):
yield self.error('T801', 'Method name overrides a builtin.', function_def)
# TODO(wickman) Only enforce this for classes that derive from object. If they
# don't derive object, it's possible that the superclass naming is out of its
# control.
for function_def in all_methods:
if not any((is_lower_snake(function_def.name),
is_builtin_name(function_def.name),
is_reserved_with_trailing_underscore(function_def.name))):
yield self.error('T002', 'Method names must be lower_snake_cased', function_def)
def count_lines(self,node):
childnodes = list(ast.walk(node))
lines = set()
for n in childnodes:
if hasattr(n,'lineno'):
lines.add(n.lineno)
return len(lines)
def looks_like_teardown_function(node):
returns = [x for x in ast.walk(node) if isinstance(x, ast.Return)]
if len(returns) != 1:
return
return_def = returns[0]
resp_name = node.args.args[0]
if not isinstance(return_def.value, ast.Name) or return_def.value.id != resp_name.id:
return
for body_node in node.body:
for child in ast.walk(body_node):
if isinstance(child, ast.Name) and child.id == resp_name.id:
if child is not return_def.value:
return
return resp_name.id
def extract_strings_from_file(fn):
filedata = open(fn, 'r', encoding="utf8")
root_node = ast.parse(filedata.read(), fn, 'exec')
filedata.close()
for node in ast.walk(root_node):
if type(node) == ast.Call:
# print("found function at")
# print("%s:%d" % (fn, node.lineno))
# lambda's
if type(node.func) == ast.Name:
continue
# getattr(self, con.type)(context, box, con)
if not hasattr(node.func, "attr"):
continue
translate_args = func_translate_args.get(node.func.attr, ())
# do nothing if not found
for arg_kw, arg_pos in translate_args:
if arg_pos < len(node.args):
extract_strings(fn, node.args[arg_pos])
else:
for kw in node.keywords:
if kw.arg == arg_kw:
extract_strings(fn, kw.value)
def root_package_name(name):
p = ast.parse(name)
for n in ast.walk(p):
if isinstance(n, ast.Name):
return n.id
else:
return None
def ClassDef_default(t, x):
"""Converts a class to an ES6 class."""
_class_guards(t, x)
name = x.name
body = x.body
if len(x.bases) > 0:
super_name = x.bases[0].id
else:
super_name = None
# strip docs from body
body = [e for e in body if isinstance(e, (ast.FunctionDef, ast.AsyncFunctionDef))]
# * Each FunctionDef must have self as its first arg
# silly check for methods
for node in body:
arg_names = [arg.arg for arg in node.args.args]
t.unsupported(node, len(arg_names) == 0 or arg_names[0] != 'self',
"First arg on method must be 'self'")
if len(body) > 0 and body[0].name == '__init__':
init = body[0]
# * __init__ may not contain a return statement
# silly check
init_args = [arg.arg for arg in init.args.args]
init_body = init.body
for stmt in ast.walk(init):
assert not isinstance(stmt, ast.Return)
if super_name:
superclass = JSName(super_name)
else:
superclass = None
return JSClass(JSName(name), superclass, body)
def run(self):
for node in ast.walk(self.tree):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
num_positional_args = len(node.args)
if (num_positional_args == 1
and isinstance(node.args[0], ast.GeneratorExp)
and node.func.id in ("list", "set")):
msg_key = {"list": "C400", "set": "C401"}[node.func.id]
yield (
node.lineno,
node.col_offset,
self.messages[msg_key],
type(self),
)
elif (num_positional_args == 1
and isinstance(node.args[0],
(ast.GeneratorExp, ast.ListComp))
and isinstance(node.args[0].elt, ast.Tuple)
and len(node.args[0].elt.elts) == 2
and node.func.id == "dict"):
if isinstance(node.args[0], ast.GeneratorExp):
msg = "C402"
else:
msg = "C404"
yield (
node.lineno,
node.col_offset,
self.messages[msg],
type(self),
)
elif (num_positional_args == 1
and isinstance(node.args[0], ast.ListComp)
and node.func.id in ("list", "set")):
msg_key = {"list": "C411", "set": "C403"}[node.func.id]
yield (
node.lineno,
node.col_offset,
self.messages[msg_key],
type(self),
)
elif num_positional_args == 1 and (
isinstance(node.args[0], ast.Tuple) and node.func.id
== "tuple" or isinstance(node.args[0], ast.List)
and node.func.id == "list"):
suffix = "remove the outer call to {func}()."
msg_key = {"tuple": "C409", "list": "C410"}[node.func.id]
msg = self.messages[msg_key] + suffix
yield (
node.lineno,
node.col_offset,
msg.format(type=type(node.args[0]).__name__.lower(),
func=node.func.id),
type(self),
)
elif (num_positional_args == 1
and isinstance(node.args[0], (ast.Tuple, ast.List))
and node.func.id in ("tuple", "list", "set", "dict")):
suffix = "rewrite as a {func} literal."
msg_key = {
"tuple": "C409",
"list": "C410",
"set": "C405",
"dict": "C406",
}[node.func.id]
msg = self.messages[msg_key] + suffix
yield (
node.lineno,
node.col_offset,
msg.format(type=type(node.args[0]).__name__.lower(),
func=node.func.id),
type(self),
)
elif (num_positional_args == 1
# These take 1 positional argument + some keyword arguments
and (node.func.id in (
"all",
"any",
"frozenset",
"tuple",
"max",
"min",
"sorted",
)) and isinstance(node.args[0], ast.ListComp)):
yield (
node.lineno,
node.col_offset,
self.messages["C407"].format(func=node.func.id),
type(self),
)
elif (num_positional_args in (1, 2)
# These can take a second positional argument
and (node.func.id in (
"enumerate",
"sum",
)) and isinstance(node.args[0], ast.ListComp)):
yield (
node.lineno,
node.col_offset,
self.messages["C407"].format(func=node.func.id),
type(self),
)
elif (num_positional_args == 2 and node.func.id == "filter"
and isinstance(node.args[1], ast.ListComp)):
yield (
node.lineno,
node.col_offset,
self.messages["C407"].format(func=node.func.id),
type(self),
)
elif (num_positional_args >= 2 and node.func.id == "map"
and any(
isinstance(a, ast.ListComp) for a in node.args[1:])):
yield (
node.lineno,
node.col_offset,
self.messages["C407"].format(func=node.func.id),
type(self),
)
elif (num_positional_args == 0 and not has_star_args(node)
and not has_keyword_args(node)
and node.func.id in ("tuple", "list", "dict")):
yield (
node.lineno,
node.col_offset,
self.messages["C408"].format(type=node.func.id),
type(self),
)
elif (node.func.id in {"list", "reversed"}
and num_positional_args > 0
and isinstance(node.args[0], ast.Call)
and isinstance(node.args[0].func, ast.Name)
and node.args[0].func.id == "sorted"):
remediation = ""
if node.func.id == "reversed":
reverse_flag_value = False
for keyword in node.args[0].keywords:
if keyword.arg != "reverse":
continue
if isinstance(keyword.value, ast.NameConstant):
reverse_flag_value = keyword.value.value
elif isinstance(keyword.value, ast.Num):
reverse_flag_value = bool(keyword.value.n)
else:
# Complex value
reverse_flag_value = None
if reverse_flag_value is None:
remediation = " - toggle reverse argument to sorted()"
else:
remediation = " - use sorted(..., reverse={!r})".format(
not reverse_flag_value)
msg = self.messages["C413"].format(
inner=node.args[0].func.id,
outer=node.func.id,
remediation=remediation,
)
yield (
node.lineno,
node.col_offset,
msg,
type(self),
)
elif (num_positional_args > 0
and isinstance(node.args[0], ast.Call)
and isinstance(node.args[0].func, ast.Name)
and ((node.func.id in {"set", "sorted"}
and node.args[0].func.id
in {"list", "reversed", "sorted", "tuple"}) or
(node.func.id in {"list", "tuple"}
and node.args[0].func.id in {"list", "tuple"}) or
(node.func.id == "set"
and node.args[0].func.id == "set"))):
yield (
node.lineno,
node.col_offset,
self.messages["C414"].format(
inner=node.args[0].func.id, outer=node.func.id),
type(self),
)
elif (node.func.id in {"reversed", "set", "sorted"}
and num_positional_args > 0
and isinstance(node.args[0], ast.Subscript)
and isinstance(node.args[0].slice, ast.Slice)
and node.args[0].slice.lower is None
and node.args[0].slice.upper is None
and isinstance(node.args[0].slice.step, ast.UnaryOp)
and isinstance(node.args[0].slice.step.op, ast.USub)
and isinstance(node.args[0].slice.step.operand, ast.Num)
and node.args[0].slice.step.operand.n == 1):
yield (
node.lineno,
node.col_offset,
self.messages["C415"].format(func=node.func.id),
type(self),
)
elif isinstance(node, ast.Compare):
if (len(node.ops) == 1 and isinstance(node.ops[0], ast.In)
and len(node.comparators) == 1 and isinstance(
node.comparators[0],
(ast.DictComp, ast.ListComp, ast.SetComp))):
yield (
node.lineno,
node.col_offset,
self.messages["C412"].format(
type=comp_type[node.comparators[0].__class__]),
type(self),
)
elif isinstance(node, (ast.ListComp, ast.SetComp)):
if (len(node.generators) == 1 and not node.generators[0].ifs
and not is_async_generator(node.generators[0]) and
(isinstance(node.elt, ast.Name)
and isinstance(node.generators[0].target, ast.Name)
and node.elt.id == node.generators[0].target.id)):
yield (
node.lineno,
node.col_offset,
self.messages["C416"].format(
type=comp_type[node.__class__]),
type(self),
)
def getCallSourceLines(callFrame, icNames, icMethod):
"""Raises NoSourceAvailableError."""
code = callFrame.f_code
# inspect.getblock(), which is called internally by inspect.getsource(),
# only returns the first line of <code> when <code> represents a top-level
# module, not the entire module's source, as needed here. The
#
# if ismodule(object):
# return lines, 0
#
# check in inspect.py doesn't account for code objects of modules, only
# actual module objects themselves.
#
# A workaround is to call findsource() directly on code objects of modules,
# which bypasses getblock().
#
# Also, the errors raised differ between Python2 and Python3 . In Python2,
# inspect.findsource() and inspect.getsource() raise IOErrors. In Python3,
# inspect.findsource() and inspect.getsource() raise OSErrors.
try:
if code.co_name == '<module>': # Module -> use workaround above.
parentBlockStartLine = 1
lines = inspect.findsource(code)[0] # Raises [IO/OS]Error.
parentBlockSource = ''.join(lines)
else: # Not a module -> use inspect.getsource() normally.
parentBlockStartLine = code.co_firstlineno
parentBlockSource = inspect.getsource(code) # Raises [IO/OS]Error.
except (IOError, OSError) as err:
if 'source code' in err.args[0]:
raise NoSourceAvailableError()
else:
raise
lineno = inspect.getframeinfo(callFrame)[1]
linenoRelativeToParent = lineno - parentBlockStartLine + 1
# There could be multiple ic() calls on the same line(s), like
#
# ic(1); ic(2); ic(3,
# 4,
# 5); ic(6)
#
# so include all of them. Which invocation is the appropriate one will be
# determined later via bytecode offset calculations.
#
# TODO(grun): Support invocations of ic() where ic() is an attribute chain
# in the AST. For example, support
#
# import icecream
# icecream.ic()
#
# and
#
# class Foo:
# blah = ic
# Foo.blah()
#
parentBlockSource = textwrap.dedent(parentBlockSource)
potentialCalls = [
node for node in ast.walk(ast.parse(parentBlockSource))
if isAstNodeIceCreamCall(node, icNames, icMethod) and (
node.lineno == linenoRelativeToParent or any(
arg.lineno == linenoRelativeToParent for arg in node.args))
]
if not potentialCalls:
# TODO(grun): Add note that to NoSourceAvailableError that this
# situation can occur when the underlying source changed during
# execution.
raise NoSourceAvailableError()
endLine = lineno - parentBlockStartLine + 1
startLine = min(call.lineno for call in potentialCalls)
lines = parentBlockSource.splitlines()[startLine - 1:endLine]
# inspect's lineno attribute doesn't point to the closing right parenthesis
# if the closing right parenthesis is on its own line without any
# arguments. E.g.
#
# ic(1,
# 2 <--- inspect's reported lineno.
# ) <--- Should be the reported lineno.
#
# Detect this situation and add the missing right parenthesis.
if isCallStrMissingClosingRightParenthesis('\n'.join(lines).strip()):
lines.append(')')
source = stripCommentsAndNewlines('\n'.join(lines)).strip()
absoluteStartLineNum = parentBlockStartLine + startLine - 1
startLineOffset = calculateLineOffsets(code)[absoluteStartLineNum]
return source, absoluteStartLineNum, startLineOffset
def dump_py_messages_from_files(msgs, reports, files, settings):
"""
Dump text inlined in the python files given, e.g. 'My Name' in:
layout.prop("someprop", text="My Name")
"""
import ast
bpy_struct = bpy.types.ID.__base__
i18n_contexts = bpy.app.translations.contexts
root_paths = tuple(
bpy.utils.resource_path(t) for t in ('USER', 'LOCAL', 'SYSTEM'))
def make_rel(path):
for rp in root_paths:
if path.startswith(rp):
try: # can't always find the relative path (between drive letters on windows)
return os.path.relpath(path, rp)
except ValueError:
return path
# Use binary's dir as fallback...
try: # can't always find the relative path (between drive letters on windows)
return os.path.relpath(path, os.path.dirname(bpy.app.binary_path))
except ValueError:
return path
# Helper function
def extract_strings_ex(node, is_split=False):
"""
Recursively get strings, needed in case we have "Blah" + "Blah", passed as an argument in that case it won't
evaluate to a string. However, break on some kind of stopper nodes, like e.g. Subscript.
"""
if type(node) == ast.Str:
eval_str = ast.literal_eval(node)
if eval_str:
yield (is_split, eval_str, (node, ))
else:
is_split = (type(node) in separate_nodes)
for nd in ast.iter_child_nodes(node):
if type(nd) not in stopper_nodes:
yield from extract_strings_ex(nd, is_split=is_split)
def _extract_string_merge(estr_ls, nds_ls):
return "".join(s for s in estr_ls
if s is not None), tuple(n for n in nds_ls
if n is not None)
def extract_strings(node):
estr_ls = []
nds_ls = []
for is_split, estr, nds in extract_strings_ex(node):
estr_ls.append(estr)
nds_ls.extend(nds)
ret = _extract_string_merge(estr_ls, nds_ls)
return ret
def extract_strings_split(node):
"""
Returns a list args as returned by 'extract_strings()', but split into groups based on separate_nodes, this way
expressions like ("A" if test else "B") won't be merged but "A" + "B" will.
"""
estr_ls = []
nds_ls = []
bag = []
for is_split, estr, nds in extract_strings_ex(node):
if is_split:
bag.append((estr_ls, nds_ls))
estr_ls = []
nds_ls = []
estr_ls.append(estr)
nds_ls.extend(nds)
bag.append((estr_ls, nds_ls))
return [
_extract_string_merge(estr_ls, nds_ls) for estr_ls, nds_ls in bag
]
i18n_ctxt_ids = {v for v in bpy.app.translations.contexts_C_to_py.values()}
def _ctxt_to_ctxt(node):
# We must try, to some extend, to get contexts from vars instead of only literal strings...
ctxt = extract_strings(node)[0]
if ctxt:
return ctxt
# Basically, we search for attributes matching py context names, for now.
# So non-literal contexts should be used that way:
# i18n_ctxt = bpy.app.translations.contexts
# foobar(text="Foo", text_ctxt=i18n_ctxt.id_object)
if type(node) == ast.Attribute:
if node.attr in i18n_ctxt_ids:
#print(node, node.attr, getattr(i18n_contexts, node.attr))
return getattr(i18n_contexts, node.attr)
return i18n_contexts.default
def _op_to_ctxt(node):
# Some smart coders like things like:
# >>> row.operator("preferences.addon_disable" if is_enabled else "preferences.addon_enable", ...)
# We only take first arg into account here!
bag = extract_strings_split(node)
opname, _ = bag[0]
if not opname:
return i18n_contexts.default
op = bpy.ops
for n in opname.split('.'):
op = getattr(op, n)
try:
return op.get_rna_type().translation_context
except Exception as e:
default_op_context = i18n_contexts.operator_default
print("ERROR: ", str(e))
print(" Assuming default operator context '{}'".format(
default_op_context))
return default_op_context
# Gather function names.
# In addition of UI func, also parse pgettext ones...
# Tuples of (module name, (short names, ...)).
pgettext_variants = (
("pgettext", ("_", )),
("pgettext_iface", ("iface_", )),
("pgettext_tip", ("tip_", )),
("pgettext_data", ("data_", )),
)
pgettext_variants_args = {"msgid": (0, {"msgctxt": 1})}
# key: msgid keywords.
# val: tuples of ((keywords,), context_getter_func) to get a context for that msgid.
# Note: order is important, first one wins!
translate_kw = {
"text": (
(("text_ctxt", ), _ctxt_to_ctxt),
(("operator", ), _op_to_ctxt),
),
"msgid": ((("msgctxt", ), _ctxt_to_ctxt), ),
"message": (),
}
context_kw_set = {}
for k, ctxts in translate_kw.items():
s = set()
for c, _ in ctxts:
s |= set(c)
context_kw_set[k] = s
# {func_id: {msgid: (arg_pos,
# {msgctxt: arg_pos,
# ...
# }
# ),
# ...
# },
# ...
# }
func_translate_args = {}
# First, functions from UILayout
# First loop is for msgid args, second one is for msgctxt args.
for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
# check it has one or more arguments as defined in translate_kw
for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
if ((arg_kw in translate_kw) and (not arg.is_output)
and (arg.type == 'STRING')):
func_translate_args.setdefault(func_id,
{})[arg_kw] = (arg_pos, {})
for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
if func_id not in func_translate_args:
continue
for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
if (not arg.is_output) and (arg.type == 'STRING'):
for msgid, msgctxts in context_kw_set.items():
if arg_kw in msgctxts:
func_translate_args[func_id][msgid][1][
arg_kw] = arg_pos
# The report() func of operators.
for func_id, func in bpy.types.Operator.bl_rna.functions.items():
# check it has one or more arguments as defined in translate_kw
for arg_pos, (arg_kw, arg) in enumerate(func.parameters.items()):
if ((arg_kw in translate_kw) and (not arg.is_output)
and (arg.type == 'STRING')):
func_translate_args.setdefault(func_id,
{})[arg_kw] = (arg_pos, {})
# We manually add funcs from bpy.app.translations
for func_id, func_ids in pgettext_variants:
func_translate_args[func_id] = pgettext_variants_args
for func_id in func_ids:
func_translate_args[func_id] = pgettext_variants_args
# print(func_translate_args)
# Break recursive nodes look up on some kind of nodes.
# E.g. we don't want to get strings inside subscripts (blah["foo"])!
# we don't want to get strings from comparisons (foo.type == 'BAR').
stopper_nodes = {ast.Subscript, ast.Compare}
# Consider strings separate: ("a" if test else "b")
separate_nodes = {ast.IfExp}
check_ctxt_py = None
if reports["check_ctxt"]:
check_ctxt = reports["check_ctxt"]
check_ctxt_py = {
"py_in_rna": (check_ctxt.get("py_in_rna"), set(msgs.keys())),
"multi_lines": check_ctxt.get("multi_lines"),
"not_capitalized": check_ctxt.get("not_capitalized"),
"end_point": check_ctxt.get("end_point"),
"spell_checker": check_ctxt.get("spell_checker"),
"spell_errors": check_ctxt.get("spell_errors"),
}
for fp in files:
with open(fp, 'r', encoding="utf8") as filedata:
root_node = ast.parse(filedata.read(), fp, 'exec')
fp_rel = make_rel(fp)
for node in ast.walk(root_node):
if type(node) == ast.Call:
# print("found function at")
# print("%s:%d" % (fp, node.lineno))
# We can't skip such situations! from blah import foo\nfoo("bar") would also be an ast.Name func!
if type(node.func) == ast.Name:
func_id = node.func.id
elif hasattr(node.func, "attr"):
func_id = node.func.attr
# Ugly things like getattr(self, con.type)(context, box, con)
else:
continue
func_args = func_translate_args.get(func_id, {})
# First try to get i18n contexts, for every possible msgid id.
msgctxts = dict.fromkeys(func_args.keys(), "")
for msgid, (_, context_args) in func_args.items():
context_elements = {}
for arg_kw, arg_pos in context_args.items():
if arg_pos < len(node.args):
context_elements[arg_kw] = node.args[arg_pos]
else:
for kw in node.keywords:
if kw.arg == arg_kw:
context_elements[arg_kw] = kw.value
break
# print(context_elements)
for kws, proc in translate_kw[msgid]:
if set(kws) <= context_elements.keys():
args = tuple(context_elements[k] for k in kws)
#print("running ", proc, " with ", args)
ctxt = proc(*args)
if ctxt:
msgctxts[msgid] = ctxt
break
# print(translate_args)
# do nothing if not found
for arg_kw, (arg_pos, _) in func_args.items():
msgctxt = msgctxts[arg_kw]
estr_lst = [(None, ())]
if arg_pos < len(node.args):
estr_lst = extract_strings_split(node.args[arg_pos])
#print(estr, nds)
else:
for kw in node.keywords:
if kw.arg == arg_kw:
estr_lst = extract_strings_split(kw.value)
break
#print(estr, nds)
for estr, nds in estr_lst:
if estr:
if nds:
msgsrc = "{}:{}".format(
fp_rel,
sorted({nd.lineno
for nd in nds})[0])
else:
msgsrc = "{}:???".format(fp_rel)
process_msg(msgs, msgctxt, estr, msgsrc, reports,
check_ctxt_py, settings)
reports["py_messages"].append(
(msgctxt, estr, msgsrc))
def _find_decorators_node_to_validate(self) -> None:
for node in ast.walk(self.tree):
if isinstance(node, ast.FunctionDef):
decorators = node.decorator_list
self._validate_decorators(decorators) # type: ignore
def debug(victim=None, ignore_exceptions=(BdbQuit,),
catch_exception=None, depth=0):
"""A decorator function to catch exceptions and enter debug mode.
Args:
victim (typing.Union(type, function)): either a class or function to
wrap and debug.
ignore_exceptions (list): list of classes of exceptions not to catch.
catch_exception (type): class of exception to catch and debug.
default is None, meaning catch all exceptions.
depth (number): how many levels of inner function calls to propagate.
Returns:
object. wrapped class or function.
Note:
This wrapper avoids recursion by setting a flag to each wrapped item.
"""
if victim is None:
# Debug is used as a decorator so we need to return wrap function to
# get the real victim
def wrapper(real_victim):
return debug(real_victim, ignore_exceptions,
catch_exception, depth)
return wrapper
register_break_signal()
if inspect.isfunction(victim):
if hasattr(victim, '_ipdebug_wrapped'):
# Don't wrap the function more than once
return victim
_transformer = ErrorsCatchTransformer(
ignore_exceptions=ignore_exceptions,
catch_exception=catch_exception,
depth=depth)
try:
# Try to get the source code of the wrapped object.
sourcelines, start_num = inspect.getsourcelines(victim.__code__)
indent = re.match(r'\s*', sourcelines[0]).group()
source = ''.join(l.replace(indent, '', 1) for l in sourcelines)
except IOError:
# Worst-case scenario we can only catch errors at a granularity
# of the whole function
return victim
else:
# If we have access to the source, we can silence errors on a
# per-expression basis, which is "better"
old_code_tree = ast.parse(source)
# Reposition the line numbers to their original value
ast.increment_lineno(old_code_tree, start_num - 1)
tree = _transformer.visit(old_code_tree)
import_debug_cmd = ast.ImportFrom(
__name__, [ast.alias("start_debugging", None),
ast.alias("debug", None)], 0)
# Add import to the debugger as first command
tree.body[0].body.insert(0, import_debug_cmd)
# Add import to the exception classes
if catch_exception is not None:
import_exception_cmd = ast.ImportFrom(
catch_exception.__module__,
[ast.alias(catch_exception.__name__, None)], 0)
tree.body[0].body.insert(1, import_exception_cmd)
if ignore_exceptions is not None:
for exception_class in ignore_exceptions:
import_exception_cmd = ast.ImportFrom(
exception_class.__module__,
[ast.alias(exception_class.__name__, None)], 0)
tree.body[0].body.insert(1, import_exception_cmd)
# Delete the debugger decorator of the function
del tree.body[0].decorator_list[:]
# Add pass at the end (to enable debugging the last command)
pass_cmd = ast.Pass()
func_body = tree.body[0].body
pass_cmd.lineno = get_last_lineno(func_body[-1]) + 1
pass_cmd.col_offset = func_body[-1].col_offset
func_body.append(pass_cmd)
# Fix missing line numbers and column offsets before compiling
for node in ast.walk(tree):
if not hasattr(node, 'lineno'):
node.lineno = 0
ast.fix_missing_locations(tree)
# Define the wrapping function object
function_definition = "def _free_vars_wrapper(): pass"
wrapping_function = ast.parse(function_definition).body[0]
# Initialize closure's variables to None
body_list = [ast.parse("{var} = None".format(var=free_var)).body[0]
for free_var in victim.__code__.co_freevars]
# Add the original function ("victim") to the wrapping function
body_list.append(tree.body[0])
wrapping_function.body = body_list
# Replace original function ("victim") with the wrapping function
tree.body[0] = wrapping_function
# Create a new runnable code object to replace the original code
code = compile(tree, victim.__code__.co_filename, 'exec')
victim.__code__ = code.co_consts[0].co_consts[1]
# Set a flag to indicate that the method was wrapped
victim._ipdebug_wrapped = True
return victim
elif inspect.ismethod(victim):
debug(victim.__func__, ignore_exceptions, catch_exception)
return victim
elif isinstance(victim, type):
# Wrap each method of the class with the debugger
for name, member in vars(victim).items():
if isinstance(member, (type, types.FunctionType,
types.LambdaType, types.MethodType)):
setattr(victim, name,
debug(member, ignore_exceptions, catch_exception))
return victim
else:
raise TypeError(
"Debugger can only wrap functions and classes. "
"Got object {!r} of type {}".format(victim, type(victim).__name__))
async def meval(code, globs, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globs.copy()
# This code saves __name__ and __package into a kwarg passed to the func.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, "exec")
code = root.body
ret_name = "_ret"
ok = False
while True:
if ret_name in globs.keys():
ret_name = "_" + ret_name
continue
for node in ast.walk(root):
if isinstance(node, ast.Name) and node.id == ret_name:
ret_name = "_" + ret_name
break
ok = True
if ok:
break
if not code:
return None
if not any(isinstance(node, ast.Return) for node in code):
for i in range(len(code)):
if isinstance(code[i], ast.Expr):
if (i == len(code) - 1
or not isinstance(code[i].value, ast.Call)):
code[i] = ast.copy_location(
ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(
id=ret_name, ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[code[i].value],
keywords=[])), code[-1])
else:
for node in code:
if isinstance(node, ast.Return):
node.value = ast.List(elts=[node.value], ctx=ast.Load())
code.append(
ast.copy_location(
ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1]))
# globals().update(**<global_args>)
glob_copy = ast.Expr(
ast.Call(
func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals",
ctx=ast.Load()),
args=[],
keywords=[]),
attr="update",
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))
]))
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())],
value=ast.List(elts=[], ctx=ast.Load()))
ast.fix_missing_locations(ret_decl)
code.insert(1, ret_decl)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
ast.fix_missing_locations(a)
args += [a]
args = ast.arguments(args=[],
vararg=None,
kwonlyargs=args,
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))])
args.posonlyargs = []
fun = ast.AsyncFunctionDef(name="tmp",
args=args,
body=code,
decorator_list=[],
returns=None)
ast.fix_missing_locations(fun)
mod = ast.parse("")
mod.body = [fun]
comp = compile(mod, "<string>", "exec")
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
for i in range(len(r)):
if hasattr(r[i], "__await__"):
r[i] = await r[i] # workaround for 3.5
i = 0
while i < len(r) - 1:
if r[i] is None:
del r[i]
else:
i += 1
if len(r) == 1:
[r] = r
elif not r:
r = None
return r
one += one
one -= one
one | one
a[0] += 100
5 < 3
not True
del apple["Hearted"]
import garbage
8 is 7
""")
print("iter_fields:", ast.iter_fields(multiline.body[0]))
print("iter_child_nodes:", ast.iter_child_nodes(multiline))
print("walk:", ast.walk(multiline))
print(ast.dump(multiline, True, False))
print("*" * 40)
class VisitStuff(ast.NodeVisitor):
def __init__(self):
self.nums = 0
def visit_Num(self, node):
print("Found a ", node.n)
self.nums += 1
vs = VisitStuff()
def get_all_imports(path,
encoding=None,
extra_ignore_dirs=None,
follow_links=True):
imports = set()
raw_imports = set()
candidates = []
ignore_errors = False
ignore_dirs = [".hg", ".svn", ".git", ".tox", "__pycache__", "env", "venv"]
if extra_ignore_dirs:
ignore_dirs_parsed = []
for e in extra_ignore_dirs:
ignore_dirs_parsed.append(os.path.basename(os.path.realpath(e)))
ignore_dirs.extend(ignore_dirs_parsed)
walk = os.walk(path, followlinks=follow_links)
for root, dirs, files in walk:
dirs[:] = [d for d in dirs if d not in ignore_dirs]
candidates.append(os.path.basename(root))
files = [fn for fn in files if os.path.splitext(fn)[1] == ".py"]
candidates += [os.path.splitext(fn)[0] for fn in files]
for file_name in files:
file_name = os.path.join(root, file_name)
with open_func(file_name, "r", encoding=encoding) as f:
contents = f.read()
try:
tree = ast.parse(contents)
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for subnode in node.names:
raw_imports.add(subnode.name)
elif isinstance(node, ast.ImportFrom):
raw_imports.add(node.module)
except Exception as exc:
if ignore_errors:
traceback.print_exc(exc)
logging.warn("Failed on file: %s" % file_name)
continue
else:
logging.error("Failed on file: %s" % file_name)
raise exc
# Clean up imports
for name in [n for n in raw_imports if n]:
# Sanity check: Name could have been None if the import
# statement was as ``from . import X``
# Cleanup: We only want to first part of the import.
# Ex: from django.conf --> django.conf. But we only want django
# as an import.
cleaned_name, _, _ = name.partition('.')
imports.add(cleaned_name)
packages = imports - (set(candidates) & imports)
logging.debug('Found packages: {0}'.format(packages))
with open(join("stdlib"), "r") as f:
data = {x.strip() for x in f}
data = {x for x in data if x not in py2_exclude} if py2 else data
return list(packages - data)
counts = {}
for subdir, dirs, files in os.walk('/home/u180133/PROGECT/qualitas/'):
for file in files:
#print os.path.join(subdir, file)
filepath = subdir + os.sep + file
if filepath.endswith(".py"):
print filepath + '\n'
try:
tree = ast.parse(open(filepath).read())
except SyntaxError:
print 'Syntax Error Keep Her Lit'
pass
for node in ast.walk(tree):
#print type(node).__name__
nt = type(node).__name__
if nt not in counts:
counts[nt] = 0
counts[nt] += 1
print '\n'
print counts
with open('/home/u180133/PROGECT/allNodeCount.txt', 'a') as bean:
#bean.write('Hello World' +'\n')
#bean.write( filepath+'\n')
json.dump(counts, bean)
def find_unprefixed_string_literals(filename):
u"""
Find unprefixed string literals in a Python source file.
Returns a list of ``(line_number, column)`` tuples (both 1-based) of
positions where string literals without a ``u`` or ``b`` prefix
start.
Note: Due to limitations in Python's ``ast`` module this does not
check the rear parts of auto-concatenated string literals
(``'foo' 'bar'``).
"""
with io.open(filename, encoding=u"utf-8") as f:
lines = f.readlines()
# In some versions of Python, the ast module cannot deal with
# encoding declarations (http://bugs.python.org/issue22221). We
# therefore replace all comment lines at the beginning of the file
# with empty lines (to keep the line numbers correct).
for i, line in enumerate(lines):
line = line.strip()
if line.startswith(u"#"):
lines[i] = u"\n"
elif line:
break
root = ast.parse(u"".join(lines), filename.encode(FILESYSTEM_ENCODING))
problems = []
for node in ast.walk(root):
if isinstance(node, ast.Str):
lineno = node.lineno - 1
col_offset = node.col_offset
if col_offset == -1:
# `lineno` and `col_offset` are broken for literals that span
# multiple lines: For these, `lineno` contains the line of the
# *closing* quotes, and `col_offset` is always -1, see
# https://bugs.python.org/issue16806. We therefore have to
# find the start of the literal manually, which is difficult
# since '''-literals can contain """ and vice versa. The
# following code assumes that no ''' or """ literal begins on
# the same line where a multi-line literal ends.
last_line = lines[lineno]
if last_line.rfind(u'"""') > last_line.rfind(u"'''"):
quotes = u'"""'
else:
quotes = u"'''"
for lineno, line in renumerate(lines[:lineno]):
try:
i = line.rindex(quotes)
if (i > 1) and (line[i - 2:i].lower() == u"ur"):
col_offset = i - 2
elif (i > 0) and (line[i - 1].lower() in u"rbu"):
col_offset = i - 1
else:
col_offset = 0
break
except ValueError:
continue
leading = lines[lineno][col_offset - 1:col_offset + 1]
if leading[:-1] == u"[": # data['id'] is unambiguous, ignore these
continue
if leading[-1:] not in u"ub": # Don't allow capital U and B either
problems.append((lineno + 1, col_offset + 1))
return sorted(problems)
def find_classes(module):
class_nodes = [c for c in ast.walk(module) if isinstance(c, ast.ClassDef)]
return class_nodes
def gather_names(node):
"""Returns the set of all names present in the node's tree."""
rtn = set(map(get_id, walk(node)))
rtn.discard(None)
return rtn
def get_config_assignments(config: Path) -> Dict[str, LINES_RANGE]:
ast_nodes = ast.walk(ast.parse(config.read_text()))
assignments = dict(
extract_assignment(ast_node) for ast_node in ast_nodes
if isinstance(ast_node, ast.Assign))
return assignments
def min_line(node):
"""Computes the minimum lineno."""
node_line = get_lineno(node)
return min(map(get_lineno, walk(node), itertools.repeat(node_line)))
def check_expression( text ):
"""
>>> check_expression("c1=='chr1' and c3-c2>=2000 and c6=='+'")
True
>>> check_expression("eval('1+1')")
False
>>> check_expression("import sys")
False
>>> check_expression("[].__str__")
False
>>> check_expression("__builtins__")
False
>>> check_expression("'x' in globals")
False
>>> check_expression("'x' in [1,2,3]")
True
>>> check_expression("c3=='chr1' and c5>5")
True
>>> check_expression("c3=='chr1' and d5>5") # Invalid d5 reference
False
>>> check_expression("c3=='chr1' and c5>5 or exec")
False
>>> check_expression("type(c1) != type(1)")
True
>>> check_expression("c1.split(',')[1] == '1'")
True
>>> check_expression("exec 1")
False
>>> check_expression("str(c2) in [\\\"a\\\",\\\"b\\\"]")
True
"""
try:
module = parse( text )
except SyntaxError:
return False
if not isinstance(module, Module):
return False
statements = module.body
if not len( statements ) == 1:
return False
expression = statements[0]
if expression.__class__.__name__ != 'Expr':
return False
for ast_node in walk( expression ):
ast_node_class = ast_node.__class__.__name__
# Toss out everything that is not a "simple" expression,
# imports, error handling, etc...
if ast_node_class not in AST_NODE_TYPE_WHITELIST:
return False
# White-list more potentially dangerous types AST elements.
if ast_node_class == 'Name':
# In order to prevent loading 'exec', 'eval', etc...
# put string restriction on names allowed.
if not __check_name( ast_node ):
return False
# Check only valid, white-listed functions are called.
elif ast_node_class == 'Call':
if not __check_call( ast_node ):
return False
# Check only valid, white-listed attributes are accessed
elif ast_node_class == 'Attribute':
if not __check_attribute( ast_node ):
return False
return True
def max_line(node):
"""Computes the maximum lineno."""
return max(map(get_lineno, walk(node)))
def _update(self):
"""update tkk
"""
# we don't need to update the base TKK value when it is still valid
now = math.floor(int(time.time() * 1000) / 3600000.0)
if self.tkk and int(self.tkk.split('.')[0]) == now:
return
r = self.session.get(self.host)
rawtkk = self.RE_RAWTKK.search(r.text)
if rawtkk:
self.tkk = rawtkk.group(1)
return
# this will be the same as python code after stripping out a reserved word 'var'
code = unicode(self.RE_TKK.search(r.text).group(1)).replace('var ', '')
# unescape special ascii characters such like a \x3d(=)
if PY3: # pragma: no cover
code = code.encode().decode('unicode-escape')
else: # pragma: no cover
code = code.decode('string_escape')
if code:
tree = ast.parse(code)
visit_return = False
operator = '+'
n, keys = 0, dict(a=0, b=0)
for node in ast.walk(tree):
if isinstance(node, ast.Assign):
name = node.targets[0].id
if name in keys:
if isinstance(node.value, ast.Num):
keys[name] = node.value.n
# the value can sometimes be negative
elif isinstance(node.value, ast.UnaryOp) and \
isinstance(node.value.op, ast.USub): # pragma: nocover
keys[name] = -node.value.operand.n
elif isinstance(node, ast.Return):
# parameters should be set after this point
visit_return = True
elif visit_return and isinstance(node, ast.Num):
n = node.n
elif visit_return and n > 0:
# the default operator is '+' but implement some more for
# all possible scenarios
if isinstance(node, ast.Add): # pragma: nocover
pass
elif isinstance(node, ast.Sub): # pragma: nocover
operator = '-'
elif isinstance(node, ast.Mult): # pragma: nocover
operator = '*'
elif isinstance(node, ast.Pow): # pragma: nocover
operator = '**'
elif isinstance(node, ast.BitXor): # pragma: nocover
operator = '^'
# a safety way to avoid Exceptions
clause = compile(
'{1}{0}{2}'.format(operator, keys['a'], keys['b']), '', 'eval')
value = eval(clause, dict(__builtin__={}))
result = '{}.{}'.format(n, value)
self.tkk = result
def _test_sort(elements, _type):
values = [e.value for e in elements]
for wrong, right in zip(values, natsorted(values)):
if wrong != right:
print(
f"{_type} is not sorted, {right!r} should come before {wrong!r}"
)
return True
return False
if len(sys.argv) == 1:
print("Usage: sort.py [filename]")
sys.exit(1)
with open(sys.argv[1]) as f:
contents = f.read()
fail = False
for node in ast.walk(ast.parse(contents)):
if type(node) == ast.List:
fail = _test_sort(node.elts, "List") or fail
if type(node) == ast.Set:
fail = _test_sort(node.elts, "Set") or fail
if type(node) == ast.Dict:
fail = _test_sort(node.keys, "Dict") or fail
sys.exit(fail)
def run(self):
for node in ast.walk(self.tree):
for rule in ('I200', 'I201'):
for err in getattr(self, 'rule_{}'.format(rule))(node):
yield err
def _maybe_ignore_child(self, node: ast.AST) -> bool:
if isinstance(node, ast.AnnAssign):
self._to_ignore.extend(ast.walk(node.annotation))
return node in self._to_ignore
def iter_excepts(cls, tree):
for ast_node in ast.walk(tree):
if isinstance(ast_node, ast.TryExcept):
yield ast_node
def read_project(root_path, verbose=False, ignore=None, encoding=None):
"""
Reads project into an AST and transforms imports into Nodes
:param root_path: String
:return: Node
"""
nodes = {}
root_node = None
errors = False
ignore_files = set(
[".hg", ".svn", ".git", ".tox", "__pycache__", "env",
"venv"]) # python 2.6 comp
if ignore:
for ignored_file in ignore:
ignore_files.add(os.path.basename(os.path.realpath(ignored_file)))
# traverse root directory, and list directories as dirs and files as files
for root, dirs, files in os.walk(root_path):
dirs[:] = [d for d in dirs if d not in ignore_files]
files = [
fn for fn in files
if os.path.splitext(fn)[1] == ".py" and fn not in ignore_files
]
for file_name in files:
full_path = os.path.join(root, file_name)
with open_func(full_path, "r", encoding=encoding) as f:
try:
# fails on empty files
file_data = f.read()
lines = file_data.splitlines()
tree = ast.parse(file_data)
if verbose:
click.echo(
crayons.yellow(
'Trying to parse file: {}'.format(full_path)))
if full_path in nodes:
node = nodes[full_path]
else:
node = Node(file_name[:-3], full_path=full_path)
nodes[full_path] = node
if not root_node:
root_node = node
for ast_node in ast.walk(tree):
if isinstance(ast_node, ast.Import) and ast_node.names:
for subnode in ast_node.names:
if not subnode.name:
continue
path_to_module = get_path_from_package_name(
root_path, subnode.name)
if path_to_module in nodes:
new_node = nodes[path_to_module]
else:
new_node = Node(subnode.name,
full_path=path_to_module)
nodes[path_to_module] = new_node
new_node.is_imported_from[full_path].append(
ast_node.lineno)
node.add(new_node)
elif isinstance(ast_node,
ast.ImportFrom) and ast_node.module:
current_path = root_path
if 0 <= ast_node.lineno - 1 < len(lines) and\
REGEX_RELATIVE_PATTERN.findall(lines[ast_node.lineno - 1]):
current_path = root
path_to_module = get_path_from_package_name(
current_path, ast_node.module)
if path_to_module in nodes:
new_node = nodes[path_to_module]
else:
new_node = Node(ast_node.module,
full_path=path_to_module)
nodes[path_to_module] = new_node
for obj_import in ast_node.names:
if ast_node.lineno not in node.func_imports:
node.func_imports[ast_node.lineno] = [
obj_import.name
]
else:
node.func_imports[ast_node.lineno].append(
obj_import.name)
new_node.is_imported_from[full_path].append(
ast_node.lineno)
node.add(new_node)
elif isinstance(ast_node,
(ast.ClassDef, ast.FunctionDef)):
node.func_defs[ast_node.name] = ast_node.lineno
except Exception as e:
errors = True
click.echo(
crayons.yellow(
'Parsing of file failed: {}'.format(full_path)))
if verbose:
click.echo(crayons.red(traceback.format_exc(e)))
if errors:
click.echo(
crayons.red(
'There were errors during the operation, perhaps you are trying to parse python 3 project, '
'with python 2 version of the script? (or vice versa)'))
return root_node
def visit_Call(self, node):
for child in ast.walk(node):
if isinstance(child, ast.Name):
self.cache.append(child.id)
def _check_contains_yield(self, node: AnyComprehension) -> None:
for sub_node in ast.walk(node):
if isinstance(sub_node, ast.Yield): # pragma: py-gte-38
self.add_violation(YieldInComprehensionViolation(node))
def find_decorated_funcs(tree):
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef):
if hasattr(node, "decorator_list"):
yield node
def getCallSourceLines(funcName, callFrame):
code = callFrame.f_code
# inspect.getblock(), which is called internally by inspect.getsource(),
# only returns the first line of <code> when <code> represents a top-level
# module, not the entire module's source, as needed here. The
#
# if ismodule(object): return lines, 0
#
# check in inspect.py doesn't account for code objects that represent
# modules, only module objects themselves.
#
# A workaround is to call findsource() directly on top-level modules, which
# bypasses getblock().
if code.co_name == '<module>': # Module -> use workaround explained above.
parentBlockStartLine = 1
parentBlockSource = ''.join(inspect.findsource(code)[0])
else: # Not a module -> use inspect.getsource() normally.
parentBlockStartLine = code.co_firstlineno
parentBlockSource = inspect.getsource(code)
lineno = inspect.getframeinfo(callFrame)[1]
linenoRelativeToParent = lineno - parentBlockStartLine + 1
# There could be multiple ic() calls on the same line(s), like
#
# ic(1); ic(2); ic(3,
# 4,
# 5); ic(6)
#
# so include all of them. Which invocation is the appropriate one will be
# determined later via bytecode offset calculations.
#
# TODO(grun): Support invocations of ic() where ic() is an attribute chain
# in the AST. For example, support
#
# import icecream
# icecream.ic()
#
# and
#
# class Foo:
# blah = ic
# Foo.blah()
#
parentBlockSource = textwrap.dedent(parentBlockSource)
potentialCalls = [
node for node in ast.walk(ast.parse(parentBlockSource))
if nodeIsIcCall(node, funcName) and (
node.lineno == linenoRelativeToParent or any(
arg.lineno == linenoRelativeToParent for arg in node.args))
]
endLine = lineno - parentBlockStartLine + 1
startLine = min(call.lineno for call in potentialCalls)
lines = parentBlockSource.splitlines()[startLine - 1:endLine]
source = ' '.join(line.strip() for line in lines)
callOffset = callFrame.f_lasti
absoluteStartLineNum = parentBlockStartLine + startLine - 1
startLineOffset = calculateLineOffsets(code)[absoluteStartLineNum]
return source, absoluteStartLineNum, startLineOffset
def calls(abstree):
return [
node.func.id for node in ast.walk(abstree)
if type(node) is ast.Call and type(node.func) is ast.Name
]
def definitions(abstree):
return [
node.name for node in ast.walk(abstree)
if type(node) is ast.FunctionDef
]
def min_col(node):
"""Computes the minimum col_offset."""
return min(map(get_col, walk(node), itertools.repeat(node.col_offset)))
def find_loop(f):
for n in ast.walk(ast.parse(_func_source(f))):
if type(n) == ast.For:
return n
if type(n) == ast.While:
return n
