def AST_FunctionDataExtract(f, fullCode):
''' Extracts function data from AST '''
# Init
FunctionData = {
"language": "c",
"name": '',
"params": {},
"desc": "",
"imports": [],
"code": "",
"return": {}
}
# Extract
# Name
FunctionData['name'] = f.decl.name
# Desc
# FunctionData['desc'] = [x.s for x in f.body]
# Imports
ImportsData = []
FunctionData['imports'] = ImportsData
# Params
FunctionData['params'] = []
if f.decl.type.args is not None:
params = f.decl.type.args.params
for i in range(len(params)):
param = params[i]
d = param.init
d_dict = {"value": d}
t = AST_GetType(param.type)
paramData = {
"name": param.name,
"desc": "",
"default_value": json.dumps(d_dict),
"type": t
}
FunctionData['params'].append(paramData)
# Code
FunctionData['code'] = ast.get_source_segment(fullCode, f)
# Return
FunctionData['return'] = {"type": AST_GetType(f.decl)}
return FunctionData
def lazyimport(line, cell):
modules = {}
for node in ast.iter_child_nodes(ast.parse(cell)):
if isinstance(node, (ast.ImportFrom, ast.Import)):
for name in [n.asname or n.name for n in node.names]:
source = ast.get_source_segment(cell, node)
setattr(builtins, name, DummyImport(name, source))
modules[name] = source
else:
raise RuntimeError(f'Unrecognized import: "{source}"')
if 'debug' in line.lower():
print("Lazily imported packages are (name -> import statement):")
print(json.dumps(modules, indent=2))
def skip_import(self, node: Union[ast.Import, ast.ImportFrom]) -> bool:
if C.PY38_PLUS:
lines = ast.get_source_segment(self.source, node).splitlines()
for lineno, comment in self._comments.items():
with contextlib.suppress(IndexError):
lines[lineno - node.lineno] += " " + comment
source_segment = "".join(lines) # NOTE "\n".join(lines)
else:
source_segment = self.source.splitlines()[node.lineno - 1]
return (bool(
re.search(
self.skip_comments_regex,
source_segment,
re.IGNORECASE,
)) or self.any_import_error)
def safe_visit(self, node, *args, **kwargs):
try:
method(self, node, *args, **kwargs)
except Exception as e:
node_repr = ast.dump(node)
if sys.version_info >= (3, 8) and self.source_code:
node_repr = ast.get_source_segment(self.source_code, node)
node_repr = "\n# ".join(node_repr.split("\n"))
warning = f"# could not {action} on node:\n" \
f"# {node_repr}\n" \
f"# {str(e)}"
logger.warning(warning, exc_info=True)
self.warnings.append(warning)
if call_generic_visit:
self.generic_visit(node)
def get_raw_metrics(self, node):
# astunparse.unparse() parses triple quote strings
# a single quote strings. A single quote string is
# interpreted as a sloc instead of a multi.
# source_segement = unparse(node)
source_segment = get_source_segment(self.code, node)
raw_metrics = analyze(source_segment)
raw_metrics_dict = raw_to_dict(raw_metrics)
self.loc = raw_metrics_dict["loc"]
self.lloc = raw_metrics_dict["lloc"]
self.sloc = raw_metrics_dict["sloc"]
self.comments = raw_metrics_dict["comments"]
self.multi = raw_metrics_dict["multi"]
self.blank = raw_metrics_dict["blank"]
self.single_comments = raw_metrics_dict["single_comments"]
def get_value(v):
if isinstance(v, ast.BinOp):
a = get_value(v.left)
b = get_value(v.right)
return a
elif isinstance(v, ast.Name):
return loc.get(v.id)
elif isinstance(v, ast.Call):
args = [get_value(a) for a in v.args]
func = loc.get(v.func.id, None) or glob.get(v.func.id, None)
return func(*args)
elif isinstance(v, ast.List):
return [get_value(e) for e in v.elts]
elif isinstance(v, ast.Constant):
return v.value
seg = get_source_segment(source, v)
return eval(seg, glob, loc)
def __init__(self, code, lamb, context=None):
if context is not None:
self.lamb = lamb
self.context = context
childs = ast.walk(lamb.body)
num_expr = 0
for child in childs:
if (type(child) in expr):
num_expr += 1
self.expressions = num_expr
self.linecontent = ast.get_source_segment(code, lamb)
else:
self.id = code[0]
self.arguments = code[3]
self.expressions = code[4]
self.line = code[5]
self.context = code[6]
self.linecontent = code[7]
def visit_Call(self, node):
"""
Instrument all function calls
"""
# pylint: disable=invalid-name
ast.NodeTransformer.generic_visit(self, node)
call_code = ast.get_source_segment(self.source_code, node)
self.add_before_call_used_value_capturing_call(call_code, node,
self.source_code)
self.add_before_call_used_args_capturing_call(call_code, node,
self.source_code)
self.add_before_call_used_kwargs_capturing_call(
call_code, node, self.source_code)
instrumented_call_node = self.add_after_call_used_capturing(
call_code, node, False)
return instrumented_call_node
def execute_patched_func(original_func, execute_inspections_func, *args,
**kwargs):
"""
Detects whether the function call comes directly from user code and decides whether to execute the original
function or the patched variant.
"""
# Performance aspects: https://gist.github.com/JettJones/c236494013f22723c1822126df944b12
# CPython implementation detail: This function should be used for internal and specialized purposes only.
# It is not guaranteed to exist in all implementations of Python.
# inspect.getcurrentframe() also only does return `sys._getframe(1) if hasattr(sys, "_getframe") else None`
# We can execute one hasattr check right at the beginning of the mlinspect execution
caller_filename = sys._getframe(2).f_code.co_filename # pylint: disable=protected-access
if caller_filename != singleton.source_code_path:
result = original_func(*args, **kwargs)
elif singleton.track_code_references:
call_ast_node = ast.Call(
lineno=singleton.lineno_next_call_or_subscript,
col_offset=singleton.col_offset_next_call_or_subscript,
end_lineno=singleton.end_lineno_next_call_or_subscript,
end_col_offset=singleton.end_col_offset_next_call_or_subscript)
caller_source_code = ast.get_source_segment(singleton.source_code,
node=call_ast_node)
caller_lineno = singleton.lineno_next_call_or_subscript
op_id = singleton.get_next_op_id()
caller_code_reference = CodeReference(
singleton.lineno_next_call_or_subscript,
singleton.col_offset_next_call_or_subscript,
singleton.end_lineno_next_call_or_subscript,
singleton.end_col_offset_next_call_or_subscript)
result = execute_inspections_func(op_id, caller_filename,
caller_lineno, caller_code_reference,
caller_source_code)
else:
op_id = singleton.get_next_op_id()
caller_lineno = sys._getframe(2).f_lineno # pylint: disable=protected-access
result = execute_inspections_func(op_id, caller_filename,
caller_lineno, None, None)
return result
def add_before_call_used_value_capturing_call(call_code, node,
all_source_code):
"""
When the method of some object is called, capture the value of the object before executing the method
"""
if hasattr(node.func, "value"):
old_value_node = node.func.value
value_code = ast.get_source_segment(all_source_code,
old_value_node)
new_value_node = ast.Call(
func=ast.Name(id='before_call_used_value', ctx=ast.Load()),
args=[
ast.Constant(n=False, kind=None),
ast.Constant(n=call_code, kind=None),
ast.Constant(n=value_code, kind=None), old_value_node,
ast.Constant(n=node.lineno, kind=None),
ast.Constant(n=node.col_offset, kind=None),
ast.Constant(n=node.end_lineno, kind=None),
ast.Constant(n=node.end_col_offset, kind=None)
],
keywords=[])
node.func.value = new_value_node
def get_calls(func: ast.FunctionDef,
code: str,
end_calls_only: bool = False) -> List[str]:
calls = []
if not end_calls_only:
# return the entire call string
for child in ast.walk(func):
if isinstance(child, ast.Call):
calls.append(ast.get_source_segment(code, child))
else:
# return just the name of the function that was called
for child in ast.walk(func):
if isinstance(child, ast.Call):
if isinstance(child.func, ast.Name):
calls.append(child.func.id)
elif isinstance(child.func, ast.Attribute):
calls.append(child.func.attr)
calls = list(set(calls))
calls.sort()
return calls # easy way to remove duplicates
def test_AssignedVariablesCollector_single_assignment_separate_variable_from_instance_attribute(
class_self_id: str, assignment_code: str, annotation_as_str: str,
self_attributes: list, variables: list):
# the assignment is the first line of the body
assignment_ast: AST = parse(assignment_code).body[0]
# assignment without annotation (multiple targets, but only one in these test cases)
if annotation_as_str is None:
annotation = None
assert len(assignment_ast.targets) == 1, 'unit test consistency'
assignment_target = assignment_ast.targets[0]
# assignment with annotation (only one target)
else:
annotation = assignment_ast.annotation
assert get_source_segment(
assignment_code,
annotation) == annotation_as_str, 'unit test consistency'
assignment_target = assignment_ast.target
assignment_collector = AssignedVariablesCollector(class_self_id,
annotation)
assignment_collector.visit(assignment_target)
# detection of self attributes
assert len(assignment_collector.self_attributes) == len(self_attributes)
for self_attribute, (variable_id, variable_type_str) in zip(
assignment_collector.self_attributes, self_attributes):
assert_Variable(self_attribute, variable_id, variable_type_str,
assignment_code)
# detection of new variables occupying the memory scope
assert len(assignment_collector.variables) == len(variables)
for variable, (variable_id,
variable_type_str) in zip(assignment_collector.variables,
variables):
assert_Variable(variable, variable_id, variable_type_str,
assignment_code)
def add_before_call_used_kwargs_capturing_call(call_code, node,
all_source_code):
"""
When a method is called, capture the keyword arguments of the method before executing it
"""
old_kwargs_nodes_ast = node.keywords # old_kwargs_nodes_ast = ast.List(node.keywords, ctx=ast.Load())
old_kwargs_code = ast.List([
ast.Constant(n=ast.get_source_segment(all_source_code, kwarg),
kind=None) for kwarg in node.keywords
],
ctx=ast.Load())
new_kwargs_node = ast.keyword(value=ast.Call(
func=ast.Name(id='before_call_used_kwargs', ctx=ast.Load()),
args=[
ast.Constant(n=False, kind=None),
ast.Constant(n=call_code, kind=None), old_kwargs_code,
ast.Constant(n=node.lineno, kind=None),
ast.Constant(n=node.col_offset, kind=None),
ast.Constant(n=node.end_lineno, kind=None),
ast.Constant(n=node.end_col_offset, kind=None)
],
keywords=old_kwargs_nodes_ast),
arg=None)
node.keywords = [new_kwargs_node]
def replace_fluent_alias(source, fluent_mapping):
fluent_mapping = {a: b for a, b in fluent_mapping}
new_src = source
for _ in range(100): # 100 is a (random) big enough number
replaced = False
tree = ast.parse(new_src)
for node in ast.walk(tree):
if (isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == 'd2l'
and node.func.attr in fluent_mapping):
new_node = ast.Call(
ast.Attribute(value=node.args[0],
attr=fluent_mapping[node.func.attr]),
node.args[1:], node.keywords)
new_src = new_src.replace(
ast.get_source_segment(new_src, node),
astor.to_source(new_node).rstrip())
replaced = True
break
if not replaced:
break
return new_src
def node_to_readable_file_location(code, node, child_node=None):
location = ""
if isinstance(node.parent, ast.Module):
# The next node up is the root, don't go higher.
pass
else:
location += node_to_readable_file_location(code, node.parent, node)
location += " > "
if isinstance(node, ast.Module):
raise Exception("We shouldn't see a Module")
elif isinstance(node, ast.If):
assert child_node
if child_node in node.body:
location += "if " + ast.get_source_segment(code, node.test)
else:
location += "else-of-if " + ast.get_source_segment(code, node.test)
elif isinstance(node, ast.For):
location += ("for " + ast.get_source_segment(code, node.target) +
" in " + ast.get_source_segment(code, node.iter))
elif isinstance(node, ast.AugAssign):
if isinstance(node.target, ast.Name):
location += node.target.id
else:
location += ast.get_source_segment(code, node.target)
elif isinstance(node, ast.Assign):
# This assert would fire if we did e.g. some_sources = all_sources = [ ... ]
assert len(
node.targets) == 1, "Assignment node contains more than one target"
if isinstance(node.targets[0], ast.Name):
location += node.targets[0].id
else:
location += ast.get_source_segment(code, node.targets[0])
else:
raise Exception("Got a node type I don't know how to handle: " +
str(node))
return location
def fstring_parts(node, source):
return {
ast.get_source_segment(source, part.value) # noqa
for part in node.values if isinstance(part, ast.FormattedValue)
}
def flatten_into(self, output_file):
code_lines = _read_file_contents_as_lines(self.script_path)
code = "".join(code_lines)
tree = ast.parse(code, self.script_path)
# insert a backward pointer in every node
for node in ast.walk(tree):
for child in ast.iter_child_nodes(node):
child.parent = node
imports = [] # list of all imports to be performed
names = [
] # list of all symbolic names that might need to be substituted
definitions = [
] # list of all class and function definitions that will need to be exported
assigns = [
] # list of all assignments to variables that will need to be exported
base_path = os.path.dirname(self.script_path)
for node in ast.walk(tree):
if isinstance(node, ast.Import):
if node.col_offset == 0:
for name in node.names:
imports.append(
_ImportStatement(
name.name,
name.asname,
node.lineno - 1,
base_path,
self.other_base_path,
))
elif isinstance(node, ast.Name):
symbol = node.id
parent = node.parent
end_offset = node.end_col_offset
names.append(
_Symbol(symbol, node.lineno - 1, node.col_offset,
end_offset))
while isinstance(parent, ast.Attribute):
symbol = symbol + "." + parent.attr
end_offset = parent.end_col_offset
names.append(
_Symbol(symbol, node.lineno - 1, node.col_offset,
end_offset))
parent = parent.parent
elif not self.is_script():
# only look for things to export if this is not the top level script
if isinstance(node, ast.ClassDef) or isinstance(
node, ast.FunctionDef):
# don't record class/function definitions made inside other definitions
if node.col_offset == 0:
name_offset = 6 if isinstance(node,
ast.ClassDef) else 4
definitions.append(
_DefStatement(node.name, self.module_name))
# the name of the class/function is not an AST 'name' so it won't
# be added to the list to substitute automatically - add it here
names.append(
_Symbol(
node.name,
node.lineno - 1,
name_offset,
name_offset + len(node.name),
))
elif isinstance(node, ast.Assign):
if node.col_offset == 0:
assigns.append(
_AssignmentStatement(
ast.get_source_segment(code, node),
self.module_name))
for an_import in imports:
previous_import = self.imports_done.get(an_import.module_path)
if previous_import is None:
an_import.flatten(into=output_file,
imports_done=self.imports_done)
self.imports_done[an_import.module_path] = an_import
else:
if previous_import.alias != an_import.alias:
raise ImportError(
f"Module '{an_import.module_path}' has already been inlined using alias '{previous_import.alias}' so cannot be inlined using alias '{an_import.alias}'",
name=an_import.module_path,
)
# always load the exported symbols into the current context, even if the import has already been done
self.local_symbol_mappings.update(
self.imports_done[an_import.module_path].exports)
for a_definition in definitions:
self.exported_symbol_mappings.update(
a_definition.get_exported_symbol_mapping())
self.local_symbol_mappings.update(
a_definition.get_local_symbol_mapping())
for an_assign in assigns:
self.exported_symbol_mappings.update(
an_assign.get_exported_symbol_mapping())
self.local_symbol_mappings.update(
an_assign.get_local_symbol_mapping())
# sort the symbols by line number and then start column number
names.sort(key=lambda x: (x.line_number * 10000) + x.start_col_offset)
current_line_num = 0
import_lines = [x.line_number for x in imports]
# copy all the lines up to next 'name' line
while len(names) > 0 and current_line_num < len(code_lines):
while current_line_num < names[0].line_number:
if current_line_num not in import_lines:
self.write_with_reference(output_file,
code_lines[current_line_num],
current_line_num)
current_line_num += 1
# replace 'names' in the line
up_to = 0
result = ""
current_line = code_lines[current_line_num]
while len(names) > 0 and current_line_num == names[0].line_number:
next_symbol = names[0]
del names[0]
result += current_line[up_to:next_symbol.start_col_offset]
up_to = max(up_to, next_symbol.start_col_offset)
if next_symbol.name in self.local_symbol_mappings:
result += self.local_symbol_mappings[next_symbol.name]
up_to = next_symbol.end_col_offset
elif next_symbol.name in self.exported_symbol_mappings:
result += self.exported_symbol_mappings[next_symbol.name]
up_to = next_symbol.end_col_offset
elif len(names) > 0 and current_line_num == names[
0].line_number and next_symbol.start_col_offset == names[
0].start_col_offset:
# There's another overlapping name that might match, so skip this one
pass
else:
result += current_line[up_to:next_symbol.end_col_offset]
up_to = next_symbol.end_col_offset
result += current_line[up_to:]
# write the updated line
self.write_with_reference(output_file, result, current_line_num)
current_line_num += 1
# copy any remaining lines
while current_line_num < len(code_lines):
self.write_with_reference(output_file,
code_lines[current_line_num],
current_line_num)
current_line_num += 1
return self.exported_symbol_mappings
def get_exports(source, root):
all_exports = [node
for node in ast.walk(root)
if isinstance(node, ast.AugAssign)
and (
(isinstance(node.target, ast.Name) and node.target.id == "EXPORTS")
or
(isinstance(node.target, ast.Attribute) and "EXPORTS" in ast.get_source_segment(source, node))
)
]
all_exports.extend([node
for node in ast.walk(root)
if isinstance(node, ast.Assign)
and (
(isinstance(node.targets[0], ast.Name) and node.targets[0].id in ["EXPORTS", "h_and_cpp"])
or
(isinstance(node.targets[0], ast.Attribute) and "EXPORTS" in ast.get_source_segment(source, node))
)
])
all_exports.extend([node
for node in ast.walk(root)
if isinstance(node, ast.Assign)
and (
(isinstance(node.targets[0], ast.Subscript) and isinstance(node.targets[0].value, ast.Name) and node.targets[0].value.id == "EXPORTS")
and
(node.targets[0].slice.value.value == "double-conversion") # Limit it to just this one case to be safe
)
])
linux_exports = []
# For each export, see if it is contained within an if statement
# whose conditional includes OS_ARCH or OS_TARGET and if so, only include it if it's Linux
for e in all_exports:
e_source = ast.get_source_segment(source, e)
#if "XXXXXXX" in e_source:
# pdb.set_trace()
if e.parent == root:
linux_exports.append((e, e_source))
continue
node = e
should_include = True
while node.parent != root:
code = ast.get_source_segment(source, node.parent)
# This test will fail for a conditional like "OS_ARCH != Linux"
# but presently we don't have any of those...
if isinstance(node.parent, ast.If):
conditional = ast.get_source_segment(source, node.parent.test)
# We want to ensure that the node is excluded if it is directly underneath a conditional
# that looks like it is for another platform
if ("OS_ARCH" in conditional or "OS_TARGET" in conditional) and "Linux" not in conditional:
if e in node.parent.body:
should_include = False
if "MOZ_WIDGET_TOOLKIT" in conditional and "gtk" not in conditional:
if e in node.parent.body:
should_include = False
if "MOZ_BUILD_APP" in conditional and '== "memory"' in conditional:
if e in node.parent.body:
should_include = False
node = node.parent
if should_include:
linux_exports.append((e, e_source))
return linux_exports
def process_calls(self, source: str):
for call in self.expression_calls:
for index, call_arg in enumerate(call['args']):
if not isinstance(call_arg, _ast.Constant):
source_call = ast.get_source_segment(source, call_arg)
call['args'][index] = source_call
def fetch(filename, **loc_data):
with tokenize.open(DATA_PATH / filename) as file:
source = file.read()
loc_node = LocationNode(**loc_data)
return ast.get_source_segment(source, loc_node, padded=True)
def modify(
dir_: Path,
paths: List[Path],
default_level: str = 'info',
accept_all: bool = False,
context_lines: int = 13,
comment_sep: str = ' \\ ',
**_kwargs,
) -> None:
for path in paths:
fpath = str(path.relative_to(dir_))
text = path.read_text()
lines = text.splitlines()
edited = False # Is there an accepted edit to write to disk?
tree = ast.parse(text)
# Make backlinks in order to later determine the context (function or class name)
# of the print statement
for node in ast.walk(tree):
for child in ast.iter_child_nodes(node):
child.parent = node
# Find all the nodes in the AST with a print statement,
print_nodes = [
node for node in ast.walk(tree)
if (isinstance(node, ast.Call)
and getattr(node.func, 'id', None) == 'print')
]
if not print_nodes:
continue
# Fetch comments
lineix_to_comment = dict() # Line index to comment
with path.open() as fil:
for toktype, tok, (start_rowno,
_), _, _ in tokenize.generate_tokens(
fil.readline):
if toktype == tokenize.COMMENT:
lineix_to_comment[start_rowno - 1] = re.sub(
r'#\s*', '', tok)
print_statements = []
for node in print_nodes:
has_vars = False
terms = [] # (term, is_var)
for arg in node.args:
term = ast.get_source_segment(text, arg)
if isinstance(arg, ast.JoinedStr):
# Is an f-string
has_vars = True
n_quotes = len(
re.match(r'^f("""|\'\'\'|"|\')', term).group(1))
# Remove 'f' and quotes from string
term = term[1 + n_quotes:-n_quotes]
# Escape newlines etc
term = term.encode('unicode_escape').decode('utf-8')
terms.append((term, True))
elif term.startswith('"') or term.startswith("'"):
term = arg.value.encode('unicode_escape').decode('utf-8')
terms.append((term, False))
else:
terms.append(('{' + term + '}', True))
has_vars = True
# Escape {} in non-f strings
if has_vars:
for i, (term, is_var) in enumerate(terms):
if not is_var:
terms[i] = (term.replace('{',
'{{').replace('}',
'}}'), is_var)
# Use custom separator from print statement
sep = ' '
for kw in node.keywords:
if kw.arg == 'sep':
sep = getattr(
kw.value, "id",
getattr(kw.value, "value",
' ')).encode('unicode_escape').decode('utf-8')
arg_line = sep.join(t[0] for t in terms)
# Line numbers are 1-indexed
lineix, end_lineix = node.lineno - 1, node.end_lineno - 1
# Find the whitespace predecing 'print' in the orginal statement
old_line = lines[lineix]
whitespace = re.match(r'(\s*)print', old_line).group(1)
comments = [
lineix_to_comment[i] for i in range(lineix, end_lineix + 1)
if i in lineix_to_comment
]
comment = comment_sep.join(comments) or None
source_context = node
while not isinstance(source_context,
(ast.AsyncFunctionDef, ast.ClassDef,
ast.FunctionDef, ast.Module)):
source_context = source_context.parent
if isinstance(source_context, ast.Module):
source_context = ''
else:
source_context = '/' + source_context.name
print_statements.append(
PrintStatement(lineix, end_lineix, whitespace, arg_line,
has_vars, comment, source_context))
# Find out if logging is already imported, and if so, as what.
has_logging_imported = False
import_nodes = []
logging_asname = 'logging'
for node in ast.iter_child_nodes(tree):
if isinstance(node, ast.Import):
import_nodes.append(node)
if node.names[0].name == 'logging':
has_logging_imported = True
logging_asname = node.names[0].asname or 'logging'
first_import_node = min(import_nodes,
default=None,
key=attrgetter('lineno'))
first_import_lineno = first_import_node.lineno or 1
def get_line(stmt: PrintStatement, level: str) -> str:
"""
Construct the full source code line with whitespace,
logging statement and optionally a comment
"""
f_string = 'f' if stmt.f_string else ''
comment = f' # {stmt.comment}' if stmt.comment else ''
return stmt.whitespace + f'{logging_asname}.{level}({f_string}"' \
+ stmt.arg_line + '")' + comment
print_statements.sort(key=lambda st: st.lineix)
# Multiline print statements are always squashed to a single-line statement,
# so there might be superfluous lines to delete
to_del_lineixs = []
n_changes = 0
for stmt in print_statements:
line = get_line(stmt, default_level)
if not accept_all:
clear_terminal()
print(
Bcolor.HEADER, f"{fpath}{stmt.source_context}:"
f"{stmt.lineix+1}-{stmt.end_lineix+1}", Bcolor.ENDC)
print()
print_context(lines, stmt.lineix, stmt.end_lineix, line,
context_lines)
print()
inp = None
while inp not in [
'', 'y', 'n', 'A', 'i', 'w', 'e', 'c', 'x', 'q'
]:
inp = input(Bcolor.OKCYAN + "Accept change? ("
f"y = yes ({default_level}) [default], "
"n = no, "
"A = yes to all, "
"i = yes (info), "
"w = yes (warning), "
"e = yes (error), "
"c = yes (critical), "
"x = yes (exception), "
"q = quit): " + Bcolor.ENDC)
if inp in ('q', 'Q'):
sys.exit(0)
elif inp in ['i', 'w', 'e', 'c', 'x']:
level = levels[inp]
line = get_line(stmt, level)
elif inp == 'A':
accept_all = True
if accept_all or inp in ['', 'y', 'A', 'i', 'w', 'e', 'c', 'e']:
lines[stmt.lineix] = line
edited = True
to_del_lineixs.extend(
range(stmt.lineix + 1, stmt.end_lineix + 1))
n_changes += 1
for index in sorted(to_del_lineixs, reverse=True):
del lines[index]
if edited:
# insert import statement, if necessacy:
if not has_logging_imported:
lines.insert(first_import_lineno - 1, 'import logging')
print("Added `import logging`. And ", end=' ')
path.write_text('\n'.join(lines))
print(f"Wrote {n_changes} changes to {fpath}")
sys.exit(1)
path = sys.argv[1]
print("# %s" % path)
code = open(path, 'r').read()
tree = ast.parse(code)
found_ConanFile = False
for node in ast.walk(tree):
if isinstance(node, ast.ClassDef):
baseIdList = [b.id for b in node.bases if hasattr(b, "id")]
# class must inherit from class ConanFile
if "ConanFile" not in baseIdList:
continue
print("class %s (%s):" % (node.name, ", ".join(baseIdList)))
found_ConanFile = True
if isinstance(node, ast.FunctionDef) and node.name == 'source':
#print(node.name)
print(ast.get_source_segment(code, node))
"""
for child in node.body:
print("child:")
print(ast.get_source_segment(code, child))
"""
if found_ConanFile == False:
print("error: not implemented: no `class SomeClass(ConanFile)` was found")
def get_source_segment(self, *args):
return Code(ast.get_source_segment(self._src, args[0]))
def __waiting(self, ttype, tstring, lineno):
opts = self.__options
# Do docstring extractions, if enabled
if opts.docstrings:
# Module docstring?
if self.__fresh_module:
if ttype == tokenize.STRING and is_literal_string(tstring):
self.__add_entry(
safe_eval(tstring), lineno, is_docstring=True
)
self.__fresh_module = False
elif ttype not in (
tokenize.ENCODING,
tokenize.COMMENT,
tokenize.NL,
):
self.__fresh_module = False
return
# Class or func/method docstring?
if ttype == tokenize.NAME and tstring in ('class', 'def'):
self.__state = self.__suite_seen
return
if opts.cmd_docstrings:
if ttype == tokenize.OP and tstring == '@':
self.__state = self.__decorator_seen
return
elif ttype == tokenize.NAME and tstring == 'class':
self.__state = self.__class_seen
return
if ttype == tokenize.NAME and tstring in opts.keywords:
self.__state = self.__keyword_seen
return
if ttype == tokenize.COMMENT:
if self.__comment and lineno == self.__comment[1] + 1:
self.__comment[0] += tstring[1:]
self.__comment[1] = lineno
else:
comment = tstring[1:].strip().split(' ', 1)
if comment[0].rstrip(':') in opts.c_keywords:
self.__comment = [comment[1], lineno]
if ttype == tokenize.STRING:
maybe_fstring = ast.parse(tstring, mode='eval').body
if not isinstance(maybe_fstring, ast.JoinedStr):
return
for value in filter(
lambda node: isinstance(node, ast.FormattedValue),
maybe_fstring.values,
):
for call in filter(
lambda node: isinstance(node, ast.Call), ast.walk(value)
):
func = call.func
if isinstance(func, ast.Name):
func_name = func.id
elif isinstance(func, ast.Attribute):
func_name = func.attr
else:
continue
if func_name not in opts.keywords:
continue
if len(call.args) != 1 and not opts.multiple_args:
print(
(
'*** %(file)s:%(lineno)s: Seen unexpected amount of'
' positional arguments in gettext call: %(source_segment)s'
)
% {
'source_segment': ast.get_source_segment(
tstring, call
)
or tstring,
'file': self.__cur_infile,
'lineno': lineno,
},
file=sys.stderr,
)
continue
if call.keywords and not opts.multiple_args:
print(
(
'*** %(file)s:%(lineno)s: Seen unexpected keyword arguments'
' in gettext call: %(source_segment)s'
)
% {
'source_segment': ast.get_source_segment(
tstring, call
)
or tstring,
'file': self.__cur_infile,
'lineno': lineno,
},
file=sys.stderr,
)
continue
arg = call.args[0]
if not isinstance(arg, ast.Constant):
print(
(
'*** %(file)s:%(lineno)s: Seen unexpected argument type'
' in gettext call: %(source_segment)s'
)
% {
'source_segment': ast.get_source_segment(
tstring, call
)
or tstring,
'file': self.__cur_infile,
'lineno': lineno,
},
file=sys.stderr,
)
continue
if isinstance(arg.value, str):
self.__add_entry(arg.value, lineno)
def __waiting(self, ttype, tstring, lineno):
opts = self.__options
# Do docstring extractions, if enabled
if opts.docstrings and not opts.nodocstrings.get(self.__curfile):
# module docstring?
if self.__freshmodule:
if ttype == tokenize.STRING and is_literal_string(tstring):
self.__addentry(safe_eval(tstring), lineno, isdocstring=1)
self.__freshmodule = 0
elif ttype not in (tokenize.COMMENT, tokenize.NL):
self.__freshmodule = 0
return
# class or func/method docstring?
if ttype == tokenize.NAME and tstring in ('class', 'def'):
self.__state = self.__suiteseen
return
if ttype == tokenize.NAME and tstring in opts.keywords:
self.__state = self.__keywordseen
return
if ttype == tokenize.STRING:
maybe_fstring = ast.parse(tstring, mode='eval').body
if not isinstance(maybe_fstring, ast.JoinedStr):
return
for value in filter(
lambda node: isinstance(node, ast.FormattedValue),
maybe_fstring.values):
for call in filter(lambda node: isinstance(node, ast.Call),
ast.walk(value)):
func = call.func
if isinstance(func, ast.Name):
func_name = func.id
elif isinstance(func, ast.Attribute):
func_name = func.attr
else:
continue
if func_name not in opts.keywords:
continue
if len(call.args) != 1:
print(_(
'*** %(file)s:%(lineno)s: Seen unexpected amount of'
' positional arguments in gettext call: %(source_segment)s'
) % {
'source_segment':
ast.get_source_segment(tstring, call) or tstring,
'file':
self.__curfile,
'lineno':
lineno
},
file=sys.stderr)
continue
if call.keywords:
print(_(
'*** %(file)s:%(lineno)s: Seen unexpected keyword arguments'
' in gettext call: %(source_segment)s') % {
'source_segment':
ast.get_source_segment(tstring, call)
or tstring,
'file':
self.__curfile,
'lineno':
lineno
},
file=sys.stderr)
continue
arg = call.args[0]
if not isinstance(arg, ast.Constant):
print(_(
'*** %(file)s:%(lineno)s: Seen unexpected argument type'
' in gettext call: %(source_segment)s') % {
'source_segment':
ast.get_source_segment(tstring, call)
or tstring,
'file':
self.__curfile,
'lineno':
lineno
},
file=sys.stderr)
continue
if isinstance(arg.value, str):
self.__addentry(arg.value, lineno)
def update_event(self, inp=-1):
self.set_output_val(
0, ast.get_source_segment(self.input(0), self.input(1)))
def visit_Assign(self, node):
expression = AssignExpression([t.id for t in node.targets],
ast.get_source_segment(self.__sourcecode, node, padded=True))
self.__current = expression
self.__expressions.append(self.__current)
self.generic_visit(node)
def segment(self):
return ast.get_source_segment(self.source.text,
self.node,
padded=False)
def source(self, node):
return ast.get_source_segment(self.code, node).strip('\n')
def AST_FunctionDataExtract(f, fullCode):
''' Extracts function data from AST '''
# Init
FunctionData = {"language": "python", "name": '', "params": '', "desc": "", "imports": [], "code": ""}
# Extract
# Name
FunctionData['name'] = f.name
# Desc
# FunctionData['desc'] = [x.s for x in f.body]
# Imports
ImportsData = []
Imports = [x for x in f.body if isinstance(x, ast.Import)]
FromImports = [x for x in f.body if isinstance(x, ast.ImportFrom)]
for impG in Imports:
impsDicts = []
for imp in impG.names:
name = imp.name.split(".")[-1]
module = imp.name.rstrip(name).rstrip(".")
impData = {
"name": name,
"dependency_path": module.split(".")
}
impsDicts.append(impData)
ImportsData.extend(impsDicts)
for impG in FromImports:
impsDicts = []
for imp in impG.names:
name = imp.name.split(".")[-1]
module = ".".join([impG.module, imp.name.rstrip(name).rstrip(".")]).rstrip(".")
impData = {
"name": name,
"dependency_path": module.split(".")
}
impsDicts.append(impData)
ImportsData.extend(impsDicts)
FunctionData['imports'] = ImportsData
# Params
FunctionData['params'] = []
defaultCount = len(f.args.defaults)
argsCount = len(f.args.args)
for i in range(len(f.args.args)):
a = f.args.args[i]
d = None
if i >= argsCount - defaultCount:
d = f.args.defaults[i - (argsCount - defaultCount)].value
d_dict = {"value": d}
t = a.annotation.id if a.annotation is not None else None
paramData = {
"name": a.arg,
"desc": "",
"default_value": json.dumps(d_dict),
"type": t
}
FunctionData['params'].append(paramData)
# Code
FunctionData['code'] = ast.get_source_segment(fullCode, f)
return FunctionData
