def handle_or_else(self, orelse, test):
"""Handle the orelse part of an if or try node.
Args:
orelse(list[Node])
test(Node)
Returns:
The last nodes of the orelse branch.
"""
if isinstance(orelse[0], ast.If):
control_flow_node = self.visit(orelse[0])
if isinstance(control_flow_node, IgnoredNode):
return IgnoredNode()
# Prefix the if label with 'el'
control_flow_node.test.label = "el" + control_flow_node.test.label
if test is not None:
test.connect(control_flow_node.test)
return control_flow_node.last_nodes
else:
else_connect_statements = self.stmt_star_handler(
orelse, prev_node_to_avoid=self.nodes[-1])
if isinstance(else_connect_statements, IgnoredNode):
return IgnoredNode()
if test is not None:
test.connect(else_connect_statements.first_statement)
return else_connect_statements.last_statements
def visit_ImportFrom_deep(self, node):
# Is it relative?
if node.level > 0:
return self.handle_relative_import(node)
# not relative
for module in self.local_modules:
if node.module == module[0]:
if os.path.isdir(module[1]):
return self.from_directory_import(
module,
not_as_alias_handler(node.names),
as_alias_handler(node.names),
)
return self.add_module(
module=module,
module_or_package_name=None,
local_names=as_alias_handler(node.names),
import_alias_mapping=retrieve_import_alias_mapping(
node.names),
from_from=True,
)
for module in self.project_modules:
name = module[0]
if node.level == 0:
break
if node.module == name:
if os.path.isdir(module[1]):
if visited_module_paths.get(module[1]):
return IgnoredNode()
# Break recursion
visited_module_paths[module[1]] = True
return self.from_directory_import(
module,
not_as_alias_handler(node.names),
as_alias_handler(node.names),
retrieve_import_alias_mapping(node.names),
)
return self.add_module(
module=module,
module_or_package_name=None,
local_names=as_alias_handler(node.names),
import_alias_mapping=retrieve_import_alias_mapping(
node.names),
from_from=True,
)
# Remember aliases for uninspectable modules such that we can label them fully qualified
# e.g. we want a call to "os.system" be recognised, even if we do "from os import system"
# from os import system as mysystem -> module=os, name=system, asname=mysystem
for name in node.names:
local_definitions = self.module_definitions_stack[-1]
local_definitions.import_alias_mapping[
name.asname
or name.name] = "{}.{}".format(node.module, name.name)
if node.module not in uninspectable_modules:
uninspectable_modules.add(node.module)
return IgnoredNode()
def loop_node_skeleton(self, test, node):
"""Common handling of looped structures, while and for."""
body_connect_stmts = self.stmt_star_handler(
node.body, prev_node_to_avoid=self.nodes[-1]
)
if isinstance(body_connect_stmts, IgnoredNode):
return IgnoredNode()
if test is not None:
test.connect(body_connect_stmts.first_statement)
test.connect_predecessors(body_connect_stmts.last_statements)
# last_nodes is used for making connections to the next node in the parent node
# this is handled in stmt_star_handler
last_nodes = list()
last_nodes.extend(body_connect_stmts.break_statements)
if node.orelse:
orelse_connect_stmts = self.stmt_star_handler(
node.orelse, prev_node_to_avoid=self.nodes[-1]
)
if not isinstance(orelse_connect_stmts, IgnoredNode):
if test is not None:
test.connect(orelse_connect_stmts.first_statement)
last_nodes.extend(orelse_connect_stmts.last_statements)
else:
last_nodes.append(
test
) # if there is no orelse, test needs an edge to the next_node
return ControlFlowNode(test, last_nodes, list())
def visit_AnnAssign(self, node):
if node.value is None:
return IgnoredNode()
else:
assign = ast.Assign(targets=[node.target], value=node.value)
ast.copy_location(assign, node)
return self.visit(assign)
def visit_If(self, node):
test = self.append_node(IfNode(node.test, node, path=self.filenames[-1]))
body_connect_stmts = self.stmt_star_handler(node.body)
if isinstance(body_connect_stmts, IgnoredNode):
body_connect_stmts = ConnectStatements(
first_statement=test, last_statements=[], break_statements=[]
)
if test is not None:
test.connect(body_connect_stmts.first_statement)
if node.orelse:
orelse_last_nodes = self.handle_or_else(node.orelse, test)
if isinstance(orelse_last_nodes, IgnoredNode):
return IgnoredNode()
body_connect_stmts.last_statements.extend(orelse_last_nodes)
else:
body_connect_stmts.last_statements.append(
test
) # if there is no orelse, test needs an edge to the next_node
last_statements = remove_breaks(body_connect_stmts.last_statements)
return ControlFlowNode(
test, last_statements, break_statements=body_connect_stmts.break_statements
)
def visit_ImportFrom(self, node):
for name in node.names:
local_definitions = self.module_definitions_stack[-1]
local_definitions.import_alias_mapping[
name.asname or name.name
] = "{}.{}".format(node.module, name.name)
if node.module not in uninspectable_modules:
uninspectable_modules.add(node.module)
return IgnoredNode()
def handle_relative_import(self, node):
"""
from A means node.level == 0
from . import B means node.level == 1
from .A means node.level == 1
"""
no_file = os.path.abspath(os.path.join(self.filenames[-1], os.pardir))
skip_init = False
if node.level == 1:
# Same directory as current file
if node.module:
name_with_dir = os.path.join(no_file,
node.module.replace(".", "/"))
if not os.path.isdir(name_with_dir):
name_with_dir = name_with_dir + ".py"
# e.g. from . import X
else:
name_with_dir = no_file
# We do not want to analyse the init file of the current directory
skip_init = True
else:
parent = os.path.abspath(os.path.join(no_file, os.pardir))
if node.level > 2:
# Perform extra `cd ..` however many times
for _ in range(0, node.level - 2):
parent = os.path.abspath(os.path.join(parent, os.pardir))
if node.module:
name_with_dir = os.path.join(parent,
node.module.replace(".", "/"))
if not os.path.isdir(name_with_dir):
name_with_dir = name_with_dir + ".py"
# e.g. from .. import X
else:
name_with_dir = parent
# Is it a file?
if name_with_dir.endswith(".py"):
if visited_module_paths.get(name_with_dir):
return IgnoredNode()
visited_module_paths[name_with_dir] = True
return self.add_module(
module=(node.module, name_with_dir),
module_or_package_name=None,
local_names=as_alias_handler(node.names),
import_alias_mapping=retrieve_import_alias_mapping(node.names),
from_from=True,
)
return self.from_directory_import(
(node.module, name_with_dir),
not_as_alias_handler(node.names),
as_alias_handler(node.names),
retrieve_import_alias_mapping(node.names),
skip_init=skip_init,
)
def visit_Import_deep(self, node):
for name in node.names:
if not hasattr(name, "name"):
continue
if name.name in BUILTIN_PKGS or visited_module_paths.get(
name.name):
continue
visited_module_paths[name.name] = True
for module in self.local_modules:
if name.name == module[0]:
if os.path.isdir(module[1]):
return self.import_package(
module,
name,
name.asname,
retrieve_import_alias_mapping(node.names),
)
return self.add_module(
module=module,
module_or_package_name=name.name,
local_names=name.asname,
import_alias_mapping=retrieve_import_alias_mapping(
node.names),
)
for module in self.project_modules:
if name.name == module[0]:
if os.path.isdir(module[1]):
return self.import_package(
module,
name,
name.asname,
retrieve_import_alias_mapping(node.names),
)
return self.add_module(
module=module,
module_or_package_name=name.name,
local_names=name.asname,
import_alias_mapping=retrieve_import_alias_mapping(
node.names),
)
for alias in node.names:
# The module is uninspectable (so blackbox or built-in). If it has an alias, we remember
# the alias so we can do fully qualified name resolution for blackbox- and built-in trigger words
# e.g. we want a call to "os.system" be recognised, even if we do "import os as myos"
if alias.asname is not None and alias.asname != alias.name:
local_definitions = self.module_definitions_stack[-1]
local_definitions.import_alias_mapping[
name.asname] = alias.name
if alias.name not in uninspectable_modules:
uninspectable_modules.add(
alias.name) # Don't repeatedly warn about this
return IgnoredNode()
def visit_Import(self, node):
for alias in node.names:
# The module is uninspectable (so blackbox or built-in). If it has an alias, we remember
# the alias so we can do fully qualified name resolution for blackbox- and built-in trigger words
# e.g. we want a call to "os.system" be recognised, even if we do "import os as myos"
if alias.asname is not None and alias.asname != alias.name:
local_definitions = self.module_definitions_stack[-1]
local_definitions.import_alias_mapping[alias.asname] = alias.name
if alias.name not in uninspectable_modules:
uninspectable_modules.add(
alias.name
) # Don't repeatedly warn about this
return IgnoredNode()
def stmt_star_handler(self, stmts, prev_node_to_avoid=None):
"""Handle stmt* expressions in an AST node.
Links all statements together in a list of statements, accounting for statements with multiple last nodes.
"""
break_nodes = list()
cfg_statements = list()
self.prev_nodes_to_avoid.append(prev_node_to_avoid)
self.last_control_flow_nodes.append(None)
first_node = None
node_not_to_step_past = self.nodes[-1]
for stmt in stmts:
node = self.visit(stmt)
if isinstance(node, IgnoredNode):
continue
if isinstance(node, ControlFlowNode) and not isinstance(
node.test, TryNode):
self.last_control_flow_nodes.append(node.test)
else:
self.last_control_flow_nodes.append(None)
if isinstance(node, ControlFlowNode):
break_nodes.extend(node.break_statements)
elif isinstance(node, BreakNode):
break_nodes.append(node)
cfg_statements.append(node)
if not first_node:
if isinstance(node, ControlFlowNode):
first_node = node.test
else:
first_node = get_first_node(node, node_not_to_step_past)
self.prev_nodes_to_avoid.pop()
self.last_control_flow_nodes.pop()
if cfg_statements:
connect_nodes(cfg_statements)
if first_node:
first_statement = first_node
else:
first_statement = get_first_statement(cfg_statements[0])
last_statements = get_last_statements(cfg_statements)
return ConnectStatements(
first_statement=first_statement,
last_statements=last_statements,
break_statements=break_nodes,
)
else: # When body of module only contains ignored nodes
return IgnoredNode()
def from_directory_import(self,
module,
real_names,
local_names,
import_alias_mapping,
skip_init=False):
"""
Directories don't need to be packages.
"""
module_path = module[1]
init_file_location = os.path.join(module_path, "__init__.py")
init_exists = os.path.isfile(init_file_location)
if init_exists and not skip_init:
package_name = os.path.split(module_path)[1]
return self.add_module(
module=(module[0], init_file_location),
module_or_package_name=package_name,
local_names=local_names,
import_alias_mapping=import_alias_mapping,
is_init=True,
from_from=True,
)
for real_name in real_names:
full_name = os.path.join(module_path, real_name)
if os.path.isdir(full_name):
new_init_file_location = os.path.join(full_name, "__init__.py")
if os.path.isfile(new_init_file_location):
self.add_module(
module=(real_name, new_init_file_location),
module_or_package_name=real_name,
local_names=local_names,
import_alias_mapping=import_alias_mapping,
is_init=True,
from_from=True,
from_fdid=True,
)
else:
continue
else:
file_module = (real_name, full_name + ".py")
self.add_module(
module=file_module,
module_or_package_name=real_name,
local_names=local_names,
import_alias_mapping=import_alias_mapping,
from_from=True,
)
return IgnoredNode()
def visit_With(self, node):
label_visitor = LabelVisitor()
label_visitor.visit(node.items[0])
with_node = self.append_node(
Node(label_visitor.result, node, path=self.filenames[-1]))
connect_statements = self.stmt_star_handler(node.body)
if isinstance(connect_statements, IgnoredNode):
return IgnoredNode()
if with_node is not None:
with_node.connect(connect_statements.first_statement)
return ControlFlowNode(
with_node,
connect_statements.last_statements,
connect_statements.break_statements,
)
def visit_ClassDef(self, node):
self.add_to_definitions(node)
local_definitions = self.module_definitions_stack[-1]
local_definitions.classes.append(node.name)
parent_definitions = self.get_parent_definitions()
if parent_definitions:
parent_definitions.classes.append(node.name)
self.stmt_star_handler(node.body)
local_definitions.classes.pop()
if parent_definitions:
parent_definitions.classes.pop()
return IgnoredNode()
def add_module( # noqa: C901
self,
module,
module_or_package_name,
local_names,
import_alias_mapping,
is_init=False,
from_from=False,
from_fdid=False,
):
"""
Returns:
The ExitNode that gets attached to the CFG of the class.
"""
module_path = module[1]
if module_or_package_name in BUILTIN_PKGS:
uninspectable_modules.add(module_or_package_name)
return IgnoredNode()
if visited_module_paths.get(module[0]) or visited_module_paths.get(
module_or_package_name
):
return IgnoredNode()
visited_module_paths[module[0]] = True
visited_module_paths[module_or_package_name] = True
parent_definitions = self.module_definitions_stack[-1]
# Here, in `visit_Import` and in `visit_ImportFrom` are the only places the `import_alias_mapping` is updated
parent_definitions.import_alias_mapping.update(import_alias_mapping)
parent_definitions.import_names = local_names
new_module_definitions = ModuleDefinitions(local_names, module_or_package_name)
new_module_definitions.is_init = is_init
self.module_definitions_stack.append(new_module_definitions)
# Analyse the file
self.filenames.append(module_path)
self.local_modules = (
get_directory_modules(module_path) if self._allow_local_modules else []
)
tree = generate_ast(module_path)
if not tree:
return IgnoredNode()
# module[0] is None during e.g. "from . import foo", so we must str()
self.nodes.append(EntryOrExitNode("Module Entry " + str(module[0])))
self.visit(tree)
exit_node = self.append_node(EntryOrExitNode("Module Exit " + str(module[0])))
# Done analysing, pop the module off
self.module_definitions_stack.pop()
self.filenames.pop()
if new_module_definitions.is_init:
for def_ in new_module_definitions.definitions:
module_def_alias = handle_aliases_in_init_files(
def_.name, new_module_definitions.import_alias_mapping
)
parent_def_alias = handle_aliases_in_init_files(
def_.name, parent_definitions.import_alias_mapping
)
# They should never both be set
assert not (module_def_alias and parent_def_alias)
def_name = def_.name
if parent_def_alias:
def_name = parent_def_alias
if module_def_alias:
def_name = module_def_alias
local_definitions = self.module_definitions_stack[-1]
if local_definitions != parent_definitions:
continue
if not isinstance(module_or_package_name, str):
module_or_package_name = module_or_package_name.name
if module_or_package_name:
if from_from:
qualified_name = def_name
if from_fdid:
alias = handle_fdid_aliases(
module_or_package_name, import_alias_mapping
)
if alias:
module_or_package_name = alias
parent_definition = ModuleDefinition(
parent_definitions,
qualified_name,
module_or_package_name,
self.filenames[-1],
)
else:
parent_definition = ModuleDefinition(
parent_definitions,
qualified_name,
None,
self.filenames[-1],
)
else:
qualified_name = module_or_package_name + "." + def_name
parent_definition = ModuleDefinition(
parent_definitions,
qualified_name,
parent_definitions.module_name,
self.filenames[-1],
)
parent_definition.node = def_.node
parent_definitions.definitions.append(parent_definition)
else:
parent_definition = ModuleDefinition(
parent_definitions,
def_name,
parent_definitions.module_name,
self.filenames[-1],
)
parent_definition.node = def_.node
parent_definitions.definitions.append(parent_definition)
return exit_node
def visit_FunctionDef(self, node):
self.add_to_definitions(node)
return IgnoredNode()
def visit_Str(self, node):
return IgnoredNode()
