def visit_For(self, node):
# only handle name targets
if isinstance(node.target, ast.Name):
target_def = self.def_manager.get(
utils.join_ns(self.current_ns, node.target.id))
# if the target definition exists
if target_def:
iter_decoded = self.decode_node(node.iter)
# assign the target to the return value
# of the next function
for item in iter_decoded:
if not isinstance(item, Definition):
continue
# return value for generators
for name in self.closured.get(item.get_ns(), []):
# If there exists a next method on the iterable
# and if yes, add a pointer to it
next_defi = self.def_manager.get(
utils.join_ns(name, utils.constants.NEXT_METHOD,
utils.constants.RETURN_NAME))
if next_defi:
for name in self.closured.get(
next_defi.get_ns(), []):
target_def.get_name_pointer().add(name)
else: # otherwise, add a pointer to the name (e.g. a yield)
target_def.get_name_pointer().add(name)
super().visit_For(node)
def _retrieve_attribute_names(self, node):
if not getattr(self, "closured", None):
return set()
parent_names = self._retrieve_parent_names(node)
names = set()
for parent_name in parent_names:
for name in self.closured.get(parent_name, []):
defi = self.def_manager.get(name)
if not defi:
continue
if defi.get_type() == utils.constants.CLS_DEF:
cls_names = self.find_cls_fun_ns(defi.get_ns(), node.attr)
if cls_names:
names = names.union(cls_names)
if defi.get_type() in [utils.constants.FUN_DEF, utils.constants.MOD_DEF]:
names.add(utils.join_ns(name, node.attr))
if defi.get_type() == utils.constants.EXT_DEF:
# HACK: extenral attributes can lead to infinite loops
# Identify them here
if node.attr in name:
continue
ext_name = utils.join_ns(name, node.attr)
if not self.def_manager.get(ext_name):
self.def_manager.create(ext_name, utils.constants.EXT_DEF)
names.add(ext_name)
return names
def visit_Dict(self, node):
# 1. create a scope using a counter
# 2. Iterate keys and add them as children of the scope
# 3. Iterate values and makes a points to connection with the keys
current_scope = self.scope_manager.get_scope(self.current_ns)
dict_counter = current_scope.inc_dict_counter()
dict_name = utils.get_dict_name(dict_counter)
dict_full_ns = utils.join_ns(self.current_ns, dict_name)
# create a scope for the dict
dict_scope = self.scope_manager.create_scope(dict_full_ns,
current_scope)
# Create a dict definition
dict_def = self.def_manager.get(dict_full_ns)
if not dict_def:
dict_def = self.def_manager.create(dict_full_ns,
utils.constants.NAME_DEF)
# add it to the current scope
current_scope.add_def(dict_name, dict_def)
self.name_stack.append(dict_name)
for key, value in zip(node.keys, node.values):
if key:
self.visit(key)
if value:
self.visit(value)
decoded_key = self.decode_node(key)
decoded_value = self.decode_node(value)
# iterate decoded keys and values
# to do the assignment operation
for k in decoded_key:
if isinstance(k, Definition):
# get literal pointer
names = k.get_lit_pointer().get()
else:
names = set()
if isinstance(k, list):
continue
names.add(k)
for name in names:
# create a definition for the key
if isinstance(name, int):
name = utils.get_int_name(name)
key_full_ns = utils.join_ns(dict_def.get_ns(), str(name))
key_def = self.def_manager.get(key_full_ns)
if not key_def:
key_def = self.def_manager.create(
key_full_ns, utils.constants.NAME_DEF)
dict_scope.add_def(str(name), key_def)
for v in decoded_value:
if isinstance(v, Definition):
key_def.get_name_pointer().add(v.get_ns())
else:
key_def.get_lit_pointer().add(v)
self.name_stack.pop()
def handle_function_def(self, parent_ns, fn_name):
full_ns = utils.join_ns(parent_ns, fn_name)
defi = self.get(full_ns)
if not defi:
defi = self.create(full_ns, utils.constants.FUN_DEF)
defi.decorator_names = set()
return_ns = utils.join_ns(full_ns, utils.constants.RETURN_NAME)
if not self.get(return_ns):
self.create(return_ns, utils.constants.NAME_DEF)
return defi
def _get_target_ns(self, target):
if isinstance(target, ast.Name):
return [utils.join_ns(self.current_ns, target.id)]
if isinstance(target, ast.Attribute):
bases = self._retrieve_base_names(target)
res = []
for base in bases:
res.append(utils.join_ns(base, target.attr))
return res
if isinstance(target, ast.Subscript):
return self.retrieve_subscript_names(target)
return []
def assign(self, ns, defi):
self.defs[ns] = Definition(ns, defi.get_type())
self.defs[ns].merge(defi)
# if it is a function def, we need to create a return pointer
if defi.is_function_def():
return_ns = utils.join_ns(ns, utils.constants.RETURN_NAME)
self.defs[return_ns] = Definition(return_ns,
utils.constants.NAME_DEF)
self.defs[return_ns].get_name_pointer().add(
utils.join_ns(defi.get_ns(), utils.constants.RETURN_NAME))
return self.defs[ns]
def test_uri(self):
self.cg_generator.functions = ['mod1.mod2.myfunc']
formatter = self.get_formatter()
### Internal uri check
# test modname without method
self.assertEqual(formatter.to_uri('mymod'), '/mymod/')
self.assertEqual(formatter.to_uri('mymod.mod1'), '/mymod.mod1/')
# test method starting with modname
self.assertEqual(formatter.to_uri('mymod.mod1', 'mymod.mod1.fn'), '/mymod.mod1/fn')
self.assertEqual(formatter.to_uri('mymod.mod1', 'mymod.mod1.cls.fn'), '/mymod.mod1/cls.fn')
# test method starting with modname but without . inbetween
with self.assertRaises(Exception):
formatter.to_uri('mymod.mod1', 'mymod.mod1cls.fn')
# test method being in functions
self.assertEqual(formatter.to_uri('mod1.mod2', 'mod1.mod2.myfunc'), '/mod1.mod2/myfunc()')
### External uri check
# test modname builtin
self.assertEqual(
formatter.to_external_uri(utils.constants.BUILTIN_NAME, utils.join_ns(utils.constants.BUILTIN_NAME, 'cls1.fn1')),
'//.builtin//cls1.fn1'
)
# test modname not builtin
self.assertEqual(formatter.to_external_uri('requests', 'requests.Request.get'), '//requests//requests.Request.get')
def update_parent_classes(self, defi):
cls = self.class_manager.get(defi.get_ns())
if not cls:
return
current_scope = self.scope_manager.get_scope(defi.get_ns())
for parent in cls.get_mro():
parent_def = self.def_manager.get(parent)
if not parent_def:
continue
parent_scope = self.scope_manager.get_scope(parent)
if not parent_scope:
continue
parent_items = list(parent_scope.get_defs().keys())
for key, child_def in current_scope.get_defs().items():
if key == "__init__":
continue
# resolve name from the parent_def
names = self.find_cls_fun_ns(parent_def.get_ns(), key)
new_ns = utils.join_ns(parent_def.get_ns(), key)
new_def = self.def_manager.get(new_ns)
if not new_def:
new_def = self.def_manager.create(new_ns,
utils.constants.NAME_DEF)
new_def.get_name_pointer().add_set(names)
new_def.get_name_pointer().add(child_def.get_ns())
def retrieve_call_names(self, node):
names = set()
if isinstance(node.func, ast.Name):
defi = self.scope_manager.get_def(self.current_ns, node.func.id)
if defi:
names = self.closured.get(defi.get_ns(), None)
elif isinstance(node.func, ast.Call) and self.last_called_names:
for name in self.last_called_names:
return_ns = utils.join_ns(name, utils.constants.RETURN_NAME)
returns = self.closured.get(return_ns)
if not returns:
continue
for ret in returns:
defi = self.def_manager.get(ret)
names.add(defi.get_ns())
elif isinstance(node.func, ast.Attribute):
names = self._retrieve_attribute_names(node.func)
elif isinstance(node.func, ast.Subscript):
# Calls can be performed only on single indices, not ranges
full_names = self.retrieve_subscript_names(node.func)
for n in full_names:
if self.closured.get(n, None):
names |= self.closured.get(n)
return names
def handle_class_def(self, parent_ns, cls_name):
full_ns = utils.join_ns(parent_ns, cls_name)
defi = self.get(full_ns)
if not defi:
defi = self.create(full_ns, utils.constants.CLS_DEF)
return defi
def find_cls_fun_ns(self, cls_name, fn):
cls = self.class_manager.get(cls_name)
if not cls:
return set()
ext_names = set()
for item in cls.get_mro():
ns = utils.join_ns(item, fn)
names = set()
if getattr(self, "closured", None) and self.closured.get(ns, None):
names = self.closured[ns]
else:
names.add(ns)
if self.def_manager.get(ns):
return names
parent = self.def_manager.get(item)
if parent and parent.get_type() == utils.constants.EXT_DEF:
ext_names.add(ns)
for name in ext_names:
self.def_manager.create(name, utils.constants.EXT_DEF)
self.add_ext_mod_node(name)
return ext_names
def visit_Call(self, node):
def create_ext_edge(name, ext_modname):
self.add_ext_mod_node(name)
self.call_graph.add_node(name, ext_modname)
self.call_graph.add_edge(self.current_method, name)
# First visit the child function so that on the case of
# func()()()
# we first visit the call to func and then the other calls
for arg in node.args:
self.visit(arg)
for keyword in node.keywords:
self.visit(keyword.value)
self.visit(node.func)
names = self.retrieve_call_names(node)
if not names:
if isinstance(node.func, ast.Attribute) and self.has_ext_parent(
node.func):
# TODO: This doesn't work for cases where there is an assignment of an attribute
# i.e. import os; lala = os.path; lala.dirname()
for name in self.get_full_attr_names(node.func):
ext_modname = name.split(".")[0]
create_ext_edge(name, ext_modname)
elif getattr(node.func, "id", None) and self.is_builtin(
node.func.id):
name = utils.join_ns(utils.constants.BUILTIN_NAME,
node.func.id)
create_ext_edge(name, utils.constants.BUILTIN_NAME)
return
self.last_called_names = names
for pointer in names:
pointer_def = self.def_manager.get(pointer)
if not pointer_def or not isinstance(pointer_def, Definition):
continue
if pointer_def.is_callable():
if pointer_def.get_type() == utils.constants.EXT_DEF:
ext_modname = pointer.split(".")[0]
create_ext_edge(pointer, ext_modname)
continue
self.call_graph.add_edge(self.current_method, pointer)
# TODO: This doesn't work and leads to calls from the decorators
# themselves to the function, creating edges to the first decorator
#for decorator in pointer_def.decorator_names:
# dec_names = self.closured.get(decorator, [])
# for dec_name in dec_names:
# if self.def_manager.get(dec_name).get_type() == utils.constants.FUN_DEF:
# self.call_graph.add_edge(self.current_ns, dec_name)
if pointer_def.get_type() == utils.constants.CLS_DEF:
init_ns = self.find_cls_fun_ns(pointer,
utils.constants.CLS_INIT)
for ns in init_ns:
self.call_graph.add_edge(self.current_method, ns)
def create_def(scope, name, imported_def):
if not name in scope.get_defs():
def_ns = utils.join_ns(scope.get_ns(), name)
defi = self.def_manager.get(def_ns)
if not defi:
defi = self.def_manager.assign(def_ns, imported_def)
defi.get_name_pointer().add(imported_def.get_ns())
current_scope.add_def(name, defi)
def visit_FunctionDef(self, node):
# here we iterate decorators
if node.decorator_list:
fn_def = self.def_manager.get(
utils.join_ns(self.current_ns, node.name))
reversed_decorators = list(reversed(node.decorator_list))
# add to the name pointer of the function definition
# the return value of the first decorator
# since, now the function is a namespace to that point
if hasattr(fn_def, "decorator_names") and reversed_decorators:
last_decoded = self.decode_node(reversed_decorators[-1])
for d in last_decoded:
if not isinstance(d, Definition):
continue
fn_def.decorator_names.add(
utils.join_ns(d.get_ns(), utils.constants.RETURN_NAME))
previous_names = self.closured.get(fn_def.get_ns(), set())
for decorator in reversed_decorators:
# assign the previous_def as the first parameter of the decorator
decoded = self.decode_node(decorator)
new_previous_names = set()
for d in decoded:
if not isinstance(d, Definition):
continue
for name in self.closured.get(d.get_ns(), []):
return_ns = utils.join_ns(name,
utils.constants.RETURN_NAME)
if self.closured.get(return_ns, None) == None:
continue
new_previous_names = new_previous_names.union(
self.closured.get(return_ns))
for prev_name in previous_names:
pos_arg_names = d.get_name_pointer().get_pos_arg(0)
if not pos_arg_names:
continue
for name in pos_arg_names:
arg_def = self.def_manager.get(name)
arg_def.get_name_pointer().add(prev_name)
previous_names = new_previous_names
super().visit_FunctionDef(node)
def _visit_return(self, node):
if not node or not node.value:
return
self.visit(node.value)
return_ns = utils.join_ns(self.current_ns, utils.constants.RETURN_NAME)
self._handle_assign(return_ns, self.decode_node(node.value))
def visit_Lambda(self, node):
counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter()
lambda_name = utils.get_lambda_name(counter)
lambda_fullns = utils.join_ns(self.current_ns, lambda_name)
self.call_graph.add_node(lambda_fullns, self.modname)
super().visit_Lambda(node, lambda_name)
def visit_For(self, node):
# just create the definition for target
if isinstance(node.target, ast.Name):
target_ns = utils.join_ns(self.current_ns, node.target.id)
if not self.def_manager.get(target_ns):
defi = self.def_manager.create(target_ns,
utils.constants.NAME_DEF)
self.scope_manager.get_scope(self.current_ns).add_def(
node.target.id, defi)
super().visit_For(node)
def visit_Lambda(self, node, lambda_name=None):
lambda_ns = utils.join_ns(self.current_ns, lambda_name)
if not self.scope_manager.get_scope(lambda_ns):
self.scope_manager.create_scope(lambda_ns,
self.scope_manager.get_scope(self.current_ns))
self.name_stack.append(lambda_name)
self.method_stack.append(lambda_name)
self.visit(node.body)
self.method_stack.pop()
self.name_stack.pop()
def visit_For(self, node):
self.visit(node.iter)
self.visit(node.target)
# assign target.id to the return value of __next__ of node.iter.it
# we need to have a visit for on the postprocessor also
iter_decoded = self.decode_node(node.iter)
for item in iter_decoded:
if not isinstance(item, Definition):
continue
names = self.closured.get(item.get_ns(), [])
for name in names:
iter_ns = utils.join_ns(name, utils.constants.ITER_METHOD)
next_ns = utils.join_ns(name, utils.constants.NEXT_METHOD)
if self.def_manager.get(iter_ns):
self.call_graph.add_edge(self.current_method, iter_ns)
if self.def_manager.get(next_ns):
self.call_graph.add_edge(self.current_method, next_ns)
super().visit_For(node)
def handle_import(self, name, level):
# We currently don't support builtin modules because they're frozen.
# Add an edge and continue.
# TODO: identify a way to include frozen modules
root = name.split(".")[0]
if root in sys.builtin_module_names:
self.create_edge(root)
return
# Import the module
try:
mod_name, package = self._handle_import_level(name, level)
except ImportError:
return
parent = ".".join(mod_name.split(".")[:-1])
parent_name = ".".join(name.split(".")[:-1])
combos = [(mod_name, package),
(parent, package),
(utils.join_ns(package, name), ""),
(utils.join_ns(package, parent_name), "")]
mod = None
for mn, pkg in combos:
try:
mod = self._do_import(mn, pkg)
break
except:
continue
if not mod:
return
if not hasattr(mod, "__file__") or not mod.__file__:
return
if self.mod_dir not in mod.__file__:
return
fname = mod.__file__
if fname.endswith("__init__.py"):
fname = os.path.split(fname)[0]
return utils.to_mod_name(
os.path.relpath(fname, self.mod_dir))
def visit_Call(self, node):
self.visit(node.func)
# if it is not a name there's nothing we can do here
# ModuleVisitor will be able to resolve those calls
# since it'll have the name tracking information
if not isinstance(node.func, ast.Name):
return
fullns = utils.join_ns(self.current_ns, node.func.id)
defi = self.scope_manager.get_def(self.current_ns, node.func.id)
if not defi:
return
if defi.get_type() == utils.constants.CLS_DEF:
defi = self.def_manager.get(
utils.join_ns(defi.get_ns(), utils.constants.CLS_INIT))
if not defi:
return
self.iterate_call_args(defi, node)
def visit_List(self, node):
counter = self.scope_manager.get_scope(self.current_ns).inc_list_counter()
list_name = utils.get_list_name(counter)
sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, list_name))
if not sc:
return
self.name_stack.append(list_name)
sc.reset_counters()
for elt in node.elts:
self.visit(elt)
self.name_stack.pop()
def visit_FunctionDef(self, node):
for decorator in node.decorator_list:
self.visit(decorator)
decoded = self.decode_node(decorator)
for d in decoded:
if not isinstance(d, Definition):
continue
names = self.closured.get(d.get_ns(), [])
for name in names:
self.call_graph.add_edge(self.current_method, name)
self.call_graph.add_node(utils.join_ns(self.current_ns, node.name), self.modname)
super().visit_FunctionDef(node)
def visit_Lambda(self, node):
# The name of a lambda is defined by the counter of the current scope
current_scope = self.scope_manager.get_scope(self.current_ns)
lambda_counter = current_scope.inc_lambda_counter()
lambda_name = utils.get_lambda_name(lambda_counter)
lambda_full_ns = utils.join_ns(self.current_ns, lambda_name)
# create a scope for the lambda
self.scope_manager.create_scope(lambda_full_ns, current_scope)
lambda_def = self._handle_function_def(node, lambda_name)
# add it to the current scope
current_scope.add_def(lambda_name, lambda_def)
super().visit_Lambda(node, lambda_name)
def visit_List(self, node):
# Works similarly with dicts
current_scope = self.scope_manager.get_scope(self.current_ns)
list_counter = current_scope.inc_list_counter()
list_name = utils.get_list_name(list_counter)
list_full_ns = utils.join_ns(self.current_ns, list_name)
# create a scope for the list
list_scope = self.scope_manager.create_scope(list_full_ns,
current_scope)
# create a list definition
list_def = self.def_manager.get(list_full_ns)
if not list_def:
list_def = self.def_manager.create(list_full_ns,
utils.constants.NAME_DEF)
current_scope.add_def(list_name, list_def)
self.name_stack.append(list_name)
for idx, elt in enumerate(node.elts):
self.visit(elt)
key_full_ns = utils.join_ns(list_def.get_ns(),
utils.get_int_name(idx))
key_def = self.def_manager.get(key_full_ns)
if not key_def:
key_def = self.def_manager.create(key_full_ns,
utils.constants.NAME_DEF)
decoded_elt = self.decode_node(elt)
for v in decoded_elt:
if isinstance(v, Definition):
key_def.get_name_pointer().add(v.get_ns())
else:
key_def.get_lit_pointer().add(v)
self.name_stack.pop()
def add_external_def(name, target):
# add an external def for the name
defi = self.def_manager.get(name)
if not defi:
defi = self.def_manager.create(name, utils.constants.EXT_DEF)
scope = self.scope_manager.get_scope(self.current_ns)
if target != "*":
# add a def for the target that points to the name
tgt_ns = utils.join_ns(scope.get_ns(), target)
tgt_defi = self.def_manager.get(tgt_ns)
if not tgt_defi:
tgt_defi = self.def_manager.create(tgt_ns,
utils.constants.EXT_DEF)
tgt_defi.get_name_pointer().add(defi.get_ns())
scope.add_def(target, tgt_defi)
def visit_Dict(self, node):
counter = self.scope_manager.get_scope(self.current_ns).inc_dict_counter()
dict_name = utils.get_dict_name(counter)
sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, dict_name))
if not sc:
return
self.name_stack.append(dict_name)
sc.reset_counters()
for key, val in zip(node.keys, node.values):
if key:
self.visit(key)
if val:
self.visit(val)
self.name_stack.pop()
def find_cls_fun_ns(self, cls_name, fn):
cls = self.class_manager.get(cls_name)
if not cls:
return set()
for item in cls.get_mro():
ns = utils.join_ns(item, fn)
names = set()
if getattr(self, "closured", None) and self.closured.get(ns, None):
names = self.closured[ns]
else:
names.add(ns)
if self.def_manager.get(ns):
return names
return set()
def _retrieve_attribute_names(self, node):
parent_names = self._retrieve_parent_names(node)
names = set()
for name in parent_names:
defi = self.def_manager.get(name)
if not defi:
continue
if defi.get_type() == utils.constants.CLS_DEF:
cls_names = self.find_cls_fun_ns(defi.get_ns(), node.attr)
if cls_names:
names = names.union(cls_names)
if defi.get_type() in [
utils.constants.FUN_DEF, utils.constants.MOD_DEF
]:
names.add(utils.join_ns(name, node.attr))
return names
def process(namespace, parent, table):
name = table.get_name() if table.get_name() != 'top' else ''
if name:
fullns = utils.join_ns(namespace, name)
else:
fullns = namespace
if table.get_type() == "function":
functions.append(fullns)
if table.get_type() == "class":
classes.append(fullns)
sc = self.create_scope(fullns, parent)
for t in table.get_children():
process(fullns, sc, t)
