def visit_For(self, node): # only handle name targets if isinstance(node.target, ast.Name): target_def = self.def_manager.get( utils.join_ns(self.current_ns, node.target.id)) # if the target definition exists if target_def: iter_decoded = self.decode_node(node.iter) # assign the target to the return value # of the next function for item in iter_decoded: if not isinstance(item, Definition): continue # return value for generators for name in self.closured.get(item.get_ns(), []): # If there exists a next method on the iterable # and if yes, add a pointer to it next_defi = self.def_manager.get( utils.join_ns(name, utils.constants.NEXT_METHOD, utils.constants.RETURN_NAME)) if next_defi: for name in self.closured.get( next_defi.get_ns(), []): target_def.get_name_pointer().add(name) else: # otherwise, add a pointer to the name (e.g. a yield) target_def.get_name_pointer().add(name) super().visit_For(node)
def _retrieve_attribute_names(self, node): if not getattr(self, "closured", None): return set() parent_names = self._retrieve_parent_names(node) names = set() for parent_name in parent_names: for name in self.closured.get(parent_name, []): defi = self.def_manager.get(name) if not defi: continue if defi.get_type() == utils.constants.CLS_DEF: cls_names = self.find_cls_fun_ns(defi.get_ns(), node.attr) if cls_names: names = names.union(cls_names) if defi.get_type() in [utils.constants.FUN_DEF, utils.constants.MOD_DEF]: names.add(utils.join_ns(name, node.attr)) if defi.get_type() == utils.constants.EXT_DEF: # HACK: extenral attributes can lead to infinite loops # Identify them here if node.attr in name: continue ext_name = utils.join_ns(name, node.attr) if not self.def_manager.get(ext_name): self.def_manager.create(ext_name, utils.constants.EXT_DEF) names.add(ext_name) return names
def visit_Dict(self, node): # 1. create a scope using a counter # 2. Iterate keys and add them as children of the scope # 3. Iterate values and makes a points to connection with the keys current_scope = self.scope_manager.get_scope(self.current_ns) dict_counter = current_scope.inc_dict_counter() dict_name = utils.get_dict_name(dict_counter) dict_full_ns = utils.join_ns(self.current_ns, dict_name) # create a scope for the dict dict_scope = self.scope_manager.create_scope(dict_full_ns, current_scope) # Create a dict definition dict_def = self.def_manager.get(dict_full_ns) if not dict_def: dict_def = self.def_manager.create(dict_full_ns, utils.constants.NAME_DEF) # add it to the current scope current_scope.add_def(dict_name, dict_def) self.name_stack.append(dict_name) for key, value in zip(node.keys, node.values): if key: self.visit(key) if value: self.visit(value) decoded_key = self.decode_node(key) decoded_value = self.decode_node(value) # iterate decoded keys and values # to do the assignment operation for k in decoded_key: if isinstance(k, Definition): # get literal pointer names = k.get_lit_pointer().get() else: names = set() if isinstance(k, list): continue names.add(k) for name in names: # create a definition for the key if isinstance(name, int): name = utils.get_int_name(name) key_full_ns = utils.join_ns(dict_def.get_ns(), str(name)) key_def = self.def_manager.get(key_full_ns) if not key_def: key_def = self.def_manager.create( key_full_ns, utils.constants.NAME_DEF) dict_scope.add_def(str(name), key_def) for v in decoded_value: if isinstance(v, Definition): key_def.get_name_pointer().add(v.get_ns()) else: key_def.get_lit_pointer().add(v) self.name_stack.pop()
def handle_function_def(self, parent_ns, fn_name): full_ns = utils.join_ns(parent_ns, fn_name) defi = self.get(full_ns) if not defi: defi = self.create(full_ns, utils.constants.FUN_DEF) defi.decorator_names = set() return_ns = utils.join_ns(full_ns, utils.constants.RETURN_NAME) if not self.get(return_ns): self.create(return_ns, utils.constants.NAME_DEF) return defi
def _get_target_ns(self, target): if isinstance(target, ast.Name): return [utils.join_ns(self.current_ns, target.id)] if isinstance(target, ast.Attribute): bases = self._retrieve_base_names(target) res = [] for base in bases: res.append(utils.join_ns(base, target.attr)) return res if isinstance(target, ast.Subscript): return self.retrieve_subscript_names(target) return []
def assign(self, ns, defi): self.defs[ns] = Definition(ns, defi.get_type()) self.defs[ns].merge(defi) # if it is a function def, we need to create a return pointer if defi.is_function_def(): return_ns = utils.join_ns(ns, utils.constants.RETURN_NAME) self.defs[return_ns] = Definition(return_ns, utils.constants.NAME_DEF) self.defs[return_ns].get_name_pointer().add( utils.join_ns(defi.get_ns(), utils.constants.RETURN_NAME)) return self.defs[ns]
def test_uri(self): self.cg_generator.functions = ['mod1.mod2.myfunc'] formatter = self.get_formatter() ### Internal uri check # test modname without method self.assertEqual(formatter.to_uri('mymod'), '/mymod/') self.assertEqual(formatter.to_uri('mymod.mod1'), '/mymod.mod1/') # test method starting with modname self.assertEqual(formatter.to_uri('mymod.mod1', 'mymod.mod1.fn'), '/mymod.mod1/fn') self.assertEqual(formatter.to_uri('mymod.mod1', 'mymod.mod1.cls.fn'), '/mymod.mod1/cls.fn') # test method starting with modname but without . inbetween with self.assertRaises(Exception): formatter.to_uri('mymod.mod1', 'mymod.mod1cls.fn') # test method being in functions self.assertEqual(formatter.to_uri('mod1.mod2', 'mod1.mod2.myfunc'), '/mod1.mod2/myfunc()') ### External uri check # test modname builtin self.assertEqual( formatter.to_external_uri(utils.constants.BUILTIN_NAME, utils.join_ns(utils.constants.BUILTIN_NAME, 'cls1.fn1')), '//.builtin//cls1.fn1' ) # test modname not builtin self.assertEqual(formatter.to_external_uri('requests', 'requests.Request.get'), '//requests//requests.Request.get')
def update_parent_classes(self, defi): cls = self.class_manager.get(defi.get_ns()) if not cls: return current_scope = self.scope_manager.get_scope(defi.get_ns()) for parent in cls.get_mro(): parent_def = self.def_manager.get(parent) if not parent_def: continue parent_scope = self.scope_manager.get_scope(parent) if not parent_scope: continue parent_items = list(parent_scope.get_defs().keys()) for key, child_def in current_scope.get_defs().items(): if key == "__init__": continue # resolve name from the parent_def names = self.find_cls_fun_ns(parent_def.get_ns(), key) new_ns = utils.join_ns(parent_def.get_ns(), key) new_def = self.def_manager.get(new_ns) if not new_def: new_def = self.def_manager.create(new_ns, utils.constants.NAME_DEF) new_def.get_name_pointer().add_set(names) new_def.get_name_pointer().add(child_def.get_ns())
def retrieve_call_names(self, node): names = set() if isinstance(node.func, ast.Name): defi = self.scope_manager.get_def(self.current_ns, node.func.id) if defi: names = self.closured.get(defi.get_ns(), None) elif isinstance(node.func, ast.Call) and self.last_called_names: for name in self.last_called_names: return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) returns = self.closured.get(return_ns) if not returns: continue for ret in returns: defi = self.def_manager.get(ret) names.add(defi.get_ns()) elif isinstance(node.func, ast.Attribute): names = self._retrieve_attribute_names(node.func) elif isinstance(node.func, ast.Subscript): # Calls can be performed only on single indices, not ranges full_names = self.retrieve_subscript_names(node.func) for n in full_names: if self.closured.get(n, None): names |= self.closured.get(n) return names
def handle_class_def(self, parent_ns, cls_name): full_ns = utils.join_ns(parent_ns, cls_name) defi = self.get(full_ns) if not defi: defi = self.create(full_ns, utils.constants.CLS_DEF) return defi
def find_cls_fun_ns(self, cls_name, fn): cls = self.class_manager.get(cls_name) if not cls: return set() ext_names = set() for item in cls.get_mro(): ns = utils.join_ns(item, fn) names = set() if getattr(self, "closured", None) and self.closured.get(ns, None): names = self.closured[ns] else: names.add(ns) if self.def_manager.get(ns): return names parent = self.def_manager.get(item) if parent and parent.get_type() == utils.constants.EXT_DEF: ext_names.add(ns) for name in ext_names: self.def_manager.create(name, utils.constants.EXT_DEF) self.add_ext_mod_node(name) return ext_names
def visit_Call(self, node): def create_ext_edge(name, ext_modname): self.add_ext_mod_node(name) self.call_graph.add_node(name, ext_modname) self.call_graph.add_edge(self.current_method, name) # First visit the child function so that on the case of # func()()() # we first visit the call to func and then the other calls for arg in node.args: self.visit(arg) for keyword in node.keywords: self.visit(keyword.value) self.visit(node.func) names = self.retrieve_call_names(node) if not names: if isinstance(node.func, ast.Attribute) and self.has_ext_parent( node.func): # TODO: This doesn't work for cases where there is an assignment of an attribute # i.e. import os; lala = os.path; lala.dirname() for name in self.get_full_attr_names(node.func): ext_modname = name.split(".")[0] create_ext_edge(name, ext_modname) elif getattr(node.func, "id", None) and self.is_builtin( node.func.id): name = utils.join_ns(utils.constants.BUILTIN_NAME, node.func.id) create_ext_edge(name, utils.constants.BUILTIN_NAME) return self.last_called_names = names for pointer in names: pointer_def = self.def_manager.get(pointer) if not pointer_def or not isinstance(pointer_def, Definition): continue if pointer_def.is_callable(): if pointer_def.get_type() == utils.constants.EXT_DEF: ext_modname = pointer.split(".")[0] create_ext_edge(pointer, ext_modname) continue self.call_graph.add_edge(self.current_method, pointer) # TODO: This doesn't work and leads to calls from the decorators # themselves to the function, creating edges to the first decorator #for decorator in pointer_def.decorator_names: # dec_names = self.closured.get(decorator, []) # for dec_name in dec_names: # if self.def_manager.get(dec_name).get_type() == utils.constants.FUN_DEF: # self.call_graph.add_edge(self.current_ns, dec_name) if pointer_def.get_type() == utils.constants.CLS_DEF: init_ns = self.find_cls_fun_ns(pointer, utils.constants.CLS_INIT) for ns in init_ns: self.call_graph.add_edge(self.current_method, ns)
def create_def(scope, name, imported_def): if not name in scope.get_defs(): def_ns = utils.join_ns(scope.get_ns(), name) defi = self.def_manager.get(def_ns) if not defi: defi = self.def_manager.assign(def_ns, imported_def) defi.get_name_pointer().add(imported_def.get_ns()) current_scope.add_def(name, defi)
def visit_FunctionDef(self, node): # here we iterate decorators if node.decorator_list: fn_def = self.def_manager.get( utils.join_ns(self.current_ns, node.name)) reversed_decorators = list(reversed(node.decorator_list)) # add to the name pointer of the function definition # the return value of the first decorator # since, now the function is a namespace to that point if hasattr(fn_def, "decorator_names") and reversed_decorators: last_decoded = self.decode_node(reversed_decorators[-1]) for d in last_decoded: if not isinstance(d, Definition): continue fn_def.decorator_names.add( utils.join_ns(d.get_ns(), utils.constants.RETURN_NAME)) previous_names = self.closured.get(fn_def.get_ns(), set()) for decorator in reversed_decorators: # assign the previous_def as the first parameter of the decorator decoded = self.decode_node(decorator) new_previous_names = set() for d in decoded: if not isinstance(d, Definition): continue for name in self.closured.get(d.get_ns(), []): return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) if self.closured.get(return_ns, None) == None: continue new_previous_names = new_previous_names.union( self.closured.get(return_ns)) for prev_name in previous_names: pos_arg_names = d.get_name_pointer().get_pos_arg(0) if not pos_arg_names: continue for name in pos_arg_names: arg_def = self.def_manager.get(name) arg_def.get_name_pointer().add(prev_name) previous_names = new_previous_names super().visit_FunctionDef(node)
def _visit_return(self, node): if not node or not node.value: return self.visit(node.value) return_ns = utils.join_ns(self.current_ns, utils.constants.RETURN_NAME) self._handle_assign(return_ns, self.decode_node(node.value))
def visit_Lambda(self, node): counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() lambda_name = utils.get_lambda_name(counter) lambda_fullns = utils.join_ns(self.current_ns, lambda_name) self.call_graph.add_node(lambda_fullns, self.modname) super().visit_Lambda(node, lambda_name)
def visit_For(self, node): # just create the definition for target if isinstance(node.target, ast.Name): target_ns = utils.join_ns(self.current_ns, node.target.id) if not self.def_manager.get(target_ns): defi = self.def_manager.create(target_ns, utils.constants.NAME_DEF) self.scope_manager.get_scope(self.current_ns).add_def( node.target.id, defi) super().visit_For(node)
def visit_Lambda(self, node, lambda_name=None): lambda_ns = utils.join_ns(self.current_ns, lambda_name) if not self.scope_manager.get_scope(lambda_ns): self.scope_manager.create_scope(lambda_ns, self.scope_manager.get_scope(self.current_ns)) self.name_stack.append(lambda_name) self.method_stack.append(lambda_name) self.visit(node.body) self.method_stack.pop() self.name_stack.pop()
def visit_For(self, node): self.visit(node.iter) self.visit(node.target) # assign target.id to the return value of __next__ of node.iter.it # we need to have a visit for on the postprocessor also iter_decoded = self.decode_node(node.iter) for item in iter_decoded: if not isinstance(item, Definition): continue names = self.closured.get(item.get_ns(), []) for name in names: iter_ns = utils.join_ns(name, utils.constants.ITER_METHOD) next_ns = utils.join_ns(name, utils.constants.NEXT_METHOD) if self.def_manager.get(iter_ns): self.call_graph.add_edge(self.current_method, iter_ns) if self.def_manager.get(next_ns): self.call_graph.add_edge(self.current_method, next_ns) super().visit_For(node)
def handle_import(self, name, level): # We currently don't support builtin modules because they're frozen. # Add an edge and continue. # TODO: identify a way to include frozen modules root = name.split(".")[0] if root in sys.builtin_module_names: self.create_edge(root) return # Import the module try: mod_name, package = self._handle_import_level(name, level) except ImportError: return parent = ".".join(mod_name.split(".")[:-1]) parent_name = ".".join(name.split(".")[:-1]) combos = [(mod_name, package), (parent, package), (utils.join_ns(package, name), ""), (utils.join_ns(package, parent_name), "")] mod = None for mn, pkg in combos: try: mod = self._do_import(mn, pkg) break except: continue if not mod: return if not hasattr(mod, "__file__") or not mod.__file__: return if self.mod_dir not in mod.__file__: return fname = mod.__file__ if fname.endswith("__init__.py"): fname = os.path.split(fname)[0] return utils.to_mod_name( os.path.relpath(fname, self.mod_dir))
def visit_Call(self, node): self.visit(node.func) # if it is not a name there's nothing we can do here # ModuleVisitor will be able to resolve those calls # since it'll have the name tracking information if not isinstance(node.func, ast.Name): return fullns = utils.join_ns(self.current_ns, node.func.id) defi = self.scope_manager.get_def(self.current_ns, node.func.id) if not defi: return if defi.get_type() == utils.constants.CLS_DEF: defi = self.def_manager.get( utils.join_ns(defi.get_ns(), utils.constants.CLS_INIT)) if not defi: return self.iterate_call_args(defi, node)
def visit_List(self, node): counter = self.scope_manager.get_scope(self.current_ns).inc_list_counter() list_name = utils.get_list_name(counter) sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, list_name)) if not sc: return self.name_stack.append(list_name) sc.reset_counters() for elt in node.elts: self.visit(elt) self.name_stack.pop()
def visit_FunctionDef(self, node): for decorator in node.decorator_list: self.visit(decorator) decoded = self.decode_node(decorator) for d in decoded: if not isinstance(d, Definition): continue names = self.closured.get(d.get_ns(), []) for name in names: self.call_graph.add_edge(self.current_method, name) self.call_graph.add_node(utils.join_ns(self.current_ns, node.name), self.modname) super().visit_FunctionDef(node)
def visit_Lambda(self, node): # The name of a lambda is defined by the counter of the current scope current_scope = self.scope_manager.get_scope(self.current_ns) lambda_counter = current_scope.inc_lambda_counter() lambda_name = utils.get_lambda_name(lambda_counter) lambda_full_ns = utils.join_ns(self.current_ns, lambda_name) # create a scope for the lambda self.scope_manager.create_scope(lambda_full_ns, current_scope) lambda_def = self._handle_function_def(node, lambda_name) # add it to the current scope current_scope.add_def(lambda_name, lambda_def) super().visit_Lambda(node, lambda_name)
def visit_List(self, node): # Works similarly with dicts current_scope = self.scope_manager.get_scope(self.current_ns) list_counter = current_scope.inc_list_counter() list_name = utils.get_list_name(list_counter) list_full_ns = utils.join_ns(self.current_ns, list_name) # create a scope for the list list_scope = self.scope_manager.create_scope(list_full_ns, current_scope) # create a list definition list_def = self.def_manager.get(list_full_ns) if not list_def: list_def = self.def_manager.create(list_full_ns, utils.constants.NAME_DEF) current_scope.add_def(list_name, list_def) self.name_stack.append(list_name) for idx, elt in enumerate(node.elts): self.visit(elt) key_full_ns = utils.join_ns(list_def.get_ns(), utils.get_int_name(idx)) key_def = self.def_manager.get(key_full_ns) if not key_def: key_def = self.def_manager.create(key_full_ns, utils.constants.NAME_DEF) decoded_elt = self.decode_node(elt) for v in decoded_elt: if isinstance(v, Definition): key_def.get_name_pointer().add(v.get_ns()) else: key_def.get_lit_pointer().add(v) self.name_stack.pop()
def add_external_def(name, target): # add an external def for the name defi = self.def_manager.get(name) if not defi: defi = self.def_manager.create(name, utils.constants.EXT_DEF) scope = self.scope_manager.get_scope(self.current_ns) if target != "*": # add a def for the target that points to the name tgt_ns = utils.join_ns(scope.get_ns(), target) tgt_defi = self.def_manager.get(tgt_ns) if not tgt_defi: tgt_defi = self.def_manager.create(tgt_ns, utils.constants.EXT_DEF) tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi)
def visit_Dict(self, node): counter = self.scope_manager.get_scope(self.current_ns).inc_dict_counter() dict_name = utils.get_dict_name(counter) sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, dict_name)) if not sc: return self.name_stack.append(dict_name) sc.reset_counters() for key, val in zip(node.keys, node.values): if key: self.visit(key) if val: self.visit(val) self.name_stack.pop()
def find_cls_fun_ns(self, cls_name, fn): cls = self.class_manager.get(cls_name) if not cls: return set() for item in cls.get_mro(): ns = utils.join_ns(item, fn) names = set() if getattr(self, "closured", None) and self.closured.get(ns, None): names = self.closured[ns] else: names.add(ns) if self.def_manager.get(ns): return names return set()
def _retrieve_attribute_names(self, node): parent_names = self._retrieve_parent_names(node) names = set() for name in parent_names: defi = self.def_manager.get(name) if not defi: continue if defi.get_type() == utils.constants.CLS_DEF: cls_names = self.find_cls_fun_ns(defi.get_ns(), node.attr) if cls_names: names = names.union(cls_names) if defi.get_type() in [ utils.constants.FUN_DEF, utils.constants.MOD_DEF ]: names.add(utils.join_ns(name, node.attr)) return names
def process(namespace, parent, table): name = table.get_name() if table.get_name() != 'top' else '' if name: fullns = utils.join_ns(namespace, name) else: fullns = namespace if table.get_type() == "function": functions.append(fullns) if table.get_type() == "class": classes.append(fullns) sc = self.create_scope(fullns, parent) for t in table.get_children(): process(fullns, sc, t)