def is_load(children): """ Returns whether children has Load op Note: children[0] indexes to first child, children[0][0] indexes to node """ return children and node_type(children[0][0]) == "Load"
def symbify(nodes): """ get representation of nodes where each element is represented by its unique id """ if not nodes: return None root, children = nodes return [(unique_id(root), node_type(root)), [symbify(child) for child in children]]
def next(self): if not self.stack: raise StopIteration else: self.node, self.children = self.stack.pop() self.ntype = node_type(self.node) #remove any stale scopes while self.scopes: #check `depth` of closest scope if self.node.depth <= self.scopes[-1].depth: self.scopes.pop() else: break return self.node, self.children, self.ntype
def next(self): if not self.stack: raise StopIteration else: self.node, self.children = self.stack.pop() self.ntype = node_type(self.node) #remove any stale scopes while self.scopes: #check `depth` of closest scope if self.node.depth <= self.scopes[-1].depth: self.scopes.pop() else: break return self.node, self.children, self.ntype
def resolve_attr_chain(node): """ Returns array of identifiers of attributes of `node`. Arguments:- node: an Attribute ast node. """ chain = [node.attr] ptr = node.value while True: ntype = node_type(ptr) if ntype == "Attribute": chain.append(ptr.attr) ptr = ptr.value elif ntype == "Call": ptr = ptr.func elif ntype == "Name": chain.append(ptr) break else: break #reverse chain, since resolution happens attribute first return chain[::-1]
def resolve_attr_chain(node): """ Returns array of identifiers of attributes of `node`. Arguments:- node: an Attribute ast node. """ chain = [node.attr] ptr = node.value while True: ntype = node_type(ptr) if ntype == "Attribute": chain.append(ptr.attr) ptr = ptr.value elif ntype == "Call": ptr = ptr.func elif ntype == "Name": chain.append(ptr) break else: break #reverse chain, since resolution happens attribute first return chain[::-1]
def create_dependency_tree(root, symtable): """ Returns a map of all the dependencies. Similar to create_symbol_table since scopes are some what like dependencies, minus the hierarchical scope info. """ deptree = DTree() #stack of nodes nodes = Stack() nodes.push(root) #stack of scopes scopestack = Stack() #stack of assigned names assignments = Stack() for node in nodes: ntype = node_type(node) #remove stale scoping nodes scopestack.predpop(lambda scopenode: scopenode.depth >= node.depth) children = get_children(node) if ntype == "Name" and is_load(children): src, dst = process_name_node(node, scopestack, symtable) deptree.add_link(src=map(unique_id, src), dst=map(unique_id, dst)) elif ntype == "Attribute": pdb.set_trace() src, dst = process_attribute_node(node, scopestack, symtable) deptree.add_link(src = map(unique_id, src), dst = map(unique_id, dst)) #don't need to add children since we resolved the whole subtree here #e.g. pdb.set_trace, is an Attribute node with children value (Name= pdb) and attr (str = 'set_trace') #adding the child Name node could lead to redundant (incorrect) dependencies continue elif ntype == "Assign": #Assigns consist of list of LHS values (targets), and a RHS types (value) #resolve the value val_type = node_type(node.value) if val_type == "Name": value = node.value.id elif val_type == "Attribute": value = resolve_attr_chain(node.value) else: create_and_raise("UnknownRHSException", "Unknown RHS, '{}' in Assign".format(val_type)) #resolve the target(s) for target in node.targets: target_type = node_type(target) if target_type == "Name": #the process func return a (src, dst) depenendency pair, ignore src since that's just the current context _, dependency = process_name_node(target, scopestack, symtable) elif target_type == "Attribute": _, dependency = process_attribute_node(target, scopestack, symtable) else: create_and_raise("UnknownLHSException", "Unknown LHS, '{}' in Assign".format(target_type)) #attach the value mapping to scopestack.get_tail() (scopetail) #these will be automatically evicted when scopetail goes out of scope #TODO: handles globals set_assignment(scopestack.get_tail(), dependency, value) continue #push nodes onto the stack; depth is already set from create_symbol_tree() #needs to be done here (i.e. after specific ast type logic) since not all #children need to put on stack nodes.pushmany(reversed(children)) if ntype in scoping_types: scopestack.push(node) return deptree
def create_symbol_table(root): """ Creates a symbol table. Arguments:- root: root ast node to be analyzed (typically a module node). """ #symbol table #creates mapping from name to scopes symtable = Multidict() #stack of nodes nodes = Stack() set_depth(root, 0) nodes.push(root) #stack of scopes scopestack = Stack() #Iterate over all children node for node in nodes: ntype = node_type(node) #remove any scope nodes that have depth >= node scopestack.predpop(lambda scopenode: scopenode.depth >= node.depth) children = get_children(node) #add children to stack in reverse order for child in reversed(children): #set depth on children nodes set_depth(child, node.depth + 1) nodes.push(child) #set lineno property of children #Not sure if there is a better way to scope objects, since #objects can be redefined, i.e. def foo(): pass\n def foo():pass is valid Python #set_lineno(node, children) #add entries to symbol table if ntype == "ClassDef" or ntype == "FunctionDef": identifier = unique_id(node) symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=node) elif ntype == "Import": for name in node.names: identifier = name.asname or name.name #Set srcmodule property of ast node `name` set_src(name, name.name) set_is_src(name) #symtable mapping should contain the node itself symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=name) elif ntype == "ImportFrom": if node.names[0].name == '*': try: imported = importlib.import_module(node.module) #add all names in imported module, except those starting with '_' for attr in dir(imported): if attr[0] != '_': symtable[attr] = scopemap(scope=scopestack.get_state(), astnode=ast_name_node(name=attr, srcmodule=node.module)) except ImportError: print "Error: local system does not have {}. Skipping!".format(node.module) else: for name in node.names: identifier = name.asname or name.name set_src(name, node.module) symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=name) elif ntype == "arguments": if node.vararg: symtable[node.vararg] = scopemap(scope=scopestack.get_state(), astnode=node) if node.kwarg: symtable[node.kwarg] = scopemap(scope=scopestack.get_state(), astnode=node) #if a name is being loaded then it must already exist in symtable elif ntype == "Name" and not is_load(children) and not has_global(scopestack.get_tail(), node.id): symtable[node.id] = scopemap(scope=scopestack.get_state(), astnode=node) elif ntype == "Global": #add a list global vars on node on the top of scope stack #nonlocal could be handled in similar way #FIXME: ensure this is correct set_globals(scopestack.get_tail(), node.names) #add any scoping nodes #Need to do this after ntype == '...' blocks otherwise scoping nodes #would show up in their own scope mapping. if ntype in scoping_types: scopestack.push(node) return symtable
def is_store(children): return children and node_type(children[0]) == "Store"
def is_load(children): """ Called on the children nodes of "Name" node. Determines if node is being loaded """ return children and node_type(children[0]) == "Load"
def is_store(children): return children and node_type(children[0]) == "Store"
def is_load(children): """ Called on the children nodes of "Name" node. Determines if node is being loaded """ return children and node_type(children[0]) == "Load"