def is_load(children):
"""
Returns whether children has Load op
Note: children[0] indexes to first child,
children[0][0] indexes to node
"""
return children and node_type(children[0][0]) == "Load"
def symbify(nodes):
"""
get representation of nodes
where each element is represented by
its unique id
"""
if not nodes: return None
root, children = nodes
return [(unique_id(root), node_type(root)),
[symbify(child) for child in children]]
def next(self):
if not self.stack:
raise StopIteration
else:
self.node, self.children = self.stack.pop()
self.ntype = node_type(self.node)
#remove any stale scopes
while self.scopes:
#check `depth` of closest scope
if self.node.depth <= self.scopes[-1].depth:
self.scopes.pop()
else:
break
return self.node, self.children, self.ntype
def next(self):
if not self.stack:
raise StopIteration
else:
self.node, self.children = self.stack.pop()
self.ntype = node_type(self.node)
#remove any stale scopes
while self.scopes:
#check `depth` of closest scope
if self.node.depth <= self.scopes[-1].depth:
self.scopes.pop()
else:
break
return self.node, self.children, self.ntype
def resolve_attr_chain(node):
"""
Returns array of identifiers of attributes of `node`.
Arguments:-
node: an Attribute ast node.
"""
chain = [node.attr]
ptr = node.value
while True:
ntype = node_type(ptr)
if ntype == "Attribute":
chain.append(ptr.attr)
ptr = ptr.value
elif ntype == "Call":
ptr = ptr.func
elif ntype == "Name":
chain.append(ptr)
break
else:
break
#reverse chain, since resolution happens attribute first
return chain[::-1]
def resolve_attr_chain(node):
"""
Returns array of identifiers of attributes of `node`.
Arguments:-
node: an Attribute ast node.
"""
chain = [node.attr]
ptr = node.value
while True:
ntype = node_type(ptr)
if ntype == "Attribute":
chain.append(ptr.attr)
ptr = ptr.value
elif ntype == "Call":
ptr = ptr.func
elif ntype == "Name":
chain.append(ptr)
break
else:
break
#reverse chain, since resolution happens attribute first
return chain[::-1]
def create_dependency_tree(root, symtable):
"""
Returns a map of all the dependencies.
Similar to create_symbol_table since scopes are some what
like dependencies, minus the hierarchical scope info.
"""
deptree = DTree()
#stack of nodes
nodes = Stack()
nodes.push(root)
#stack of scopes
scopestack = Stack()
#stack of assigned names
assignments = Stack()
for node in nodes:
ntype = node_type(node)
#remove stale scoping nodes
scopestack.predpop(lambda scopenode: scopenode.depth >= node.depth)
children = get_children(node)
if ntype == "Name" and is_load(children):
src, dst = process_name_node(node, scopestack, symtable)
deptree.add_link(src=map(unique_id, src), dst=map(unique_id, dst))
elif ntype == "Attribute":
pdb.set_trace()
src, dst = process_attribute_node(node, scopestack, symtable)
deptree.add_link(src = map(unique_id, src), dst = map(unique_id, dst))
#don't need to add children since we resolved the whole subtree here
#e.g. pdb.set_trace, is an Attribute node with children value (Name= pdb) and attr (str = 'set_trace')
#adding the child Name node could lead to redundant (incorrect) dependencies
continue
elif ntype == "Assign":
#Assigns consist of list of LHS values (targets), and a RHS types (value)
#resolve the value
val_type = node_type(node.value)
if val_type == "Name":
value = node.value.id
elif val_type == "Attribute":
value = resolve_attr_chain(node.value)
else:
create_and_raise("UnknownRHSException",
"Unknown RHS, '{}' in Assign".format(val_type))
#resolve the target(s)
for target in node.targets:
target_type = node_type(target)
if target_type == "Name":
#the process func return a (src, dst) depenendency pair, ignore src since that's just the current context
_, dependency = process_name_node(target, scopestack, symtable)
elif target_type == "Attribute":
_, dependency = process_attribute_node(target, scopestack, symtable)
else:
create_and_raise("UnknownLHSException",
"Unknown LHS, '{}' in Assign".format(target_type))
#attach the value mapping to scopestack.get_tail() (scopetail)
#these will be automatically evicted when scopetail goes out of scope
#TODO: handles globals
set_assignment(scopestack.get_tail(), dependency, value)
continue
#push nodes onto the stack; depth is already set from create_symbol_tree()
#needs to be done here (i.e. after specific ast type logic) since not all
#children need to put on stack
nodes.pushmany(reversed(children))
if ntype in scoping_types:
scopestack.push(node)
return deptree
def create_symbol_table(root):
"""
Creates a symbol table.
Arguments:-
root: root ast node to be analyzed (typically a module node).
"""
#symbol table
#creates mapping from name to scopes
symtable = Multidict()
#stack of nodes
nodes = Stack()
set_depth(root, 0)
nodes.push(root)
#stack of scopes
scopestack = Stack()
#Iterate over all children node
for node in nodes:
ntype = node_type(node)
#remove any scope nodes that have depth >= node
scopestack.predpop(lambda scopenode: scopenode.depth >= node.depth)
children = get_children(node)
#add children to stack in reverse order
for child in reversed(children):
#set depth on children nodes
set_depth(child, node.depth + 1)
nodes.push(child)
#set lineno property of children
#Not sure if there is a better way to scope objects, since
#objects can be redefined, i.e. def foo(): pass\n def foo():pass is valid Python
#set_lineno(node, children)
#add entries to symbol table
if ntype == "ClassDef" or ntype == "FunctionDef":
identifier = unique_id(node)
symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=node)
elif ntype == "Import":
for name in node.names:
identifier = name.asname or name.name
#Set srcmodule property of ast node `name`
set_src(name, name.name)
set_is_src(name)
#symtable mapping should contain the node itself
symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=name)
elif ntype == "ImportFrom":
if node.names[0].name == '*':
try:
imported = importlib.import_module(node.module)
#add all names in imported module, except those starting with '_'
for attr in dir(imported):
if attr[0] != '_':
symtable[attr] = scopemap(scope=scopestack.get_state(),
astnode=ast_name_node(name=attr, srcmodule=node.module))
except ImportError:
print "Error: local system does not have {}. Skipping!".format(node.module)
else:
for name in node.names:
identifier = name.asname or name.name
set_src(name, node.module)
symtable[identifier] = scopemap(scope=scopestack.get_state(), astnode=name)
elif ntype == "arguments":
if node.vararg:
symtable[node.vararg] = scopemap(scope=scopestack.get_state(), astnode=node)
if node.kwarg:
symtable[node.kwarg] = scopemap(scope=scopestack.get_state(), astnode=node)
#if a name is being loaded then it must already exist in symtable
elif ntype == "Name" and not is_load(children) and not has_global(scopestack.get_tail(), node.id):
symtable[node.id] = scopemap(scope=scopestack.get_state(), astnode=node)
elif ntype == "Global":
#add a list global vars on node on the top of scope stack
#nonlocal could be handled in similar way
#FIXME: ensure this is correct
set_globals(scopestack.get_tail(), node.names)
#add any scoping nodes
#Need to do this after ntype == '...' blocks otherwise scoping nodes
#would show up in their own scope mapping.
if ntype in scoping_types:
scopestack.push(node)
return symtable
def is_store(children):
return children and node_type(children[0]) == "Store"
def is_load(children):
"""
Called on the children nodes of "Name" node.
Determines if node is being loaded
"""
return children and node_type(children[0]) == "Load"
def is_store(children):
return children and node_type(children[0]) == "Store"
def is_load(children):
"""
Called on the children nodes of "Name" node.
Determines if node is being loaded
"""
return children and node_type(children[0]) == "Load"