def __init__(self, manager=None):
if manager is None:
from clonedigger.logilab.astng import MANAGER as manager
self._manager = manager
self._module = None
self._file = None
self._done = None
self._stack, self._par_stack = None, None
self._metaclass = None
self._walker = ASTWalker(self)
self._dyn_modname_map = {'gtk': 'gtk._gtk'}
self._delayed = []
def __init__(self, manager=None):
if manager is None:
from clonedigger.logilab.astng import MANAGER as manager
self._manager = manager
self._module = None
self._file = None
self._done = None
self._stack, self._par_stack = None, None
self._metaclass = None
self._walker = ASTWalker(self)
self._dyn_modname_map = {'gtk': 'gtk._gtk'}
self._delayed = []
class ASTNGBuilder(object):
"""provide astng building methods
"""
def __init__(self, manager=None):
if manager is None:
from clonedigger.logilab.astng import MANAGER as manager
self._manager = manager
self._module = None
self._file = None
self._done = None
self._stack, self._par_stack = None, None
self._metaclass = None
self._walker = ASTWalker(self)
self._dyn_modname_map = {'gtk': 'gtk._gtk'}
self._delayed = []
def module_build(self, module, modname=None):
"""build an astng from a living module instance
"""
node = None
self._module = module
path = getattr(module, '__file__', None)
if path is not None:
path_, ext = splitext(module.__file__)
if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'):
node = self.file_build(path_ + '.py', modname)
if node is None:
# this is a built-in module
# get a partial representation by introspection
node = self.inspect_build(module, modname=modname, path=path)
return node
def inspect_build(self, module, modname=None, path=None):
"""build astng from a living module (i.e. using inspect)
this is used when there is no python source code available (either
because it's a built-in module or because the .py is not available)
"""
self._module = module
node = build_module(modname or module.__name__, module.__doc__)
node.file = node.path = path and abspath(path) or path
if self._manager is not None:
self._manager._cache[node.file] = self._manager._cache[node.name] = node
node.package = hasattr(module, '__path__')
attach___dict__(node)
self._done = {}
self.object_build(node, module)
return node
def file_build(self, path, modname=None):
"""build astng from a source code file (i.e. from an ast)
path is expected to be a python source file
"""
try:
data = norm_read(path)
except IOError as ex:
msg = 'Unable to load file %r (%s)' % (path, ex)
raise ASTNGBuildingException(msg)
self._file = path
# get module name if necessary, *before modifying sys.path*
if modname is None:
try:
modname = '.'.join(modpath_from_file(path))
except ImportError:
modname = splitext(basename(path))[0]
# build astng representation
try:
sys.path.insert(0, dirname(path))
node = self.string_build(data, modname, path)
node.file = abspath(path)
finally:
self._file = None
sys.path.pop(0)
return node
def string_build(self, data, modname='', path=None):
"""build astng from a source code stream (i.e. from an ast)"""
return self.ast_build(parse(data + '\n'), modname, path)
def ast_build(self, node, modname=None, path=None):
"""recurse on the ast (soon ng) to add some arguments et method
"""
if path is not None:
node.file = node.path = abspath(path)
else:
node.file = node.path = '<?>'
if modname.endswith('.__init__'):
modname = modname[:-9]
node.package = True
else:
node.package = path and path.find('__init__.py') > -1 or False
node.name = modname
node.pure_python = True
if self._manager is not None:
self._manager._cache[node.file] = node
if self._file:
self._manager._cache[abspath(self._file)] = node
self._walker.walk(node)
while self._delayed:
dnode = self._delayed.pop(0)
getattr(self, 'delayed_visit_%s' % dnode.__class__.__name__.lower())(dnode)
return node
# callbacks to build from an existing compiler.ast tree ###################
def visit_module(self, node):
"""visit a stmt.Module node -> init node and push the corresponding
object or None on the top of the stack
"""
self._stack = [self._module]
self._par_stack = [node]
self._metaclass = ['']
self._global_names = []
node.parent = None
node.globals = node.locals = {}
for name, value in ( ('__name__', node.name),
('__file__', node.path),
('__doc__', node.doc) ):
const = nodes.Const(value)
const.parent = node
node.locals[name] = [const]
attach___dict__(node)
if node.package:
# FIXME: List(Const())
const = nodes.Const(dirname(node.path))
const.parent = node
node.locals['__path__'] = [const]
def leave_module(self, _):
"""leave a stmt.Module node -> pop the last item on the stack and check
the stack is empty
"""
self._stack.pop()
assert not self._stack, 'Stack is not empty : %s' % self._stack
self._par_stack.pop()
assert not self._par_stack, \
'Parent stack is not empty : %s' % self._par_stack
def visit_class(self, node):
"""visit a stmt.Class node -> init node and push the corresponding
object or None on the top of the stack
"""
self.visit_default(node)
node.instance_attrs = {}
node.basenames = [b_node for b_node in node.bases]
self._push(node)
for name, value in ( ('__name__', node.name),
('__module__', node.root().name),
('__doc__', node.doc) ):
const = nodes.Const(value)
const.parent = node
node.locals[name] = [const]
attach___dict__(node)
self._metaclass.append(self._metaclass[-1])
def leave_class(self, node):
"""leave a stmt.Class node -> pop the last item on the stack
"""
self.leave_default(node)
self._stack.pop()
metaclass = self._metaclass.pop()
if not node.bases:
# no base classes, detect new / style old style according to
# current scope
node._newstyle = metaclass == 'type'
def visit_function(self, node):
"""visit a stmt.Function node -> init node and push the corresponding
object or None on the top of the stack
"""
self.visit_default(node)
self._global_names.append({})
node.argnames = list(node.argnames)
if isinstance(node.parent.frame(), nodes.Class):
node.type = 'method'
if node.name == '__new__':
node.type = 'classmethod'
self._push(node)
register_arguments(node, node.argnames)
def leave_function(self, node):
"""leave a stmt.Function node -> pop the last item on the stack
"""
self.leave_default(node)
self._stack.pop()
self._global_names.pop()
def visit_lambda(self, node):
"""visit a stmt.Lambda node -> init node locals
"""
self.visit_default(node)
node.argnames = list(node.argnames)
node.locals = {}
register_arguments(node, node.argnames)
def visit_genexpr(self, node):
"""visit a stmt.GenExpr node -> init node locals
"""
self.visit_default(node)
node.locals = {}
def visit_global(self, node):
"""visit a stmt.Global node -> add declared names to locals
"""
self.visit_default(node)
if not self._global_names: # global at the module level, no effect
return
for name in node.names:
self._global_names[-1].setdefault(name, []).append(node)
# node.parent.set_local(name, node)
# module = node.root()
# if module is not node.frame():
# for name in node.names:
# module.set_local(name, node)
def visit_import(self, node):
"""visit a stmt.Import node -> add imported names to locals
"""
self.visit_default(node)
for (name, asname) in node.names:
name = asname or name
node.parent.set_local(name.split('.')[0], node)
def visit_from(self, node):
"""visit a stmt.From node -> add imported names to locals
"""
self.visit_default(node)
# add names imported by the import to locals
for (name, asname) in node.names:
if name == '*':
try:
imported = node.root().import_module(node.modname)
except ASTNGBuildingException:
#import traceback
#traceback.print_exc()
continue
# FIXME: log error
#print >> sys.stderr, \
# 'Unable to get imported names for %r line %s"' % (
# node.modname, node.lineno)
for name in imported.wildcard_import_names():
node.parent.set_local(name, node)
else:
node.parent.set_local(asname or name, node)
def leave_decorators(self, node):
"""python >= 2.4
visit a stmt.Decorator node -> check for classmethod and staticmethod
"""
func = node.parent
for decorator_expr in node.nodes:
if isinstance(decorator_expr, nodes.Name) and \
decorator_expr.name in ('classmethod', 'staticmethod'):
func.type = decorator_expr.name
self.leave_default(node)
def visit_assign(self, node):
"""visit a stmt.Assign node -> check for classmethod and staticmethod
+ __metaclass__
"""
self.visit_default(node)
klass = node.parent.frame()
#print node
if isinstance(klass, nodes.Class) and \
isinstance(node.expr, nodes.CallFunc) and \
isinstance(node.expr.node, nodes.Name):
func_name = node.expr.node.name
if func_name in ('classmethod', 'staticmethod'):
for ass_node in node.nodes:
if isinstance(ass_node, nodes.AssName):
try:
meth = klass[ass_node.name]
if isinstance(meth, nodes.Function):
meth.type = func_name
#else:
# print >> sys.stderr, 'FIXME 1', meth
except KeyError:
#print >> sys.stderr, 'FIXME 2', ass_node.name
continue
elif (isinstance(node.nodes[0], nodes.AssName)
and node.nodes[0].name == '__metaclass__'): # XXX check more...
self._metaclass[-1] = 'type' # XXX get the actual metaclass
def visit_assname(self, node):
"""visit a stmt.AssName node -> add name to locals
"""
self.visit_default(node)
self._add_local(node, node.name)
def visit_augassign(self, node):
"""visit a stmt.AssName node -> add name to locals
"""
self.visit_default(node)
if not isinstance(node.node, nodes.Name):
return # XXX
self._add_local(node, node.node.name)
def _add_local(self, node, name):
if self._global_names and name in self._global_names[-1]:
node.root().set_local(name, node)
else:
node.parent.set_local(name, node)
def visit_assattr(self, node):
"""visit a stmt.AssAttr node -> delay it to handle members
definition later
"""
self.visit_default(node)
self._delayed.append(node)
def delayed_visit_assattr(self, node):
"""visit a stmt.AssAttr node -> add name to locals, handle members
definition
"""
try:
frame = node.frame()
for infered in node.expr.infer():
if infered is YES:
continue
try:
if infered.__class__ is Instance:
infered = infered._proxied
iattrs = infered.instance_attrs
else:
iattrs = infered.locals
except AttributeError:
continue
values = iattrs.setdefault(node.attrname, [])
if node in values:
continue
# get assign in __init__ first XXX useful ?
if frame.name == '__init__' and values and not \
values[0].frame().name == '__init__':
values.insert(0, node)
else:
values.append(node)
#print node.attrname, infered, values
except InferenceError:
#print frame, node
pass
def visit_default(self, node):
"""default visit method, handle the parent attribute
"""
node.parent = self._par_stack[-1]
assert node.parent is not node
self._par_stack.append(node)
def leave_default(self, _):
"""default leave method, handle the parent attribute
"""
self._par_stack.pop()
def _push(self, node):
"""update the stack and init some parts of the Function or Class node
"""
obj = getattr(self._stack[-1], node.name, None)
self._stack.append(obj)
node.locals = {}
node.parent.frame().set_local(node.name, node)
# astng from living objects ###############################################
#
# this is actually a really minimal representation, including only Module,
# Function and Class nodes and some others as guessed
def object_build(self, node, obj):
"""recursive method which create a partial ast from real objects
(only function, class, and method are handled)
"""
if obj in self._done:
return self._done[obj]
self._done[obj] = node
modname = self._module.__name__
modfile = getattr(self._module, '__file__', None)
for name in dir(obj):
try:
member = getattr(obj, name)
except AttributeError:
# damned ExtensionClass.Base, I know you're there !
attach_dummy_node(node, name)
continue
if ismethod(member):
member = member.__func__
if isfunction(member):
# verify this is not an imported function
if member.__code__.co_filename != modfile:
attach_dummy_node(node, name, member)
continue
object_build_function(node, member)
elif isbuiltin(member):
# verify this is not an imported member
if self._member_module(member) != modname:
imported_member(node, member, name)
continue
object_build_methoddescriptor(node, member)
elif isclass(member):
# verify this is not an imported class
if self._member_module(member) != modname:
imported_member(node, member, name)
continue
if member in self._done:
class_node = self._done[member]
node.add_local_node(class_node, name)
else:
class_node = object_build_class(node, member)
# recursion
self.object_build(class_node, member)
elif ismethoddescriptor(member):
assert isinstance(member, object)
object_build_methoddescriptor(node, member)
elif isdatadescriptor(member):
assert isinstance(member, object)
object_build_datadescriptor(node, member, name)
elif isinstance(member, (int, int, float, str, str)) or member is None:
attach_const_node(node, name, member)
else:
# create an empty node so that the name is actually defined
attach_dummy_node(node, name, member)
def _member_module(self, member):
modname = getattr(member, '__module__', None)
return self._dyn_modname_map.get(modname, modname)
class ASTNGBuilder(object):
"""provide astng building methods
"""
def __init__(self, manager=None):
if manager is None:
from clonedigger.logilab.astng import MANAGER as manager
self._manager = manager
self._module = None
self._file = None
self._done = None
self._stack, self._par_stack = None, None
self._metaclass = None
self._walker = ASTWalker(self)
self._dyn_modname_map = {'gtk': 'gtk._gtk'}
self._delayed = []
def module_build(self, module, modname=None):
"""build an astng from a living module instance
"""
node = None
self._module = module
path = getattr(module, '__file__', None)
if path is not None:
path_, ext = splitext(module.__file__)
if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'):
node = self.file_build(path_ + '.py', modname)
if node is None:
# this is a built-in module
# get a partial representation by introspection
node = self.inspect_build(module, modname=modname, path=path)
return node
def inspect_build(self, module, modname=None, path=None):
"""build astng from a living module (i.e. using inspect)
this is used when there is no python source code available (either
because it's a built-in module or because the .py is not available)
"""
self._module = module
node = build_module(modname or module.__name__, module.__doc__)
node.file = node.path = path and abspath(path) or path
if self._manager is not None:
self._manager._cache[node.file] = self._manager._cache[
node.name] = node
node.package = hasattr(module, '__path__')
attach___dict__(node)
self._done = {}
self.object_build(node, module)
return node
def file_build(self, path, modname=None):
"""build astng from a source code file (i.e. from an ast)
path is expected to be a python source file
"""
try:
data = norm_read(path)
except IOError as ex:
msg = 'Unable to load file %r (%s)' % (path, ex)
raise ASTNGBuildingException(msg)
self._file = path
# get module name if necessary, *before modifying sys.path*
if modname is None:
try:
modname = '.'.join(modpath_from_file(path))
except ImportError:
modname = splitext(basename(path))[0]
# build astng representation
try:
sys.path.insert(0, dirname(path))
node = self.string_build(data, modname, path)
node.file = abspath(path)
finally:
self._file = None
sys.path.pop(0)
return node
def string_build(self, data, modname='', path=None):
"""build astng from a source code stream (i.e. from an ast)"""
return self.ast_build(parse(data + '\n'), modname, path)
def ast_build(self, node, modname=None, path=None):
"""recurse on the ast (soon ng) to add some arguments et method
"""
if path is not None:
node.file = node.path = abspath(path)
else:
node.file = node.path = '<?>'
if modname.endswith('.__init__'):
modname = modname[:-9]
node.package = True
else:
node.package = path and path.find('__init__.py') > -1 or False
node.name = modname
node.pure_python = True
if self._manager is not None:
self._manager._cache[node.file] = node
if self._file:
self._manager._cache[abspath(self._file)] = node
self._walker.walk(node)
while self._delayed:
dnode = self._delayed.pop(0)
getattr(self, 'delayed_visit_%s' %
dnode.__class__.__name__.lower())(dnode)
return node
# callbacks to build from an existing compiler.ast tree ###################
def visit_module(self, node):
"""visit a stmt.Module node -> init node and push the corresponding
object or None on the top of the stack
"""
self._stack = [self._module]
self._par_stack = [node]
self._metaclass = ['']
self._global_names = []
node.parent = None
node.globals = node.locals = {}
for name, value in (('__name__', node.name), ('__file__', node.path),
('__doc__', node.doc)):
const = nodes.Const(value)
const.parent = node
node.locals[name] = [const]
attach___dict__(node)
if node.package:
# FIXME: List(Const())
const = nodes.Const(dirname(node.path))
const.parent = node
node.locals['__path__'] = [const]
def leave_module(self, _):
"""leave a stmt.Module node -> pop the last item on the stack and check
the stack is empty
"""
self._stack.pop()
assert not self._stack, 'Stack is not empty : %s' % self._stack
self._par_stack.pop()
assert not self._par_stack, \
'Parent stack is not empty : %s' % self._par_stack
def visit_class(self, node):
"""visit a stmt.Class node -> init node and push the corresponding
object or None on the top of the stack
"""
self.visit_default(node)
node.instance_attrs = {}
node.basenames = [b_node for b_node in node.bases]
self._push(node)
for name, value in (('__name__', node.name),
('__module__', node.root().name), ('__doc__',
node.doc)):
const = nodes.Const(value)
const.parent = node
node.locals[name] = [const]
attach___dict__(node)
self._metaclass.append(self._metaclass[-1])
def leave_class(self, node):
"""leave a stmt.Class node -> pop the last item on the stack
"""
self.leave_default(node)
self._stack.pop()
metaclass = self._metaclass.pop()
if not node.bases:
# no base classes, detect new / style old style according to
# current scope
node._newstyle = metaclass == 'type'
def visit_function(self, node):
"""visit a stmt.Function node -> init node and push the corresponding
object or None on the top of the stack
"""
self.visit_default(node)
self._global_names.append({})
node.argnames = list(node.argnames)
if isinstance(node.parent.frame(), nodes.Class):
node.type = 'method'
if node.name == '__new__':
node.type = 'classmethod'
self._push(node)
register_arguments(node, node.argnames)
def leave_function(self, node):
"""leave a stmt.Function node -> pop the last item on the stack
"""
self.leave_default(node)
self._stack.pop()
self._global_names.pop()
def visit_lambda(self, node):
"""visit a stmt.Lambda node -> init node locals
"""
self.visit_default(node)
node.argnames = list(node.argnames)
node.locals = {}
register_arguments(node, node.argnames)
def visit_genexpr(self, node):
"""visit a stmt.GenExpr node -> init node locals
"""
self.visit_default(node)
node.locals = {}
def visit_global(self, node):
"""visit a stmt.Global node -> add declared names to locals
"""
self.visit_default(node)
if not self._global_names: # global at the module level, no effect
return
for name in node.names:
self._global_names[-1].setdefault(name, []).append(node)
# node.parent.set_local(name, node)
# module = node.root()
# if module is not node.frame():
# for name in node.names:
# module.set_local(name, node)
def visit_import(self, node):
"""visit a stmt.Import node -> add imported names to locals
"""
self.visit_default(node)
for (name, asname) in node.names:
name = asname or name
node.parent.set_local(name.split('.')[0], node)
def visit_from(self, node):
"""visit a stmt.From node -> add imported names to locals
"""
self.visit_default(node)
# add names imported by the import to locals
for (name, asname) in node.names:
if name == '*':
try:
imported = node.root().import_module(node.modname)
except ASTNGBuildingException:
#import traceback
#traceback.print_exc()
continue
# FIXME: log error
#print >> sys.stderr, \
# 'Unable to get imported names for %r line %s"' % (
# node.modname, node.lineno)
for name in imported.wildcard_import_names():
node.parent.set_local(name, node)
else:
node.parent.set_local(asname or name, node)
def leave_decorators(self, node):
"""python >= 2.4
visit a stmt.Decorator node -> check for classmethod and staticmethod
"""
func = node.parent
for decorator_expr in node.nodes:
if isinstance(decorator_expr, nodes.Name) and \
decorator_expr.name in ('classmethod', 'staticmethod'):
func.type = decorator_expr.name
self.leave_default(node)
def visit_assign(self, node):
"""visit a stmt.Assign node -> check for classmethod and staticmethod
+ __metaclass__
"""
self.visit_default(node)
klass = node.parent.frame()
#print node
if isinstance(klass, nodes.Class) and \
isinstance(node.expr, nodes.CallFunc) and \
isinstance(node.expr.node, nodes.Name):
func_name = node.expr.node.name
if func_name in ('classmethod', 'staticmethod'):
for ass_node in node.nodes:
if isinstance(ass_node, nodes.AssName):
try:
meth = klass[ass_node.name]
if isinstance(meth, nodes.Function):
meth.type = func_name
#else:
# print >> sys.stderr, 'FIXME 1', meth
except KeyError:
#print >> sys.stderr, 'FIXME 2', ass_node.name
continue
elif (isinstance(node.nodes[0], nodes.AssName)
and node.nodes[0].name == '__metaclass__'): # XXX check more...
self._metaclass[-1] = 'type' # XXX get the actual metaclass
def visit_assname(self, node):
"""visit a stmt.AssName node -> add name to locals
"""
self.visit_default(node)
self._add_local(node, node.name)
def visit_augassign(self, node):
"""visit a stmt.AssName node -> add name to locals
"""
self.visit_default(node)
if not isinstance(node.node, nodes.Name):
return # XXX
self._add_local(node, node.node.name)
def _add_local(self, node, name):
if self._global_names and name in self._global_names[-1]:
node.root().set_local(name, node)
else:
node.parent.set_local(name, node)
def visit_assattr(self, node):
"""visit a stmt.AssAttr node -> delay it to handle members
definition later
"""
self.visit_default(node)
self._delayed.append(node)
def delayed_visit_assattr(self, node):
"""visit a stmt.AssAttr node -> add name to locals, handle members
definition
"""
try:
frame = node.frame()
for infered in node.expr.infer():
if infered is YES:
continue
try:
if infered.__class__ is Instance:
infered = infered._proxied
iattrs = infered.instance_attrs
else:
iattrs = infered.locals
except AttributeError:
continue
values = iattrs.setdefault(node.attrname, [])
if node in values:
continue
# get assign in __init__ first XXX useful ?
if frame.name == '__init__' and values and not \
values[0].frame().name == '__init__':
values.insert(0, node)
else:
values.append(node)
#print node.attrname, infered, values
except InferenceError:
#print frame, node
pass
def visit_default(self, node):
"""default visit method, handle the parent attribute
"""
node.parent = self._par_stack[-1]
assert node.parent is not node
self._par_stack.append(node)
def leave_default(self, _):
"""default leave method, handle the parent attribute
"""
self._par_stack.pop()
def _push(self, node):
"""update the stack and init some parts of the Function or Class node
"""
obj = getattr(self._stack[-1], node.name, None)
self._stack.append(obj)
node.locals = {}
node.parent.frame().set_local(node.name, node)
# astng from living objects ###############################################
#
# this is actually a really minimal representation, including only Module,
# Function and Class nodes and some others as guessed
def object_build(self, node, obj):
"""recursive method which create a partial ast from real objects
(only function, class, and method are handled)
"""
if obj in self._done:
return self._done[obj]
self._done[obj] = node
modname = self._module.__name__
modfile = getattr(self._module, '__file__', None)
for name in dir(obj):
try:
member = getattr(obj, name)
except AttributeError:
# damned ExtensionClass.Base, I know you're there !
attach_dummy_node(node, name)
continue
if ismethod(member):
member = member.__func__
if isfunction(member):
# verify this is not an imported function
if member.__code__.co_filename != modfile:
attach_dummy_node(node, name, member)
continue
object_build_function(node, member)
elif isbuiltin(member):
# verify this is not an imported member
if self._member_module(member) != modname:
imported_member(node, member, name)
continue
object_build_methoddescriptor(node, member)
elif isclass(member):
# verify this is not an imported class
if self._member_module(member) != modname:
imported_member(node, member, name)
continue
if member in self._done:
class_node = self._done[member]
node.add_local_node(class_node, name)
else:
class_node = object_build_class(node, member)
# recursion
self.object_build(class_node, member)
elif ismethoddescriptor(member):
assert isinstance(member, object)
object_build_methoddescriptor(node, member)
elif isdatadescriptor(member):
assert isinstance(member, object)
object_build_datadescriptor(node, member, name)
elif isinstance(member,
(int, int, float, str, str)) or member is None:
attach_const_node(node, name, member)
else:
# create an empty node so that the name is actually defined
attach_dummy_node(node, name, member)
def _member_module(self, member):
modname = getattr(member, '__module__', None)
return self._dyn_modname_map.get(modname, modname)
