def leave_module(self, node):
"""leave module: check globals
"""
assert len(self._to_consume) == 1
not_consumed = self._to_consume.pop()[0]
# attempt to check for __all__ if defined
if '__all__' in node.locals:
assigned = node.igetattr('__all__').next()
if assigned is not astroid.YES:
for elt in getattr(assigned, 'elts', ()):
try:
elt_name = elt.infer().next()
except astroid.InferenceError:
continue
if not isinstance(elt_name, astroid.Const) or not isinstance(elt_name.value, basestring):
self.add_message('E0604', args=elt.as_string(), node=elt)
continue
elt_name = elt.value
# If elt is in not_consumed, remove it from not_consumed
if elt_name in not_consumed:
del not_consumed[elt_name]
continue
if elt_name not in node.locals:
if not node.package:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
else:
basename = os.path.splitext(node.file)[0]
if os.path.basename(basename) == '__init__':
name = node.name + "." + elt_name
try:
file_from_modpath(name.split("."))
except ImportError:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
except SyntaxError as exc:
# don't yield an syntax-error warning,
# because it will be later yielded
# when the file will be checked
pass
# don't check unused imports in __init__ files
if not self.config.init_import and node.package:
return
for name, stmts in not_consumed.iteritems():
stmt = stmts[0]
if isinstance(stmt, astroid.Import):
self.add_message('W0611', args=name, node=stmt)
elif isinstance(stmt, astroid.From) and stmt.modname != '__future__':
if stmt.names[0][0] == '*':
self.add_message('W0614', args=name, node=stmt)
else:
self.add_message('W0611', args=name, node=stmt)
del self._to_consume
def leave_module(self, node):
"""leave module: check globals
"""
assert len(self._to_consume) == 1
not_consumed = self._to_consume.pop()[0]
# attempt to check for __all__ if defined
if '__all__' in node.locals:
assigned = next(node.igetattr('__all__'))
if assigned is not astroid.YES:
for elt in getattr(assigned, 'elts', ()):
try:
elt_name = next(elt.infer())
except astroid.InferenceError:
continue
if not isinstance(elt_name, astroid.Const) \
or not isinstance(elt_name.value, six.string_types):
self.add_message('invalid-all-object',
args=elt.as_string(),
node=elt)
continue
elt_name = elt_name.value
# If elt is in not_consumed, remove it from not_consumed
if elt_name in not_consumed:
del not_consumed[elt_name]
continue
if elt_name not in node.locals:
if not node.package:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
else:
basename = os.path.splitext(node.file)[0]
if os.path.basename(basename) == '__init__':
name = node.name + "." + elt_name
try:
file_from_modpath(name.split("."))
except ImportError:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
except SyntaxError:
# don't yield an syntax-error warning,
# because it will be later yielded
# when the file will be checked
pass
# don't check unused imports in __init__ files
if not self.config.init_import and node.package:
return
self._check_imports(not_consumed)
def getPathList(self, filesOrModules):
"""
Transform a list of modules to path.
@param filesOrModules: a list of modules (may be foo/bar.py or
foo.bar)
"""
pathList = []
for fileOrMod in filesOrModules:
if not os.path.exists(fileOrMod):
# May be given module is not not a path,
# then transform it to a path.
try:
filepath = file_from_modpath(fileOrMod.split('.'))
except (ImportError, SyntaxError):
# Could not load this module.
continue
if not os.path.exists(filepath):
# Could not find this module in file system.
continue
if os.path.basename(filepath) == "__init__.py":
filepath = os.path.dirname(filepath)
else:
filepath = fileOrMod
pathList.append(filepath)
return pathList
def getPathList(self, filesOrModules):
"""
Transform a list of modules to path.
@param filesOrModules: a list of modules (may be foo/bar.py or
foo.bar)
"""
pathList = []
for fileOrMod in filesOrModules:
if not os.path.exists(fileOrMod):
# May be given module is not not a path,
# then transform it to a path.
try:
filepath = file_from_modpath(fileOrMod.split('.'))
except (ImportError, SyntaxError):
# Could not load this module.
continue
if not os.path.exists(filepath):
# Could not find this module in file system.
continue
if os.path.basename(filepath) == "__init__.py":
filepath = os.path.dirname(filepath)
else:
filepath = fileOrMod
pathList.append(filepath)
return pathList
def project_from_files(self,
files,
func_wrapper=astng_wrapper,
project_name=None,
black_list=None):
"""return a Project from a list of files or modules"""
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None:
continue
# XXX why is first file defining the project.path ?
project.path = project.path or astng.file
project.add_module(astng)
base_name = astng.name
# recurse in package except if __init__ was explicitly given
if astng.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in get_module_files(dirname(astng.file), black_list):
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None or astng.name == base_name:
continue
project.add_module(astng)
return project
def test_knownValues_file_from_modpath_2(self):
from os import path
self.assertEqual(
path.realpath(
modutils.file_from_modpath(['os',
'path']).replace('.pyc', '.py')),
path.realpath(path.__file__.replace('.pyc', '.py')))
def test_knownValues_file_from_modpath_2(self):
from os import path
self.assertEqual(
path.realpath(modutils.file_from_modpath(["os", "path"]).replace(".pyc", ".py")),
path.realpath(path.__file__.replace(".pyc", ".py")),
)
def project_from_files(self, files, func_wrapper=astroid_wrapper,
project_name=None, black_list=None):
"""return a Project from a list of files or modules"""
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None:
continue
# XXX why is first file defining the project.path ?
project.path = project.path or astroid.file
project.add_module(astroid)
base_name = astroid.name
# recurse in package except if __init__ was explicitly given
if astroid.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in get_module_files(dirname(astroid.file),
black_list):
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None or astroid.name == base_name:
continue
project.add_module(astroid)
return project
def test_knownValues_file_from_modpath_3(self):
try:
# don't fail if pyxml isn't installed
from xml.dom import ext
except ImportError:
pass
else:
self.assertEqual(path.realpath(modutils.file_from_modpath(['xml', 'dom', 'ext']).replace('.pyc', '.py')),
path.realpath(ext.__file__.replace('.pyc', '.py')))
def test_knownValues_file_from_modpath_3(self):
try:
# don't fail if pyxml isn't installed
from xml.dom import ext
except ImportError:
pass
else:
self.assertEqual(path.realpath(modutils.file_from_modpath(['xml', 'dom', 'ext']).replace('.pyc', '.py')),
path.realpath(ext.__file__.replace('.pyc', '.py')))
def leave_module(self, node):
"""leave module: check globals
"""
assert len(self._to_consume) == 1
not_consumed = self._to_consume.pop()[0]
# attempt to check for __all__ if defined
if '__all__' in node.locals:
assigned = node.igetattr('__all__').next()
if assigned is not astroid.YES:
for elt in getattr(assigned, 'elts', ()):
try:
elt_name = elt.infer().next()
except astroid.InferenceError:
continue
if not isinstance(elt_name, astroid.Const) \
or not isinstance(elt_name.value, basestring):
self.add_message('invalid-all-object', args=elt.as_string(), node=elt)
continue
elt_name = elt_name.value
# If elt is in not_consumed, remove it from not_consumed
if elt_name in not_consumed:
del not_consumed[elt_name]
continue
if elt_name not in node.locals:
if not node.package:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
else:
basename = os.path.splitext(node.file)[0]
if os.path.basename(basename) == '__init__':
name = node.name + "." + elt_name
try:
file_from_modpath(name.split("."))
except ImportError:
self.add_message('undefined-all-variable',
args=elt_name,
node=elt)
except SyntaxError, exc:
# don't yield an syntax-error warning,
# because it will be later yielded
# when the file will be checked
pass
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
real_stuff = []
for file_or_module_name in files_or_modules:
if not any([
re.search(pattern, file_or_module_name)
for pattern in black_list
]):
real_stuff.append(file_or_module_name)
result = []
errors = []
for something in real_stuff:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
errors.append({'key': 'F0003', 'mod': modname})
continue
except (ImportError, SyntaxError), ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'F0001', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
result.append({
'path': filepath,
'name': modname,
'basepath': filepath,
'basename': modname
})
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append({
'path': subfilepath,
'name': submodname,
'basepath': filepath,
'basename': modname
})
def file_from_module_name(self, modname, contextfile):
try:
value = self._mod_file_cache[(modname, contextfile)]
except KeyError:
try:
value = file_from_modpath(modname.split('.'),
context_file=contextfile)
except ImportError, ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
value = AstroidBuildingException(msg)
self._mod_file_cache[(modname, contextfile)] = value
def file_from_module_name(self, modname, contextfile):
try:
value = self._mod_file_cache[(modname, contextfile)]
except KeyError:
try:
value = file_from_modpath(modname.split('.'),
context_file=contextfile)
except ImportError, ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
value = ASTNGBuildingException(msg)
self._mod_file_cache[(modname, contextfile)] = value
def test_knownValues_file_from_modpath_3(self):
try:
# don't fail if pyxml isn't installed
from xml.dom import ext
except ImportError:
pass
else:
self.assertEqual(
path.realpath(modutils.file_from_modpath(["xml", "dom", "ext"]).replace(".pyc", ".py")),
path.realpath(ext.__file__.replace(".pyc", ".py")),
)
def get_path_containing(file_or_module):
"""
Returns the directory that contains the specified file or module.
This is used to determine the location of the patchfile.
"""
if os.path.exists(file_or_module):
filepath = os.path.realpath(file_or_module)
if os.path.isdir(filepath):
return filepath
else:
return os.path.dirname(filepath)
else:
filepath = file_from_modpath(file_or_module.split('.'))
return os.path.dirname(os.path.realpath(filepath))
def get_path_containing(file_or_module):
"""
Returns the directory that contains the specified file or module.
This is used to determine the location of the patchfile.
"""
if os.path.exists(file_or_module):
filepath = os.path.realpath(file_or_module)
if os.path.isdir(filepath):
return filepath
else:
return os.path.dirname(filepath)
else:
filepath = file_from_modpath(file_or_module.split('.'))
return os.path.dirname(os.path.realpath(filepath))
def file_from_module_name(self, modname, contextfile):
try:
value = self._mod_file_cache[(modname, contextfile)]
except KeyError:
try:
value = file_from_modpath(modname.split('.'),
context_file=contextfile)
except ImportError as ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
value = ASTNGBuildingException(msg)
self._mod_file_cache[(modname, contextfile)] = value
if isinstance(value, ASTNGBuildingException):
raise value
return value
def file_from_modname(modname, path=None, context_file=None):
"""
handle namespace package
"""
try:
return file_from_modpath(modname.split("."), path, context_file)
except ImportError:
loader = pkgutil.find_loader(modname)
if loader is None:
raise
if not hasattr(loader, "filename"):
raise
init_file = os.path.join(loader.filename, "__init__.py")
if os.path.exists(init_file):
return init_file
else:
raise
def file_from_modname(modname, path=None, context_file=None):
"""
handle namespace package
"""
try:
return file_from_modpath(modname.split("."), path, context_file)
except ImportError:
loader = pkgutil.find_loader(modname)
if loader is None:
raise
if not hasattr(loader, "filename"):
raise
init_file = os.path.join(loader.filename, "__init__.py")
if os.path.exists(init_file):
return init_file
else:
raise
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
real_stuff = []
for file_or_module_name in files_or_modules:
if not any([re.search(pattern, file_or_module_name) for pattern in black_list]):
real_stuff.append(file_or_module_name)
result = []
errors = []
for something in real_stuff:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
errors.append( {'key' : 'F0003', 'mod': modname} )
continue
except (ImportError, SyntaxError), ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append( {'key': 'F0001', 'mod': modname, 'ex': ex} )
continue
filepath = normpath(filepath)
result.append( {'path': filepath, 'name': modname,
'basepath': filepath, 'basename': modname} )
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append( {'path': subfilepath, 'name': submodname,
'basepath': filepath, 'basename': modname} )
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = ".".join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split("."))
if filepath is None:
errors.append({"key": "ignored-builtin-module", "mod": modname})
continue
except (ImportError, SyntaxError), ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({"key": "fatal", "mod": modname, "ex": ex})
continue
filepath = normpath(filepath)
result.append({"path": filepath, "name": modname, "isarg": True, "basepath": filepath, "basename": modname})
if not (modname.endswith(".__init__") or modname == "__init__") and "__init__.py" in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = ".".join(modpath_from_file(subfilepath))
result.append(
{"path": subfilepath, "name": submodname, "isarg": False, "basepath": filepath, "basename": modname}
)
def project_from_files(self,
files,
func_wrapper=astng_wrapper,
project_name=None,
black_list=None):
"""return a Project from a list of files or modules"""
# insert current working directory to the python path to have a correct
# behaviour
sys.path.insert(0, os.getcwd())
try:
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None:
continue
project.path = project.path or astng.file
project.add_module(astng)
base_name = astng.name
# recurse in package except if __init__ was explicitly given
if astng.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in get_module_files(dirname(astng.file),
black_list):
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None or astng.name == base_name:
continue
project.add_module(astng)
return project
finally:
sys.path.pop(0)
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
errors.append( {'key' : 'F0003', 'mod': modname} )
continue
except ImportError, ex:
errors.append( {'key': 'F0001', 'mod': modname, 'ex': ex} )
continue
filepath = normpath(filepath)
result.append( {'path': filepath, 'name': modname,
'basepath': filepath, 'basename': modname} )
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append( {'path': subfilepath, 'name': submodname,
'basepath': filepath, 'basename': modname} )
def is_standard_module(self,modname, std_path=(STD_LIB_DIR,)):
"""try to guess if a module is a standard python module (by default,
see `std_path` parameter's description)
:type modname: str
:param modname: name of the module we are interested in
:type std_path: list(str) or tuple(str)
:param std_path: list of path considered has standard
:rtype: bool
:return:
true if the module:
- is located on the path listed in one of the directory in `std_path`
- is a built-in module"""
modname = modname.split('.')[0]
try:
filename = file_from_modpath([modname])
except ImportError, ex:
# import failed, i'm probably not so wrong by supposing it's
# not standard...
print "Unresolved Import, module name - ", modname
self.bad_imports += 1
return 0
def project_from_files(self, files, func_wrapper=astng_wrapper,
project_name=None, black_list=None):
"""return a Project from a list of files or modules"""
# insert current working directory to the python path to have a correct
# behaviour
sys.path.insert(0, os.getcwd())
try:
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None:
continue
project.path = project.path or astng.file
project.add_module(astng)
base_name = astng.name
# recurse in package except if __init__ was explicitly given
if astng.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in get_module_files(dirname(astng.file),
black_list):
astng = func_wrapper(self.astng_from_file, fpath)
if astng is None or astng.name == base_name:
continue
project.add_module(astng)
return project
finally:
sys.path.pop(0)
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = ".".join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split("."))
if filepath is None:
errors.append({"key": "F0003", "mod": modname})
continue
except ImportError, ex:
errors.append({"key": "F0001", "mod": modname, "ex": ex})
continue
filepath = normpath(filepath)
result.append({"path": filepath, "name": modname, "basepath": filepath, "basename": modname})
if not (modname.endswith(".__init__") or modname == "__init__") and "__init__.py" in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = ".".join(modpath_from_file(subfilepath))
result.append({"path": subfilepath, "name": submodname, "basepath": filepath, "basename": modname})
def test_knownValues_file_from_modpath_4(self):
self.assertEqual(modutils.file_from_modpath(['sys']), None)
def test_site_packages(self):
self.assertEqual(
path.realpath(
modutils.file_from_modpath(['logilab', 'common', 'modutils'])),
path.realpath(modutils.__file__.replace('.pyc', '.py')))
def test_builtin(self):
self.assertEqual(modutils.file_from_modpath(['sys']), None)
def test_site_packages(self):
self.assertEqual(path.realpath(modutils.file_from_modpath(['logilab', 'common', 'modutils'])),
path.realpath(modutils.__file__.replace('.pyc', '.py')))
def test_builtin(self):
self.assertEqual(modutils.file_from_modpath(['sys']),
None)
def test_knownValues_file_from_modpath_4(self):
self.assertEqual(modutils.file_from_modpath(['sys']),
None)
def test_knownValues_file_from_modpath_2(self):
from os import path
self.assertEqual(path.realpath(modutils.file_from_modpath(['os', 'path']).replace('.pyc', '.py')),
path.realpath(path.__file__.replace('.pyc', '.py')))
def test_knownValues_file_from_modpath_1(self):
self.assertEqual(path.realpath(modutils.file_from_modpath(['logilab', 'common', 'modutils'])),
path.realpath(modutils.__file__.replace('.pyc', '.py')))
def test_knownValues_file_from_modpath_1(self):
self.assertEqual(
path.realpath(
modutils.file_from_modpath(['logilab', 'common', 'modutils'])),
path.realpath(modutils.__file__.replace('.pyc', '.py')))
def test_site_packages(self):
from pytz import tzinfo
self.assertEqual(path.realpath(modutils.file_from_modpath(['pytz', 'tzinfo'])),
path.realpath(tzinfo.__file__.replace('.pyc', '.py')))
def test_knownValues_file_from_modpath_1(self):
self.assertEqual(
path.realpath(modutils.file_from_modpath(["logilab", "common", "modutils"])),
path.realpath(modutils.__file__.replace(".pyc", ".py")),
)