def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
errors.append({
'key': 'ignored-builtin-module',
'mod': modname
})
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
result.append({
'path': filepath,
'name': modname,
'isarg': True,
'basepath': filepath,
'basename': modname
})
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append({
'path': subfilepath,
'name': submodname,
'isarg': False,
'basepath': filepath,
'basename': modname
})
return result, errors
def test_knownValues_modpath_from_file_2(self):
self.assertEqual(
modutils.modpath_from_file(
"unittest_modutils.py", {os.getcwd(): "arbitrary.pkg"}
),
["arbitrary", "pkg", "unittest_modutils"],
)
def ast_from_file(self,
filepath,
modname=None,
fallback=True,
source=False):
"""given a module name, return the astroid object"""
try:
filepath = modutils.get_source_file(filepath, include_no_ext=True)
source = True
except modutils.NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[
modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
elif fallback and modname:
return self.ast_from_module_name(modname)
raise AstroidBuildingException('unable to get astroid for file %s' %
filepath)
def ast_from_file(self,
filepath,
modname=None,
fallback=True,
source=False):
"""given a module name, return the astroid object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
pass
if modname is None:
try:
modname = ".".join(modpath_from_file(filepath))
except ImportError:
modname = filepath
if (modname in self.astroid_cache
and self.astroid_cache[modname].file == filepath):
return self.astroid_cache[modname]
if source:
# pylint: disable=import-outside-toplevel; circular import
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
if fallback and modname:
return self.ast_from_module_name(modname)
raise AstroidBuildingError("Unable to build an AST for {path}.",
path=filepath)
def file_build(self, path, modname=None):
"""Build astroid from a source code file (i.e. from an ast)
*path* is expected to be a python source file
"""
try:
stream, encoding, data = open_source_file(path)
except IOError as exc:
util.reraise(exceptions.AstroidBuildingError(
'Unable to load file {path}:\n{error}',
modname=modname, path=path, error=exc))
except (SyntaxError, LookupError) as exc:
util.reraise(exceptions.AstroidSyntaxError(
'Python 3 encoding specification error or unknown encoding:\n'
'{error}', modname=modname, path=path, error=exc))
except UnicodeError: # wrong encoding
# detect_encoding returns utf-8 if no encoding specified
util.reraise(exceptions.AstroidBuildingError(
'Wrong or no encoding specified for {filename}.',
filename=path))
with stream:
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(path))
except ImportError:
modname = os.path.splitext(os.path.basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
def test_load_from_module_symlink_on_symlinked_paths_in_syspath(self):
# constants
tmp = tempfile.gettempdir()
deployment_path = os.path.join(tmp, 'deployment')
path_to_include = os.path.join(tmp, 'path_to_include')
real_secret_path = os.path.join(tmp, 'secret.py')
symlink_secret_path = os.path.join(path_to_include, 'secret.py')
# setup double symlink
# /tmp/deployment
# /tmp/path_to_include (symlink to /tmp/deployment)
# /tmp/secret.py
# /tmp/deployment/secret.py (points to /tmp/secret.py)
os.mkdir(deployment_path)
self.addCleanup(shutil.rmtree, deployment_path)
os.symlink(deployment_path, path_to_include)
self.addCleanup(os.remove, path_to_include)
with open(real_secret_path, "w"):
pass
os.symlink(real_secret_path, symlink_secret_path)
self.addCleanup(os.remove, real_secret_path)
# add the symlinked path to sys.path
sys.path.append(path_to_include)
self.addCleanup(sys.path.pop)
# this should be equivalent to: import secret
self.assertEqual(
modutils.modpath_from_file(symlink_secret_path),
['secret'])
def test_load_from_module_symlink_on_symlinked_paths_in_syspath(
self) -> None:
# constants
tmp = tempfile.gettempdir()
deployment_path = os.path.join(tmp, "deployment")
path_to_include = os.path.join(tmp, "path_to_include")
real_secret_path = os.path.join(tmp, "secret.py")
symlink_secret_path = os.path.join(path_to_include, "secret.py")
# setup double symlink
# /tmp/deployment
# /tmp/path_to_include (symlink to /tmp/deployment)
# /tmp/secret.py
# /tmp/deployment/secret.py (points to /tmp/secret.py)
try:
os.mkdir(deployment_path)
self.addCleanup(shutil.rmtree, deployment_path)
os.symlink(deployment_path, path_to_include)
self.addCleanup(os.remove, path_to_include)
except OSError:
pass
with open(real_secret_path, "w", encoding="utf-8"):
pass
os.symlink(real_secret_path, symlink_secret_path)
self.addCleanup(os.remove, real_secret_path)
# add the symlinked path to sys.path
sys.path.append(path_to_include)
self.addCleanup(sys.path.pop)
# this should be equivalent to: import secret
self.assertEqual(modutils.modpath_from_file(symlink_secret_path),
["secret"])
def test_load_from_module_symlink_on_symlinked_paths_in_syspath(self):
# constants
tmp = tempfile.gettempdir()
deployment_path = os.path.join(tmp, 'deployment')
path_to_include = os.path.join(tmp, 'path_to_include')
real_secret_path = os.path.join(tmp, 'secret.py')
symlink_secret_path = os.path.join(path_to_include, 'secret.py')
# setup double symlink
# /tmp/deployment
# /tmp/path_to_include (symlink to /tmp/deployment)
# /tmp/secret.py
# /tmp/deployment/secret.py (points to /tmp/secret.py)
os.mkdir(deployment_path)
self.addCleanup(shutil.rmtree, deployment_path)
os.symlink(deployment_path, path_to_include)
self.addCleanup(os.remove, path_to_include)
with open(real_secret_path, "w"):
pass
os.symlink(real_secret_path, symlink_secret_path)
self.addCleanup(os.remove, real_secret_path)
# add the symlinked path to sys.path
sys.path.append(path_to_include)
self.addCleanup(sys.path.pop)
# this should be equivalent to: import secret
self.assertEqual(
modutils.modpath_from_file(symlink_secret_path),
['secret'])
def test_load_packages_without_init(self) -> None:
"""Test that we correctly find packages with an __init__.py file.
Regression test for issue reported in:
https://github.com/PyCQA/astroid/issues/1327
"""
tmp_dir = Path(tempfile.gettempdir())
self.addCleanup(os.chdir, os.getcwd())
os.chdir(tmp_dir)
self.addCleanup(shutil.rmtree, tmp_dir / "src")
os.mkdir(tmp_dir / "src")
os.mkdir(tmp_dir / "src" / "package")
with open(tmp_dir / "src" / "__init__.py", "w", encoding="utf-8"):
pass
with open(tmp_dir / "src" / "package" / "file.py",
"w",
encoding="utf-8"):
pass
# this should be equivalent to: import secret
self.assertEqual(
modutils.modpath_from_file(str(Path("src") / "package"), ["."]),
["src", "package"],
)
def test_knownValues_modpath_from_file_1(self):
from xml.etree import ElementTree
self.assertEqual(
modutils.modpath_from_file(ElementTree.__file__),
["xml", "etree", "ElementTree"],
)
def file_build(self, path, modname=None):
"""Build astroid from a source code file (i.e. from an ast)
*path* is expected to be a python source file
"""
try:
stream, encoding, data = open_source_file(path)
except IOError as exc:
raise exceptions.AstroidBuildingError(
'Unable to load file {path}:\n{error}',
modname=modname,
path=path,
error=exc) from exc
except (SyntaxError, LookupError) as exc:
raise exceptions.AstroidSyntaxError(
'Python 3 encoding specification error or unknown encoding:\n'
'{error}',
modname=modname,
path=path,
error=exc) from exc
except UnicodeError as exc: # wrong encoding
# detect_encoding returns utf-8 if no encoding specified
raise exceptions.AstroidBuildingError(
'Wrong or no encoding specified for {filename}.',
filename=path) from exc
with stream:
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(path))
except ImportError:
modname = os.path.splitext(os.path.basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
def test_import_symlink_with_source_outside_of_path(self) -> None:
with tempfile.NamedTemporaryFile() as tmpfile:
linked_file_name = "symlinked_file.py"
try:
os.symlink(tmpfile.name, linked_file_name)
self.assertEqual(modutils.modpath_from_file(linked_file_name),
["symlinked_file"])
finally:
os.remove(linked_file_name)
def test_import_symlink_with_source_outside_of_path(self):
with tempfile.NamedTemporaryFile() as tmpfile:
linked_file_name = 'symlinked_file.py'
try:
os.symlink(tmpfile.name, linked_file_name)
self.assertEqual(modutils.modpath_from_file(linked_file_name),
['symlinked_file'])
finally:
os.remove(linked_file_name)
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
result.append({'path': filepath, 'name': modname, 'isarg': True,
'basepath': filepath, 'basename': modname})
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append({'path': subfilepath, 'name': submodname,
'isarg': False,
'basepath': filepath, 'basename': modname})
return result, errors
def _get_file_descr_from_stdin(filepath):
"""Return file description (tuple of module name, file path, base name) from given file path
This method is used for creating suitable file description for _check_files when the
source is standard input.
"""
try:
# Note that this function does not really perform an
# __import__ but may raise an ImportError exception, which
# we want to catch here.
modname = ".".join(modutils.modpath_from_file(filepath))
except ImportError:
modname = os.path.splitext(os.path.basename(filepath))[0]
return (modname, filepath, filepath)
def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
"""given a module name, return the astroid object"""
try:
filepath = modutils.get_source_file(filepath, include_no_ext=True)
source = True
except modutils.NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
if fallback and modname:
return self.ast_from_module_name(modname)
raise exceptions.AstroidBuildingError(
'Unable to build an AST for {path}.', path=filepath)
def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
"""given a module name, return the astroid object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
elif fallback and modname:
return self.ast_from_module_name(modname)
raise AstroidBuildingException('unable to get astroid for file %s' %
filepath)
def file_build(self, path, modname=None):
"""build astroid from a source code file (i.e. from an ast)
path is expected to be a python source file
"""
try:
_, encoding, data = open_source_file(path)
except IOError as exc:
msg = 'Unable to load file %r (%s)' % (path, exc)
raise AstroidBuildingException(msg)
except SyntaxError as exc: # py3k encoding specification error
raise AstroidBuildingException(exc)
except LookupError as exc: # unknown encoding
raise AstroidBuildingException(exc)
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modpath_from_file(path))
except ImportError:
modname = splitext(basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
def file_build(self, path, modname=None):
"""Build astroid from a source code file (i.e. from an ast)
*path* is expected to be a python source file
"""
try:
stream, encoding, data = open_source_file(path)
except IOError as exc:
msg = 'Unable to load file %r (%s)' % (path, exc)
raise exceptions.AstroidBuildingException(msg)
except SyntaxError as exc: # py3k encoding specification error
raise exceptions.AstroidBuildingException(exc)
except LookupError as exc: # unknown encoding
raise exceptions.AstroidBuildingException(exc)
with stream:
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(path))
except ImportError:
modname = os.path.splitext(os.path.basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
path is expected to be a python source file
"""
try:
_, encoding, data = open_source_file(path)
except IOError, exc:
msg = 'Unable to load file %r (%s)' % (path, exc)
raise AstroidBuildingException(msg)
except SyntaxError, exc: # py3k encoding specification error
raise AstroidBuildingException(exc)
except LookupError, exc: # unknown encoding
raise AstroidBuildingException(exc)
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modpath_from_file(path))
except ImportError:
modname = splitext(basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
def string_build(self, data, modname='', path=None):
"""build astroid from source code string and return rebuilded astroid"""
module = self._data_build(data, modname, path)
module.file_bytes = data.encode('utf-8')
return self._post_build(module, 'utf-8')
def _post_build(self, module, encoding):
"""handles encoding and delayed nodes
after a module has been built
def test_knownValues_modpath_from_file_1(self):
from xml.etree import ElementTree
self.assertEqual(modutils.modpath_from_file(ElementTree.__file__),
['xml', 'etree', 'ElementTree'])
def test_knownValues_modpath_from_file_2(self):
self.assertEqual(modutils.modpath_from_file('unittest_modutils.py',
{os.getcwd(): 'arbitrary.pkg'}),
['arbitrary', 'pkg', 'unittest_modutils'])
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
path = sys.path.copy()
for something in files_or_modules:
if os.path.basename(something) in black_list:
continue
if _basename_in_blacklist_re(os.path.basename(something),
black_list_re):
continue
module_path = get_python_path(something)
additional_search_path = [".", module_path] + path
if os.path.exists(something):
# this is a file or a directory
try:
modname = ".".join(
modutils.modpath_from_file(something,
path=additional_search_path))
except ImportError:
modname = os.path.splitext(os.path.basename(something))[0]
if os.path.isdir(something):
filepath = os.path.join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(
modname.split("."), path=additional_search_path)
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# The SyntaxError is a Python bug and should be
# removed once we move away from imp.find_module: https://bugs.python.org/issue10588
errors.append({"key": "fatal", "mod": modname, "ex": ex})
continue
filepath = os.path.normpath(filepath)
modparts = (modname or something).split(".")
try:
spec = modutils.file_info_from_modpath(modparts,
path=additional_search_path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = os.path.isdir(something)
else:
is_namespace = modutils.is_namespace(spec)
is_directory = modutils.is_directory(spec)
if not is_namespace:
result.append({
"path": filepath,
"name": modname,
"isarg": True,
"basepath": filepath,
"basename": modname,
})
has_init = (
not (modname.endswith(".__init__") or modname == "__init__")
and os.path.basename(filepath) == "__init__.py")
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
os.path.dirname(filepath), black_list,
list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(os.path.basename(subfilepath),
black_list_re):
continue
modpath = _modpath_from_file(subfilepath,
is_namespace,
path=additional_search_path)
submodname = ".".join(modpath)
result.append({
"path": subfilepath,
"name": submodname,
"isarg": False,
"basepath": filepath,
"basename": modname,
})
return result, errors
def test_knownValues_modpath_from_file_1(self):
self.assertEqual(modutils.modpath_from_file(modutils.__file__),
['astroid', 'modutils'])
def test_knownValues_modpath_from_file_1(self):
self.assertEqual(modutils.modpath_from_file(configuration.__file__),
['logilab', 'common', 'configuration'])
def test_knownValues_modpath_from_file_1(self):
self.assertEqual(modutils.modpath_from_file(configuration.__file__),
['logilab', 'common', 'configuration'])
def _apply_pylint_run_to_cache(projroot: Path, run: Any, dirtyfiles: List[str],
allfiles: List[str], cache: FileCache) -> int:
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
from astroid import modutils
from efrotools import getconfig
from efro.error import CleanError
# First off, build a map of dirtyfiles to module names
# (and the corresponding reverse map).
paths_to_names: Dict[str, str] = {}
names_to_paths: Dict[str, str] = {}
for fname in allfiles:
try:
mpath = modutils.modpath_from_file(fname)
mpath = _filter_module_name('.'.join(mpath))
paths_to_names[fname] = mpath
except ImportError:
# This probably means its a tool or something not in our
# standard path. In this case just use its base name.
# (seems to be what pylint does)
dummyname = os.path.splitext(os.path.basename(fname))[0]
paths_to_names[fname] = dummyname
for key, val in paths_to_names.items():
names_to_paths[val] = key
# If there's any cyclic-import errors, just mark all deps as dirty;
# don't want to add the logic to figure out which ones the cycles cover
# since they all seems to appear as errors for the last file in the list.
cycles: int = run.linter.stats.get('by_msg', {}).get('cyclic-import', 0)
have_dep_cycles: bool = cycles > 0
if have_dep_cycles:
print(f'Found {cycles} cycle-errors; keeping all dirty files dirty.')
# Update dependencies for what we just ran.
# A run leaves us with a map of modules to a list of the modules that
# imports them. We want the opposite though: for each of our modules
# we want a list of the modules it imports.
reversedeps = {}
# Make sure these are all proper module names; no foo.bar.__init__ stuff.
for key, val in run.linter.stats['dependencies'].items():
sval = [_filter_module_name(m) for m in val]
reversedeps[_filter_module_name(key)] = sval
deps: Dict[str, Set[str]] = {}
untracked_deps = set()
for mname, mallimportedby in reversedeps.items():
for mimportedby in mallimportedby:
if mname in names_to_paths:
deps.setdefault(mimportedby, set()).add(mname)
else:
untracked_deps.add(mname)
ignored_untracked_deps: List[str] = getconfig(projroot).get(
'pylint_ignored_untracked_deps', [])
# Add a few that this package itself triggers.
ignored_untracked_deps += ['pylint.lint', 'astroid.modutils', 'astroid']
# Ignore some specific untracked deps; complain about any others.
untracked_deps = set(dep for dep in untracked_deps
if dep not in ignored_untracked_deps)
if untracked_deps:
raise CleanError(
f'Pylint found untracked dependencies: {untracked_deps}.'
' If these are external to your project, add them to'
' "pylint_ignored_untracked_deps" in the project config.')
# Finally add the dependency lists to our entries (operate on
# everything in the run; it may not be mentioned in deps).
no_deps_modules = set()
for fname in dirtyfiles:
fmod = paths_to_names[fname]
if fmod not in deps:
# Since this code is a bit flaky, lets always announce when we
# come up empty and keep a whitelist of expected values to ignore.
no_deps_modules.add(fmod)
depsval: List[str] = []
else:
# Our deps here are module names; store paths.
depsval = [names_to_paths[dep] for dep in deps[fmod]]
cache.entries[fname]['deps'] = depsval
# Let's print a list of modules with no detected deps so we can make sure
# this is behaving.
if no_deps_modules:
if bool(False):
print('NOTE: no dependencies found for:',
', '.join(no_deps_modules))
# Ok, now go through all dirtyfiles involved in this run.
# Mark them as either errored or clean depending on whether there's
# error info for them in the run stats.
# Once again need to convert any foo.bar.__init__ to foo.bar.
stats_by_module: Dict[str, Any] = {
_filter_module_name(key): val
for key, val in run.linter.stats['by_module'].items()
}
errcount = 0
for fname in dirtyfiles:
mname2 = paths_to_names.get(fname)
if mname2 is None:
raise Exception('unable to get module name for "' + fname + '"')
counts = stats_by_module.get(mname2)
# 'statement' count seems to be new and always non-zero; ignore it
if counts is not None:
counts = {c: v for c, v in counts.items() if c != 'statement'}
if (counts is not None and any(counts.values())) or have_dep_cycles:
# print('GOT FAIL FOR', fname, counts)
if 'hash' in cache.entries[fname]:
del cache.entries[fname]['hash']
errcount += 1
else:
# print('MARKING FILE CLEAN', mname2, fname)
cache.entries[fname]['hash'] = (cache.curhashes[fname])
return errcount
def test_knownValues_modpath_from_file_1(self):
from xml.etree import ElementTree
self.assertEqual(modutils.modpath_from_file(ElementTree.__file__),
['xml', 'etree', 'ElementTree'])
def test_knownValues_modpath_from_file_2(self):
self.assertEqual(modutils.modpath_from_file('unittest_modutils.py',
{os.getcwd(): 'arbitrary.pkg'}),
['arbitrary', 'pkg', 'unittest_modutils'])
def test_known_values_modpath_from_file_1(self) -> None:
self.assertEqual(
modutils.modpath_from_file(ElementTree.__file__),
["xml", "etree", "ElementTree"],
)
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split('.')
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = spec.type == modutils.ModuleType.PY_NAMESPACE
is_directory = spec.type == modutils.ModuleType.PKG_DIRECTORY
if not is_namespace:
result.append({'path': filepath, 'name': modname, 'isarg': True,
'basepath': filepath, 'basename': modname})
has_init = (not (modname.endswith('.__init__') or modname == '__init__')
and '__init__.py' in filepath)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(dirname(filepath), black_list,
list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = '.'.join(modpath)
result.append({'path': subfilepath, 'name': submodname,
'isarg': False,
'basepath': filepath, 'basename': modname})
return result, errors
def _apply_pylint_run_to_cache(projroot: Path, run: Any, dirtyfiles: list[str],
allfiles: list[str], cache: FileCache) -> int:
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
from astroid import modutils
from efrotools import getconfig
from efro.error import CleanError
# First off, build a map of dirtyfiles to module names
# (and the corresponding reverse map).
paths_to_names: dict[str, str] = {}
names_to_paths: dict[str, str] = {}
for fname in allfiles:
try:
mpath = modutils.modpath_from_file(fname)
mpath = _filter_module_name('.'.join(mpath))
paths_to_names[fname] = mpath
except ImportError:
# This probably means its a tool or something not in our
# standard path. In this case just use its base name.
# (seems to be what pylint does)
dummyname = os.path.splitext(os.path.basename(fname))[0]
paths_to_names[fname] = dummyname
for key, val in paths_to_names.items():
names_to_paths[val] = key
# If there's any cyclic-import errors, just mark all deps as dirty;
# don't want to add the logic to figure out which ones the cycles cover
# since they all seems to appear as errors for the last file in the list.
cycles: int = run.linter.stats.by_msg.get('cyclic-import', 0)
have_dep_cycles: bool = cycles > 0
if have_dep_cycles:
print(f'Found {cycles} cycle-errors; keeping all dirty files dirty.')
# Update dependencies for what we just ran.
# A run leaves us with a map of modules to a list of the modules that
# imports them. We want the opposite though: for each of our modules
# we want a list of the modules it imports.
reversedeps = {}
# Make sure these are all proper module names; no foo.bar.__init__ stuff.
for key, val in run.linter.stats.dependencies.items():
sval = [_filter_module_name(m) for m in val]
reversedeps[_filter_module_name(key)] = sval
deps: dict[str, set[str]] = {}
untracked_deps = set()
for mname, mallimportedby in reversedeps.items():
for mimportedby in mallimportedby:
if mname in names_to_paths:
deps.setdefault(mimportedby, set()).add(mname)
else:
untracked_deps.add(mname)
ignored_untracked_deps: set[str] = set(
getconfig(projroot).get('pylint_ignored_untracked_deps', []))
# Add a few that this package itself triggers.
ignored_untracked_deps |= {'pylint.lint', 'astroid.modutils', 'astroid'}
# EW; as of Python 3.9, suddenly I'm seeing system modules showing up
# here where I wasn't before. I wonder what changed. Anyway, explicitly
# suppressing them here but should come up with a more robust system
# as I feel this will get annoying fast.
ignored_untracked_deps |= {
're', 'importlib', 'os', 'xml.dom', 'weakref', 'random',
'collections.abc', 'textwrap', 'webbrowser', 'signal', 'pathlib',
'zlib', 'json', 'pydoc', 'base64', 'functools', 'asyncio', 'xml',
'__future__', 'traceback', 'typing', 'urllib.parse', 'ctypes.wintypes',
'code', 'urllib.error', 'threading', 'xml.etree.ElementTree', 'pickle',
'dataclasses', 'enum', 'py_compile', 'urllib.request', 'math',
'multiprocessing', 'socket', 'getpass', 'hashlib', 'ctypes', 'inspect',
'rlcompleter', 'http.client', 'readline', 'platform', 'datetime',
'copy', 'concurrent.futures', 'ast', 'subprocess', 'numbers',
'logging', 'xml.dom.minidom', 'uuid', 'types', 'tempfile', 'shutil',
'shlex', 'stat', 'wave', 'html', 'binascii'
}
# Ignore some specific untracked deps; complain about any others.
untracked_deps = set(dep for dep in untracked_deps
if dep not in ignored_untracked_deps
and not dep.startswith('bametainternal'))
if untracked_deps:
raise CleanError(
f'Pylint found untracked dependencies: {untracked_deps}.'
' If these are external to your project, add them to'
' "pylint_ignored_untracked_deps" in the project config.')
# Finally add the dependency lists to our entries (operate on
# everything in the run; it may not be mentioned in deps).
no_deps_modules = set()
for fname in dirtyfiles:
fmod = paths_to_names[fname]
if fmod not in deps:
# Since this code is a bit flaky, lets always announce when we
# come up empty and keep a whitelist of expected values to ignore.
no_deps_modules.add(fmod)
depsval: list[str] = []
else:
# Our deps here are module names; store paths.
depsval = [names_to_paths[dep] for dep in deps[fmod]]
cache.entries[fname]['deps'] = depsval
# Let's print a list of modules with no detected deps so we can make sure
# this is behaving.
if no_deps_modules:
if bool(False):
print('NOTE: no dependencies found for:',
', '.join(no_deps_modules))
# Ok, now go through all dirtyfiles involved in this run.
# Mark them as either errored or clean depending on whether there's
# error info for them in the run stats.
# Once again need to convert any foo.bar.__init__ to foo.bar.
stats_by_module: dict[str, Any] = {
_filter_module_name(key): val
for key, val in run.linter.stats.by_module.items()
}
errcount = 0
for fname in dirtyfiles:
mname2 = paths_to_names.get(fname)
if mname2 is None:
raise Exception('unable to get module name for "' + fname + '"')
counts = stats_by_module.get(mname2)
# 'statement' count seems to be new and always non-zero; ignore it
if counts is not None:
counts = {c: v for c, v in counts.items() if c != 'statement'}
if (counts is not None and any(counts.values())) or have_dep_cycles:
# print('GOT FAIL FOR', fname, counts)
if 'hash' in cache.entries[fname]:
del cache.entries[fname]['hash']
errcount += 1
else:
# print('MARKING FILE CLEAN', mname2, fname)
cache.entries[fname]['hash'] = (cache.curhashes[fname])
return errcount
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if basename(something) in black_list:
continue
if _basename_in_blacklist_re(basename(something), black_list_re):
continue
if exists(something):
# this is a file or a directory
try:
modname = ".".join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split("."))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({"key": "fatal", "mod": modname, "ex": ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split(".")
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = modutils.is_namespace(spec)
is_directory = modutils.is_directory(spec)
if not is_namespace:
result.append(
{
"path": filepath,
"name": modname,
"isarg": True,
"basepath": filepath,
"basename": modname,
}
)
has_init = (
not (modname.endswith(".__init__") or modname == "__init__")
and basename(filepath) == "__init__.py"
)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
dirname(filepath), black_list, list_all=is_namespace
):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = ".".join(modpath)
result.append(
{
"path": subfilepath,
"name": submodname,
"isarg": False,
"basepath": filepath,
"basename": modname,
}
)
return result, errors
path is expected to be a python source file
"""
try:
_, encoding, data = open_source_file(path)
except IOError, exc:
msg = 'Unable to load file %r (%s)' % (path, exc)
raise AstroidBuildingException(msg)
except SyntaxError, exc: # py3k encoding specification error
raise AstroidBuildingException(exc)
except LookupError, exc: # unknown encoding
raise AstroidBuildingException(exc)
# get module name if necessary
if modname is None:
try:
modname = '.'.join(modpath_from_file(path))
except ImportError:
modname = splitext(basename(path))[0]
# build astroid representation
module = self._data_build(data, modname, path)
return self._post_build(module, encoding)
def string_build(self, data, modname='', path=None):
"""build astroid from source code string and return rebuilded astroid"""
module = self._data_build(data, modname, path)
module.file_bytes = data.encode('utf-8')
return self._post_build(module, 'utf-8')
def _post_build(self, module, encoding):
"""handles encoding and delayed nodes
after a module has been built
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split('.')
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = spec.type == modutils.ModuleType.PY_NAMESPACE
is_directory = spec.type == modutils.ModuleType.PKG_DIRECTORY
if not is_namespace:
result.append({
'path': filepath,
'name': modname,
'isarg': True,
'basepath': filepath,
'basename': modname
})
has_init = (
not (modname.endswith('.__init__') or modname == '__init__')
and '__init__.py' in filepath)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
dirname(filepath), black_list, list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath),
black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = '.'.join(modpath)
result.append({
'path': subfilepath,
'name': submodname,
'isarg': False,
'basepath': filepath,
'basename': modname
})
return result, errors
def test_knownValues_modpath_from_file_1(self):
self.assertEqual(modutils.modpath_from_file(modutils.__file__),
['astroid', 'modutils'])