def test_builtins(self):
if sys.version_info < (3, 0):
self.assertTrue(modutils.is_standard_module('__builtin__'))
self.assertFalse(modutils.is_standard_module('builtins'))
else:
self.assertFalse(modutils.is_standard_module('__builtin__'))
self.assertTrue(modutils.is_standard_module('builtins'))
def test_custom_path(self):
datadir = resources.find('')
if datadir.startswith(modutils.EXT_LIB_DIR):
self.skipTest('known breakage of is_standard_module on installed package')
self.assertTrue(modutils.is_standard_module('data.module', (datadir,)))
self.assertTrue(modutils.is_standard_module('data.module', (os.path.abspath(datadir),)))
def test_builtins(self):
if sys.version_info < (3, 0):
self.assertEqual(modutils.is_standard_module('__builtin__'), True)
self.assertEqual(modutils.is_standard_module('builtins'), False)
else:
self.assertEqual(modutils.is_standard_module('__builtin__'), False)
self.assertEqual(modutils.is_standard_module('builtins'), True)
def test_custom_path(self):
if DATADIR.startswith(modutils.EXT_LIB_DIR):
self.skipTest('known breakage of is_standard_module on installed package')
print(repr(DATADIR))
print(modutils.EXT_LIB_DIR)
self.assertEqual(modutils.is_standard_module('data.module', (DATADIR,)), True)
self.assertEqual(modutils.is_standard_module('data.module', (path.abspath(DATADIR),)), True)
def test_builtins(self):
if sys.version_info < (3, 0):
self.assertEqual(modutils.is_standard_module('__builtin__'), True)
self.assertEqual(modutils.is_standard_module('builtins'), False)
else:
self.assertEqual(modutils.is_standard_module('__builtin__'), False)
self.assertEqual(modutils.is_standard_module('builtins'), True)
def test_custom_path(self):
datadir = resources.find('')
if datadir.startswith(modutils.EXT_LIB_DIR):
self.skipTest('known breakage of is_standard_module on installed package')
self.assertTrue(modutils.is_standard_module('data.module', (datadir,)))
self.assertTrue(modutils.is_standard_module('data.module', (os.path.abspath(datadir),)))
def test_builtins(self):
if sys.version_info < (3, 0):
self.assertTrue(modutils.is_standard_module('__builtin__'))
self.assertFalse(modutils.is_standard_module('builtins'))
else:
self.assertFalse(modutils.is_standard_module('__builtin__'))
self.assertTrue(modutils.is_standard_module('builtins'))
def test_failing_edge_cases(self):
# using a subpackage/submodule path as std_path argument
self.assertFalse(modutils.is_standard_module('xml.etree', etree.__path__))
# using a module + object name as modname argument
self.assertTrue(modutils.is_standard_module('sys.path'))
# this is because only the first package/module is considered
self.assertTrue(modutils.is_standard_module('sys.whatever'))
self.assertFalse(modutils.is_standard_module('xml.whatever', etree.__path__))
def test_failing_edge_cases(self):
# using a subpackage/submodule path as std_path argument
self.assertFalse(modutils.is_standard_module("xml.etree", etree.__path__))
# using a module + object name as modname argument
self.assertTrue(modutils.is_standard_module("sys.path"))
# this is because only the first package/module is considered
self.assertTrue(modutils.is_standard_module("sys.whatever"))
self.assertFalse(modutils.is_standard_module("xml.whatever", etree.__path__))
def test_failing_edge_cases(self):
from logilab import common
# using a subpackage/submodule path as std_path argument
self.assertEqual(modutils.is_standard_module('logilab.common', common.__path__), False)
# using a module + object name as modname argument
self.assertEqual(modutils.is_standard_module('sys.path'), True)
# this is because only the first package/module is considered
self.assertEqual(modutils.is_standard_module('sys.whatever'), True)
self.assertEqual(modutils.is_standard_module('logilab.whatever', common.__path__), False)
def test_custom_path(self):
datadir = resources.find("")
if any(datadir.startswith(p) for p in modutils.EXT_LIB_DIRS):
self.skipTest("known breakage of is_standard_module on installed package")
self.assertTrue(modutils.is_standard_module("data.module", (datadir,)))
self.assertTrue(
modutils.is_standard_module("data.module", (os.path.abspath(datadir),))
)
def test_custom_path(self):
if DATADIR.startswith(modutils.EXT_LIB_DIR):
self.skipTest(
'known breakage of is_standard_module on installed package')
print repr(DATADIR)
print modutils.EXT_LIB_DIR
self.assertEqual(
modutils.is_standard_module('data.module', (DATADIR, )), True)
self.assertEqual(
modutils.is_standard_module('data.module',
(path.abspath(DATADIR), )), True)
def test_failing_edge_cases(self):
from xml import etree
# using a subpackage/submodule path as std_path argument
self.assertEqual(
modutils.is_standard_module('xml.etree', etree.__path__), False)
# using a module + object name as modname argument
self.assertEqual(modutils.is_standard_module('sys.path'), True)
# this is because only the first package/module is considered
self.assertEqual(modutils.is_standard_module('sys.whatever'), True)
self.assertEqual(
modutils.is_standard_module('xml.whatever', etree.__path__), False)
def test_4(self):
import astroid
if sys.version_info > (3, 0):
skip = sys.platform.startswith('win') or '.tox' in astroid.__file__
if skip:
self.skipTest('imp module has a broken behaviour in Python 3 on '
'Windows, returning the module path with different '
'case than it should be.')
self.assertEqual(modutils.is_standard_module('hashlib'), True)
self.assertEqual(modutils.is_standard_module('pickle'), True)
self.assertEqual(modutils.is_standard_module('email'), True)
self.assertEqual(modutils.is_standard_module('io'), sys.version_info >= (2, 6))
self.assertEqual(modutils.is_standard_module('StringIO'), sys.version_info < (3, 0))
def test_4(self):
import astroid
if sys.version_info > (3, 0):
skip = sys.platform.startswith('win') or '.tox' in astroid.__file__
if skip:
self.skipTest(
'imp module has a broken behaviour in Python 3 on '
'Windows, returning the module path with different '
'case than it should be.')
self.assertEqual(modutils.is_standard_module('hashlib'), True)
self.assertEqual(modutils.is_standard_module('pickle'), True)
self.assertEqual(modutils.is_standard_module('email'), True)
self.assertEqual(modutils.is_standard_module('io'), sys.version_info >=
(2, 6))
self.assertEqual(modutils.is_standard_module('StringIO'),
sys.version_info < (3, 0))
def _can_load_extension(self, modname: str) -> bool:
if self.always_load_extensions:
return True
if is_standard_module(modname):
return True
return is_module_name_part_of_extension_package_whitelist(
modname, self.extension_package_whitelist)
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
module_file = node.root().file
context_name = node.root().name
base = os.path.splitext(os.path.basename(module_file))[0]
try:
importedmodname = get_module_part(importedmodname, module_file)
except ImportError:
pass
if context_name == importedmodname:
self.add_message("import-self", node=node)
elif not is_standard_module(importedmodname):
# if this is not a package __init__ module
if base != "__init__" and context_name not in self._module_pkg:
# record the module's parent, or the module itself if this is
# a top level module, as the package it belongs to
self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
# handle dependencies
importedmodnames = self.stats["dependencies"].setdefault(
importedmodname, set()
)
if context_name not in importedmodnames:
importedmodnames.add(context_name)
# update import graph
self.import_graph[context_name].add(importedmodname)
if not self.linter.is_message_enabled("cyclic-import", line=node.lineno):
self._excluded_edges[context_name].add(importedmodname)
def _can_load_extension(self, modname):
if self.always_load_extensions:
return True
if modutils.is_standard_module(modname):
return True
parts = modname.split(".")
return any(".".join(parts[:x]) in self.extension_package_whitelist for x in range(1, len(parts) + 1))
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
module_file = node.root().file
context_name = node.root().name
base = os.path.splitext(os.path.basename(module_file))[0]
# Determine if we have a `from .something import` in a package's
# __init__. This means the module will never be able to import
# itself using this condition (the level will be bigger or
# if the same module is named as the package, it will be different
# anyway).
if isinstance(node, astroid.ImportFrom):
if node.level and node.level > 0 and base == '__init__':
return
try:
importedmodname = get_module_part(importedmodname, module_file)
except ImportError:
pass
if context_name == importedmodname:
self.add_message('import-self', node=node)
elif not is_standard_module(importedmodname):
# handle dependencies
importedmodnames = self.stats['dependencies'].setdefault(
importedmodname, set())
if context_name not in importedmodnames:
importedmodnames.add(context_name)
# update import graph
self.import_graph[context_name].add(importedmodname)
if not self.linter.is_message_enabled('cyclic-import',
line=node.lineno):
self._excluded_edges[context_name].add(importedmodname)
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
module_file = node.root().file
context_name = node.root().name
base = os.path.splitext(os.path.basename(module_file))[0]
try:
importedmodname = modutils.get_module_part(importedmodname,
module_file)
except ImportError:
pass
if context_name == importedmodname:
self.add_message("import-self", node=node)
elif not modutils.is_standard_module(importedmodname):
# if this is not a package __init__ module
if base != "__init__" and context_name not in self._module_pkg:
# record the module's parent, or the module itself if this is
# a top level module, as the package it belongs to
self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
# handle dependencies
importedmodnames = self.stats["dependencies"].setdefault(
importedmodname, set())
if context_name not in importedmodnames:
importedmodnames.add(context_name)
# update import graph
self.import_graph[context_name].add(importedmodname)
if not self.linter.is_message_enabled("cyclic-import",
line=node.lineno):
self._excluded_edges[context_name].add(importedmodname)
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
module_file = node.root().file
context_name = node.root().name
base = os.path.splitext(os.path.basename(module_file))[0]
# Determine if we have a `from .something import` in a package's
# __init__. This means the module will never be able to import
# itself using this condition (the level will be bigger or
# if the same module is named as the package, it will be different
# anyway).
if isinstance(node, astroid.ImportFrom):
if node.level and node.level > 0 and base == "__init__":
return
try:
importedmodname = get_module_part(importedmodname, module_file)
except ImportError:
pass
if context_name == importedmodname:
self.add_message("import-self", node=node)
elif not is_standard_module(importedmodname):
# handle dependencies
importedmodnames = self.stats["dependencies"].setdefault(importedmodname, set())
if context_name not in importedmodnames:
importedmodnames.add(context_name)
# update import graph
mgraph = self.import_graph[context_name]
if importedmodname not in mgraph:
mgraph.add(importedmodname)
def compute_module(self, context_name, mod_path):
"""return true if the module should be added to dependencies"""
package_dir = os.path.dirname(self.project.path)
if context_name == mod_path:
return 0
elif modutils.is_standard_module(mod_path, (package_dir,)):
return 1
return 0
def compute_module(self, context_name, mod_path):
"""return true if the module should be added to dependencies"""
package_dir = os.path.dirname(self.project.path)
if context_name == mod_path:
return 0
if modutils.is_standard_module(mod_path, (package_dir, )):
return 1
return 0
def _can_load_extension(self, modname):
if self.always_load_extensions:
return True
if modutils.is_standard_module(modname):
return True
parts = modname.split('.')
return any('.'.join(parts[:x]) in self.extension_package_whitelist
for x in range(1,
len(parts) + 1))
def test_4(self):
self.assertEqual(modutils.is_standard_module('hashlib'), True)
self.assertEqual(modutils.is_standard_module('pickle'), True)
self.assertEqual(modutils.is_standard_module('email'), True)
self.assertEqual(modutils.is_standard_module('io'), sys.version_info >= (2, 6))
self.assertEqual(modutils.is_standard_module('StringIO'), sys.version_info < (3, 0))
self.assertEqual(modutils.is_standard_module('unicodedata'), True)
def test_4(self) -> None:
self.assertTrue(modutils.is_standard_module("hashlib"))
self.assertTrue(modutils.is_standard_module("pickle"))
self.assertTrue(modutils.is_standard_module("email"))
self.assertTrue(modutils.is_standard_module("io"))
self.assertFalse(modutils.is_standard_module("StringIO"))
self.assertTrue(modutils.is_standard_module("unicodedata"))
def test_clear_cache_clears_other_lru_caches(self) -> None:
lrus = (
astroid.nodes.node_classes.LookupMixIn.lookup,
astroid.modutils._cache_normalize_path_,
util.is_namespace,
astroid.interpreter.objectmodel.ObjectModel.attributes,
)
# Get a baseline for the size of the cache after simply calling bootstrap()
baseline_cache_infos = [lru.cache_info() for lru in lrus]
# Generate some hits and misses
ClassDef().lookup("garbage")
is_standard_module("unittest", std_path=["garbage_path"])
util.is_namespace("unittest")
astroid.interpreter.objectmodel.ObjectModel().attributes()
# Did the hits or misses actually happen?
incremented_cache_infos = [lru.cache_info() for lru in lrus]
for incremented_cache, baseline_cache in zip(incremented_cache_infos,
baseline_cache_infos):
with self.subTest(incremented_cache=incremented_cache):
self.assertGreater(
incremented_cache.hits + incremented_cache.misses,
baseline_cache.hits + baseline_cache.misses,
)
astroid.MANAGER.clear_cache() # also calls bootstrap()
# The cache sizes are now as low or lower than the original baseline
cleared_cache_infos = [lru.cache_info() for lru in lrus]
for cleared_cache, baseline_cache in zip(cleared_cache_infos,
baseline_cache_infos):
with self.subTest(cleared_cache=cleared_cache):
# less equal because the "baseline" might have had multiple calls to bootstrap()
self.assertLessEqual(cleared_cache.currsize,
baseline_cache.currsize)
def test_4(self):
self.assertTrue(modutils.is_standard_module("hashlib"))
self.assertTrue(modutils.is_standard_module("pickle"))
self.assertTrue(modutils.is_standard_module("email"))
self.assertEqual(modutils.is_standard_module("io"), sys.version_info >= (2, 6))
self.assertEqual(
modutils.is_standard_module("StringIO"), sys.version_info < (3, 0)
)
self.assertTrue(modutils.is_standard_module("unicodedata"))
def test_4(self):
self.assertTrue(modutils.is_standard_module('hashlib'))
self.assertTrue(modutils.is_standard_module('pickle'))
self.assertTrue(modutils.is_standard_module('email'))
self.assertEqual(modutils.is_standard_module('io'),
sys.version_info >= (2, 6))
self.assertEqual(modutils.is_standard_module('StringIO'),
sys.version_info < (3, 0))
self.assertTrue(modutils.is_standard_module('unicodedata'))
def get_shape_color(self, obj: DiagramEntity) -> str:
"""Get shape color."""
qualified_name = obj.node.qname()
if modutils.is_standard_module(
qualified_name.split(".", maxsplit=1)[0]):
return "grey"
if isinstance(obj.node, nodes.ClassDef):
package = qualified_name.rsplit(".", maxsplit=2)[0]
elif obj.node.package:
package = qualified_name
else:
package = qualified_name.rsplit(".", maxsplit=1)[0]
base_name = ".".join(package.split(".", self.depth)[:self.depth])
if base_name not in self.used_colors:
self.used_colors[base_name] = next(self.available_colors)
return self.used_colors[base_name]
def _check_imports_order(self, node):
"""Checks imports of module `node` are grouped by category
Imports must follow this order: standard, 3rd party, local
"""
extern_imports = []
local_imports = []
std_imports = []
stdlib_paths = [sys.prefix, self.ext_lib_dir]
real_prefix = getattr(sys, "real_prefix", None)
if real_prefix is not None:
stdlib_paths.append(real_prefix)
for node, modname in self._imports_stack:
package = modname.split(".")[0]
if is_standard_module(modname):
std_imports.append((node, package))
wrong_import = extern_imports or local_imports
if not wrong_import:
continue
self.add_message(
"wrong-import-order",
node=node,
args=('standard import "%s"' % node.as_string(), '"%s"' % wrong_import[0][0].as_string()),
)
else:
try:
filename = file_from_modpath([package])
except ImportError:
extern_imports.append((node, package))
continue
if not filename:
extern_imports.append((node, package))
continue
filename = os.path.normcase(os.path.abspath(filename))
if not any(filename.startswith(path) for path in stdlib_paths):
local_imports.append((node, package))
continue
extern_imports.append((node, package))
if not local_imports:
continue
self.add_message(
"wrong-import-order",
node=node,
args=('external import "%s"' % node.as_string(), '"%s"' % local_imports[0][0].as_string()),
)
return std_imports, extern_imports, local_imports
def _check_imports_order(self, node):
"""Checks imports of module `node` are grouped by category
Imports must follow this order: standard, 3rd party, local
"""
extern_imports = []
local_imports = []
std_imports = []
stdlib_paths = [sys.prefix, self.ext_lib_dir]
real_prefix = getattr(sys, 'real_prefix', None)
if real_prefix is not None:
stdlib_paths.append(real_prefix)
for node, modname in self._imports_stack:
package = modname.split('.')[0]
if is_standard_module(modname):
std_imports.append((node, package))
wrong_import = extern_imports or local_imports
if not wrong_import:
continue
self.add_message(
'wrong-import-order',
node=node,
args=('standard import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string()))
else:
try:
filename = file_from_modpath([package])
except ImportError:
extern_imports.append((node, package))
continue
if not filename:
extern_imports.append((node, package))
continue
filename = os.path.normcase(os.path.abspath(filename))
if not any(filename.startswith(path) for path in stdlib_paths):
local_imports.append((node, package))
continue
extern_imports.append((node, package))
if not local_imports:
continue
self.add_message(
'wrong-import-order',
node=node,
args=('external import "%s"' % node.as_string(),
'"%s"' % local_imports[0][0].as_string()))
return std_imports, extern_imports, local_imports
def _inspect_call(call: Call, used_symbols: Set[str], pkgfiles: Set[str]):
try:
for funcdef in call.func.infer():
orig_mod = funcdef.root()
# ignore inferred call that does not generate dependencies
# (uninferable or call to stdlib or internal call)
if (orig_mod == Uninferable or is_standard_module(orig_mod.name)
or _is_module_packaged(orig_mod, pkgfiles)):
continue
used_symbols.add(funcdef.qname())
# As python is a dynamic language, uninferable name lookup or uninferable
# object can be common (when it highly depends on the context that we
# cannot have without executing the code) Hence, it is safer to ignore.
except (NameInferenceError, InferenceError):
pass
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
try:
importedmodname = get_module_part(importedmodname)
except ImportError:
pass
context_name = node.root().name
if context_name == importedmodname:
# module importing itself !
self.add_message('import-self', node=node)
elif not is_standard_module(importedmodname):
# handle dependencies
importedmodnames = self.stats['dependencies'].setdefault(
importedmodname, set())
if not context_name in importedmodnames:
importedmodnames.add(context_name)
# update import graph
mgraph = self.import_graph[context_name]
if importedmodname not in mgraph:
mgraph.add(importedmodname)
def _add_imported_module(self, node, importedmodname):
"""notify an imported module, used to analyze dependencies"""
try:
importedmodname = get_module_part(importedmodname)
except ImportError:
pass
context_name = node.root().name
if context_name == importedmodname:
# module importing itself !
self.add_message('import-self', node=node)
elif not is_standard_module(importedmodname):
# handle dependencies
importedmodnames = self.stats['dependencies'].setdefault(
importedmodname, set())
if not context_name in importedmodnames:
importedmodnames.add(context_name)
# update import graph
mgraph = self.import_graph.setdefault(context_name, set())
if not importedmodname in mgraph:
mgraph.add(importedmodname)
def _check_imports_order(self, node):
"""Checks imports of module `node` are grouped by category
Imports must follow this order: standard, 3rd party, local
"""
extern_imports = []
local_imports = []
std_imports = []
for node, modname in self._imports_stack:
package = modname.split('.')[0]
if is_standard_module(modname):
std_imports.append((node, package))
wrong_import = extern_imports or local_imports
if not wrong_import:
continue
if self._is_fallback_import(node, wrong_import):
continue
self.add_message('wrong-import-order', node=node,
args=('standard import "%s"' % node.as_string(),
'"%s"' % wrong_import[0][0].as_string()))
else:
try:
filename = file_from_modpath([package])
except ImportError:
continue
if not filename:
continue
filename = os.path.normcase(os.path.abspath(filename))
if not any(filename.startswith(path) for path in self._site_packages):
local_imports.append((node, package))
continue
extern_imports.append((node, package))
if not local_imports:
continue
self.add_message('wrong-import-order', node=node,
args=('external import "%s"' % node.as_string(),
'"%s"' % local_imports[0][0].as_string()))
return std_imports, extern_imports, local_imports
def test_nonstandard(self) -> None:
self.assertFalse(modutils.is_standard_module("astroid"))
def test_unknown(self) -> None:
self.assertFalse(modutils.is_standard_module("unknown"))
def test_unknown(self):
self.assertFalse(modutils.is_standard_module('unknown'))
def test_nonstandard(self):
self.assertFalse(modutils.is_standard_module('astroid'))
def test_unknown(self):
self.assertEqual(modutils.is_standard_module('unknown'), False)
def test_builtin(self):
self.assertEqual(modutils.is_standard_module('marshal'), True)
def test_datetime(self):
# This is an interesting example, since datetime, on pypy,
# is under lib_pypy, rather than the usual Lib directory.
self.assertTrue(modutils.is_standard_module('datetime'))
def test_builtin(self):
self.assertTrue(modutils.is_standard_module('sys'))
self.assertTrue(modutils.is_standard_module('marshal'))
def test_datetime(self) -> None:
# This is an interesting example, since datetime, on pypy,
# is under lib_pypy, rather than the usual Lib directory.
self.assertTrue(modutils.is_standard_module("datetime"))
def test_builtin(self):
self.assertEqual(modutils.is_standard_module('marshal'), True)
def test_builtins(self) -> None:
self.assertFalse(modutils.is_standard_module("__builtin__"))
self.assertTrue(modutils.is_standard_module("builtins"))
def test_builtin(self):
self.assertTrue(modutils.is_standard_module('sys'))
self.assertTrue(modutils.is_standard_module('marshal'))
def test_unknown(self):
self.assertFalse(modutils.is_standard_module('unknown'))
def test_nonstandard(self):
self.assertEqual(modutils.is_standard_module('logilab'), False)
def test_builtin(self) -> None:
self.assertTrue(modutils.is_standard_module("sys"))
self.assertTrue(modutils.is_standard_module("marshal"))
def test_nonstandard(self):
self.assertFalse(modutils.is_standard_module('astroid'))