def test_find_imported_modules_sets_encoding_to_utf8_when_reading(tmp_path):
(tmp_path / 'module.py').touch()
class options:
paths = [tmp_path]
def ignore_files(*_):
return False
expected_encoding = 'utf-8'
used_encoding = None
original_open = common.__builtins__['open']
def mocked_open(*args, **kwargs):
# As of Python 3.9, the args to open() are as follows:
# file, mode, buffering, encoding, erorrs, newline, closedf, opener
nonlocal used_encoding
if 'encoding' in kwargs:
used_encoding = kwargs['encoding']
return original_open(*args, **kwargs)
common.__builtins__['open'] = mocked_open
common.find_imported_modules(options)
common.__builtins__['open'] = original_open
assert used_encoding == expected_encoding
def find_extra_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug("installed package: %s (at %s)", package["name"], package["location"])
for f in package.get("files", []):
path = os.path.realpath(os.path.join(package["location"], f))
installed_files[path] = package["name"]
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package["name"]
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug("used module: %s (from package %s)", modname, installed_files[info.filename])
used[used_name].append(info)
else:
log.debug("used module: %s (from file %s, assuming stdlib or local)", modname, info.filename)
# 4. compare with requirements.txt
explicit = common.find_required_modules(options)
return [name for name in explicit if name not in used]
def find_missing_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for package_file in package.get('files', []) or []:
path = os.path.realpath(
os.path.join(package['location'], package_file),
)
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements(
requirements_filename,
session=PipSession(),
):
try:
requirement_name = requirement.name
# The type of "requirement" changed between pip versions.
# We exclude the "except" from coverage so that on any pip version we
# can report 100% coverage.
except AttributeError: # pragma: no cover
from pip._internal.req.constructors import install_req_from_line
requirement_name = install_req_from_line(
requirement.requirement,
).name
log.debug('found requirement: %s', requirement_name)
explicit.add(canonicalize_name(requirement_name))
return [(name, used[name]) for name in used if name not in explicit]
def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib,
expect, locs):
monkeypatch.setattr(common, 'pyfiles',
pretend.call_recorder(lambda x: ['spam.py', 'ham.py']))
if sys.version_info[0] == 2:
# py2 will find sys module but py3k won't
expect.append('sys')
class FakeFile():
contents = [
'from os import path\nimport ast, hashlib',
'from __future__ import braces\nimport ast, sys\n'
'from . import friend',
]
def __init__(self, filename):
pass
def read(self):
return self.contents.pop()
def __enter__(self):
return self
def __exit__(self, *args):
pass
monkeypatch.setattr(common, 'open', FakeFile, raising=False)
caplog.set_level(logging.INFO)
class options:
paths = ['dummy']
verbose = True
@staticmethod
def ignore_files(path):
if path == 'ham.py' and ignore_ham:
return True
return False
@staticmethod
def ignore_mods(module):
if module == 'hashlib' and ignore_hashlib:
return True
return False
result = common.find_imported_modules(options)
assert set(result) == set(expect)
assert result['ast'].locations == locs
if ignore_ham:
assert caplog.records[0].message == 'ignoring: ham.py'
def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib,
expect, locs):
monkeypatch.setattr(common, 'pyfiles',
pretend.call_recorder(lambda x: ['spam.py', 'ham.py']))
if sys.version_info[0] == 2:
# py2 will find sys module but py3k won't
expect.append('sys')
class FakeFile():
contents = [
'from os import path\nimport ast, hashlib',
'from __future__ import braces\nimport ast, sys\n'
'from . import friend',
]
def __init__(self, filename):
pass
def read(self):
return self.contents.pop()
def __enter__(self):
return self
def __exit__(self, *args):
pass
monkeypatch.setattr(common, 'open', FakeFile, raising=False)
caplog.set_level(logging.INFO)
class options:
paths = ['dummy']
verbose = True
@staticmethod
def ignore_files(path):
if path == 'ham.py' and ignore_ham:
return True
return False
@staticmethod
def ignore_mods(module):
if module == 'hashlib' and ignore_hashlib:
return True
return False
result = common.find_imported_modules(options)
assert set(result) == set(expect)
assert result['ast'].locations == locs
if ignore_ham:
assert caplog.records[0].message == 'ignoring: ham.py'
def find_extra_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for package_file in package.get('files', []) or []:
path = os.path.realpath(
os.path.join(package['location'], package_file),
)
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = common.find_required_modules(
options=options,
requirements_filename=requirements_filename,
)
return [name for name in explicit if name not in used]
def find_missing_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for file in package['files'] or []:
path = os.path.realpath(os.path.join(package['location'], file))
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = normalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
log.debug('found requirement: %s', requirement.name)
explicit.add(normalize_name(requirement.name))
return [(name, used[name]) for name in used
if name not in explicit]
def find_extra_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
if isinstance(package, dict): # pragma: no cover
package_name = package['name']
package_location = package['location']
package_files = package.get('files', []) or []
else: # pragma: no cover
package_name = package.name
package_location = package.location
package_files = []
for item in (package.files or []):
here = pathlib.Path('.').resolve()
item_location_rel = (pathlib.Path(package_location) / item)
item_location = item_location_rel.resolve()
try:
relative_item_location = item_location.relative_to(here)
except ValueError:
# Ideally we would use Pathlib.is_relative_to rather than
# checking for a ValueError, but that is only available in
# Python 3.9+.
relative_item_location = item_location
package_files.append(str(relative_item_location))
log.debug('installed package: %s (at %s)', package_name,
package_location)
for package_file in package_files:
path = os.path.realpath(
os.path.join(package_location, package_file),
)
installed_files[path] = package_name
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package_name
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements
explicit = common.find_required_modules(
options=options,
requirements_filename=requirements_filename,
)
return [name for name in explicit if name not in used]
