def find_extra_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug("installed package: %s (at %s)", package["name"], package["location"])
for f in package.get("files", []):
path = os.path.realpath(os.path.join(package["location"], f))
installed_files[path] = package["name"]
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package["name"]
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug("used module: %s (from package %s)", modname, installed_files[info.filename])
used[used_name].append(info)
else:
log.debug("used module: %s (from file %s, assuming stdlib or local)", modname, info.filename)
# 4. compare with requirements.txt
explicit = common.find_required_modules(options)
return [name for name in explicit if name not in used]
def test_find_required_modules(monkeypatch, tmp_path: Path):
class options:
skip_incompatible = False
options.ignore_reqs = common.ignorer(ignore_cfg=['barfoo'])
fake_requirements_file = tmp_path / 'requirements.txt'
fake_requirements_file.write_text('foobar==1\nbarfoo==2')
reqs = common.find_required_modules(
options=options,
requirements_filename=str(fake_requirements_file),
)
assert reqs == set(['foobar'])
def test_find_required_modules(monkeypatch):
class options:
@staticmethod
def ignore_reqs(req):
if req.name == 'barfoo':
return True
return False
FakeReq = collections.namedtuple('FakeReq', ['name'])
requirements = [FakeReq('foobar'), FakeReq('barfoo')]
monkeypatch.setattr(common, 'parse_requirements',
pretend.call_recorder(lambda a, session=None: requirements))
reqs = common.find_required_modules(options)
assert reqs == set(['foobar'])
def test_find_required_modules(monkeypatch):
class options:
@staticmethod
def ignore_reqs(req):
if req.name == 'barfoo':
return True
return False
FakeReq = collections.namedtuple('FakeReq', ['name'])
requirements = [FakeReq('foobar'), FakeReq('barfoo')]
monkeypatch.setattr(
common, 'parse_requirements',
pretend.call_recorder(lambda a, session=None: requirements))
reqs = common.find_required_modules(options)
assert reqs == set(['foobar'])
def test_find_required_modules_env_markers(monkeypatch, tmp_path):
class options:
skip_incompatible = True
def ignore_reqs(self, modname):
return False
fake_requirements_file = tmp_path / 'requirements.txt'
fake_requirements_file.write_text('spam==1; python_version<"2.0"\n'
'ham==2;\n'
'eggs==3\n')
reqs = common.find_required_modules(
options=options(),
requirements_filename=str(fake_requirements_file),
)
assert reqs == {'ham', 'eggs'}
def find_extra_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for package_file in package.get('files', []) or []:
path = os.path.realpath(
os.path.join(package['location'], package_file),
)
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = common.find_required_modules(
options=options,
requirements_filename=requirements_filename,
)
return [name for name in explicit if name not in used]
def find_extra_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
if isinstance(package, dict): # pragma: no cover
package_name = package['name']
package_location = package['location']
package_files = package.get('files', []) or []
else: # pragma: no cover
package_name = package.name
package_location = package.location
package_files = []
for item in (package.files or []):
here = pathlib.Path('.').resolve()
item_location_rel = (pathlib.Path(package_location) / item)
item_location = item_location_rel.resolve()
try:
relative_item_location = item_location.relative_to(here)
except ValueError:
# Ideally we would use Pathlib.is_relative_to rather than
# checking for a ValueError, but that is only available in
# Python 3.9+.
relative_item_location = item_location
package_files.append(str(relative_item_location))
log.debug('installed package: %s (at %s)', package_name,
package_location)
for package_file in package_files:
path = os.path.realpath(
os.path.join(package_location, package_file),
)
installed_files[path] = package_name
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package_name
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements
explicit = common.find_required_modules(
options=options,
requirements_filename=requirements_filename,
)
return [name for name in explicit if name not in used]
