def find_extra_reqs(options):
    # 1. find files used by imports in the code (as best we can without
    #    executing)
    used_modules = common.find_imported_modules(options)

    # 2. find which packages provide which files
    installed_files = {}
    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
    for package in search_packages_info(all_pkgs):
        log.debug("installed package: %s (at %s)", package["name"], package["location"])
        for f in package.get("files", []):
            path = os.path.realpath(os.path.join(package["location"], f))
            installed_files[path] = package["name"]
            package_path = common.is_package_file(path)
            if package_path:
                # we've seen a package file so add the bare package directory
                # to the installed list as well as we might want to look up
                # a package by its directory path later
                installed_files[package_path] = package["name"]

    # 3. match imported modules against those packages
    used = collections.defaultdict(list)
    for modname, info in used_modules.items():
        # probably standard library if it's not in the files list
        if info.filename in installed_files:
            used_name = canonicalize_name(installed_files[info.filename])
            log.debug("used module: %s (from package %s)", modname, installed_files[info.filename])
            used[used_name].append(info)
        else:
            log.debug("used module: %s (from file %s, assuming stdlib or local)", modname, info.filename)

    # 4. compare with requirements.txt
    explicit = common.find_required_modules(options)

    return [name for name in explicit if name not in used]
def find_missing_reqs(options, requirements_filename):
    # 1. find files used by imports in the code (as best we can without
    #    executing)
    used_modules = common.find_imported_modules(options)

    # 2. find which packages provide which files
    installed_files = {}
    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
    for package in search_packages_info(all_pkgs):
        log.debug('installed package: %s (at %s)', package['name'],
                  package['location'])
        for package_file in package.get('files', []) or []:
            path = os.path.realpath(
                os.path.join(package['location'], package_file),
            )
            installed_files[path] = package['name']
            package_path = common.is_package_file(path)
            if package_path:
                # we've seen a package file so add the bare package directory
                # to the installed list as well as we might want to look up
                # a package by its directory path later
                installed_files[package_path] = package['name']

    # 3. match imported modules against those packages
    used = collections.defaultdict(list)
    for modname, info in used_modules.items():
        # probably standard library if it's not in the files list
        if info.filename in installed_files:
            used_name = canonicalize_name(installed_files[info.filename])
            log.debug('used module: %s (from package %s)', modname,
                      installed_files[info.filename])
            used[used_name].append(info)
        else:
            log.debug(
                'used module: %s (from file %s, assuming stdlib or local)',
                modname, info.filename)

    # 4. compare with requirements.txt
    explicit = set()
    for requirement in parse_requirements(
        requirements_filename,
        session=PipSession(),
    ):
        try:
            requirement_name = requirement.name
        # The type of "requirement" changed between pip versions.
        # We exclude the "except" from coverage so that on any pip version we
        # can report 100% coverage.
        except AttributeError:  # pragma: no cover
            from pip._internal.req.constructors import install_req_from_line
            requirement_name = install_req_from_line(
                requirement.requirement,
            ).name

        log.debug('found requirement: %s', requirement_name)
        explicit.add(canonicalize_name(requirement_name))

    return [(name, used[name]) for name in used if name not in explicit]
def find_extra_reqs(options, requirements_filename):
    # 1. find files used by imports in the code (as best we can without
    #    executing)
    used_modules = common.find_imported_modules(options)

    # 2. find which packages provide which files
    installed_files = {}
    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
    for package in search_packages_info(all_pkgs):
        log.debug('installed package: %s (at %s)', package['name'],
                  package['location'])
        for package_file in package.get('files', []) or []:
            path = os.path.realpath(
                os.path.join(package['location'], package_file),
            )
            installed_files[path] = package['name']
            package_path = common.is_package_file(path)
            if package_path:
                # we've seen a package file so add the bare package directory
                # to the installed list as well as we might want to look up
                # a package by its directory path later
                installed_files[package_path] = package['name']

    # 3. match imported modules against those packages
    used = collections.defaultdict(list)
    for modname, info in used_modules.items():
        # probably standard library if it's not in the files list
        if info.filename in installed_files:
            used_name = canonicalize_name(installed_files[info.filename])
            log.debug('used module: %s (from package %s)', modname,
                      installed_files[info.filename])
            used[used_name].append(info)
        else:
            log.debug(
                'used module: %s (from file %s, assuming stdlib or local)',
                modname, info.filename)

    # 4. compare with requirements.txt
    explicit = common.find_required_modules(
        options=options,
        requirements_filename=requirements_filename,
    )

    return [name for name in explicit if name not in used]
def find_missing_reqs(options):
    # 1. find files used by imports in the code (as best we can without
    #    executing)
    used_modules = common.find_imported_modules(options)

    # 2. find which packages provide which files
    installed_files = {}
    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
    for package in search_packages_info(all_pkgs):
        log.debug('installed package: %s (at %s)', package['name'],
            package['location'])
        for file in package['files'] or []:
            path = os.path.realpath(os.path.join(package['location'], file))
            installed_files[path] = package['name']
            package_path = common.is_package_file(path)
            if package_path:
                # we've seen a package file so add the bare package directory
                # to the installed list as well as we might want to look up
                # a package by its directory path later
                installed_files[package_path] = package['name']

    # 3. match imported modules against those packages
    used = collections.defaultdict(list)
    for modname, info in used_modules.items():
        # probably standard library if it's not in the files list
        if info.filename in installed_files:
            used_name = normalize_name(installed_files[info.filename])
            log.debug('used module: %s (from package %s)', modname,
                installed_files[info.filename])
            used[used_name].append(info)
        else:
            log.debug(
                'used module: %s (from file %s, assuming stdlib or local)',
                modname, info.filename)

    # 4. compare with requirements.txt
    explicit = set()
    for requirement in parse_requirements('requirements.txt',
            session=PipSession()):
        log.debug('found requirement: %s', requirement.name)
        explicit.add(normalize_name(requirement.name))

    return [(name, used[name]) for name in used
        if name not in explicit]
Esempio n. 5
0
def test_is_package_file(path, result):
    assert common.is_package_file(path) == result
Esempio n. 6
0
def test_is_package_file(path, result):
    assert common.is_package_file(path) == result
Esempio n. 7
0
def find_extra_reqs(options, requirements_filename):
    # 1. find files used by imports in the code (as best we can without
    #    executing)
    used_modules = common.find_imported_modules(options)

    # 2. find which packages provide which files
    installed_files = {}
    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
    for package in search_packages_info(all_pkgs):
        if isinstance(package, dict):  # pragma: no cover
            package_name = package['name']
            package_location = package['location']
            package_files = package.get('files', []) or []
        else:  # pragma: no cover
            package_name = package.name
            package_location = package.location
            package_files = []
            for item in (package.files or []):
                here = pathlib.Path('.').resolve()
                item_location_rel = (pathlib.Path(package_location) / item)
                item_location = item_location_rel.resolve()
                try:
                    relative_item_location = item_location.relative_to(here)
                except ValueError:
                    # Ideally we would use Pathlib.is_relative_to rather than
                    # checking for a ValueError, but that is only available in
                    # Python 3.9+.
                    relative_item_location = item_location
                package_files.append(str(relative_item_location))

        log.debug('installed package: %s (at %s)', package_name,
                  package_location)
        for package_file in package_files:
            path = os.path.realpath(
                os.path.join(package_location, package_file),
            )
            installed_files[path] = package_name
            package_path = common.is_package_file(path)
            if package_path:
                # we've seen a package file so add the bare package directory
                # to the installed list as well as we might want to look up
                # a package by its directory path later
                installed_files[package_path] = package_name

    # 3. match imported modules against those packages
    used = collections.defaultdict(list)

    for modname, info in used_modules.items():
        # probably standard library if it's not in the files list
        if info.filename in installed_files:
            used_name = canonicalize_name(installed_files[info.filename])
            log.debug('used module: %s (from package %s)', modname,
                      installed_files[info.filename])
            used[used_name].append(info)
        else:
            log.debug(
                'used module: %s (from file %s, assuming stdlib or local)',
                modname, info.filename)

    # 4. compare with requirements
    explicit = common.find_required_modules(
        options=options,
        requirements_filename=requirements_filename,
    )

    return [name for name in explicit if name not in used]