def parse_requirements(directory): # Parse the requirements.txt requirement_packages = [] parser_options = optparse.Values( { "skip_requirements_regex": JINJA_DELIMITER_IGNORE_REGEX, # pip._internal assumes parse_requirements will be called from # CLI, which sets default values. When passing parser options, # need to explicitly set those defaults. "isolated_mode": False, "format_control": FormatControl(), } ) requirement_files = glob.glob(os.path.join(directory, '*.txt')) \ + glob.glob(os.path.join(directory, '**', '*.txt')) pip_compile_files = glob.glob(os.path.join(directory, '*.in')) \ + glob.glob(os.path.join(directory, '**', '*.in')) def version_from_install_req(install_req): if install_req.is_pinned: return next(iter(install_req.specifier)).version for reqs_file in requirement_files + pip_compile_files: try: requirements = pip._internal.req.req_file.parse_requirements( reqs_file, options=parser_options, session=PipSession() ) for install_req in requirements: if install_req.original_link: continue pattern = r"-[cr] (.*) \(line \d+\)" abs_path = re.search(pattern, install_req.comes_from).group(1) rel_path = os.path.relpath(abs_path, directory) requirement_packages.append({ "name": install_req.req.name, "version": version_from_install_req(install_req), "markers": str(install_req.markers) or None, "file": rel_path, "requirement": str(install_req.specifier) or None, "extras": sorted(list(install_req.extras)) }) except Exception as e: print(json.dumps({ "error": repr(e) })) exit(1) return json.dumps({ "result": requirement_packages })
def no_binary(): format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", default=format_control, help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " "more package names with commas between them. Note that some " "packages are tricky to compile and may fail to install when " "this option is used on them.", )
def only_binary(): format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", default=format_control, help="Do not use source packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all source packages, :none: to empty the set, or one or " "more package names with commas between them. Packages without " "binary distributions will fail to install when this option is " "used on them.", )
def test_write_format_controls(writer): """ Tests --no-binary/--only-binary options. """ writer.format_control = FormatControl(no_binary=["psycopg2", "click"], only_binary=["pytz", "django"]) lines = list(writer.write_format_controls()) assert "--no-binary psycopg2" in lines assert "--no-binary click" in lines assert "--only-binary pytz" in lines assert "--only-binary django" in lines
def test_get_with_legacy_entry_only(tmpdir): """ Test that an existing cache entry that was created with the legacy hashing mechanism is actually returned in WheelCache.get(). """ wc = WheelCache(tmpdir, FormatControl()) link = Link("https://g.c/o/r") legacy_path = wc.get_path_for_link_legacy(link) ensure_dir(legacy_path) with open(os.path.join(legacy_path, "test-1.0.0-py3-none-any.whl"), "w"): pass cached_link = wc.get(link, "test", [Tag("py3", "none", "any")]) assert (os.path.normcase(os.path.dirname( cached_link.file_path)) == os.path.normcase(legacy_path))
def only_binary(): # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", default=format_control, help='Do not use source packages. Can be supplied multiple times, and ' 'each time adds to the existing value. Accepts either ":all:" to ' 'disable all source packages, ":none:" to empty the set, or one ' 'or more package names with commas between them. Packages ' 'without binary distributions will fail to install when this ' 'option is used on them.', )
def no_binary(): # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", default=format_control, help='Do not use binary packages. Can be supplied multiple times, and ' 'each time adds to the existing value. Accepts either ":all:" to ' 'disable all binary packages, ":none:" to empty the set (notice ' 'the colons), or one or more package names with commas between ' 'them (no colons). Note that some packages are tricky to compile ' 'and may fail to install when this option is used on them.', )
def test_wheel_name_filter(tmpdir): """ Test the wheel cache filters on wheel name when several wheels for different package are stored under the same cache directory. """ wc = WheelCache(tmpdir, FormatControl()) link = Link("https://g.c/package.tar.gz") cache_path = wc.get_path_for_link(link) ensure_dir(cache_path) with open(os.path.join(cache_path, "package-1.0-py3-none-any.whl"), "w"): pass # package matches wheel name assert wc.get(link, "package", [("py3", "none", "any")]) is not link # package2 does not match wheel name assert wc.get(link, "package2", [("py3", "none", "any")]) is link
def check_dist_restriction(options, check_target=False): # type: (Values, bool) -> None """Function for determining if custom platform options are allowed. :param options: The OptionParser options. :param check_target: Whether or not to check if --target is being used. """ dist_restriction_set = any([ options.python_version, options.platforms, options.abis, options.implementation, ]) binary_only = FormatControl(set(), {':all:'}) sdist_dependencies_allowed = (options.format_control != binary_only and not options.ignore_dependencies) # Installations or downloads using dist restrictions must not combine # source distributions and dist-specific wheels, as they are not # guaranteed to be locally compatible. if dist_restriction_set and sdist_dependencies_allowed: raise CommandError( "When restricting platform and interpreter constraints using " "--python-version, --platform, --abi, or --implementation, " "either --no-deps must be set, or --only-binary=:all: must be " "set and --no-binary must not be set (or must be set to " ":none:).") if check_target: if dist_restriction_set and not options.target_dir: raise CommandError( "Can not use any platform or abi specific options unless " "installing via '--target'")
def test_get_path_for_link_legacy(tmpdir): """ Test that an existing cache entry that was created with the legacy hashing mechanism is used. """ wc = WheelCache(tmpdir, FormatControl()) link = Link("https://g.c/o/r") path = wc.get_path_for_link(link) legacy_path = wc.get_path_for_link_legacy(link) assert path != legacy_path ensure_dir(path) with open(os.path.join(path, "test-pyz-none-any.whl"), "w"): pass ensure_dir(legacy_path) with open(os.path.join(legacy_path, "test-pyx-none-any.whl"), "w"): pass expected_candidates = {"test-pyx-none-any.whl", "test-pyz-none-any.whl"} assert set(wc._get_candidates(link, "test")) == expected_candidates
def test_get_cache_entry(tmpdir): wc = WheelCache(tmpdir, FormatControl()) persi_link = Link("https://g.c/o/r/persi") persi_path = wc.get_path_for_link(persi_link) ensure_dir(persi_path) with open(os.path.join(persi_path, "persi-1.0.0-py3-none-any.whl"), "w"): pass ephem_link = Link("https://g.c/o/r/ephem") ephem_path = wc.get_ephem_path_for_link(ephem_link) ensure_dir(ephem_path) with open(os.path.join(ephem_path, "ephem-1.0.0-py3-none-any.whl"), "w"): pass other_link = Link("https://g.c/o/r/other") supported_tags = [Tag("py3", "none", "any")] assert (wc.get_cache_entry(persi_link, "persi", supported_tags).persistent) assert (not wc.get_cache_entry(ephem_link, "ephem", supported_tags).persistent) assert wc.get_cache_entry(other_link, "other", supported_tags) is None
def test_write_format_controls(writer): """ Tests --no-binary/--only-binary options. """ # FormatControl actually expects sets, but we give it lists here to # ensure that we are sorting them when writing. writer.format_control = FormatControl(no_binary=["psycopg2", "click"], only_binary=["pytz", "django"]) lines = list(writer.write_format_controls()) expected_lines = [ "--no-binary click", "--no-binary psycopg2", "--only-binary django", "--only-binary pytz", ] assert lines == expected_lines
def test_download_info_archive_legacy_cache(self, tmp_path: Path, shared_data: TestData) -> None: """Test download_info hash is not set for an archive with legacy cache entry.""" url = shared_data.packages.joinpath("simple-1.0.tar.gz").as_uri() finder = make_test_finder() wheel_cache = WheelCache(str(tmp_path / "cache"), FormatControl()) cache_entry_dir = wheel_cache.get_path_for_link(Link(url)) Path(cache_entry_dir).mkdir(parents=True) wheel.make_wheel(name="simple", version="1.0").save_to_dir(cache_entry_dir) with self._basic_resolver(finder, wheel_cache=wheel_cache) as resolver: ireq = get_processed_req_from_line(f"simple @ {url}") reqset = resolver.resolve([ireq], True) assert len(reqset.all_requirements) == 1 req = reqset.all_requirements[0] assert req.original_link_is_in_wheel_cache assert req.download_info assert req.download_info.url == url assert isinstance(req.download_info.info, ArchiveInfo) assert not req.download_info.info.hash
def __init__( self, candidate_evaluator, # type: CandidateEvaluator search_scope, # type: SearchScope session, # type: PipSession target_python, # type: TargetPython allow_yanked, # type: bool format_control=None, # type: Optional[FormatControl] trusted_hosts=None, # type: Optional[List[str]] ignore_requires_python=None, # type: Optional[bool] ): # type: (...) -> None """ This constructor is primarily meant to be used by the create() class method and from tests. :param candidate_evaluator: A CandidateEvaluator object. :param session: The Session to use to make requests. :param format_control: A FormatControl object, used to control the selection of source packages / binary packages when consulting the index and links. """ if trusted_hosts is None: trusted_hosts = [] format_control = format_control or FormatControl(set(), set()) self._allow_yanked = allow_yanked self._ignore_requires_python = ignore_requires_python self._target_python = target_python self.candidate_evaluator = candidate_evaluator self.search_scope = search_scope self.session = session self.format_control = format_control self.trusted_hosts = trusted_hosts # These are boring links that have already been logged somehow. self._logged_links = set() # type: Set[Link]
def run(self, options, args): format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: skip.update(DEV_PKGS) cmdoptions.check_list_path_option(options) freeze_kwargs = dict( requirement=options.requirements, find_links=options.find_links, local_only=options.local, user_only=options.user, paths=options.path, isolated=options.isolated_mode, wheel_cache=wheel_cache, skip=skip, exclude_editable=options.exclude_editable, ) for line in freeze(**freeze_kwargs): sys.stdout.write(line + "\n")
def test_download_info_archive_cache_with_origin( self, tmp_path: Path, shared_data: TestData) -> None: """Test download_info hash is set for a web archive with cache entry that has origin.json.""" url = shared_data.packages.joinpath("simple-1.0.tar.gz").as_uri() hash = "sha256=ad977496000576e1b6c41f6449a9897087ce9da6db4f15b603fe8372af4bf3c6" finder = make_test_finder() wheel_cache = WheelCache(str(tmp_path / "cache"), FormatControl()) cache_entry_dir = wheel_cache.get_path_for_link(Link(url)) Path(cache_entry_dir).mkdir(parents=True) Path(cache_entry_dir).joinpath("origin.json").write_text( DirectUrl(url, ArchiveInfo(hash=hash)).to_json()) wheel.make_wheel(name="simple", version="1.0").save_to_dir(cache_entry_dir) with self._basic_resolver(finder, wheel_cache=wheel_cache) as resolver: ireq = get_processed_req_from_line(f"simple @ {url}") reqset = resolver.resolve([ireq], True) assert len(reqset.all_requirements) == 1 req = reqset.all_requirements[0] assert req.original_link_is_in_wheel_cache assert req.download_info assert req.download_info.url == url assert isinstance(req.download_info.info, ArchiveInfo) assert req.download_info.info.hash == hash
def __init__( self, link_collector: LinkCollector, target_python: TargetPython, allow_yanked: bool, use_deprecated_html5lib: bool, format_control: Optional[FormatControl] = None, candidate_prefs: Optional[CandidatePreferences] = None, ignore_requires_python: Optional[bool] = None, ) -> None: """ This constructor is primarily meant to be used by the create() class method and from tests. :param format_control: A FormatControl object, used to control the selection of source packages / binary packages when consulting the index and links. :param candidate_prefs: Options to use when creating a CandidateEvaluator object. """ if candidate_prefs is None: candidate_prefs = CandidatePreferences() format_control = format_control or FormatControl(set(), set()) self._allow_yanked = allow_yanked self._candidate_prefs = candidate_prefs self._ignore_requires_python = ignore_requires_python self._link_collector = link_collector self._target_python = target_python self._use_deprecated_html5lib = use_deprecated_html5lib self.format_control = format_control # These are boring links that have already been logged somehow. self._logged_links: Set[Tuple[Link, LinkType, str]] = set()
def run(self, options, args): # type: (Values, List[str]) -> int format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: skip.update(DEV_PKGS) if options.excludes: skip.update(options.excludes) cmdoptions.check_list_path_option(options) if options.find_links: deprecated( "--find-links option in pip freeze is deprecated.", replacement=None, gone_in="21.2", issue=9069, ) freeze_kwargs = dict( requirement=options.requirements, find_links=options.find_links, local_only=options.local, user_only=options.user, paths=options.path, isolated=options.isolated_mode, wheel_cache=wheel_cache, skip=skip, exclude_editable=options.exclude_editable, ) for line in freeze(**freeze_kwargs): sys.stdout.write(line + '\n') return SUCCESS
def run(self, options, args): format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: skip.update(DEV_PKGS) freeze_kwargs = dict( requirement=options.requirements, find_links=options.find_links, local_only=options.local, user_only=options.user, skip_regex=options.skip_requirements_regex, isolated=options.isolated_mode, wheel_cache=wheel_cache, skip=skip, exclude_editable=options.exclude_editable, ) try: for line in freeze(**freeze_kwargs): sys.stdout.write(line + '\n') finally: wheel_cache.cleanup()
def test_fmt_ctl_matches(no_binary, only_binary, argument, expected): fmt = FormatControl(no_binary, only_binary) assert fmt.get_allowed_formats(argument) == expected
def test_none_resets(): cmd = SimpleCommand() cmd.main(['fake', '--no-binary=:all:', '--no-binary=:none:']) format_control = FormatControl(set(), set()) assert cmd.options.format_control == format_control
def test_comma_separated_values(): cmd = SimpleCommand() cmd.main(['fake', '--no-binary=1,2,3']) format_control = FormatControl({'1', '2', '3'}, set()) assert cmd.options.format_control == format_control
def test_only_binary_overrides(): cmd = SimpleCommand() cmd.main(['fake', '--no-binary=:all:', '--only-binary=fred']) format_control = FormatControl({':all:'}, {'fred'}) assert cmd.options.format_control == format_control
def _handle_no_binary(option, opt_str, value, parser): existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, )
def _handle_only_binary(option, opt_str, value, parser): # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, )
def test_falsey_path_none() -> None: wc = WheelCache("", FormatControl()) assert wc.cache_dir is None
def options(session): return stub(isolated_mode=False, index_url='default_url', skip_requirements_regex=False, format_control=FormatControl(set(), set()))
def check_dist_restriction(options, check_target=False): # type: (Values, bool) -> None """Function for determining if custom platform options are allowed. :param options: The OptionParser options. :param check_target: Whether or not to check if --target is being used. """ dist_restriction_set = any([ options.python_version, options.platform, options.abi, options.implementation, ]) binary_only = FormatControl(set(), {':all:'}) sdist_dependencies_allowed = ( options.format_control != binary_only and not options.ignore_dependencies ) # Installations or downloads using dist restrictions must not combine # source distributions and dist-specific wheels, as they are not # guaranteed to be locally compatible. if dist_restriction_set and sdist_dependencies_allowed: raise CommandError( "When restricting platform and interpreter constraints using " "--python-version, --platform, --abi, or --implementation, " "either --no-deps must be set, or --only-binary=:all: must be " "set and --no-binary must not be set (or must be set to " ":none:)."
def only_binary(): # type: () -> Option format_control = FormatControl(set(), set())
def __init__( self, find_links, # type: List[str] index_urls, # type: List[str] allow_all_prereleases=False, # type: bool trusted_hosts=None, # type: Optional[Iterable[str]] session=None, # type: Optional[PipSession] format_control=None, # type: Optional[FormatControl] platform=None, # type: Optional[str] versions=None, # type: Optional[List[str]] abi=None, # type: Optional[str] implementation=None, # type: Optional[str] prefer_binary=False # type: bool ): # type: (...) -> None """Create a PackageFinder. :param format_control: A FormatControl object or None. Used to control the selection of source packages / binary packages when consulting the index and links. :param platform: A string or None. If None, searches for packages that are supported by the current system. Otherwise, will find packages that can be built on the platform passed in. These packages will only be downloaded for distribution: they will not be built locally. :param versions: A list of strings or None. This is passed directly to pep425tags.py in the get_supported() method. :param abi: A string or None. This is passed directly to pep425tags.py in the get_supported() method. :param implementation: A string or None. This is passed directly to pep425tags.py in the get_supported() method. :param prefer_binary: Whether to prefer an old, but valid, binary dist over a new source dist. """ if session is None: raise TypeError( "PackageFinder() missing 1 required keyword argument: " "'session'") # Build find_links. If an argument starts with ~, it may be # a local file relative to a home directory. So try normalizing # it and if it exists, use the normalized version. # This is deliberately conservative - it might be fine just to # blindly normalize anything starting with a ~... self.find_links = [] # type: List[str] for link in find_links: if link.startswith('~'): new_link = normalize_path(link) if os.path.exists(new_link): link = new_link self.find_links.append(link) self.index_urls = index_urls self.format_control = format_control or FormatControl(set(), set()) # Domains that we won't emit warnings for when not using HTTPS self.secure_origins = [ ("*", host, "*") for host in (trusted_hosts if trusted_hosts else []) ] # type: List[SecureOrigin] # Do we want to allow _all_ pre-releases? self.allow_all_prereleases = allow_all_prereleases # The Session we'll use to make requests self.session = session # The valid tags to check potential found wheel candidates against valid_tags = get_supported( versions=versions, platform=platform, abi=abi, impl=implementation, ) self.candidate_evaluator = CandidateEvaluator( valid_tags=valid_tags, prefer_binary=prefer_binary, ) # If we don't have TLS enabled, then WARN if anyplace we're looking # relies on TLS. if not HAS_TLS: for link in itertools.chain(self.index_urls, self.find_links): parsed = urllib_parse.urlparse(link) if parsed.scheme == "https": logger.warning( "pip is configured with locations that require " "TLS/SSL, however the ssl module in Python is not " "available.") break
def options(session): return stub( isolated_mode=False, index_url='default_url', format_control=FormatControl(set(), set()), )