def parse_requirements(directory):
# Parse the requirements.txt
requirement_packages = []
parser_options = optparse.Values(
{
"skip_requirements_regex": JINJA_DELIMITER_IGNORE_REGEX,
# pip._internal assumes parse_requirements will be called from
# CLI, which sets default values. When passing parser options,
# need to explicitly set those defaults.
"isolated_mode": False,
"format_control": FormatControl(),
}
)
requirement_files = glob.glob(os.path.join(directory, '*.txt')) \
+ glob.glob(os.path.join(directory, '**', '*.txt'))
pip_compile_files = glob.glob(os.path.join(directory, '*.in')) \
+ glob.glob(os.path.join(directory, '**', '*.in'))
def version_from_install_req(install_req):
if install_req.is_pinned:
return next(iter(install_req.specifier)).version
for reqs_file in requirement_files + pip_compile_files:
try:
requirements = pip._internal.req.req_file.parse_requirements(
reqs_file,
options=parser_options,
session=PipSession()
)
for install_req in requirements:
if install_req.original_link:
continue
pattern = r"-[cr] (.*) \(line \d+\)"
abs_path = re.search(pattern, install_req.comes_from).group(1)
rel_path = os.path.relpath(abs_path, directory)
requirement_packages.append({
"name": install_req.req.name,
"version": version_from_install_req(install_req),
"markers": str(install_req.markers) or None,
"file": rel_path,
"requirement": str(install_req.specifier) or None,
"extras": sorted(list(install_req.extras))
})
except Exception as e:
print(json.dumps({ "error": repr(e) }))
exit(1)
return json.dumps({ "result": requirement_packages })
def no_binary():
format_control = FormatControl(set(), set())
return Option(
"--no-binary", dest="format_control", action="callback",
callback=_handle_no_binary, type="str",
default=format_control,
help="Do not use binary packages. Can be supplied multiple times, and "
"each time adds to the existing value. Accepts either :all: to "
"disable all binary packages, :none: to empty the set, or one or "
"more package names with commas between them. Note that some "
"packages are tricky to compile and may fail to install when "
"this option is used on them.",
)
def only_binary():
format_control = FormatControl(set(), set())
return Option(
"--only-binary", dest="format_control", action="callback",
callback=_handle_only_binary, type="str",
default=format_control,
help="Do not use source packages. Can be supplied multiple times, and "
"each time adds to the existing value. Accepts either :all: to "
"disable all source packages, :none: to empty the set, or one or "
"more package names with commas between them. Packages without "
"binary distributions will fail to install when this option is "
"used on them.",
)
def test_write_format_controls(writer):
"""
Tests --no-binary/--only-binary options.
"""
writer.format_control = FormatControl(no_binary=["psycopg2", "click"],
only_binary=["pytz", "django"])
lines = list(writer.write_format_controls())
assert "--no-binary psycopg2" in lines
assert "--no-binary click" in lines
assert "--only-binary pytz" in lines
assert "--only-binary django" in lines
def test_get_with_legacy_entry_only(tmpdir):
"""
Test that an existing cache entry that was created with the legacy hashing
mechanism is actually returned in WheelCache.get().
"""
wc = WheelCache(tmpdir, FormatControl())
link = Link("https://g.c/o/r")
legacy_path = wc.get_path_for_link_legacy(link)
ensure_dir(legacy_path)
with open(os.path.join(legacy_path, "test-1.0.0-py3-none-any.whl"), "w"):
pass
cached_link = wc.get(link, "test", [Tag("py3", "none", "any")])
assert (os.path.normcase(os.path.dirname(
cached_link.file_path)) == os.path.normcase(legacy_path))
def only_binary():
# type: () -> Option
format_control = FormatControl(set(), set())
return Option(
"--only-binary", dest="format_control", action="callback",
callback=_handle_only_binary, type="str",
default=format_control,
help='Do not use source packages. Can be supplied multiple times, and '
'each time adds to the existing value. Accepts either ":all:" to '
'disable all source packages, ":none:" to empty the set, or one '
'or more package names with commas between them. Packages '
'without binary distributions will fail to install when this '
'option is used on them.',
)
def no_binary():
# type: () -> Option
format_control = FormatControl(set(), set())
return Option(
"--no-binary", dest="format_control", action="callback",
callback=_handle_no_binary, type="str",
default=format_control,
help='Do not use binary packages. Can be supplied multiple times, and '
'each time adds to the existing value. Accepts either ":all:" to '
'disable all binary packages, ":none:" to empty the set (notice '
'the colons), or one or more package names with commas between '
'them (no colons). Note that some packages are tricky to compile '
'and may fail to install when this option is used on them.',
)
def test_wheel_name_filter(tmpdir):
"""
Test the wheel cache filters on wheel name when several wheels
for different package are stored under the same cache directory.
"""
wc = WheelCache(tmpdir, FormatControl())
link = Link("https://g.c/package.tar.gz")
cache_path = wc.get_path_for_link(link)
ensure_dir(cache_path)
with open(os.path.join(cache_path, "package-1.0-py3-none-any.whl"), "w"):
pass
# package matches wheel name
assert wc.get(link, "package", [("py3", "none", "any")]) is not link
# package2 does not match wheel name
assert wc.get(link, "package2", [("py3", "none", "any")]) is link
def check_dist_restriction(options, check_target=False):
# type: (Values, bool) -> None
"""Function for determining if custom platform options are allowed.
:param options: The OptionParser options.
:param check_target: Whether or not to check if --target is being used.
"""
dist_restriction_set = any([
options.python_version,
options.platforms,
options.abis,
options.implementation,
])
binary_only = FormatControl(set(), {':all:'})
sdist_dependencies_allowed = (options.format_control != binary_only
and not options.ignore_dependencies)
# Installations or downloads using dist restrictions must not combine
# source distributions and dist-specific wheels, as they are not
# guaranteed to be locally compatible.
if dist_restriction_set and sdist_dependencies_allowed:
raise CommandError(
"When restricting platform and interpreter constraints using "
"--python-version, --platform, --abi, or --implementation, "
"either --no-deps must be set, or --only-binary=:all: must be "
"set and --no-binary must not be set (or must be set to "
":none:).")
if check_target:
if dist_restriction_set and not options.target_dir:
raise CommandError(
"Can not use any platform or abi specific options unless "
"installing via '--target'")
def test_get_path_for_link_legacy(tmpdir):
"""
Test that an existing cache entry that was created with the legacy hashing
mechanism is used.
"""
wc = WheelCache(tmpdir, FormatControl())
link = Link("https://g.c/o/r")
path = wc.get_path_for_link(link)
legacy_path = wc.get_path_for_link_legacy(link)
assert path != legacy_path
ensure_dir(path)
with open(os.path.join(path, "test-pyz-none-any.whl"), "w"):
pass
ensure_dir(legacy_path)
with open(os.path.join(legacy_path, "test-pyx-none-any.whl"), "w"):
pass
expected_candidates = {"test-pyx-none-any.whl", "test-pyz-none-any.whl"}
assert set(wc._get_candidates(link, "test")) == expected_candidates
def test_get_cache_entry(tmpdir):
wc = WheelCache(tmpdir, FormatControl())
persi_link = Link("https://g.c/o/r/persi")
persi_path = wc.get_path_for_link(persi_link)
ensure_dir(persi_path)
with open(os.path.join(persi_path, "persi-1.0.0-py3-none-any.whl"), "w"):
pass
ephem_link = Link("https://g.c/o/r/ephem")
ephem_path = wc.get_ephem_path_for_link(ephem_link)
ensure_dir(ephem_path)
with open(os.path.join(ephem_path, "ephem-1.0.0-py3-none-any.whl"), "w"):
pass
other_link = Link("https://g.c/o/r/other")
supported_tags = [Tag("py3", "none", "any")]
assert (wc.get_cache_entry(persi_link, "persi", supported_tags).persistent)
assert (not wc.get_cache_entry(ephem_link, "ephem",
supported_tags).persistent)
assert wc.get_cache_entry(other_link, "other", supported_tags) is None
def test_write_format_controls(writer):
"""
Tests --no-binary/--only-binary options.
"""
# FormatControl actually expects sets, but we give it lists here to
# ensure that we are sorting them when writing.
writer.format_control = FormatControl(no_binary=["psycopg2", "click"],
only_binary=["pytz", "django"])
lines = list(writer.write_format_controls())
expected_lines = [
"--no-binary click",
"--no-binary psycopg2",
"--only-binary django",
"--only-binary pytz",
]
assert lines == expected_lines
def test_download_info_archive_legacy_cache(self, tmp_path: Path,
shared_data: TestData) -> None:
"""Test download_info hash is not set for an archive with legacy cache entry."""
url = shared_data.packages.joinpath("simple-1.0.tar.gz").as_uri()
finder = make_test_finder()
wheel_cache = WheelCache(str(tmp_path / "cache"), FormatControl())
cache_entry_dir = wheel_cache.get_path_for_link(Link(url))
Path(cache_entry_dir).mkdir(parents=True)
wheel.make_wheel(name="simple",
version="1.0").save_to_dir(cache_entry_dir)
with self._basic_resolver(finder, wheel_cache=wheel_cache) as resolver:
ireq = get_processed_req_from_line(f"simple @ {url}")
reqset = resolver.resolve([ireq], True)
assert len(reqset.all_requirements) == 1
req = reqset.all_requirements[0]
assert req.original_link_is_in_wheel_cache
assert req.download_info
assert req.download_info.url == url
assert isinstance(req.download_info.info, ArchiveInfo)
assert not req.download_info.info.hash
def __init__(
self,
candidate_evaluator, # type: CandidateEvaluator
search_scope, # type: SearchScope
session, # type: PipSession
target_python, # type: TargetPython
allow_yanked, # type: bool
format_control=None, # type: Optional[FormatControl]
trusted_hosts=None, # type: Optional[List[str]]
ignore_requires_python=None, # type: Optional[bool]
):
# type: (...) -> None
"""
This constructor is primarily meant to be used by the create() class
method and from tests.
:param candidate_evaluator: A CandidateEvaluator object.
:param session: The Session to use to make requests.
:param format_control: A FormatControl object, used to control
the selection of source packages / binary packages when consulting
the index and links.
"""
if trusted_hosts is None:
trusted_hosts = []
format_control = format_control or FormatControl(set(), set())
self._allow_yanked = allow_yanked
self._ignore_requires_python = ignore_requires_python
self._target_python = target_python
self.candidate_evaluator = candidate_evaluator
self.search_scope = search_scope
self.session = session
self.format_control = format_control
self.trusted_hosts = trusted_hosts
# These are boring links that have already been logged somehow.
self._logged_links = set() # type: Set[Link]
def run(self, options, args):
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
skip = set(stdlib_pkgs)
if not options.freeze_all:
skip.update(DEV_PKGS)
cmdoptions.check_list_path_option(options)
freeze_kwargs = dict(
requirement=options.requirements,
find_links=options.find_links,
local_only=options.local,
user_only=options.user,
paths=options.path,
isolated=options.isolated_mode,
wheel_cache=wheel_cache,
skip=skip,
exclude_editable=options.exclude_editable,
)
for line in freeze(**freeze_kwargs):
sys.stdout.write(line + "\n")
def test_download_info_archive_cache_with_origin(
self, tmp_path: Path, shared_data: TestData) -> None:
"""Test download_info hash is set for a web archive with cache entry
that has origin.json."""
url = shared_data.packages.joinpath("simple-1.0.tar.gz").as_uri()
hash = "sha256=ad977496000576e1b6c41f6449a9897087ce9da6db4f15b603fe8372af4bf3c6"
finder = make_test_finder()
wheel_cache = WheelCache(str(tmp_path / "cache"), FormatControl())
cache_entry_dir = wheel_cache.get_path_for_link(Link(url))
Path(cache_entry_dir).mkdir(parents=True)
Path(cache_entry_dir).joinpath("origin.json").write_text(
DirectUrl(url, ArchiveInfo(hash=hash)).to_json())
wheel.make_wheel(name="simple",
version="1.0").save_to_dir(cache_entry_dir)
with self._basic_resolver(finder, wheel_cache=wheel_cache) as resolver:
ireq = get_processed_req_from_line(f"simple @ {url}")
reqset = resolver.resolve([ireq], True)
assert len(reqset.all_requirements) == 1
req = reqset.all_requirements[0]
assert req.original_link_is_in_wheel_cache
assert req.download_info
assert req.download_info.url == url
assert isinstance(req.download_info.info, ArchiveInfo)
assert req.download_info.info.hash == hash
def __init__(
self,
link_collector: LinkCollector,
target_python: TargetPython,
allow_yanked: bool,
use_deprecated_html5lib: bool,
format_control: Optional[FormatControl] = None,
candidate_prefs: Optional[CandidatePreferences] = None,
ignore_requires_python: Optional[bool] = None,
) -> None:
"""
This constructor is primarily meant to be used by the create() class
method and from tests.
:param format_control: A FormatControl object, used to control
the selection of source packages / binary packages when consulting
the index and links.
:param candidate_prefs: Options to use when creating a
CandidateEvaluator object.
"""
if candidate_prefs is None:
candidate_prefs = CandidatePreferences()
format_control = format_control or FormatControl(set(), set())
self._allow_yanked = allow_yanked
self._candidate_prefs = candidate_prefs
self._ignore_requires_python = ignore_requires_python
self._link_collector = link_collector
self._target_python = target_python
self._use_deprecated_html5lib = use_deprecated_html5lib
self.format_control = format_control
# These are boring links that have already been logged somehow.
self._logged_links: Set[Tuple[Link, LinkType, str]] = set()
def run(self, options, args):
# type: (Values, List[str]) -> int
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
skip = set(stdlib_pkgs)
if not options.freeze_all:
skip.update(DEV_PKGS)
if options.excludes:
skip.update(options.excludes)
cmdoptions.check_list_path_option(options)
if options.find_links:
deprecated(
"--find-links option in pip freeze is deprecated.",
replacement=None,
gone_in="21.2",
issue=9069,
)
freeze_kwargs = dict(
requirement=options.requirements,
find_links=options.find_links,
local_only=options.local,
user_only=options.user,
paths=options.path,
isolated=options.isolated_mode,
wheel_cache=wheel_cache,
skip=skip,
exclude_editable=options.exclude_editable,
)
for line in freeze(**freeze_kwargs):
sys.stdout.write(line + '\n')
return SUCCESS
def run(self, options, args):
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
skip = set(stdlib_pkgs)
if not options.freeze_all:
skip.update(DEV_PKGS)
freeze_kwargs = dict(
requirement=options.requirements,
find_links=options.find_links,
local_only=options.local,
user_only=options.user,
skip_regex=options.skip_requirements_regex,
isolated=options.isolated_mode,
wheel_cache=wheel_cache,
skip=skip,
exclude_editable=options.exclude_editable,
)
try:
for line in freeze(**freeze_kwargs):
sys.stdout.write(line + '\n')
finally:
wheel_cache.cleanup()
def test_fmt_ctl_matches(no_binary, only_binary, argument, expected):
fmt = FormatControl(no_binary, only_binary)
assert fmt.get_allowed_formats(argument) == expected
def test_none_resets():
cmd = SimpleCommand()
cmd.main(['fake', '--no-binary=:all:', '--no-binary=:none:'])
format_control = FormatControl(set(), set())
assert cmd.options.format_control == format_control
def test_comma_separated_values():
cmd = SimpleCommand()
cmd.main(['fake', '--no-binary=1,2,3'])
format_control = FormatControl({'1', '2', '3'}, set())
assert cmd.options.format_control == format_control
def test_only_binary_overrides():
cmd = SimpleCommand()
cmd.main(['fake', '--no-binary=:all:', '--only-binary=fred'])
format_control = FormatControl({':all:'}, {'fred'})
assert cmd.options.format_control == format_control
def _handle_no_binary(option, opt_str, value, parser):
existing = _get_format_control(parser.values, option)
FormatControl.handle_mutual_excludes(
value, existing.no_binary, existing.only_binary,
)
def _handle_only_binary(option, opt_str, value, parser):
# type: (Option, str, str, OptionParser) -> None
existing = _get_format_control(parser.values, option)
FormatControl.handle_mutual_excludes(
value, existing.only_binary, existing.no_binary,
)
def _handle_only_binary(option, opt_str, value, parser):
# type: (Option, str, str, OptionParser) -> None
existing = _get_format_control(parser.values, option)
FormatControl.handle_mutual_excludes(
value, existing.only_binary, existing.no_binary,
)
def test_falsey_path_none() -> None:
wc = WheelCache("", FormatControl())
assert wc.cache_dir is None
def options(session):
return stub(isolated_mode=False,
index_url='default_url',
skip_requirements_regex=False,
format_control=FormatControl(set(), set()))
def check_dist_restriction(options, check_target=False):
# type: (Values, bool) -> None
"""Function for determining if custom platform options are allowed.
:param options: The OptionParser options.
:param check_target: Whether or not to check if --target is being used.
"""
dist_restriction_set = any([
options.python_version,
options.platform,
options.abi,
options.implementation,
])
binary_only = FormatControl(set(), {':all:'})
sdist_dependencies_allowed = (
options.format_control != binary_only and
not options.ignore_dependencies
)
# Installations or downloads using dist restrictions must not combine
# source distributions and dist-specific wheels, as they are not
# guaranteed to be locally compatible.
if dist_restriction_set and sdist_dependencies_allowed:
raise CommandError(
"When restricting platform and interpreter constraints using "
"--python-version, --platform, --abi, or --implementation, "
"either --no-deps must be set, or --only-binary=:all: must be "
"set and --no-binary must not be set (or must be set to "
":none:)."
def only_binary():
# type: () -> Option
format_control = FormatControl(set(), set())
def test_fmt_ctl_matches(no_binary, only_binary, argument, expected):
fmt = FormatControl(no_binary, only_binary)
assert fmt.get_allowed_formats(argument) == expected
def __init__(
self,
find_links, # type: List[str]
index_urls, # type: List[str]
allow_all_prereleases=False, # type: bool
trusted_hosts=None, # type: Optional[Iterable[str]]
session=None, # type: Optional[PipSession]
format_control=None, # type: Optional[FormatControl]
platform=None, # type: Optional[str]
versions=None, # type: Optional[List[str]]
abi=None, # type: Optional[str]
implementation=None, # type: Optional[str]
prefer_binary=False # type: bool
):
# type: (...) -> None
"""Create a PackageFinder.
:param format_control: A FormatControl object or None. Used to control
the selection of source packages / binary packages when consulting
the index and links.
:param platform: A string or None. If None, searches for packages
that are supported by the current system. Otherwise, will find
packages that can be built on the platform passed in. These
packages will only be downloaded for distribution: they will
not be built locally.
:param versions: A list of strings or None. This is passed directly
to pep425tags.py in the get_supported() method.
:param abi: A string or None. This is passed directly
to pep425tags.py in the get_supported() method.
:param implementation: A string or None. This is passed directly
to pep425tags.py in the get_supported() method.
:param prefer_binary: Whether to prefer an old, but valid, binary
dist over a new source dist.
"""
if session is None:
raise TypeError(
"PackageFinder() missing 1 required keyword argument: "
"'session'")
# Build find_links. If an argument starts with ~, it may be
# a local file relative to a home directory. So try normalizing
# it and if it exists, use the normalized version.
# This is deliberately conservative - it might be fine just to
# blindly normalize anything starting with a ~...
self.find_links = [] # type: List[str]
for link in find_links:
if link.startswith('~'):
new_link = normalize_path(link)
if os.path.exists(new_link):
link = new_link
self.find_links.append(link)
self.index_urls = index_urls
self.format_control = format_control or FormatControl(set(), set())
# Domains that we won't emit warnings for when not using HTTPS
self.secure_origins = [
("*", host, "*")
for host in (trusted_hosts if trusted_hosts else [])
] # type: List[SecureOrigin]
# Do we want to allow _all_ pre-releases?
self.allow_all_prereleases = allow_all_prereleases
# The Session we'll use to make requests
self.session = session
# The valid tags to check potential found wheel candidates against
valid_tags = get_supported(
versions=versions,
platform=platform,
abi=abi,
impl=implementation,
)
self.candidate_evaluator = CandidateEvaluator(
valid_tags=valid_tags,
prefer_binary=prefer_binary,
)
# If we don't have TLS enabled, then WARN if anyplace we're looking
# relies on TLS.
if not HAS_TLS:
for link in itertools.chain(self.index_urls, self.find_links):
parsed = urllib_parse.urlparse(link)
if parsed.scheme == "https":
logger.warning(
"pip is configured with locations that require "
"TLS/SSL, however the ssl module in Python is not "
"available.")
break
def options(session):
return stub(
isolated_mode=False,
index_url='default_url',
format_control=FormatControl(set(), set()),
)
