def _get_finder():
try:
return PackageFinder(find_links=[],
index_urls=[],
session=PipSession())
except TypeError:
pass
# pip 19.3
from pip._internal.models.search_scope import SearchScope
from pip._internal.models.selection_prefs import SelectionPreferences
try:
return PackageFinder.create(
search_scope=SearchScope(find_links=[], index_urls=[]),
selection_prefs=SelectionPreferences(allow_yanked=False),
session=PipSession(),
)
except TypeError:
pass
from pip._internal.models.target_python import TargetPython
try:
# pip 19.3.1
from pip._internal.collector import LinkCollector
except ImportError:
from pip._internal.index.collector import LinkCollector
return PackageFinder.create(
link_collector=LinkCollector(
search_scope=SearchScope(find_links=[], index_urls=[]),
session=PipSession(),
),
selection_prefs=SelectionPreferences(allow_yanked=False),
target_python=TargetPython(),
)
def test_get_index_urls_locations(self):
"""Check that the canonical name is on all indexes"""
search_scope = SearchScope(
find_links=[],
index_urls=['file://index1/', 'file://index2'],
)
actual = search_scope.get_index_urls_locations(
install_req_from_line('Complex_Name').name
)
assert actual == [
'file://index1/complex-name/',
'file://index2/complex-name/',
]
def test_get_index_urls_locations(self) -> None:
"""Check that the canonical name is on all indexes"""
search_scope = SearchScope(
find_links=[],
index_urls=["file://index1/", "file://index2"],
)
req = install_req_from_line("Complex_Name")
assert req.name is not None
actual = search_scope.get_index_urls_locations(req.name)
assert actual == [
"file://index1/complex-name/",
"file://index2/complex-name/",
]
def make_test_finder(
find_links=None, # type: Optional[List[str]]
index_urls=None, # type: Optional[List[str]]
allow_all_prereleases=False, # type: bool
trusted_hosts=None, # type: Optional[Iterable[str]]
session=None, # type: Optional[PipSession]
target_python=None, # type: Optional[TargetPython]
):
# type: (...) -> PackageFinder
"""
Create a PackageFinder for testing purposes.
"""
if find_links is None:
find_links = []
if index_urls is None:
index_urls = []
if session is None:
session = PipSession()
search_scope = SearchScope.create(
find_links=find_links,
index_urls=index_urls,
)
return PackageFinder.create(
search_scope=search_scope,
allow_yanked=True,
allow_all_prereleases=allow_all_prereleases,
trusted_hosts=trusted_hosts,
session=session,
target_python=target_python,
)
def make_link_collector(
session, # type: PipSession
options, # type: Values
suppress_no_index=False, # type: bool
):
# type: (...) -> LinkCollector
"""
:param session: The Session to use to make requests.
:param suppress_no_index: Whether to ignore the --no-index option
when constructing the SearchScope object.
"""
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index and not suppress_no_index:
logger.debug(
'Ignoring indexes: %s',
','.join(redact_auth_from_url(url) for url in index_urls),
)
index_urls = []
# Make sure find_links is a list before passing to create().
find_links = options.find_links or []
search_scope = SearchScope.create(
find_links=find_links,
index_urls=index_urls,
)
link_collector = LinkCollector(session=session, search_scope=search_scope)
return link_collector
def test_create__candidate_prefs(
self,
allow_all_prereleases: bool,
prefer_binary: bool,
) -> None:
"""
Test that the _candidate_prefs attribute is set correctly.
"""
link_collector = LinkCollector(
session=PipSession(),
search_scope=SearchScope([], []),
)
selection_prefs = SelectionPreferences(
allow_yanked=True,
allow_all_prereleases=allow_all_prereleases,
prefer_binary=prefer_binary,
)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib=False,
)
candidate_prefs = finder._candidate_prefs
assert candidate_prefs.allow_all_prereleases == allow_all_prereleases
assert candidate_prefs.prefer_binary == prefer_binary
def patch_link_collection(
computation_backend: ComputationBackend, nightly: bool
) -> Iterator[None]:
base = "https://download.pytorch.org/whl/"
url = (
f"nightly/{computation_backend}/torch_nightly.html"
if nightly
else "torch_stable.html"
)
search_scope = SearchScope.create([urljoin(base, url)], [])
@contextlib.contextmanager
def context(args: Tuple[LinkCollector, str], kwargs: Any) -> Iterator[None]:
self, project_name, *_ = args
if project_name not in PYTORCH_DISTRIBUTIONS:
yield
return
with mock.patch.object(self, "search_scope", search_scope):
yield
with apply_patch(
"pip._internal.index.collector.LinkCollector.collect_links",
context=context, # type: ignore[arg-type]
):
yield
def handle_option_line(
opts, # type: Values
filename, # type: str
lineno, # type: int
finder=None, # type: Optional[PackageFinder]
options=None, # type: Optional[optparse.Values]
session=None, # type: Optional[PipSession]
):
# type: (...) -> None
if options:
# percolate options upward
if opts.require_hashes:
options.require_hashes = opts.require_hashes
if opts.features_enabled:
options.features_enabled.extend(
f for f in opts.features_enabled
if f not in options.features_enabled)
# set finder options
if finder:
find_links = finder.find_links
index_urls = finder.index_urls
if opts.index_url:
index_urls = [opts.index_url]
if opts.no_index is True:
index_urls = []
if opts.extra_index_urls:
index_urls.extend(opts.extra_index_urls)
if opts.find_links:
# FIXME: it would be nice to keep track of the source
# of the find_links: support a find-links local path
# relative to a requirements file.
value = opts.find_links[0]
req_dir = os.path.dirname(os.path.abspath(filename))
relative_to_reqs_file = os.path.join(req_dir, value)
if os.path.exists(relative_to_reqs_file):
value = relative_to_reqs_file
find_links.append(value)
if session:
# We need to update the auth urls in session
session.update_index_urls(index_urls)
search_scope = SearchScope(
find_links=find_links,
index_urls=index_urls,
)
finder.search_scope = search_scope
if opts.pre:
finder.set_allow_all_prereleases()
if opts.prefer_binary:
finder.set_prefer_binary()
if session:
for host in opts.trusted_hosts or []:
source = f'line {lineno} of {filename}'
session.add_trusted_host(host, source=source)
def finder(data):
session = PipSession()
scope = SearchScope([str(data.packages)], [])
collector = LinkCollector(session, scope)
prefs = SelectionPreferences(allow_yanked=False)
finder = PackageFinder.create(collector, prefs)
yield finder
def _instantiate_package_finder(
pip_session): # type: (PipSession) -> PackageFinder
"""Instantiate package finder, in a pip>=10 and pip<10 compatible way."""
try:
return PackageFinder(find_links=[],
session=pip_session,
index_urls=_DEFAULT_INDEX_URLS)
except TypeError: # API changed in pip>=10
from pip._internal.models.search_scope import SearchScope
from pip._internal.models.selection_prefs import SelectionPreferences
selection_prefs = SelectionPreferences(allow_yanked=True, )
search_scope = SearchScope([], [])
try:
from pip._internal.index.collector import LinkCollector
except ModuleNotFoundError:
try:
from pip._internal.collector import LinkCollector
except ModuleNotFoundError: # pip>=19.2<20
return PackageFinder.create(session=pip_session,
selection_prefs=selection_prefs,
search_scope=search_scope)
link_collector = LinkCollector(session=pip_session,
search_scope=search_scope)
return PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
)
def test_make_link_evaluator(
self,
allow_yanked,
ignore_requires_python,
only_binary,
expected_formats,
):
# Create a test TargetPython that we can check for.
target_python = TargetPython(py_version_info=(3, 7))
format_control = FormatControl(set(), only_binary)
finder = PackageFinder(
search_scope=SearchScope([], []),
session=PipSession(),
target_python=target_python,
allow_yanked=allow_yanked,
format_control=format_control,
ignore_requires_python=ignore_requires_python,
)
# Pass a project_name that will be different from canonical_name.
link_evaluator = finder.make_link_evaluator('Twine')
assert link_evaluator.project_name == 'Twine'
assert link_evaluator._canonical_name == 'twine'
assert link_evaluator._allow_yanked == allow_yanked
assert link_evaluator._ignore_requires_python == ignore_requires_python
assert link_evaluator._formats == expected_formats
# Test the _target_python attribute.
actual_target_python = link_evaluator._target_python
# The target_python attribute should be set as is.
assert actual_target_python is target_python
# For good measure, check that the attributes weren't reset.
assert actual_target_python._given_py_version_info == (3, 7)
assert actual_target_python.py_version_info == (3, 7, 0)
def test_make_candidate_evaluator(
self,
allow_all_prereleases,
prefer_binary,
):
target_python = TargetPython()
target_python._valid_tags = [('py36', 'none', 'any')]
candidate_prefs = CandidatePreferences(
prefer_binary=prefer_binary,
allow_all_prereleases=allow_all_prereleases,
)
finder = PackageFinder(
search_scope=SearchScope([], []),
session=PipSession(),
target_python=target_python,
allow_yanked=True,
candidate_prefs=candidate_prefs,
)
specifier = SpecifierSet()
# Pass hashes to check that _hashes is set.
hashes = Hashes({'sha256': [64 * 'a']})
evaluator = finder.make_candidate_evaluator(
'my-project',
specifier=specifier,
hashes=hashes,
)
assert evaluator._allow_all_prereleases == allow_all_prereleases
assert evaluator._hashes == hashes
assert evaluator._prefer_binary == prefer_binary
assert evaluator._project_name == 'my-project'
assert evaluator._specifier is specifier
assert evaluator._supported_tags == [('py36', 'none', 'any')]
def finder(data: TestData) -> Iterator[PackageFinder]:
session = PipSession()
scope = SearchScope([str(data.packages)], [])
collector = LinkCollector(session, scope)
prefs = SelectionPreferences(allow_yanked=False)
finder = PackageFinder.create(collector,
prefs,
use_deprecated_html5lib=False)
yield finder
def make_test_search_scope(
find_links=None, # type: Optional[List[str]]
index_urls=None, # type: Optional[List[str]]
):
if find_links is None:
find_links = []
if index_urls is None:
index_urls = []
return SearchScope.create(find_links=find_links, index_urls=index_urls)
def test_create__allow_yanked(self, allow_yanked):
"""
Test that the _allow_yanked attribute is set correctly.
"""
selection_prefs = SelectionPreferences(allow_yanked=allow_yanked)
finder = PackageFinder.create(
search_scope=SearchScope([], []),
selection_prefs=selection_prefs,
session=PipSession(),
)
assert finder._allow_yanked == allow_yanked
def test_get_formatted_locations_basic_auth(self) -> None:
"""
Test that basic authentication credentials defined in URL
is not included in formatted output.
"""
index_urls = [
"https://pypi.org/simple",
"https://*****:*****@repo.domain.com",
]
find_links = ["https://*****:*****@page.domain.com"]
search_scope = SearchScope(
find_links=find_links,
index_urls=index_urls,
)
result = search_scope.get_formatted_locations()
assert "repo-user:****@repo.domain.com" in result
assert "repo-pass" not in result
assert "links-user:****@page.domain.com" in result
assert "links-pass" not in result
def test_create__allow_yanked(self, allow_yanked):
"""
Test that allow_yanked is passed to CandidateEvaluator.
"""
search_scope = SearchScope([], [])
finder = PackageFinder.create(
search_scope=search_scope,
allow_yanked=allow_yanked,
session=object(),
)
evaluator = finder.candidate_evaluator
assert evaluator._allow_yanked == allow_yanked
def test_create__allow_yanked(self, allow_yanked: bool) -> None:
"""
Test that the _allow_yanked attribute is set correctly.
"""
link_collector = LinkCollector(
session=PipSession(),
search_scope=SearchScope([], []),
)
selection_prefs = SelectionPreferences(allow_yanked=allow_yanked)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
)
assert finder._allow_yanked == allow_yanked
def test_create__target_python_none(self):
"""
Test passing target_python=None.
"""
finder = PackageFinder.create(
search_scope=SearchScope([], []),
selection_prefs=SelectionPreferences(allow_yanked=True),
session=PipSession(),
target_python=None,
)
# Spot-check the default TargetPython object.
actual_target_python = finder._target_python
assert actual_target_python._given_py_version_info is None
assert actual_target_python.py_version_info == CURRENT_PY_VERSION_INFO
def test_create__link_collector(self):
"""
Test that the _link_collector attribute is set correctly.
"""
link_collector = LinkCollector(
session=PipSession(),
search_scope=SearchScope([], []),
)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=SelectionPreferences(allow_yanked=True),
)
assert finder._link_collector is link_collector
def test_create__ignore_requires_python(self, ignore_requires_python):
"""
Test that the _ignore_requires_python attribute is set correctly.
"""
selection_prefs = SelectionPreferences(
allow_yanked=True,
ignore_requires_python=ignore_requires_python,
)
finder = PackageFinder.create(
search_scope=SearchScope([], []),
selection_prefs=selection_prefs,
session=PipSession(),
)
assert finder._ignore_requires_python == ignore_requires_python
def test_create__target_python(self):
"""
Test that target_python is passed to CandidateEvaluator as is.
"""
search_scope = SearchScope([], [])
target_python = TargetPython(py_version_info=(3, 7, 3))
finder = PackageFinder.create(
search_scope=search_scope,
session=object(),
target_python=target_python,
)
evaluator = finder.candidate_evaluator
actual_target_python = evaluator._target_python
assert actual_target_python is target_python
assert actual_target_python.py_version_info == (3, 7, 3)
def test_create__link_collector(self):
"""
Test that the _link_collector attribute is set correctly.
"""
search_scope = SearchScope([], [])
session = PipSession()
finder = PackageFinder.create(
search_scope=search_scope,
selection_prefs=SelectionPreferences(allow_yanked=True),
session=session,
)
actual_link_collector = finder._link_collector
assert actual_link_collector.search_scope is search_scope
assert actual_link_collector.session is session
def handle_option_line(
opts, # type: Values
filename, # type: str
lineno, # type: int
finder=None, # type: Optional[PackageFinder]
options=None, # type: Optional[optparse.Values]
session=None, # type: Optional[PipSession]
):
# type: (...) -> None
# percolate hash-checking option upward
if opts.require_hashes:
options.require_hashes = opts.require_hashes
# set finder options
elif finder:
find_links = finder.find_links
index_urls = finder.index_urls
if opts.index_url:
index_urls = [opts.index_url]
if opts.no_index is True:
index_urls = []
if opts.extra_index_urls:
index_urls.extend(opts.extra_index_urls)
if opts.find_links:
# FIXME: it would be nice to keep track of the source
# of the find_links: support a find-links local path
# relative to a requirements file.
value = opts.find_links[0]
req_dir = os.path.dirname(os.path.abspath(filename))
relative_to_reqs_file = os.path.join(req_dir, value)
if os.path.exists(relative_to_reqs_file):
value = relative_to_reqs_file
find_links.append(value)
search_scope = SearchScope(
find_links=find_links,
index_urls=index_urls,
)
finder.search_scope = search_scope
if opts.pre:
finder.set_allow_all_prereleases()
if session:
for host in opts.trusted_hosts or []:
source = 'line {} of {}'.format(lineno, filename)
session.add_trusted_host(host, source=source)
def _get_finder():
try:
return PackageFinder(find_links=[],
index_urls=[],
session=PipSession())
except TypeError:
pass
from pip._internal.models.search_scope import SearchScope
from pip._internal.models.selection_prefs import SelectionPreferences
return PackageFinder.create(
search_scope=SearchScope(find_links=[], index_urls=[]),
selection_prefs=SelectionPreferences(allow_yanked=False),
session=PipSession(),
)
def test_create__target_python(self):
"""
Test that the _target_python attribute is set correctly.
"""
target_python = TargetPython(py_version_info=(3, 7, 3))
finder = PackageFinder.create(
search_scope=SearchScope([], []),
selection_prefs=SelectionPreferences(allow_yanked=True),
session=PipSession(),
target_python=target_python,
)
actual_target_python = finder._target_python
# The target_python attribute should be set as is.
assert actual_target_python is target_python
# Check that the attributes weren't reset.
assert actual_target_python.py_version_info == (3, 7, 3)
def test_iter_secure_origins__none_trusted_hosts(self):
"""
Test iter_secure_origins() after passing trusted_hosts=None.
"""
# Use PackageFinder.create() rather than make_test_finder()
# to make sure we're really passing trusted_hosts=None.
search_scope = SearchScope([], [])
finder = PackageFinder.create(
search_scope=search_scope,
trusted_hosts=None,
session=object(),
)
actual = list(finder.iter_secure_origins())
assert len(actual) == 6
# Spot-check that SECURE_ORIGINS is included.
assert actual[0] == ('https', '*', '*')
def test_create__target_python_none(self) -> None:
"""
Test passing target_python=None.
"""
link_collector = LinkCollector(
session=PipSession(),
search_scope=SearchScope([], []),
)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=SelectionPreferences(allow_yanked=True),
target_python=None,
use_deprecated_html5lib=False,
)
# Spot-check the default TargetPython object.
actual_target_python = finder._target_python
assert actual_target_python._given_py_version_info is None
assert actual_target_python.py_version_info == CURRENT_PY_VERSION_INFO
def test_create__ignore_requires_python(self, ignore_requires_python: bool) -> None:
"""
Test that the _ignore_requires_python attribute is set correctly.
"""
link_collector = LinkCollector(
session=PipSession(),
search_scope=SearchScope([], []),
)
selection_prefs = SelectionPreferences(
allow_yanked=True,
ignore_requires_python=ignore_requires_python,
)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib=False,
)
assert finder._ignore_requires_python == ignore_requires_python
def test_create__format_control(self):
"""
Test that the format_control attribute is set correctly.
"""
format_control = FormatControl(set(), {':all:'})
selection_prefs = SelectionPreferences(
allow_yanked=True,
format_control=format_control,
)
finder = PackageFinder.create(
search_scope=SearchScope([], []),
selection_prefs=selection_prefs,
session=PipSession(),
)
actual_format_control = finder.format_control
assert actual_format_control is format_control
# Check that the attributes weren't reset.
assert actual_format_control.only_binary == {':all:'}
