def test_request_retries(caplog: pytest.LogCaptureFixture) -> None:
caplog.set_level(logging.DEBUG)
link = Link("http://localhost")
session = mock.Mock(PipSession)
session.get.side_effect = requests.exceptions.RetryError("Retry error")
assert _get_html_page(link, session=session) is None
assert "Could not fetch URL http://localhost: Retry error - skipping" in caplog.text
def test_get_html_page_invalid_content_type(
mock_raise_for_status: mock.Mock,
caplog: pytest.LogCaptureFixture,
content_type: str,
) -> None:
"""`_get_html_page()` should warn if an invalid content-type is given.
Only text/html is allowed.
"""
caplog.set_level(logging.DEBUG)
url = "https://pypi.org/simple/pip"
link = Link(url)
session = mock.Mock(PipSession)
session.get.return_value = mock.Mock(**{
"request.method": "GET",
"headers": {
"Content-Type": content_type
},
})
assert _get_html_page(link, session=session) is None
mock_raise_for_status.assert_called_once_with(session.get.return_value)
assert (
"pip._internal.index.collector",
logging.WARNING,
"Skipping page {} because the GET request got Content-Type: {}."
"The only supported Content-Type is text/html".format(
url, content_type),
) in caplog.record_tuples
def test_get_html_page_windows_shared_directory_append_index(tmpdir):
"""`_get_html_page()` should append "index.html" to a shared directory URL.
"""
dirpath = r"\\mypypiserver.org\simple\repo"
dir_url = pathlib.Path(dirpath).as_uri()
expected_url = "{}/index.html".format(dir_url.rstrip("/"))
session = mock.Mock(PipSession)
fake_response = make_fake_html_response(expected_url)
mock_isdir = mock.patch("pip._internal.index.collector.os.path.isdir")
mock_func = mock.patch("pip._internal.index.collector._get_html_response")
with mock_isdir as mock_isdir:
mock_isdir.return_value = True
with mock_func as mock_func:
mock_func.return_value = fake_response
actual = _get_html_page(Link(dir_url), session=session)
assert mock_func.mock_calls == [
mock.call(expected_url, session=session),
], f'actual calls: {mock_func.mock_calls}'
assert actual.content == fake_response.content
assert actual.encoding is None
assert actual.url == expected_url
assert mock_isdir.mock_calls == [
mock.call(dirpath),
]
def test_request_retries(caplog):
caplog.set_level(logging.DEBUG)
link = Link('http://localhost')
session = Mock(PipSession)
session.get.side_effect = requests.exceptions.RetryError('Retry error')
assert _get_html_page(link, session=session) is None
assert ('Could not fetch URL http://localhost: Retry error - skipping'
in caplog.text)
def test_get_html_page_caches_same_link():
link = Link('https://example.com/link-1/')
session = mock.Mock(PipSession)
fake_response = make_fake_html_response(link.url)
mock_func = mock.patch("pip._internal.index.collector._get_html_response")
with mock_func as mock_func:
mock_func.return_value = fake_response
page_1 = _get_html_page(link, session=session)
mock_func.assert_called_once()
with mock_func as mock_func:
page_2 = _get_html_page(link, session=session)
# Assert that the result of the cached html page fetch will also then
# be cached by parse_links() and @with_cached_html_pages.
assert CacheablePageContent(page_1) == CacheablePageContent(page_2)
mock_func.assert_not_called()
def test_request_http_error(caplog):
caplog.set_level(logging.DEBUG)
link = Link('http://localhost')
session = Mock(PipSession)
session.get.return_value = resp = Mock()
resp.raise_for_status.side_effect = requests.HTTPError('Http error')
assert _get_html_page(link, session=session) is None
assert ('Could not fetch URL http://localhost: Http error - skipping'
in caplog.text)
def test_request_http_error(mock_raise_for_status: mock.Mock,
caplog: pytest.LogCaptureFixture) -> None:
caplog.set_level(logging.DEBUG)
link = Link("http://localhost")
session = mock.Mock(PipSession)
session.get.return_value = mock.Mock()
mock_raise_for_status.side_effect = NetworkConnectionError("Http error")
assert _get_html_page(link, session=session) is None
assert "Could not fetch URL http://localhost: Http error - skipping" in caplog.text
def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme):
"""`_get_html_page()` should error if an invalid scheme is given.
Only file:, http:, https:, and ftp: are allowed.
"""
with caplog.at_level(logging.DEBUG):
page = _get_html_page(Link(url), session=mock.Mock(PipSession))
assert page is None
assert caplog.record_tuples == [
(
"pip._internal.index.collector",
logging.DEBUG,
"Cannot look at {} URL {}".format(vcs_scheme, url),
),
]
def test_get_html_page_invalid_content_type_archive(caplog, url):
"""`_get_html_page()` should warn if an archive URL is not HTML
and therefore cannot be used for a HEAD request.
"""
caplog.set_level(logging.WARNING)
link = Link(url)
session = mock.Mock(PipSession)
assert _get_html_page(link, session=session) is None
assert ('pip._internal.index.collector',
logging.WARNING,
'Skipping page {} because it looks like an archive, and cannot '
'be checked by a HTTP HEAD request.'.format(
url)) \
in caplog.record_tuples
def test_get_html_page_invalid_scheme(caplog: pytest.LogCaptureFixture,
url: str, vcs_scheme: str) -> None:
"""`_get_html_page()` should error if an invalid scheme is given.
Only file:, http:, https:, and ftp: are allowed.
"""
with caplog.at_level(logging.WARNING):
page = _get_html_page(Link(url), session=mock.Mock(PipSession))
assert page is None
assert caplog.record_tuples == [
(
"pip._internal.index.collector",
logging.WARNING,
"Cannot look at {} URL {} because it does not support "
"lookup as web pages.".format(vcs_scheme, url),
),
]
def test_get_html_page_directory_append_index(tmpdir: Path) -> None:
"""`_get_html_page()` should append "index.html" to a directory URL."""
dirpath = tmpdir / "something"
dirpath.mkdir()
dir_url = dirpath.as_uri()
expected_url = "{}/index.html".format(dir_url.rstrip("/"))
session = mock.Mock(PipSession)
fake_response = make_fake_html_response(expected_url)
mock_func = mock.patch("pip._internal.index.collector._get_html_response")
with mock_func as mock_func:
mock_func.return_value = fake_response
actual = _get_html_page(Link(dir_url), session=session)
assert mock_func.mock_calls == [
mock.call(expected_url, session=session),
], f"actual calls: {mock_func.mock_calls}"
assert actual is not None
assert actual.content == fake_response.content
assert actual.encoding is None
assert actual.url == expected_url
def test_get_html_page_directory_append_index(tmpdir):
"""`_get_html_page()` should append "index.html" to a directory URL.
"""
dirpath = tmpdir / "something"
dirpath.mkdir()
dir_url = "file:///{}".format(
urllib_request.pathname2url(dirpath).lstrip("/"), )
expected_url = "{}/index.html".format(dir_url.rstrip("/"))
session = mock.Mock(PipSession)
fake_response = make_fake_html_response(expected_url)
mock_func = mock.patch("pip._internal.index.collector._get_html_response")
with mock_func as mock_func:
mock_func.return_value = fake_response
actual = _get_html_page(Link(dir_url), session=session)
assert mock_func.mock_calls == [
mock.call(expected_url, session=session),
], 'actual calls: {}'.format(mock_func.mock_calls)
assert actual.content == fake_response.content
assert actual.encoding is None
assert actual.url == expected_url
def test_get_html_page_invalid_content_type(caplog, content_type):
"""`_get_html_page()` should warn if an invalid content-type is given.
Only text/html is allowed.
"""
caplog.set_level(logging.DEBUG)
url = 'https://pypi.org/simple/pip'
link = Link(url)
session = mock.Mock(PipSession)
session.get.return_value = mock.Mock(**{
"request.method": "GET",
"headers": {
"Content-Type": content_type
},
})
assert _get_html_page(link, session=session) is None
assert ('pip._internal.index.collector',
logging.WARNING,
'Skipping page {} because the GET request got Content-Type: {}.'
'The only supported Content-Type is text/html'.format(
url, content_type)) \
in caplog.record_tuples
