def test_from_tuples_rfc2231(self):
field = RequestField.from_tuples(
u("fieldname"),
(u("filen\u00e4me"), "data"),
header_formatter=format_header_param_rfc2231,
)
cd = field.headers["Content-Disposition"]
assert cd == u("form-data; name=\"fieldname\"; filename*=utf-8''filen%C3%A4me")
def test_unicode_upload(self):
fieldname = u("myfile")
filename = u("\xe2\x99\xa5.txt")
data = u("\xe2\x99\xa5").encode("utf8")
size = len(data)
fields = {
u("upload_param"): fieldname,
u("upload_filename"): filename,
u("upload_size"): size,
fieldname: (filename, data),
}
r = self.pool.request("POST", "/upload", fields=fields)
assert r.status == 200, r.data
class TestUtil(object):
# TODO: Add more tests
@pytest.mark.parametrize(
"location",
[
"http://google.com:foo",
"http://::1/",
"http://::1:80/",
"http://google.com:-80",
six.u("http://google.com:\xb2\xb2"), # \xb2 = ^2
],
)
def test_invalid_host(self, location):
with pytest.raises(LocationParseError):
parse_url(location)
@pytest.mark.parametrize(
"url",
[
# Invalid IDNA labels
u"http://\uD7FF.com",
u"http://❤️",
# Unicode surrogates
u"http://\uD800.com",
u"http://\uDC00.com",
],
)
def test_invalid_url(self, url):
with pytest.raises(LocationParseError):
parse_url(url)
@pytest.mark.parametrize(
"url, expected_normalized_url",
[
("HTTP://GOOGLE.COM/MAIL/", "http://google.com/MAIL/"),
(
"http://[email protected]:[email protected]/~tilde@?@",
"http://user%40domain.com:[email protected]/~tilde@?@",
),
(
"HTTP://*****:*****@Example.com:8080/",
"http://*****:*****@example.com:8080/",
),
("HTTPS://Example.Com/?Key=Value", "https://example.com/?Key=Value"),
("Https://Example.Com/#Fragment", "https://example.com/#Fragment"),
("[::1%25]", "[::1%25]"),
("[::Ff%etH0%Ff]/%ab%Af", "[::ff%etH0%FF]/%AB%AF"),
(
"http://*****:*****@[AaAa::Ff%25etH0%Ff]/%ab%Af",
"http://*****:*****@[aaaa::ff%etH0%FF]/%AB%AF",
),
# Invalid characters for the query/fragment getting encoded
(
'http://google.com/p[]?parameter[]="hello"#fragment#',
"http://google.com/p%5B%5D?parameter%5B%5D=%22hello%22#fragment%23",
),
# Percent encoding isn't applied twice despite '%' being invalid
# but the percent encoding is still normalized.
(
"http://google.com/p%5B%5d?parameter%5b%5D=%22hello%22#fragment%23",
"http://google.com/p%5B%5D?parameter%5B%5D=%22hello%22#fragment%23",
),
],
)
def test_parse_url_normalization(self, url, expected_normalized_url):
"""Assert parse_url normalizes the scheme/host, and only the scheme/host"""
actual_normalized_url = parse_url(url).url
assert actual_normalized_url == expected_normalized_url
@pytest.mark.parametrize("char", [chr(i) for i in range(0x00, 0x21)] + ["\x7F"])
def test_control_characters_are_percent_encoded(self, char):
percent_char = "%" + (hex(ord(char))[2:].zfill(2).upper())
url = parse_url(
"http://user{0}@example.com/path{0}?query{0}#fragment{0}".format(char)
)
assert url == Url(
"http",
auth="user" + percent_char,
host="example.com",
path="/path" + percent_char,
query="query" + percent_char,
fragment="fragment" + percent_char,
)
parse_url_host_map = [
("http://google.com/mail", Url("http", host="google.com", path="/mail")),
("http://google.com/mail/", Url("http", host="google.com", path="/mail/")),
("http://google.com/mail", Url("http", host="google.com", path="mail")),
("google.com/mail", Url(host="google.com", path="/mail")),
("http://google.com/", Url("http", host="google.com", path="/")),
("http://google.com", Url("http", host="google.com")),
("http://google.com?foo", Url("http", host="google.com", query="foo")),
# Path/query/fragment
("", Url()),
("/", Url(path="/")),
("#?/!google.com/?foo", Url(fragment="?/!google.com/?foo")),
("/foo", Url(path="/foo")),
("/foo?bar=baz", Url(path="/foo", query="bar=baz")),
(
"/foo?bar=baz#banana?apple/orange",
Url(path="/foo", query="bar=baz", fragment="banana?apple/orange"),
),
(
"/redirect?target=http://localhost:61020/",
Url(path="redirect", query="target=http://localhost:61020/"),
),
# Port
("http://google.com/", Url("http", host="google.com", path="/")),
("http://google.com:80/", Url("http", host="google.com", port=80, path="/")),
("http://google.com:80", Url("http", host="google.com", port=80)),
# Auth
(
"http://*****:*****@localhost/",
Url("http", auth="foo:bar", host="localhost", path="/"),
),
("http://foo@localhost/", Url("http", auth="foo", host="localhost", path="/")),
(
"http://*****:*****@localhost/",
Url("http", auth="foo:bar", host="localhost", path="/"),
),
# Unicode type (Python 2.x)
(
u"http://*****:*****@localhost/",
Url(u"http", auth=u"foo:bar", host=u"localhost", path=u"/"),
),
(
"http://*****:*****@localhost/",
Url("http", auth="foo:bar", host="localhost", path="/"),
),
(
"http://*****:*****@127.0.0.1:1234",
Url(scheme="http", auth="user:password", host="127.0.0.1", port=1234),
),
(
"http://google.com/foo=http://bar:42/baz",
Url(scheme="http", host="google.com", path="/foo=http://bar:42/baz"),
),
(
"http://google.com?foo=http://bar:42/baz",
Url(scheme="http", host="google.com", query="foo=http://bar:42/baz"),
),
(
"http://google.com#foo=http://bar:42/baz",
Url(scheme="http", host="google.com", fragment="foo=http://bar:42/baz"),
),
# IPv4
("173.194.35.7", Url(scheme=None, host="173.194.35.7")),
("http://173.194.35.7", Url(scheme="http", host="173.194.35.7")),
(
"http://173.194.35.7/test",
Url(scheme="http", host="173.194.35.7", path="/test"),
),
("http://173.194.35.7:80", Url(scheme="http", host="173.194.35.7", port=80)),
(
"http://173.194.35.7:80/test",
Url(scheme="http", host="173.194.35.7", port=80, path="/test"),
),
# IPv6
("[2a00:1450:4001:c01::67]", Url(auth=None, host="[2a00:1450:4001:c01::67]")),
(
"http://[2a00:1450:4001:c01::67]",
Url(scheme="http", host="[2a00:1450:4001:c01::67]"),
),
(
"http://[2a00:1450:4001:c01::67]/test",
Url(scheme="http", host="[2a00:1450:4001:c01::67]", path="/test"),
),
(
"http://[2a00:1450:4001:c01::67]:80",
Url(scheme="http", host="[2a00:1450:4001:c01::67]", port=80),
),
(
"http://[2a00:1450:4001:c01::67]:80/test",
Url(scheme="http", host="[2a00:1450:4001:c01::67]", port=80, path="/test"),
),
# More IPv6 from http://www.ietf.org/rfc/rfc2732.txt
(
"http://[fedc:ba98:7654:3210:fedc:ba98:7654:3210]:8000/index.html",
Url(
scheme="http",
host="[fedc:ba98:7654:3210:fedc:ba98:7654:3210]",
port=8000,
path="/index.html",
),
),
(
"http://[1080:0:0:0:8:800:200c:417a]/index.html",
Url(scheme="http", host="[1080:0:0:0:8:800:200c:417a]", path="/index.html"),
),
(
"http://[3ffe:2a00:100:7031::1]",
Url(scheme="http", host="[3ffe:2a00:100:7031::1]"),
),
(
"http://[1080::8:800:200c:417a]/foo",
Url(scheme="http", host="[1080::8:800:200c:417a]", path="/foo"),
),
(
"http://[::192.9.5.5]/ipng",
Url(scheme="http", host="[::192.9.5.5]", path="/ipng"),
),
(
"http://[::ffff:129.144.52.38]:42/index.html",
Url(
scheme="http",
host="[::ffff:129.144.52.38]",
port=42,
path="/index.html",
),
),
(
"http://[2010:836b:4179::836b:4179]",
Url(scheme="http", host="[2010:836b:4179::836b:4179]"),
),
# Hosts
("google.com/mail", Url(scheme=None, host="google.com", path="/mail")),
(
"http://google.com:8000/mail/",
Url(scheme="http", host="google.com", port=8000, path="/mail/"),
),
(
"http://*****:*****@example.com:1234",
Url(scheme="http", auth="user:password", host="example.com", port=1234),
),
("173.194.35.7", Url(scheme=None, host="173.194.35.7")),
("http://173.194.35.7", Url(scheme="http", host="173.194.35.7")),
(
"http://[2a00:1450:4001:c01::67]:80/test",
Url(scheme="http", host="[2a00:1450:4001:c01::67]", port=80, path="/test"),
),
(
"http://[fedc:ba98:7654:3210:fedc:ba98:7654:3210]:8000/index.html",
Url(
scheme="http",
host="[fedc:ba98:7654:3210:fedc:ba98:7654:3210]",
port=8000,
path="/index.html",
),
),
(
"https://[1080:0:0:0:8:800:200c:417a]/index.html",
Url(
scheme="https", host="[1080:0:0:0:8:800:200c:417a]", path="/index.html"
),
),
(
"about://eXamPlE.com?info=1",
Url(scheme="about", host="eXamPlE.com", query="info=1"),
),
(
"http+unix://%2fvar%2frun%2fSOCKET/path",
Url(scheme="http+unix", host="%2fvar%2frun%2fSOCKET", path="/path"),
),
]
non_round_tripping_parse_url_host_map = [
# Path/query/fragment
("?", Url(query="")),
("#", Url(fragment="")),
# Path normalization
("/abc/../def", Url(path="/def")),
# Empty Port
("http://google.com:", Url("http", host="google.com")),
("http://google.com:/", Url("http", host="google.com", path="/")),
# Uppercase IRI
(
u"http://Königsgäßchen.de/straße",
Url("http", host="xn--knigsgchen-b4a3dun.de", path="/stra%C3%9Fe"),
),
# Percent-encode in userinfo
(
u"http://[email protected]:[email protected]/",
Url("http", auth="user%40email.com:password", host="example.com", path="/"),
),
(
u'http://user":[email protected]/',
Url("http", auth="user%22:quoted", host="example.com", path="/"),
),
# Unicode Surrogates
(u"http://google.com/\uD800", Url("http", host="google.com", path="%ED%A0%80")),
(
u"http://google.com?q=\uDC00",
Url("http", host="google.com", query="q=%ED%B0%80"),
),
(
u"http://google.com#\uDC00",
Url("http", host="google.com", fragment="%ED%B0%80"),
),
# Hosts and Schemes
(
"HTTP://GOOGLE.COM/mail/",
Url(scheme="http", host="google.com", path="/mail/"),
),
("GOogle.COM/mail", Url(scheme=None, host="google.com", path="/mail")),
(
"HTTP://GoOgLe.CoM:8000/mail/",
Url(scheme="http", host="google.com", port=8000, path="/mail/"),
),
(
"HTTP://*****:*****@EXAMPLE.COM:1234",
Url(scheme="http", auth="user:password", host="example.com", port=1234),
),
("HTTP://173.194.35.7", Url(scheme="http", host="173.194.35.7")),
(
"HTTP://[2a00:1450:4001:c01::67]:80/test",
Url(scheme="http", host="[2a00:1450:4001:c01::67]", port=80, path="/test"),
),
(
"HTTP://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8000/index.html",
Url(
scheme="http",
host="[fedc:ba98:7654:3210:fedc:ba98:7654:3210]",
port=8000,
path="/index.html",
),
),
(
"HTTPS://[1080:0:0:0:8:800:200c:417A]/index.html",
Url(
scheme="https", host="[1080:0:0:0:8:800:200c:417a]", path="/index.html"
),
),
(
"aBoUt://eXamPlE.com?info=1",
Url(scheme="about", host="eXamPlE.com", query="info=1"),
),
(
"http+UNIX://%2fvar%2frun%2fSOCKET/path",
Url(scheme="http+unix", host="%2fvar%2frun%2fSOCKET", path="/path"),
),
]
@pytest.mark.parametrize(
"url, expected_url",
chain(parse_url_host_map, non_round_tripping_parse_url_host_map),
)
def test_parse_url(self, url, expected_url):
returned_url = parse_url(url)
assert returned_url == expected_url
@pytest.mark.parametrize("url, expected_url", parse_url_host_map)
def test_unparse_url(self, url, expected_url):
assert url == expected_url.url
@pytest.mark.parametrize(
["url", "expected_url"],
[
# RFC 3986 5.2.4
("/abc/../def", Url(path="/def")),
("/..", Url(path="/")),
("/./abc/./def/", Url(path="/abc/def/")),
("/.", Url(path="/")),
("/./", Url(path="/")),
("/abc/./.././d/././e/.././f/./../../ghi", Url(path="/ghi")),
],
)
def test_parse_and_normalize_url_paths(self, url, expected_url):
actual_url = parse_url(url)
assert actual_url == expected_url
assert actual_url.url == expected_url.url
def test_parse_url_invalid_IPv6(self):
with pytest.raises(LocationParseError):
parse_url("[::1")
def test_parse_url_negative_port(self):
with pytest.raises(LocationParseError):
parse_url("https://www.google.com:-80/")
def test_Url_str(self):
U = Url("http", host="google.com")
assert str(U) == U.url
request_uri_map = [
("http://google.com/mail", "/mail"),
("http://google.com/mail/", "/mail/"),
("http://google.com/", "/"),
("http://google.com", "/"),
("", "/"),
("/", "/"),
("?", "/?"),
("#", "/"),
("/foo?bar=baz", "/foo?bar=baz"),
]
@pytest.mark.parametrize("url, expected_request_uri", request_uri_map)
def test_request_uri(self, url, expected_request_uri):
returned_url = parse_url(url)
assert returned_url.request_uri == expected_request_uri
url_netloc_map = [
("http://google.com/mail", "google.com"),
("http://google.com:80/mail", "google.com:80"),
("google.com/foobar", "google.com"),
("google.com:12345", "google.com:12345"),
]
@pytest.mark.parametrize("url, expected_netloc", url_netloc_map)
def test_netloc(self, url, expected_netloc):
assert parse_url(url).netloc == expected_netloc
url_vulnerabilities = [
# urlparse doesn't follow RFC 3986 Section 3.2
(
"http://google.com#@evil.com/",
Url("http", host="google.com", fragment="@evil.com/"),
),
# CVE-2016-5699
(
"http://127.0.0.1%0d%0aConnection%3a%20keep-alive",
Url("http", host="127.0.0.1%0d%0aconnection%3a%20keep-alive"),
),
# NodeJS unicode -> double dot
(
u"http://google.com/\uff2e\uff2e/abc",
Url("http", host="google.com", path="/%EF%BC%AE%EF%BC%AE/abc"),
),
# Scheme without ://
(
"javascript:a='@google.com:12345/';alert(0)",
Url(scheme="javascript", path="a='@google.com:12345/';alert(0)"),
),
("//google.com/a/b/c", Url(host="google.com", path="/a/b/c")),
# International URLs
(
u"http://ヒ:キ@ヒ.abc.ニ/ヒ?キ#ワ",
Url(
u"http",
host=u"xn--pdk.abc.xn--idk",
auth=u"%E3%83%92:%E3%82%AD",
path=u"/%E3%83%92",
query=u"%E3%82%AD",
fragment=u"%E3%83%AF",
),
),
# Injected headers (CVE-2016-5699, CVE-2019-9740, CVE-2019-9947)
(
"10.251.0.83:7777?a=1 HTTP/1.1\r\nX-injected: header",
Url(
host="10.251.0.83",
port=7777,
query="a=1%20HTTP/1.1%0D%0AX-injected:%20header",
),
),
(
"http://127.0.0.1:6379?\r\nSET test failure12\r\n:8080/test/?test=a",
Url(
scheme="http",
host="127.0.0.1",
port=6379,
query="%0D%0ASET%20test%20failure12%0D%0A:8080/test/?test=a",
),
),
]
@pytest.mark.parametrize("url, expected_url", url_vulnerabilities)
def test_url_vulnerabilities(self, url, expected_url):
if expected_url is False:
with pytest.raises(LocationParseError):
parse_url(url)
else:
assert parse_url(url) == expected_url
@onlyPy2
def test_parse_url_bytes_to_str_python_2(self):
url = parse_url(b"https://www.google.com/")
assert url == Url("https", host="www.google.com", path="/")
assert isinstance(url.scheme, str)
assert isinstance(url.host, str)
assert isinstance(url.path, str)
@onlyPy2
def test_parse_url_unicode_python_2(self):
url = parse_url(u"https://www.google.com/")
assert url == Url(u"https", host=u"www.google.com", path=u"/")
assert isinstance(url.scheme, six.text_type)
assert isinstance(url.host, six.text_type)
assert isinstance(url.path, six.text_type)
@onlyPy3
def test_parse_url_bytes_type_error_python_3(self):
with pytest.raises(TypeError):
parse_url(b"https://www.google.com/")
@pytest.mark.parametrize(
"kwargs, expected",
[
pytest.param(
{"accept_encoding": True},
{"accept-encoding": "gzip,deflate,br"},
marks=onlyBrotlipy(),
),
pytest.param(
{"accept_encoding": True},
{"accept-encoding": "gzip,deflate"},
marks=notBrotlipy(),
),
({"accept_encoding": "foo,bar"}, {"accept-encoding": "foo,bar"}),
({"accept_encoding": ["foo", "bar"]}, {"accept-encoding": "foo,bar"}),
pytest.param(
{"accept_encoding": True, "user_agent": "banana"},
{"accept-encoding": "gzip,deflate,br", "user-agent": "banana"},
marks=onlyBrotlipy(),
),
pytest.param(
{"accept_encoding": True, "user_agent": "banana"},
{"accept-encoding": "gzip,deflate", "user-agent": "banana"},
marks=notBrotlipy(),
),
({"user_agent": "banana"}, {"user-agent": "banana"}),
({"keep_alive": True}, {"connection": "keep-alive"}),
({"basic_auth": "foo:bar"}, {"authorization": "Basic Zm9vOmJhcg=="}),
(
{"proxy_basic_auth": "foo:bar"},
{"proxy-authorization": "Basic Zm9vOmJhcg=="},
),
({"disable_cache": True}, {"cache-control": "no-cache"}),
],
)
def test_make_headers(self, kwargs, expected):
assert make_headers(**kwargs) == expected
def test_rewind_body(self):
body = io.BytesIO(b"test data")
assert body.read() == b"test data"
# Assert the file object has been consumed
assert body.read() == b""
# Rewind it back to just be b'data'
rewind_body(body, 5)
assert body.read() == b"data"
def test_rewind_body_failed_tell(self):
body = io.BytesIO(b"test data")
body.read() # Consume body
# Simulate failed tell()
body_pos = _FAILEDTELL
with pytest.raises(UnrewindableBodyError):
rewind_body(body, body_pos)
def test_rewind_body_bad_position(self):
body = io.BytesIO(b"test data")
body.read() # Consume body
# Pass non-integer position
with pytest.raises(ValueError):
rewind_body(body, body_pos=None)
with pytest.raises(ValueError):
rewind_body(body, body_pos=object())
def test_rewind_body_failed_seek(self):
class BadSeek:
def seek(self, pos, offset=0):
raise IOError
with pytest.raises(UnrewindableBodyError):
rewind_body(BadSeek(), body_pos=2)
def test_add_stderr_logger(self):
handler = add_stderr_logger(level=logging.INFO) # Don't actually print debug
logger = logging.getLogger("hip")
assert handler in logger.handlers
logger.debug("Testing add_stderr_logger")
logger.removeHandler(handler)
def test_disable_warnings(self):
with warnings.catch_warnings(record=True) as w:
clear_warnings()
warnings.warn("This is a test.", InsecureRequestWarning)
assert len(w) == 1
disable_warnings()
warnings.warn("This is a test.", InsecureRequestWarning)
assert len(w) == 1
def _make_time_pass(self, seconds, timeout, time_mock):
""" Make some time pass for the timeout object """
time_mock.return_value = TIMEOUT_EPOCH
timeout.start_connect()
time_mock.return_value = TIMEOUT_EPOCH + seconds
return timeout
@pytest.mark.parametrize(
"kwargs, message",
[
({"total": -1}, "less than"),
({"connect": 2, "total": -1}, "less than"),
({"read": -1}, "less than"),
({"connect": False}, "cannot be a boolean"),
({"read": True}, "cannot be a boolean"),
({"connect": 0}, "less than or equal"),
({"read": "foo"}, "int, float or None"),
],
)
def test_invalid_timeouts(self, kwargs, message):
with pytest.raises(ValueError) as e:
Timeout(**kwargs)
assert message in str(e.value)
@patch("hip.util.timeout.current_time")
def test_timeout(self, current_time):
timeout = Timeout(total=3)
# make 'no time' elapse
timeout = self._make_time_pass(
seconds=0, timeout=timeout, time_mock=current_time
)
assert timeout.read_timeout == 3
assert timeout.connect_timeout == 3
timeout = Timeout(total=3, connect=2)
assert timeout.connect_timeout == 2
timeout = Timeout()
assert timeout.connect_timeout == Timeout.DEFAULT_TIMEOUT
# Connect takes 5 seconds, leaving 5 seconds for read
timeout = Timeout(total=10, read=7)
timeout = self._make_time_pass(
seconds=5, timeout=timeout, time_mock=current_time
)
assert timeout.read_timeout == 5
# Connect takes 2 seconds, read timeout still 7 seconds
timeout = Timeout(total=10, read=7)
timeout = self._make_time_pass(
seconds=2, timeout=timeout, time_mock=current_time
)
assert timeout.read_timeout == 7
timeout = Timeout(total=10, read=7)
assert timeout.read_timeout == 7
timeout = Timeout(total=None, read=None, connect=None)
assert timeout.connect_timeout is None
assert timeout.read_timeout is None
assert timeout.total is None
timeout = Timeout(5)
assert timeout.total == 5
def test_timeout_str(self):
timeout = Timeout(connect=1, read=2, total=3)
assert str(timeout) == "Timeout(connect=1, read=2, total=3)"
timeout = Timeout(connect=1, read=None, total=3)
assert str(timeout) == "Timeout(connect=1, read=None, total=3)"
@patch("hip.util.timeout.current_time")
def test_timeout_elapsed(self, current_time):
current_time.return_value = TIMEOUT_EPOCH
timeout = Timeout(total=3)
with pytest.raises(TimeoutStateError):
timeout.get_connect_duration()
timeout.start_connect()
with pytest.raises(TimeoutStateError):
timeout.start_connect()
current_time.return_value = TIMEOUT_EPOCH + 2
assert timeout.get_connect_duration() == 2
current_time.return_value = TIMEOUT_EPOCH + 37
assert timeout.get_connect_duration() == 37
@pytest.mark.parametrize(
"candidate, requirements",
[
(None, ssl.CERT_REQUIRED),
(ssl.CERT_NONE, ssl.CERT_NONE),
(ssl.CERT_REQUIRED, ssl.CERT_REQUIRED),
("REQUIRED", ssl.CERT_REQUIRED),
("CERT_REQUIRED", ssl.CERT_REQUIRED),
],
)
def test_resolve_cert_reqs(self, candidate, requirements):
assert resolve_cert_reqs(candidate) == requirements
@pytest.mark.parametrize(
"candidate, version",
[
(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1),
("PROTOCOL_TLSv1", ssl.PROTOCOL_TLSv1),
("TLSv1", ssl.PROTOCOL_TLSv1),
(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23),
],
)
def test_resolve_ssl_version(self, candidate, version):
assert resolve_ssl_version(candidate) == version
def test_ssl_wrap_socket_loads_the_cert_chain(self):
socket = object()
mock_context = Mock()
ssl_wrap_socket(
ssl_context=mock_context, sock=socket, certfile="/path/to/certfile"
)
mock_context.load_cert_chain.assert_called_once_with("/path/to/certfile", None)
@patch("hip.util.ssl_.create_ssl_context")
def test_ssl_wrap_socket_creates_new_context(self, create_ssl_context):
socket = object()
ssl_wrap_socket(sock=socket, cert_reqs="CERT_REQUIRED")
create_ssl_context.assert_called_once_with(None, "CERT_REQUIRED", ciphers=None)
def test_ssl_wrap_socket_loads_verify_locations(self):
socket = object()
mock_context = Mock()
ssl_wrap_socket(ssl_context=mock_context, ca_certs="/path/to/pem", sock=socket)
mock_context.load_verify_locations.assert_called_once_with("/path/to/pem", None)
def test_ssl_wrap_socket_loads_certificate_directories(self):
socket = object()
mock_context = Mock()
ssl_wrap_socket(
ssl_context=mock_context, ca_cert_dir="/path/to/pems", sock=socket
)
mock_context.load_verify_locations.assert_called_once_with(
None, "/path/to/pems"
)
def test_ssl_wrap_socket_with_no_sni_warns(self):
socket = object()
mock_context = Mock()
# Ugly preservation of original value
HAS_SNI = ssl_.HAS_SNI
ssl_.HAS_SNI = False
try:
with patch("warnings.warn") as warn:
ssl_wrap_socket(
ssl_context=mock_context,
sock=socket,
server_hostname="www.google.com",
)
mock_context.wrap_socket.assert_called_once_with(socket)
assert warn.call_count >= 1
warnings = [call[0][1] for call in warn.call_args_list]
assert SNIMissingWarning in warnings
finally:
ssl_.HAS_SNI = HAS_SNI
def test_const_compare_digest_fallback(self):
target = hashlib.sha256(b"abcdef").digest()
assert _const_compare_digest_backport(target, target)
prefix = target[:-1]
assert not _const_compare_digest_backport(target, prefix)
suffix = target + b"0"
assert not _const_compare_digest_backport(target, suffix)
incorrect = hashlib.sha256(b"xyz").digest()
assert not _const_compare_digest_backport(target, incorrect)
def test_has_ipv6_disabled_on_compile(self):
with patch("socket.has_ipv6", False):
assert not _has_ipv6("::1")
def test_has_ipv6_enabled_but_fails(self):
with patch("socket.has_ipv6", True):
with patch("socket.socket") as mock:
instance = mock.return_value
instance.bind = Mock(side_effect=Exception("No IPv6 here!"))
assert not _has_ipv6("::1")
def test_has_ipv6_enabled_and_working(self):
with patch("socket.has_ipv6", True):
with patch("socket.socket") as mock:
instance = mock.return_value
instance.bind.return_value = True
assert _has_ipv6("::1")
def test_ip_family_ipv6_enabled(self):
with patch("hip.util.connection.HAS_IPV6", True):
assert allowed_gai_family() == socket.AF_UNSPEC
def test_ip_family_ipv6_disabled(self):
with patch("hip.util.connection.HAS_IPV6", False):
assert allowed_gai_family() == socket.AF_INET
@pytest.mark.parametrize("value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2
def test_parse_retry_after_invalid(self, value):
retry = Retry()
with pytest.raises(InvalidHeader):
retry.parse_retry_after(value)
@pytest.mark.parametrize(
"value, expected", [("0", 0), ("1000", 1000), ("\t42 ", 42)]
)
def test_parse_retry_after(self, value, expected):
retry = Retry()
assert retry.parse_retry_after(value) == expected
def test_from_tuples_html5(self):
field = RequestField.from_tuples(u("fieldname"), (u("filen\u00e4me"), "data"))
cd = field.headers["Content-Disposition"]
assert cd == u('form-data; name="fieldname"; filename="filen\u00e4me"')
def test_render_part_html5_unicode_with_control_character(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("hello\x1A\x1B\x1C"))
assert param == u('filename="hello%1A\x1B%1C"')
def test_render_part_html5_unicode_escape(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("hello\\world\u0022"))
assert param == u('filename="hello\\\\world%22"')
def test_render_part_html5_unicode(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("n\u00e4me"))
assert param == u('filename="n\u00e4me"')
def test_render_part_rfc2231_unicode(self):
field = RequestField(
"somename", "data", header_formatter=format_header_param_rfc2231
)
param = field._render_part("filename", u("n\u00e4me"))
assert param == "filename*=utf-8''n%C3%A4me"
class TestRetry(object):
def test_string(self):
""" Retry string representation looks the way we expect """
retry = Retry()
assert (
str(retry) ==
"Retry(total=10, connect=None, read=None, redirect=None, status=None)"
)
for _ in range(3):
retry = retry.increment(method="GET")
assert (
str(retry) ==
"Retry(total=7, connect=None, read=None, redirect=None, status=None)"
)
def test_retry_both_specified(self):
"""Total can win if it's lower than the connect value"""
error = ConnectTimeoutError()
retry = Retry(connect=3, total=2)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError) as e:
retry.increment(error=error)
assert e.value.reason == error
def test_retry_higher_total_loses(self):
""" A lower connect timeout than the total is honored """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError):
retry.increment(error=error)
def test_retry_higher_total_loses_vs_read(self):
""" A lower read timeout than the total is honored """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=2, total=3)
retry = retry.increment(method="GET", error=error)
retry = retry.increment(method="GET", error=error)
with pytest.raises(MaxRetryError):
retry.increment(method="GET", error=error)
def test_retry_total_none(self):
""" if Total is none, connect error should take precedence """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError) as e:
retry.increment(error=error)
assert e.value.reason == error
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(connect=2, total=None)
retry = retry.increment(method="GET", error=error)
retry = retry.increment(method="GET", error=error)
retry = retry.increment(method="GET", error=error)
assert not retry.is_exhausted()
def test_retry_default(self):
""" If no value is specified, should retry connects 3 times """
retry = Retry()
assert retry.total == 10
assert retry.connect is None
assert retry.read is None
assert retry.redirect is None
error = ConnectTimeoutError()
retry = Retry(connect=1)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError):
retry.increment(error=error)
retry = Retry(connect=1)
retry = retry.increment(error=error)
assert not retry.is_exhausted()
assert Retry(0).raise_on_redirect
assert not Retry(False).raise_on_redirect
def test_retry_read_zero(self):
""" No second chances on read timeouts, by default """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=0)
with pytest.raises(MaxRetryError) as e:
retry.increment(method="GET", error=error)
assert e.value.reason == error
def test_status_counter(self):
resp = HTTPResponse(status=400)
retry = Retry(status=2)
retry = retry.increment(response=resp)
retry = retry.increment(response=resp)
with pytest.raises(MaxRetryError) as e:
retry.increment(response=resp)
assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format(
status_code=400)
def test_backoff(self):
""" Backoff is computed correctly """
max_backoff = Retry.BACKOFF_MAX
retry = Retry(total=100, backoff_factor=0.2)
assert retry.get_backoff_time() == 0 # First request
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 0 # First retry
retry = retry.increment(method="GET")
assert retry.backoff_factor == 0.2
assert retry.total == 98
assert retry.get_backoff_time() == 0.4 # Start backoff
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 0.8
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 1.6
for _ in xrange(10):
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == max_backoff
def test_zero_backoff(self):
retry = Retry()
assert retry.get_backoff_time() == 0
retry = retry.increment(method="GET")
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 0
def test_backoff_reset_after_redirect(self):
retry = Retry(total=100, redirect=5, backoff_factor=0.2)
retry = retry.increment(method="GET")
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 0.4
redirect_response = HTTPResponse(status=302,
headers={"location": "test"})
retry = retry.increment(method="GET", response=redirect_response)
assert retry.get_backoff_time() == 0
retry = retry.increment(method="GET")
retry = retry.increment(method="GET")
assert retry.get_backoff_time() == 0.4
def test_sleep(self):
# sleep a very small amount of time so our code coverage is happy
retry = Retry(backoff_factor=0.0001)
retry = retry.increment(method="GET")
retry = retry.increment(method="GET")
retry.sleep()
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500, 600))
assert not retry.is_retry("GET", status_code=200)
assert not retry.is_retry("GET", status_code=400)
assert retry.is_retry("GET", status_code=500)
retry = Retry(total=1, status_forcelist=[418])
assert not retry.is_retry("GET", status_code=400)
assert retry.is_retry("GET", status_code=418)
# String status codes are not matched.
retry = Retry(total=1, status_forcelist=["418"])
assert not retry.is_retry("GET", status_code=418)
def test_method_whitelist_with_status_forcelist(self):
# Falsey method_whitelist means to retry on any method.
retry = Retry(status_forcelist=[500], method_whitelist=None)
assert retry.is_retry("GET", status_code=500)
assert retry.is_retry("POST", status_code=500)
# Criteria of method_whitelist and status_forcelist are ANDed.
retry = Retry(status_forcelist=[500], method_whitelist=["POST"])
assert not retry.is_retry("GET", status_code=500)
assert retry.is_retry("POST", status_code=500)
def test_exhausted(self):
assert not Retry(0).is_exhausted()
assert Retry(-1).is_exhausted()
assert Retry(1).increment(method="GET").total == 0
@pytest.mark.parametrize("total", [-1, 0])
def test_disabled(self, total):
with pytest.raises(MaxRetryError):
Retry(total).increment(method="GET")
def test_error_message(self):
retry = Retry(total=0)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment(method="GET",
error=ReadTimeoutError(
None, "/", "read timed out"))
assert "Caused by redirect" not in str(e.value)
assert str(e.value.reason) == "None: read timed out"
retry = Retry(total=1)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment("POST", "/")
retry = retry.increment("POST", "/")
assert "Caused by redirect" not in str(e.value)
assert isinstance(e.value.reason, ResponseError)
assert str(e.value.reason) == ResponseError.GENERIC_ERROR
retry = Retry(total=1)
response = HTTPResponse(status=500)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment("POST", "/", response=response)
retry = retry.increment("POST", "/", response=response)
assert "Caused by redirect" not in str(e.value)
msg = ResponseError.SPECIFIC_ERROR.format(status_code=500)
assert str(e.value.reason) == msg
retry = Retry(connect=1)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
assert "Caused by redirect" not in str(e.value)
assert str(e.value.reason) == "conntimeout"
def test_history(self):
retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"]))
assert retry.history == tuple()
connection_error = ConnectTimeoutError("conntimeout")
retry = retry.increment("GET", "/test1", None, connection_error)
history = (RequestHistory("GET", "/test1", connection_error, None,
None), )
assert retry.history == history
read_error = ReadTimeoutError(None, "/test2", "read timed out")
retry = retry.increment("POST", "/test2", None, read_error)
history = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
)
assert retry.history == history
response = HTTPResponse(status=500)
retry = retry.increment("GET", "/test3", response, None)
history = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
RequestHistory("GET", "/test3", None, 500, None),
)
assert retry.history == history
def test_retry_method_not_in_whitelist(self):
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry()
with pytest.raises(ReadTimeoutError):
retry.increment(method="POST", error=error)
def test_retry_default_remove_headers_on_redirect(self):
retry = Retry()
assert list(retry.remove_headers_on_redirect) == ["authorization"]
def test_retry_set_remove_headers_on_redirect(self):
retry = Retry(remove_headers_on_redirect=["X-API-Secret"])
assert list(retry.remove_headers_on_redirect) == ["x-api-secret"]
@pytest.mark.parametrize(
"value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2
def test_parse_retry_after_invalid(self, value):
retry = Retry()
with pytest.raises(InvalidHeader):
retry.parse_retry_after(value)
@pytest.mark.parametrize("value, expected", [("0", 0), ("1000", 1000),
("\t42 ", 42)])
def test_parse_retry_after(self, value, expected):
retry = Retry()
assert retry.parse_retry_after(value) == expected
@pytest.mark.parametrize("respect_retry_after_header", [True, False])
def test_respect_retry_after_header_propagated(self,
respect_retry_after_header):
retry = Retry(respect_retry_after_header=respect_retry_after_header)
new_retry = retry.new()
assert new_retry.respect_retry_after_header == respect_retry_after_header
@pytest.mark.parametrize(
"retry_after_header,respect_retry_after_header,sleep_duration",
[
("3600", True, 3600),
("3600", False, None),
# Will sleep due to header is 1 hour in future
("Mon, 3 Jun 2019 12:00:00 UTC", True, 3600),
# Won't sleep due to not respecting header
("Mon, 3 Jun 2019 12:00:00 UTC", False, None),
# Won't sleep due to current time reached
("Mon, 3 Jun 2019 11:00:00 UTC", True, None),
# Won't sleep due to current time reached + not respecting header
("Mon, 3 Jun 2019 11:00:00 UTC", False, None),
],
)
def test_respect_retry_after_header_sleep(self, retry_after_header,
respect_retry_after_header,
sleep_duration):
retry = Retry(respect_retry_after_header=respect_retry_after_header)
# Date header syntax can specify an absolute date; compare this to the
# time in the parametrized inputs above.
current_time = mock.MagicMock(return_value=time.mktime(
datetime.datetime(year=2019, month=6, day=3, hour=11).timetuple()))
with mock.patch("time.sleep") as sleep_mock, mock.patch(
"time.time", current_time):
# for the default behavior, it must be in RETRY_AFTER_STATUS_CODES
response = HTTPResponse(
status=503, headers={"Retry-After": retry_after_header})
retry.sleep(response)
# The expected behavior is that we'll only sleep if respecting
# this header (since we won't have any backoff sleep attempts)
if respect_retry_after_header and sleep_duration is not None:
sleep_mock.assert_called_with(sleep_duration)
else:
sleep_mock.assert_not_called()
class TestMultipartEncoding(object):
@pytest.mark.parametrize(
"fields", [dict(k="v", k2="v2"), [("k", "v"), ("k2", "v2")]])
def test_input_datastructures(self, fields):
encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY)
assert encoded.count(b(BOUNDARY)) == 3
@pytest.mark.parametrize(
"fields",
[
[("k", "v"), ("k2", "v2")],
[("k", b"v"), (u("k2"), b"v2")],
[("k", b"v"), (u("k2"), "v2")],
],
)
def test_field_encoding(self, fields):
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (b"--" + b(BOUNDARY) + b"\r\n"
b'Content-Disposition: form-data; name="k"\r\n'
b"\r\n"
b"v\r\n"
b"--" + b(BOUNDARY) + b"\r\n"
b'Content-Disposition: form-data; name="k2"\r\n'
b"\r\n"
b"v2\r\n"
b"--" + b(BOUNDARY) + b"--\r\n")
assert encoded == expected
assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY)
def test_filename(self):
fields = [("k", ("somename", b"v"))]
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (
b"--" + b(BOUNDARY) + b"\r\n"
b'Content-Disposition: form-data; name="k"; filename="somename"\r\n'
b"Content-Type: application/octet-stream\r\n"
b"\r\n"
b"v\r\n"
b"--" + b(BOUNDARY) + b"--\r\n")
assert encoded == expected
assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY)
def test_textplain(self):
fields = [("k", ("somefile.txt", b"v"))]
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (
b"--" + b(BOUNDARY) + b"\r\n"
b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
b"Content-Type: text/plain\r\n"
b"\r\n"
b"v\r\n"
b"--" + b(BOUNDARY) + b"--\r\n")
assert encoded == expected
assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY)
def test_explicit(self):
fields = [("k", ("somefile.txt", b"v", "image/jpeg"))]
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (
b"--" + b(BOUNDARY) + b"\r\n"
b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
b"Content-Type: image/jpeg\r\n"
b"\r\n"
b"v\r\n"
b"--" + b(BOUNDARY) + b"--\r\n")
assert encoded == expected
assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY)
def test_request_fields(self):
fields = [
RequestField(
"k",
b"v",
filename="somefile.txt",
headers={"Content-Type": "image/jpeg"},
)
]
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (b"--" + b(BOUNDARY) + b"\r\n"
b"Content-Type: image/jpeg\r\n"
b"\r\n"
b"v\r\n"
b"--" + b(BOUNDARY) + b"--\r\n")
assert encoded == expected