def test_dns_error(self): pool = HTTPConnectionPool("thishostdoesnotexist.invalid", self.port, timeout=0.001) with pytest.raises(MaxRetryError): pool.request("GET", "/test", retries=2)
class TestConnectionPool(HTTPDummyServerTestCase): def setup_method(self, method): self.pool = HTTPConnectionPool(self.host, self.port) def teardown_method(self): self.pool.close() def test_get(self): r = self.pool.request("GET", "/specific_method", fields={"method": "GET"}) assert r.status == 200, r.data def test_post_url(self): r = self.pool.request("POST", "/specific_method", fields={"method": "POST"}) assert r.status == 200, r.data def test_urlopen_put(self): r = self.pool.urlopen("PUT", "/specific_method?method=PUT") assert r.status == 200, r.data def test_wrong_specific_method(self): # To make sure the dummy server is actually returning failed responses r = self.pool.request("GET", "/specific_method", fields={"method": "POST"}) assert r.status == 400, r.data r = self.pool.request("POST", "/specific_method", fields={"method": "GET"}) assert r.status == 400, r.data def test_upload(self): data = "I'm in ur multipart form-data, hazing a cheezburgr" fields = { "upload_param": "filefield", "upload_filename": "lolcat.txt", "upload_size": len(data), "filefield": ("lolcat.txt", data), } r = self.pool.request("POST", "/upload", fields=fields) assert r.status == 200, r.data def test_one_name_multiple_values(self): fields = [("foo", "a"), ("foo", "b")] # urlencode r = self.pool.request("GET", "/echo", fields=fields) assert r.data == b"foo=a&foo=b" # multipart r = self.pool.request("POST", "/echo", fields=fields) assert r.data.count(b'name="foo"') == 2 def test_request_method_body(self): body = b"hi" r = self.pool.request("POST", "/echo", body=body) assert r.data == body fields = [("hi", "hello")] with pytest.raises(TypeError): self.pool.request("POST", "/echo", body=body, fields=fields) def test_unicode_upload(self): fieldname = u("myfile") filename = u("\xe2\x99\xa5.txt") data = u("\xe2\x99\xa5").encode("utf8") size = len(data) fields = { u("upload_param"): fieldname, u("upload_filename"): filename, u("upload_size"): size, fieldname: (filename, data), } r = self.pool.request("POST", "/upload", fields=fields) assert r.status == 200, r.data def test_nagle(self): """ Test that connections have TCP_NODELAY turned on """ # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port) as pool: try: conn = pool._get_conn() pool._make_request(conn, "GET", "/") tcp_nodelay_setting = conn._sock.getsockopt( socket.IPPROTO_TCP, socket.TCP_NODELAY) assert tcp_nodelay_setting finally: conn.close() def test_socket_options(self): """Test that connections accept socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries to # connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool( self.host, self.port, socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)], ) as pool: conn = pool._new_conn() conn.connect() s = conn._sock using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0 assert using_keepalive s.close() def test_disable_default_socket_options(self): """Test that passing None disables all socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port, socket_options=None) as pool: conn = pool._new_conn() conn.connect() s = conn._sock using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0 assert using_nagle s.close() def test_defaults_are_applied(self): """Test that modifying the default socket options works.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port) as pool: # Get the HTTPConnection instance conn = pool._new_conn() try: # Update the default socket options conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] conn.connect() s = conn._sock nagle_disabled = (s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0) using_keepalive = (s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0) assert nagle_disabled assert using_keepalive finally: conn.close() s.close() def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) assert type(e.value.reason) == NewConnectionError def test_timeout_success(self): timeout = Timeout(connect=3, read=5, total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: pool.request("GET", "/") # This should not raise a "Timeout already started" error pool.request("GET", "/") with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: # This should also not raise a "Timeout already started" error pool.request("GET", "/") timeout = Timeout(total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: pool.request("GET", "/") def test_bad_connect(self): with HTTPConnectionPool("badhost.invalid", self.port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=5) assert type(e.value.reason) == NewConnectionError def test_keepalive(self): with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) as pool: r = pool.request("GET", "/keepalive?close=0") r = pool.request("GET", "/keepalive?close=0") assert r.status == 200 assert pool.num_connections == 1 assert pool.num_requests == 2 def test_keepalive_close(self): with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) as pool: r = pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}) assert pool.num_connections == 1 # The dummyserver will have responded with Connection:close, # and httplib will properly cleanup the socket. # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs # pylint: disable-msg=W0212 conn = pool.pool.get() assert conn._sock is None pool._put_conn(conn) # Now with keep-alive r = pool.request( "GET", "/keepalive?close=0", retries=0, headers={"Connection": "keep-alive"}, ) # The dummyserver responded with Connection:keep-alive, the connection # persists. conn = pool.pool.get() assert conn._sock is not None pool._put_conn(conn) # Another request asking the server to close the connection. This one # should get cleaned up for the next request. r = pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}) assert r.status == 200 conn = pool.pool.get() assert conn._sock is None pool._put_conn(conn) # Next request r = pool.request("GET", "/keepalive?close=0") def test_post_with_urlencode(self): data = {"banana": "hammock", "lol": "cat"} r = self.pool.request("POST", "/echo", fields=data, encode_multipart=False) assert r.data.decode("utf-8") == urlencode(data) def test_post_with_multipart(self): data = {"banana": "hammock", "lol": "cat"} r = self.pool.request("POST", "/echo", fields=data, encode_multipart=True) body = r.data.split(b"\r\n") encoded_data = encode_multipart_formdata(data)[0] expected_body = encoded_data.split(b"\r\n") # TODO: Get rid of extra parsing stuff when you can specify # a custom boundary to encode_multipart_formdata """ We need to loop the return lines because a timestamp is attached from within encode_multipart_formdata. When the server echos back the data, it has the timestamp from when the data was encoded, which is not equivalent to when we run encode_multipart_formdata on the data again. """ for i, line in enumerate(body): if line.startswith(b"--"): continue assert body[i] == expected_body[i] def test_post_with_multipart__iter__(self): data = {"hello": "world"} r = self.pool.request( "POST", "/echo", fields=data, preload_content=False, multipart_boundary="boundary", encode_multipart=True, ) chunks = [chunk for chunk in r] assert chunks == [ b"--boundary\r\n", b'Content-Disposition: form-data; name="hello"\r\n', b"\r\n", b"world\r\n", b"--boundary--\r\n", ] def test_check_gzip(self): r = self.pool.request("GET", "/encodingrequest", headers={"accept-encoding": "gzip"}) assert r.headers.get("content-encoding") == "gzip" assert r.data == b"hello, world!" def test_check_deflate(self): r = self.pool.request("GET", "/encodingrequest", headers={"accept-encoding": "deflate"}) assert r.headers.get("content-encoding") == "deflate" assert r.data == b"hello, world!" def test_bad_decode(self): with pytest.raises(DecodeError): self.pool.request( "GET", "/encodingrequest", headers={"accept-encoding": "garbage-deflate"}, ) with pytest.raises(DecodeError): self.pool.request("GET", "/encodingrequest", headers={"accept-encoding": "garbage-gzip"}) def test_connection_count(self): with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: pool.request("GET", "/") pool.request("GET", "/") pool.request("GET", "/") assert pool.num_connections == 1 assert pool.num_requests == 3 def test_connection_count_bigpool(self): with HTTPConnectionPool(self.host, self.port, maxsize=16) as http_pool: http_pool.request("GET", "/") http_pool.request("GET", "/") http_pool.request("GET", "/") assert http_pool.num_connections == 1 assert http_pool.num_requests == 3 def test_partial_response(self): with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: req_data = {"lol": "cat"} resp_data = urlencode(req_data).encode("utf-8") r = pool.request("GET", "/echo", fields=req_data, preload_content=False) assert r.read(5) == resp_data[:5] assert r.read() == resp_data[5:] def test_lazy_load_twice(self): # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) as pool: payload_size = 1024 * 2 first_chunk = 512 boundary = "foo" req_data = {"count": "a" * payload_size} resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0] req2_data = {"count": "b" * payload_size} resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0] r1 = pool.request( "POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False, ) first_data = r1.read(first_chunk) assert len(first_data) > 0 assert first_data == resp_data[:len(first_data)] try: r2 = pool.request( "POST", "/echo", fields=req2_data, multipart_boundary=boundary, preload_content=False, pool_timeout=0.001, ) # This branch should generally bail here, but maybe someday it will # work? Perhaps by some sort of magic. Consider it a TODO. second_data = r2.read(first_chunk) assert len(second_data) > 0 assert second_data == resp2_data[:len(second_data)] assert r1.read() == resp_data[len(first_data):] assert r2.read() == resp2_data[len(second_data):] assert pool.num_requests == 2 except EmptyPoolError: assert r1.read() == resp_data[len(first_data):] assert pool.num_requests == 1 assert pool.num_connections == 1 def test_for_double_release(self): MAXSIZE = 5 # Check default state with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: assert pool.num_connections == 0 assert pool.pool.qsize() == MAXSIZE # Make an empty slot for testing pool.pool.get() assert pool.pool.qsize() == MAXSIZE - 1 # Check state after simple request pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 1 # Check state without release pool.urlopen("GET", "/", preload_content=False) assert pool.pool.qsize() == MAXSIZE - 2 pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 2 # Check state after read pool.urlopen("GET", "/").data assert pool.pool.qsize() == MAXSIZE - 2 pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 2 def test_connections_arent_released(self): MAXSIZE = 5 with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: assert pool.pool.qsize() == MAXSIZE pool.request("GET", "/", preload_content=False) assert pool.pool.qsize() == MAXSIZE - 1 def test_dns_error(self): pool = HTTPConnectionPool("thishostdoesnotexist.invalid", self.port, timeout=0.001) with pytest.raises(MaxRetryError): pool.request("GET", "/test", retries=2) def test_percent_encode_invalid_target_chars(self): with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/echo_params?q=\r&k=\n \n") assert r.data == b"[('k', '\\n \\n'), ('q', '\\r')]" def test_source_address(self): for addr, is_ipv6 in VALID_SOURCE_ADDRESSES: if is_ipv6 and not HAS_IPV6_AND_DNS: warnings.warn("No IPv6 support: skipping.", NoIPv6Warning) continue with HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) as pool: r = pool.request("GET", "/source_address") assert r.data == b(addr[0]) def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: with HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) as pool: with pytest.raises(NewConnectionError): pool.request("GET", "/source_address?{0}".format(addr)) def test_stream_keepalive(self): x = 2 for _ in range(x): response = self.pool.request( "GET", "/chunked", headers={"Connection": "keep-alive"}, preload_content=False, retries=False, ) for chunk in response.stream(3): assert chunk == b"123" assert self.pool.num_connections == 1 assert self.pool.num_requests == x def test_chunked_gzip(self): response = self.pool.request("GET", "/chunked_gzip", preload_content=False, decode_content=True) assert b"123" * 4 == response.read() def test_mixed_case_hostname(self): with HTTPConnectionPool("LoCaLhOsT", self.port) as pool: response = pool.request("GET", "http://LoCaLhOsT:%d/" % self.port) assert response.status == 200