def test_total_timeout(self): block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) wait_for_socket(ready_event) # This will get the socket to raise an EAGAIN on the read timeout = Timeout(connect=3, read=SHORT_TIMEOUT) with HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) as pool: with pytest.raises(ReadTimeoutError): pool.request("GET", "/") block_event.set() wait_for_socket(ready_event) block_event.clear() # The connect should succeed and this should hit the read timeout timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) with HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) as pool: with pytest.raises(ReadTimeoutError): pool.request("GET", "/")
def test_connect_timeout(self): url = "/" host, port = TARPIT_HOST, 80 timeout = Timeout(connect=SHORT_TIMEOUT) # Pool-global timeout with HTTPConnectionPool(host, port, timeout=timeout) as pool: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", url) # Retries retries = Retry(connect=0) with pytest.raises(MaxRetryError): pool.request("GET", url, retries=retries) # Request-specific connection timeouts big_timeout = Timeout(read=LONG_TIMEOUT, connect=LONG_TIMEOUT) with HTTPConnectionPool(host, port, timeout=big_timeout, retries=False) as pool: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", url, timeout=timeout) pool._put_conn(conn) with pytest.raises(ConnectTimeoutError): pool.request("GET", url, timeout=timeout)
def test_timeout_success(self): timeout = Timeout(connect=3, read=5, total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: pool.request("GET", "/") # This should not raise a "Timeout already started" error pool.request("GET", "/") with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: # This should also not raise a "Timeout already started" error pool.request("GET", "/") timeout = Timeout(total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: pool.request("GET", "/")
def test_lazy_load_twice(self): # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) as pool: payload_size = 1024 * 2 first_chunk = 512 boundary = "foo" req_data = {"count": "a" * payload_size} resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0] req2_data = {"count": "b" * payload_size} resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0] r1 = pool.request( "POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False, ) first_data = r1.read(first_chunk) assert len(first_data) > 0 assert first_data == resp_data[:len(first_data)] try: r2 = pool.request( "POST", "/echo", fields=req2_data, multipart_boundary=boundary, preload_content=False, pool_timeout=0.001, ) # This branch should generally bail here, but maybe someday it will # work? Perhaps by some sort of magic. Consider it a TODO. second_data = r2.read(first_chunk) assert len(second_data) > 0 assert second_data == resp2_data[:len(second_data)] assert r1.read() == resp_data[len(first_data):] assert r2.read() == resp2_data[len(second_data):] assert pool.num_requests == 2 except EmptyPoolError: assert r1.read() == resp_data[len(first_data):] assert pool.num_requests == 1 assert pool.num_connections == 1
def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) assert type(e.value.reason) == NewConnectionError
def test_for_double_release(self): MAXSIZE = 5 # Check default state with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: assert pool.num_connections == 0 assert pool.pool.qsize() == MAXSIZE # Make an empty slot for testing pool.pool.get() assert pool.pool.qsize() == MAXSIZE - 1 # Check state after simple request pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 1 # Check state without release pool.urlopen("GET", "/", preload_content=False) assert pool.pool.qsize() == MAXSIZE - 2 pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 2 # Check state after read pool.urlopen("GET", "/").data assert pool.pool.qsize() == MAXSIZE - 2 pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 2
def test_connections_arent_released(self): MAXSIZE = 5 with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: assert pool.pool.qsize() == MAXSIZE pool.request("GET", "/", preload_content=False) assert pool.pool.qsize() == MAXSIZE - 1
def test_preserve_chunked_on_retry(self): self.chunked_requests = 0 def socket_handler(listener): for _ in range(2): sock = listener.accept()[0] request = consume_socket(sock) if b"transfer-encoding: chunked" in request.split(b"\r\n"): self.chunked_requests += 1 sock.send( b"HTTP/1.1 429 Too Many Requests\r\n" b"Content-Type: text/plain\r\n" b"Retry-After: 1\r\n" b"\r\n" ) sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: retries = Retry(total=1) pool.urlopen( "GET", "/", body=iter([b"chunk1", b"chunk2"]), preload_content=False, retries=retries, ) assert self.chunked_requests == 2
def test_preserve_chunked_on_broken_connection(self): self.chunked_requests = 0 def socket_handler(listener): for i in range(2): sock = listener.accept()[0] request = consume_socket(sock) if b"transfer-encoding: chunked" in request.split(b"\r\n"): self.chunked_requests += 1 if i == 0: # Bad HTTP version will trigger a connection close sock.send(b"HTTP/0.5 200 OK\r\n\r\n") else: sock.send(b"HTTP/1.1 200 OK\r\n\r\n") sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: retries = Retry(read=1) pool.urlopen( "GET", "/", body=iter([b"chunk1", b"chunk2"]), preload_content=False, retries=retries, ) assert self.chunked_requests == 2
def test_connection_count(self): with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: pool.request("GET", "/") pool.request("GET", "/") pool.request("GET", "/") assert pool.num_connections == 1 assert pool.num_requests == 3
def test_total_applies_connect(self): host, port = TARPIT_HOST, 80 timeout = Timeout(total=None, connect=SHORT_TIMEOUT) with HTTPConnectionPool(host, port, timeout=timeout) as pool: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", "/") timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) with HTTPConnectionPool(host, port, timeout=timeout) as pool: try: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", "/") finally: conn.close()
def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: with HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) as pool: with pytest.raises(NewConnectionError): pool.request("GET", "/source_address?{0}".format(addr))
def test_connection_count_bigpool(self): with HTTPConnectionPool(self.host, self.port, maxsize=16) as http_pool: http_pool.request("GET", "/") http_pool.request("GET", "/") http_pool.request("GET", "/") assert http_pool.num_connections == 1 assert http_pool.num_requests == 3
def test_keepalive(self): with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) as pool: r = pool.request("GET", "/keepalive?close=0") r = pool.request("GET", "/keepalive?close=0") assert r.status == 200 assert pool.num_connections == 1 assert pool.num_requests == 2
def test_provides_default_host_header(self): self.start_chunked_handler() chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", chunks) header_block = self.buffer.split(b"\r\n\r\n", 1)[0].lower() header_lines = header_block.split(b"\r\n")[1:] host_headers = [x for x in header_lines if x.startswith(b"host")] assert len(host_headers) == 1
def test_source_address(self): for addr, is_ipv6 in VALID_SOURCE_ADDRESSES: if is_ipv6 and not HAS_IPV6_AND_DNS: warnings.warn("No IPv6 support: skipping.", NoIPv6Warning) continue with HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) as pool: r = pool.request("GET", "/source_address") assert r.data == b(addr[0])
def test_partial_response(self): with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: req_data = {"lol": "cat"} resp_data = urlencode(req_data).encode("utf-8") r = pool.request("GET", "/echo", fields=req_data, preload_content=False) assert r.read(5) == resp_data[:5] assert r.read() == resp_data[5:]
def test_create_connection_timeout(self): self.start_basic_handler(block_send=Event(), num=0) # needed for self.port timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT) with HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout, retries=False) as pool: conn = pool._new_conn() with pytest.raises(ConnectTimeoutError): conn.connect(connect_timeout=timeout.connect_timeout)
def test_keepalive_close(self): with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) as pool: r = pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}) assert pool.num_connections == 1 # The dummyserver will have responded with Connection:close, # and httplib will properly cleanup the socket. # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs # pylint: disable-msg=W0212 conn = pool.pool.get() assert conn._sock is None pool._put_conn(conn) # Now with keep-alive r = pool.request( "GET", "/keepalive?close=0", retries=0, headers={"Connection": "keep-alive"}, ) # The dummyserver responded with Connection:keep-alive, the connection # persists. conn = pool.pool.get() assert conn._sock is not None pool._put_conn(conn) # Another request asking the server to close the connection. This one # should get cleaned up for the next request. r = pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}) assert r.status == 200 conn = pool.pool.get() assert conn._sock is None pool._put_conn(conn) # Next request r = pool.request("GET", "/keepalive?close=0")
def test_nagle(self): """ Test that connections have TCP_NODELAY turned on """ # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port) as pool: try: conn = pool._get_conn() pool._make_request(conn, "GET", "/") tcp_nodelay_setting = conn._sock.getsockopt( socket.IPPROTO_TCP, socket.TCP_NODELAY) assert tcp_nodelay_setting finally: conn.close()
def test_disable_default_socket_options(self): """Test that passing None disables all socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port, socket_options=None) as pool: conn = pool._new_conn() conn.connect() s = conn._sock using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0 assert using_nagle s.close()
def test_chunks(self): self.start_chunked_handler() chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", chunks, headers=dict(DNT="1")) assert b"transfer-encoding" in self.buffer body = self.buffer.split(b"\r\n\r\n", 1)[1] lines = body.split(b"\r\n") # Empty chunks should have been skipped, as this could not be distinguished # from terminating the transmission for i, chunk in enumerate([c for c in chunks if c]): assert lines[i * 2] == hex(len(chunk))[2:].encode("utf-8") assert lines[i * 2 + 1] == chunk
def test_timeout_float(self): block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) with HTTPConnectionPool(self.host, self.port, retries=False) as pool: wait_for_socket(ready_event) with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release block # Shouldn't raise this time wait_for_socket(ready_event) block_event.set() # Pre-release block pool.request("GET", "/", timeout=LONG_TIMEOUT)
def test_timeout(self): # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=3) # Pool-global timeout short_timeout = Timeout(read=SHORT_TIMEOUT) with HTTPConnectionPool(self.host, self.port, timeout=short_timeout, retries=False) as pool: wait_for_socket(ready_event) block_event.clear() with pytest.raises(ReadTimeoutError): pool.request("GET", "/") block_event.set() # Release request # Request-specific timeouts should raise errors with HTTPConnectionPool(self.host, self.port, timeout=short_timeout, retries=False) as pool: wait_for_socket(ready_event) now = time.time() with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=LONG_TIMEOUT) delta = time.time() - now message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT" assert delta >= LONG_TIMEOUT, message block_event.set() # Release request # Timeout passed directly to request should raise a request timeout wait_for_socket(ready_event) with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release request
def test_socket_options(self): """Test that connections accept socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries to # connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool( self.host, self.port, socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)], ) as pool: conn = pool._new_conn() conn.connect() s = conn._sock using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0 assert using_keepalive s.close()
def _test_body(self, data): self.start_chunked_handler() with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", data) header, body = self.buffer.split(b"\r\n\r\n", 1) assert b"transfer-encoding: chunked" in header.split(b"\r\n") if data: bdata = data if isinstance(data, bytes) else data.encode("utf-8") assert b"\r\n" + bdata + b"\r\n" in body assert body.endswith(b"\r\n0\r\n\r\n") len_str = body.split(b"\r\n", 1)[0] stated_len = int(len_str, 16) assert stated_len == len(bdata) else: assert body == b"0\r\n\r\n"
def test_conn_closed(self): block_event = Event() self.start_basic_handler(block_send=block_event, num=1) with HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) as pool: conn = pool._get_conn() pool._put_conn(conn) try: with pytest.raises(ReadTimeoutError): pool.urlopen("GET", "/") if conn._sock: with pytest.raises(socket.error): conn.sock.recv(1024) finally: pool._put_conn(conn) block_event.set()
def test_defaults_are_applied(self): """Test that modifying the default socket options works.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool(self.host, self.port) as pool: # Get the HTTPConnection instance conn = pool._new_conn() try: # Update the default socket options conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] conn.connect() s = conn._sock nagle_disabled = (s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0) using_keepalive = (s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0) assert nagle_disabled assert using_keepalive finally: conn.close() s.close()
def test_queue_monkeypatching(self): with mock.patch.object(queue, "Empty", BadError): with HTTPConnectionPool(host="localhost", block=True) as http: http._get_conn() with pytest.raises(EmptyPoolError): http._get_conn(timeout=0)
def test_mixed_case_hostname(self): with HTTPConnectionPool("LoCaLhOsT", self.port) as pool: response = pool.request("GET", "http://LoCaLhOsT:%d/" % self.port) assert response.status == 200