def test_redirect_cross_host_set_removed_headers(self): with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={"X-API-Secret": "foo", "Authorization": "bar"}, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), ) assert r.status == 200 data = json.loads(r.data.decode("utf-8")) assert "X-API-Secret" not in data assert data["Authorization"] == "bar" r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={"x-api-secret": "foo", "authorization": "bar"}, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), ) assert r.status == 200 data = json.loads(r.data.decode("utf-8")) assert "x-api-secret" not in data assert "X-API-Secret" not in data assert data["Authorization"] == "bar"
def test_raise_on_status(self): with PoolManager() as http: with pytest.raises(MaxRetryError): # the default is to raise r = http.request( "GET", "%s/status" % self.base_url, fields={"status": "500 Internal Server Error"}, retries=Retry(total=1, status_forcelist=range(500, 600)), ) with pytest.raises(MaxRetryError): # raise explicitly r = http.request( "GET", "%s/status" % self.base_url, fields={"status": "500 Internal Server Error"}, retries=Retry( total=1, status_forcelist=range(500, 600), raise_on_status=True ), ) # don't raise r = http.request( "GET", "%s/status" % self.base_url, fields={"status": "500 Internal Server Error"}, retries=Retry( total=1, status_forcelist=range(500, 600), raise_on_status=False ), ) assert r.status == 500
def test_error_message(self): retry = Retry(total=0) with pytest.raises(MaxRetryError) as e: retry = retry.increment(method="GET", error=ReadTimeoutError( None, "/", "read timed out")) assert "Caused by redirect" not in str(e.value) assert str(e.value.reason) == "None: read timed out" retry = Retry(total=1) with pytest.raises(MaxRetryError) as e: retry = retry.increment("POST", "/") retry = retry.increment("POST", "/") assert "Caused by redirect" not in str(e.value) assert isinstance(e.value.reason, ResponseError) assert str(e.value.reason) == ResponseError.GENERIC_ERROR retry = Retry(total=1) response = HTTPResponse(status=500) with pytest.raises(MaxRetryError) as e: retry = retry.increment("POST", "/", response=response) retry = retry.increment("POST", "/", response=response) assert "Caused by redirect" not in str(e.value) msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) assert str(e.value.reason) == msg retry = Retry(connect=1) with pytest.raises(MaxRetryError) as e: retry = retry.increment(error=ConnectTimeoutError("conntimeout")) retry = retry.increment(error=ConnectTimeoutError("conntimeout")) assert "Caused by redirect" not in str(e.value) assert str(e.value.reason) == "conntimeout"
def test_method_whitelist_with_status_forcelist(self): # Falsey method_whitelist means to retry on any method. retry = Retry(status_forcelist=[500], method_whitelist=None) assert retry.is_retry("GET", status_code=500) assert retry.is_retry("POST", status_code=500) # Criteria of method_whitelist and status_forcelist are ANDed. retry = Retry(status_forcelist=[500], method_whitelist=["POST"]) assert not retry.is_retry("GET", status_code=500) assert retry.is_retry("POST", status_code=500)
def test_status_forcelist(self): retry = Retry(status_forcelist=xrange(500, 600)) assert not retry.is_retry("GET", status_code=200) assert not retry.is_retry("GET", status_code=400) assert retry.is_retry("GET", status_code=500) retry = Retry(total=1, status_forcelist=[418]) assert not retry.is_retry("GET", status_code=400) assert retry.is_retry("GET", status_code=418) # String status codes are not matched. retry = Retry(total=1, status_forcelist=["418"]) assert not retry.is_retry("GET", status_code=418)
def test_backoff(self): """ Backoff is computed correctly """ max_backoff = Retry.BACKOFF_MAX retry = Retry(total=100, backoff_factor=0.2) assert retry.get_backoff_time() == 0 # First request retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0 # First retry retry = retry.increment(method="GET") assert retry.backoff_factor == 0.2 assert retry.total == 98 assert retry.get_backoff_time() == 0.4 # Start backoff retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0.8 retry = retry.increment(method="GET") assert retry.get_backoff_time() == 1.6 for _ in xrange(10): retry = retry.increment(method="GET") assert retry.get_backoff_time() == max_backoff
def test_retries(self): fp = BytesIO(b"") resp = HTTPResponse(fp) assert resp.retries is None retry = Retry() resp = HTTPResponse(fp, retries=retry) assert resp.retries == retry
def test_preserve_chunked_on_broken_connection(self): self.chunked_requests = 0 def socket_handler(listener): for i in range(2): sock = listener.accept()[0] request = consume_socket(sock) if b"transfer-encoding: chunked" in request.split(b"\r\n"): self.chunked_requests += 1 if i == 0: # Bad HTTP version will trigger a connection close sock.send(b"HTTP/0.5 200 OK\r\n\r\n") else: sock.send(b"HTTP/1.1 200 OK\r\n\r\n") sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: retries = Retry(read=1) pool.urlopen( "GET", "/", body=iter([b"chunk1", b"chunk2"]), preload_content=False, retries=retries, ) assert self.chunked_requests == 2
def test_connect_timeout(self): url = "/" host, port = TARPIT_HOST, 80 timeout = Timeout(connect=SHORT_TIMEOUT) # Pool-global timeout with HTTPConnectionPool(host, port, timeout=timeout) as pool: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", url) # Retries retries = Retry(connect=0) with pytest.raises(MaxRetryError): pool.request("GET", url, retries=retries) # Request-specific connection timeouts big_timeout = Timeout(read=LONG_TIMEOUT, connect=LONG_TIMEOUT) with HTTPConnectionPool(host, port, timeout=big_timeout, retries=False) as pool: conn = pool._get_conn() with pytest.raises(ConnectTimeoutError): pool._make_request(conn, "GET", url, timeout=timeout) pool._put_conn(conn) with pytest.raises(ConnectTimeoutError): pool.request("GET", url, timeout=timeout)
def test_retry_read_zero(self): """ No second chances on read timeouts, by default """ error = ReadTimeoutError(None, "/", "read timed out") retry = Retry(read=0) with pytest.raises(MaxRetryError) as e: retry.increment(method="GET", error=error) assert e.value.reason == error
def test_disabled_retry(self): """ Disabled retries should disable redirect handling. """ with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "/"}, retries=False, ) assert r.status == 303 r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "/"}, retries=Retry(redirect=False), ) assert r.status == 303 with pytest.raises(NewConnectionError): http.request( "GET", "http://thishostdoesnotexist.invalid/", timeout=0.001, retries=False, )
def test_history(self): retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"])) assert retry.history == tuple() connection_error = ConnectTimeoutError("conntimeout") retry = retry.increment("GET", "/test1", None, connection_error) history = (RequestHistory("GET", "/test1", connection_error, None, None), ) assert retry.history == history read_error = ReadTimeoutError(None, "/test2", "read timed out") retry = retry.increment("POST", "/test2", None, read_error) history = ( RequestHistory("GET", "/test1", connection_error, None, None), RequestHistory("POST", "/test2", read_error, None, None), ) assert retry.history == history response = HTTPResponse(status=500) retry = retry.increment("GET", "/test3", response, None) history = ( RequestHistory("GET", "/test1", connection_error, None, None), RequestHistory("POST", "/test2", read_error, None, None), RequestHistory("GET", "/test3", None, 500, None), ) assert retry.history == history
def test_preserve_chunked_on_retry(self): self.chunked_requests = 0 def socket_handler(listener): for _ in range(2): sock = listener.accept()[0] request = consume_socket(sock) if b"transfer-encoding: chunked" in request.split(b"\r\n"): self.chunked_requests += 1 sock.send( b"HTTP/1.1 429 Too Many Requests\r\n" b"Content-Type: text/plain\r\n" b"Retry-After: 1\r\n" b"\r\n" ) sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: retries = Retry(total=1) pool.urlopen( "GET", "/", body=iter([b"chunk1", b"chunk2"]), preload_content=False, retries=retries, ) assert self.chunked_requests == 2
def test_retry_total_none(self): """ if Total is none, connect error should take precedence """ error = ConnectTimeoutError() retry = Retry(connect=2, total=None) retry = retry.increment(error=error) retry = retry.increment(error=error) with pytest.raises(MaxRetryError) as e: retry.increment(error=error) assert e.value.reason == error error = ReadTimeoutError(None, "/", "read timed out") retry = Retry(connect=2, total=None) retry = retry.increment(method="GET", error=error) retry = retry.increment(method="GET", error=error) retry = retry.increment(method="GET", error=error) assert not retry.is_exhausted()
def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) assert type(e.value.reason) == NewConnectionError
def test_retry_higher_total_loses_vs_read(self): """ A lower read timeout than the total is honored """ error = ReadTimeoutError(None, "/", "read timed out") retry = Retry(read=2, total=3) retry = retry.increment(method="GET", error=error) retry = retry.increment(method="GET", error=error) with pytest.raises(MaxRetryError): retry.increment(method="GET", error=error)
def test_retry_higher_total_loses(self): """ A lower connect timeout than the total is honored """ error = ConnectTimeoutError() retry = Retry(connect=2, total=3) retry = retry.increment(error=error) retry = retry.increment(error=error) with pytest.raises(MaxRetryError): retry.increment(error=error)
def test_status_counter(self): resp = HTTPResponse(status=400) retry = Retry(status=2) retry = retry.increment(response=resp) retry = retry.increment(response=resp) with pytest.raises(MaxRetryError) as e: retry.increment(response=resp) assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format( status_code=400)
def test_retry_both_specified(self): """Total can win if it's lower than the connect value""" error = ConnectTimeoutError() retry = Retry(connect=3, total=2) retry = retry.increment(error=error) retry = retry.increment(error=error) with pytest.raises(MaxRetryError) as e: retry.increment(error=error) assert e.value.reason == error
def test_retry_default(self): """ If no value is specified, should retry connects 3 times """ retry = Retry() assert retry.total == 10 assert retry.connect is None assert retry.read is None assert retry.redirect is None error = ConnectTimeoutError() retry = Retry(connect=1) retry = retry.increment(error=error) with pytest.raises(MaxRetryError): retry.increment(error=error) retry = Retry(connect=1) retry = retry.increment(error=error) assert not retry.is_exhausted() assert Retry(0).raise_on_redirect assert not Retry(False).raise_on_redirect
def test_backoff_reset_after_redirect(self): retry = Retry(total=100, redirect=5, backoff_factor=0.2) retry = retry.increment(method="GET") retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0.4 redirect_response = HTTPResponse(status=302, headers={"location": "test"}) retry = retry.increment(method="GET", response=redirect_response) assert retry.get_backoff_time() == 0 retry = retry.increment(method="GET") retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0.4
def test_read_retries(self): """ Should retry for status codes in the whitelist """ retry = Retry(read=1, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers={"test-name": "test_read_retries"}, retries=retry, ) assert resp.status == 200
def test_raise_on_redirect(self): with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={ "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) }, retries=Retry(total=None, redirect=1, raise_on_redirect=False), ) assert r.status == 303
def test_default_method_whitelist_retried(self): """Hip should retry methods in the default method whitelist""" retry = Retry(total=1, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "OPTIONS", "%s/successful_retry" % self.base_url, headers={"test-name": "test_default_whitelist"}, retries=retry, ) assert resp.status == 200
def test_retries_wrong_whitelist(self): """HTTP response w/ status code not in whitelist shouldn't be retried""" retry = Retry(total=1, status_forcelist=[202]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers={"test-name": "test_wrong_whitelist"}, retries=retry, ) assert resp.status == 418
def test_string(self): """ Retry string representation looks the way we expect """ retry = Retry() assert ( str(retry) == "Retry(total=10, connect=None, read=None, redirect=None, status=None)" ) for _ in range(3): retry = retry.increment(method="GET") assert ( str(retry) == "Retry(total=7, connect=None, read=None, redirect=None, status=None)" )
def test_retries_wrong_method_list(self): """Method not in our whitelist should not be retried, even if code matches""" headers = {"test-name": "test_wrong_method_whitelist"} retry = Retry(total=1, status_forcelist=[418], method_whitelist=["POST"]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 418
def test_read_total_retries(self): """ HTTP response w/ status code in the whitelist should be retried """ headers = {"test-name": "test_read_total_retries"} retry = Retry(total=1, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 200
def test_redirect_cross_host_no_remove_headers(self): with PoolManager() as http: r = http.request( "GET", "%s/redirect" % self.base_url, fields={"target": "%s/headers" % self.base_url_alt}, headers={"Authorization": "foo"}, retries=Retry(remove_headers_on_redirect=[]), ) assert r.status == 200 data = json.loads(r.data.decode("utf-8")) assert data["Authorization"] == "foo"
def test_retry_return_in_response(self): headers = {"test-name": "test_retry_return_in_response"} retry = Retry(total=2, status_forcelist=[418]) with PoolManager() as http: resp = http.request( "GET", "%s/successful_retry" % self.base_url, headers=headers, retries=retry, ) assert resp.status == 200 assert resp.retries.total == 1 assert resp.retries.history == ( RequestHistory("GET", "/successful_retry", None, 418, None), )