def http_request(hostname, endpoint, retries=3, retry_interval=3, **kwargs):
"""
Makes an HTTP request with the specified method to the specified hostname/endpoint. Retries
up to `retries` times if a request fails with a server error (e.g. error code 500), waiting
`retry_interval` seconds between successive retries. Parses the API response (assumed to be
JSON) into a Python object and returns it.
:param headers: Request headers to use when making the HTTP request
:param req_body_json: Dictionary containing the request body
:param params: Query parameters for the request
:return: Parsed API response
"""
cleaned_hostname = strip_suffix(hostname, '/')
url = "%s%s" % (cleaned_hostname, endpoint)
for i in range(retries):
response = requests.request(url=url, **kwargs)
if response.status_code >= 200 and response.status_code < 500:
return response
else:
eprint(
"API request to %s failed with code %s != 200, retrying up to %s more times. "
"API response body: %s" %
(url, response.status_code, retries - i - 1, response.text))
time.sleep(retry_interval)
raise Exception(
"API request to %s failed to return code 200 after %s tries" %
(url, retries))
def http_request(host_creds, endpoint, retries=3, retry_interval=3,
max_rate_limit_interval=60, **kwargs):
"""
Makes an HTTP request with the specified method to the specified hostname/endpoint. Ratelimit
error code (429) will be retried with an exponential back off (1, 2, 4, ... seconds) for at most
`max_rate_limit_interval` seconds. Internal errors (500s) will be retried up to `retries` times
, waiting `retry_interval` seconds between successive retries. Parses the API response
(assumed to be JSON) into a Python object and returns it.
:param host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing
hostname and optional authentication.
:return: Parsed API response
"""
hostname = host_creds.host
auth_str = None
if host_creds.username and host_creds.password:
basic_auth_str = ("%s:%s" % (host_creds.username, host_creds.password)).encode("utf-8")
auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8")
elif host_creds.token:
auth_str = "Bearer %s" % host_creds.token
headers = dict(_DEFAULT_HEADERS)
if auth_str:
headers['Authorization'] = auth_str
verify = not host_creds.ignore_tls_verification
def request_with_ratelimit_retries(max_rate_limit_interval, **kwargs):
response = requests.request(**kwargs)
time_left = max_rate_limit_interval
sleep = 1
while response.status_code == 429 and time_left > 0:
_logger.warning(
"API request to {path} returned status code 429 (Rate limit exceeded). "
"Retrying in %d seconds. "
"Will continue to retry 429s for up to %d seconds.",
sleep, time_left)
time.sleep(sleep)
time_left -= sleep
response = requests.request(**kwargs)
sleep = min(time_left, sleep*2) # sleep for 1, 2, 4, ... seconds;
return response
cleaned_hostname = strip_suffix(hostname, '/')
url = "%s%s" % (cleaned_hostname, endpoint)
for i in range(retries):
response = request_with_ratelimit_retries(max_rate_limit_interval,
url=url, headers=headers, verify=verify, **kwargs)
if response.status_code >= 200 and response.status_code < 500:
return response
else:
_logger.error(
"API request to %s failed with code %s != 200, retrying up to %s more times. "
"API response body: %s",
url, response.status_code, retries - i - 1, response.text)
time.sleep(retry_interval)
raise MlflowException("API request to %s failed to return code 200 after %s tries" %
(url, retries))
def http_request(host_creds, endpoint, retries=3, retry_interval=3, **kwargs):
"""
Makes an HTTP request with the specified method to the specified hostname/endpoint. Retries
up to `retries` times if a request fails with a server error (e.g. error code 500), waiting
`retry_interval` seconds between successive retries. Parses the API response (assumed to be
JSON) into a Python object and returns it.
:param host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing
hostname and optional authentication.
:return: Parsed API response
"""
hostname = host_creds.host
auth_str = None
if host_creds.username and host_creds.password:
basic_auth_str = (
"%s:%s" %
(host_creds.username, host_creds.password)).encode("utf-8")
auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode(
"utf-8")
elif host_creds.token:
auth_str = "Bearer %s" % host_creds.token
headers = {}
if auth_str:
headers['Authorization'] = auth_str
verify = not host_creds.ignore_tls_verification
cleaned_hostname = strip_suffix(hostname, '/')
url = "%s%s" % (cleaned_hostname, endpoint)
for i in range(retries):
response = requests.request(url=url,
headers=headers,
verify=verify,
**kwargs)
if response.status_code >= 200 and response.status_code < 500:
return response
else:
eprint(
"API request to %s failed with code %s != 200, retrying up to %s more times. "
"API response body: %s" %
(url, response.status_code, retries - i - 1, response.text))
time.sleep(retry_interval)
raise MlflowException(
"API request to %s failed to return code 200 after %s tries" %
(url, retries))
def test_strip_suffix(original, suffix, expected):
assert strip_suffix(original, suffix) == expected
def http_request(
host_creds,
endpoint,
method,
max_retries=5,
backoff_factor=2,
retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES,
timeout=120,
**kwargs,
):
"""
Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient
errors such as Rate-limited (429), service unavailable (503) and internal error (500) are
retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds).
The function parses the API response (assumed to be JSON) into a Python object and returns it.
:param host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing
hostname and optional authentication.
:param endpoint: a string for service endpoint, e.g. "/path/to/object".
:param method: a string indicating the method to use, e.g. "GET", "POST", "PUT".
:param max_retries: maximum number of retries before throwing an exception.
:param backoff_factor: a time factor for exponential backoff. e.g. value 5 means the HTTP
request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the
exponential backoff.
:param retry_codes: a list of HTTP response error codes that qualifies for retry.
:param timeout: wait for timeout seconds for response from remote server for connect and
read request.
:param kwargs: Additional keyword arguments to pass to `requests.Session.request()`
:return: requests.Response object.
"""
hostname = host_creds.host
auth_str = None
if host_creds.username and host_creds.password:
basic_auth_str = (
"%s:%s" %
(host_creds.username, host_creds.password)).encode("utf-8")
auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode(
"utf-8")
elif host_creds.token:
auth_str = "Bearer %s" % host_creds.token
from mlflow.tracking.request_header.registry import resolve_request_headers
headers = dict({**_DEFAULT_HEADERS, **resolve_request_headers()})
if auth_str:
headers["Authorization"] = auth_str
if host_creds.server_cert_path is None:
verify = not host_creds.ignore_tls_verification
else:
verify = host_creds.server_cert_path
if host_creds.client_cert_path is not None:
kwargs["cert"] = host_creds.client_cert_path
cleaned_hostname = strip_suffix(hostname, "/")
url = "%s%s" % (cleaned_hostname, endpoint)
try:
return _get_http_response_with_retries(
method,
url,
max_retries,
backoff_factor,
retry_codes,
headers=headers,
verify=verify,
timeout=timeout,
**kwargs,
)
except Exception as e:
raise MlflowException("API request to %s failed with exception %s" %
(url, e))
