def request(self,
url,
get_params=None,
first_request_time=None,
retry_counter=0,
requests_kwargs=None,
post_json=None,
dry_run=None):
"""Performs HTTP GET/POST with credentials, returning the body as
JSON.
:param url: URL path for the request. Should begin with a slash.
:type url: string
:param get_params: HTTP GET parameters.
:type get_params: dict or list of key/value tuples
:param first_request_time: The time of the first request (None if no
retries have occurred).
:type first_request_time: datetime.datetime
:param retry_counter: The number of this retry, or zero for first attempt.
:type retry_counter: int
:param requests_kwargs: Same extra keywords arg for requests as per
__init__, but provided here to allow overriding internally on a
per-request basis.
:type requests_kwargs: dict
:param post_json: HTTP POST parameters. Only specified by calling method.
:type post_json: dict
:param dry_run: If 'true', only prints URL and parameters. 'true' or 'false'.
:type dry_run: string
:raises ApiError: when the API returns an error.
:raises Timeout: if the request timed out.
:rtype: dict from JSON response.
"""
if not first_request_time:
first_request_time = datetime.now()
elapsed = datetime.now() - first_request_time
if elapsed > self._retry_timeout:
raise exceptions.Timeout()
if retry_counter > 0:
# 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
# starting at 0.5s when retry_counter=1. The first retry will occur
# at 1, so subtract that first.
delay_seconds = 1.5**(retry_counter - 1)
# Jitter this value by 50% and pause.
time.sleep(delay_seconds * (random.random() + 0.5))
authed_url = self._generate_auth_url(
url,
get_params,
)
# Default to the client-level self.requests_kwargs, with method-level
# requests_kwargs arg overriding.
requests_kwargs = requests_kwargs or {}
final_requests_kwargs = dict(self._requests_kwargs, **requests_kwargs)
# Determine GET/POST.
requests_method = self._session.get
if post_json is not None:
requests_method = self._session.post
final_requests_kwargs["json"] = post_json
# Only print URL and parameters for dry_run
if dry_run:
print("url:\n{}\nHeaders:\n{}".format(
self._base_url + authed_url,
json.dumps(final_requests_kwargs, indent=2)))
return
try:
response = requests_method(self._base_url + authed_url,
**final_requests_kwargs)
self._req = response.request
except requests.exceptions.Timeout:
raise exceptions.Timeout()
if response.status_code in _RETRIABLE_STATUSES:
# Retry request.
warnings.warn('Server down.\nRetrying for the {0}{1} time.'.format(
retry_counter + 1, get_ordinal(retry_counter + 1)),
UserWarning,
stacklevel=1)
return self.request(url, get_params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
try:
result = self._get_body(response)
return result
except exceptions._RetriableRequest as e:
if isinstance(e, exceptions._OverQueryLimit
) and not self._retry_over_query_limit:
raise
warnings.warn(
'Rate limit exceeded. Retrying for the {0}{1} time.'.format(
retry_counter + 1, get_ordinal(retry_counter + 1)),
UserWarning,
stacklevel=1)
# Retry request.
return self.request(url, get_params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
def request(self,
url,
params,
first_request_time=None,
retry_counter=0,
requests_kwargs=None,
post_json=None,
dry_run=None):
"""Performs HTTP GET/POST with credentials, returning the body as
JSON.
:param url: URL path for the request. Should begin with a slash.
:type url: string
:param params: HTTP GET parameters.
:type params: dict or list of key/value tuples
:param first_request_time: The time of the first request (None if no
retries have occurred).
:type first_request_time: datetime.datetime
:param retry_counter: The number of this retry, or zero for first attempt.
:type retry_counter: int
:param requests_kwargs: Same extra keywords arg for requests as per
__init__, but provided here to allow overriding internally on a
per-request basis.
:type requests_kwargs: dict
:param post_json: HTTP POST parameters. Only specified by calling method.
:type post_json: dict
:param dry_run: If 'true', only prints URL and parameters. 'true' or 'false'.
:type dry_run: string
:raises ApiError: when the API returns an error.
:raises Timeout: if the request timed out.
:raises TransportError: when something went wrong while trying to
execute a request.
:rtype: dict from JSON response.
"""
if not first_request_time:
first_request_time = datetime.now()
elapsed = datetime.now() - first_request_time
if elapsed > self.retry_timeout:
raise exceptions.Timeout()
if retry_counter > 0:
# 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
# starting at 0.5s when retry_counter=1. The first retry will occur
# at 1, so subtract that first.
delay_seconds = 1.5**(retry_counter - 1)
# Jitter this value by 50% and pause.
time.sleep(delay_seconds * (random.random() + 0.5))
authed_url = self._generate_auth_url(
url,
params,
)
# Default to the client-level self.requests_kwargs, with method-level
# requests_kwargs arg overriding.
requests_kwargs = requests_kwargs or {}
final_requests_kwargs = dict(self.requests_kwargs, **requests_kwargs)
# Check if the time of the nth previous query (where n is
# queries_per_second) is under a second ago - if so, sleep for
# the difference.
if self.sent_times and len(self.sent_times) == self.queries_per_minute:
elapsed_since_earliest = time.time() - self.sent_times[0]
if elapsed_since_earliest < 60:
print(
"Request limit of {} per minute exceeded. Wait for {} seconds"
.format(self.queries_per_minute,
60 - elapsed_since_earliest))
time.sleep(60 - elapsed_since_earliest)
# Determine GET/POST.
# post_json is so far only sent from matrix call
requests_method = self.session.get
if post_json is not None:
requests_method = self.session.post
final_requests_kwargs["json"] = post_json
# Only print URL and parameters for dry_run
if dry_run:
print("url:\n{}\nParameters:\n{}".format(
self.base_url + authed_url, final_requests_kwargs))
return
try:
response = requests_method(self.base_url + authed_url,
**final_requests_kwargs)
except requests.exceptions.Timeout:
raise exceptions.Timeout()
except Exception as e:
raise exceptions.TransportError(e)
if response.status_code in _RETRIABLE_STATUSES:
# Retry request.
print('Server down.\nRetrying for the {}th time.'.format(
retry_counter + 1))
return self.request(url, params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
try:
result = self._get_body(response)
self.sent_times.append(time.time())
return result
except exceptions._RetriableRequest as e:
if isinstance(e, exceptions._OverQueryLimit
) and not self.retry_over_query_limit:
raise
print('Rate limit exceeded.\nRetrying for the {}th time.'.format(
retry_counter + 1))
# Retry request.
return self.request(url, params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
except:
raise