def wrapper(*args, **kargs):
'''
Extend the behaviour of the decoated function, forwarding function
invocations previously called no sooner than a specified period of
time. The decorator will raise an exception if the function cannot
be called so the caller may implement a retry strategy such as an
exponential backoff.
:param args: non-keyword variable length argument list to the decorated function.
:param kargs: keyworded variable length argument list to the decorated function.
:raises: RateLimitException
'''
with self.lock:
period_remaining = self.__period_remaining()
# If the time window has elapsed then reset.
if period_remaining <= 0:
self.num_calls = 0
self.last_reset = self.clock()
# Increase the number of attempts to call the function.
self.num_calls += 1
# If the number of attempts to call the function exceeds the
# maximum then raise an exception.
if self.num_calls > self.clamped_calls:
if self.raise_on_limit:
raise RateLimitException('too many calls',
period_remaining)
return
return func(*args, **kargs)
def _do_request(self, endpoint, params=None):
"""Internal method for building request url and performing HTTP request to the
haveibeenpwned.com API
Args:
endpoint (string): API endpoint string (eg. "breachedaccount" or "breach" )
params (dict, Optional): HTTP request parameters to be passed to the HTTP
request
Returns:
json: Requests response .json() object
"""
resp = requests.get(urljoin(API_BASE_URL, endpoint),
headers=self.headers,
params=params)
print(urljoin(API_BASE_URL, endpoint))
if resp.status_code == 404:
return []
if resp.status_code == 429:
period_remaining = int(
re.match(r"\D*(\d+)\D*",
resp.json()["message"]).group(1))
raise RateLimitException(message=resp.json()["message"],
period_remaining=period_remaining)
resp.raise_for_status()
return resp.json()
def wrapper(*args, **kargs):
'''
Extend the behaviour of the decorated function, forwarding function
invocations previously called no sooner than a specified period of
time. The decorator will raise an exception if the function cannot
be called so the caller may implement a retry strategy such as an
exponential backoff.
:param args: non-keyword variable length argument list to the decorated function.
:param kargs: keyworded variable length argument list to the decorated function.
:raises: RateLimitException
'''
while True:
try:
with self.database, self.lock:
self.database.execute("BEGIN TRANSACTION")
# If the number of attempts to call the function exceeds the
# maximum then raise an exception.
if self._num_calls >= self.clamped_calls:
if self.raise_on_limit:
raise RateLimitException(
'too many calls', self._period_remaining)
return
# Clean old calls
self.database.execute("""
DELETE FROM {}
WHERE time <= julianday('now', '{}')
""".format(self.name, self._offset))
# Log call
self.database.execute(
"INSERT INTO {} DEFAULT VALUES".format(self.name))
return func(*args, **kargs)
except sqlite3.OperationalError:
pass
def _too_many_calls(self):
# If the number of attempts to call the function exceeds the
# maximum then raise an exception.
if self._num_calls >= self.clamped_calls:
if self.raise_on_limit:
raise RateLimitException("too many calls",
self._period_remaining)
return True
async def get_with_retry(url, retry_on=(), sleep_period=None, headers=None):
if not headers:
headers = {}
if settings.OSF_BEARER_TOKEN:
headers["Authorization"] = f"Bearer {settings.OSF_BEARER_TOKEN}"
async with ClientSession() as session:
async with session.get(url, headers=headers) as resp:
if resp.status in retry_on:
raise RateLimitException(
message="Too many requests, sleeping.",
period_remaining=sleep_period
or int(resp.headers.get("Retry-After") or 0),
) # This will be caught by @sleep_and_retry and retried
resp.raise_for_status()
return await resp.json()
def get_with_retry(url,
retry_on: Tuple[int] = (),
sleep_period: int = None,
headers: Dict = None) -> requests.Response:
if not headers:
headers = {}
resp = requests.get(url, headers=headers)
if resp.status_code in retry_on:
raise RateLimitException(
message='Too many requests, sleeping.',
period_remaining=sleep_period or int(
resp.headers.get('Retry-After')
or 0)) # This will be caught by @sleep_and_retry and retried
return resp
def get_with_retry(url, retry_on=(), sleep_period=None, headers=None):
if not headers:
headers = {}
if settings.OSF_BEARER_TOKEN:
headers["Authorization"] = f"Bearer {settings.OSF_BEARER_TOKEN}"
resp = requests.get(url, headers=headers, stream=True)
if resp.status_code in retry_on:
raise RateLimitException(
message="Too many requests, sleeping.",
period_remaining=sleep_period
or int(resp.headers.get("Retry-After") or 0),
) # This will be caught by @sleep_and_retry and retried
resp.raise_for_status()
return resp
def get_with_retry(url,
retry_on: Tuple[int] = (),
sleep_period: int = None,
headers: Dict = None) -> requests.Response:
if not headers:
headers = {}
if not settings.OSF_USER_THROTTLE_ENABLED:
assert settings.OSF_BEARER_TOKEN, \
'must have OSF_BEARER_TOKEN set to disable the api user throttle of the OSF'
headers['Authorization'] = settings.OSF_BEARER_TOKEN
resp = requests.get(url, headers=headers)
if resp.status_code in retry_on:
raise RateLimitException(
message='Too many requests, sleeping.',
period_remaining=sleep_period or int(
resp.headers.get('Retry-After')
or 0)) # This will be caught by @sleep_and_retry and retried
resp.raise_for_status()
return resp