def test_get_bucket_name(self):
bucket_name = 'test-bucket'
s3_key = '/some-folder/some-key.txt'
hosts = [
's3-ap-northeast-1.amazonaws.com',
's3-ap-northeast-2.amazonaws.com', 's3-ap-south-1.amazonaws.com',
's3-ap-southeast-1.amazonaws.com',
's3-ap-southeast-2.amazonaws.com', 's3-ca-central-1.amazonaws.com',
's3-eu-central-1.amazonaws.com', 's3-eu-west-1.amazonaws.com',
's3-eu-west-2.amazonaws.com', 's3-eu-west-3.amazonaws.com',
's3-external-1.amazonaws.com', 's3-sa-east-1.amazonaws.com',
's3-us-east-2.amazonaws.com', 's3-us-west-1.amazonaws.com',
's3-us-west-2.amazonaws.com', 's3.amazonaws.com',
's3.ap-northeast-1.amazonaws.com',
's3.ap-northeast-2.amazonaws.com', 's3.ap-south-1.amazonaws.com',
's3.ap-southeast-1.amazonaws.com',
's3.ap-southeast-2.amazonaws.com', 's3.ca-central-1.amazonaws.com',
's3.cn-north-1.amazonaws.com.cn',
's3.cn-northwest-1.amazonaws.com.cn',
's3.dualstack.ap-northeast-1.amazonaws.com',
's3.dualstack.ap-northeast-2.amazonaws.com',
's3.dualstack.ap-south-1.amazonaws.com',
's3.dualstack.ap-southeast-1.amazonaws.com',
's3.dualstack.ap-southeast-2.amazonaws.com',
's3.dualstack.ca-central-1.amazonaws.com',
's3.dualstack.eu-central-1.amazonaws.com',
's3.dualstack.eu-west-1.amazonaws.com',
's3.dualstack.eu-west-2.amazonaws.com',
's3.dualstack.eu-west-3.amazonaws.com',
's3.dualstack.sa-east-1.amazonaws.com',
's3.dualstack.us-east-1.amazonaws.com',
's3.dualstack.us-east-2.amazonaws.com',
's3.dualstack.us-west-1.amazonaws.com',
's3.dualstack.us-west-2.amazonaws.com',
's3.eu-central-1.amazonaws.com', 's3.eu-west-1.amazonaws.com',
's3.eu-west-2.amazonaws.com', 's3.eu-west-3.amazonaws.com',
's3.sa-east-1.amazonaws.com', 's3.us-east-1.amazonaws.com',
's3.us-east-2.amazonaws.com', 's3.us-west-1.amazonaws.com',
's3.us-west-2.amazonaws.com'
]
# test all available hosts with the bucket_name in the path
bucket_path = '/{}/{}'.format(bucket_name, s3_key)
for host in hosts:
headers = CaseInsensitiveDict({'Host': hosts[0]})
returned_bucket_name = s3_listener.get_bucket_name(
bucket_path, headers)
self.assertEqual(returned_bucket_name, bucket_name,
'Should match when bucket_name is in path')
# test all available hosts with the bucket_name in the host and the path is only the s3_key
for host in hosts:
headers = CaseInsensitiveDict(
{'Host': '{}.{}'.format(bucket_name, host)})
returned_bucket_name = s3_listener.get_bucket_name(s3_key, headers)
self.assertEqual(returned_bucket_name, bucket_name,
'Should match when bucket_name is in the host')
def requests_response(content, status_code=200, headers={}):
resp = RequestsResponse()
headers = CaseInsensitiveDict(dict(headers or {}))
if isinstance(content, dict):
content = json.dumps(content)
if not headers.get(HEADER_CONTENT_TYPE):
headers[HEADER_CONTENT_TYPE] = APPLICATION_JSON
resp._content = content
resp.status_code = int(status_code)
# Note: update headers (instead of assigning directly), to ensure we're using a case-insensitive dict
resp.headers.update(headers)
return resp
def test_business_limit():
headers = CaseInsensitiveDict({
"x-business-use-case-usage":
'{"112130216863063":[{"type":"pages","call_count":1,"total_cputime":1,"total_time":1,"estimated_time_to_regain_access":0}]}'
})
r = RateLimit()
r.set_limit(headers)
assert r.get_limit("112130216863063", "pages").call_count == 1
def test_app_limit():
headers = CaseInsensitiveDict({
"x-app-usage":
'{"call_count":91,"total_cputime":15,"total_time":12}'
})
r = RateLimit()
r.set_limit(headers)
assert r.get_limit().total_cputime == 15
assert r.get_limit().max_percent() == 91
assert r.get_max_percent() == 91
assert r.get_sleep_seconds() == 0
mapping = [PercentSecond(10, 1), PercentSecond(20, 2)]
assert r.get_sleep_seconds(sleep_data=mapping) == 600
headers = CaseInsensitiveDict({
"x-app-usage":
'{"call_count":16,"total_cputime":15,"total_time":12}'
})
r.set_limit(headers)
assert r.get_sleep_seconds(sleep_data=mapping) == 2
def parse_headers(headers: CaseInsensitiveDict, key: str) -> Optional[dict]:
"""
Get rate limit information from header for key.
:param headers: Response headers
:param key: rate limit key
:return:
"""
usage = headers.get(key)
if usage:
try:
data = json.loads(usage)
return data
except JSONDecodeError as ex:
logger.error(
f"Exception in parse {key} data error. Usage is: {usage}. errors: {ex}"
)
return None
return None
def get_content(resp: HTTPResponse, url: str):
# logic taken from HTTPAdapter.build_response (requests.adapters)
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
response.url = url
return response.content
def notify_acquirablefile(api_url: str, acquirable_id: str, datetime: datetime,
key: str, query: str) -> Response:
dtf = "%Y-%m-%dT%H:%M:%SZ"
url_parse = urlparse(api_url)
url = urlunparse(
(url_parse.scheme, url_parse.netloc, "acquirablefiles", "", query, ""))
data = {
"datetime": datetime.strftime(dtf),
"file": key,
"acquirable_id": acquirable_id,
}
headers = CaseInsensitiveDict()
headers["content-type"] = "application/json; charset=utf-8"
headers["accept"] = "application/json"
with requests.Session() as s:
s.headers = headers
return s.post(url=url, json=data, headers=headers)
def __init__(self):
self._content = False
self.status_code = None
self.multi_value_headers = CaseInsensitiveDict()
self.headers = CaseInsensitiveDict()
def test_parse_headers():
headers = CaseInsensitiveDict({"key": "None"})
assert RateLimit.parse_headers(headers, key="key") is None
assert RateLimit.parse_headers(headers, key="key-null") is None
