def update_with_304_response(cached_response: Response,
new_response_headers: Headers) -> Response:
"""On a 304 we will get a new set of headers that we want to
update our cached value with, assuming we have one.
This should only ever be called when we've sent an ETag and
gotten a 304 as the response.
"""
updated_response = copy(cached_response)
# Lets update our headers with the headers from the new request:
# http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
#
# The server isn't supposed to send headers that would make
# the cached body invalid. But... just in case, we'll be sure
# to strip out ones we know that might be problematic due to
# typical assumptions.
excluded_headers = ["content-length"]
updated_response.headers.update(
dict((k, v) for k, v in new_response_headers.items()
# TODO: Don't think .lower() is necessary
if k.lower() not in excluded_headers))
# we want a 200 b/c we have content via the cache
updated_response.status_code = 200
return updated_response
def request(
self,
method: typing.Union[str, typing.Callable],
url: typing.Optional[typing.Union[str, typing.Pattern]] = None,
status_code: typing.Optional[int] = None,
content: typing.Optional[ContentDataTypes] = None,
content_type: typing.Optional[str] = None,
headers: typing.Optional[HeaderTypes] = None,
pass_through: bool = False,
alias: typing.Optional[str] = None,
) -> RequestPattern:
"""
Adds a request pattern with given mocked response details.
"""
headers = Headers(headers or {})
if content_type:
headers["Content-Type"] = content_type
response = ResponseTemplate(status_code, headers, content)
pattern = RequestPattern(
method,
url,
response,
pass_through=pass_through,
alias=alias,
base_url=self._base_url,
)
self.add(pattern)
return pattern
def _generate_authorization_headers(self, request: Request) -> Headers:
date = self._get_formatted_date()
digest = self._compute_digest(request)
string = self._compose_string(request, date, digest)
signature = self._sign_string(string)
authorization_header = self._compose_authorization_header(signature)
headers = {'Host': request.url.host, 'Date': date, 'Digest': digest, 'Authorization': authorization_header}
return Headers(headers)
def test_last_modified_is_not_used_when_cache_control_present(self):
headers = {
"Date": self.now,
"Last-Modified": self.week_ago,
"Cache-Control": "private",
}
assert self.heuristic.update_headers(Headers(headers), 200) == {}
def __init__(self):
self.client = AsyncClient()
self.client.headers = Headers({
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 "
"Safari/537.36"
})
self.disabled_until: Optional[datetime.datetime] = None
def parse_cache_control_directives(headers: Headers):
known_directives = {
# https://tools.ietf.org/html/rfc7234#section-5.2
"max-age": (int, True),
"max-stale": (int, False),
"min-fresh": (int, True),
"no-cache": (None, False),
"no-store": (None, False),
"no-transform": (None, False),
"only-if-cached": (None, False),
"must-revalidate": (None, False),
"public": (None, False),
"private": (None, False),
"proxy-revalidate": (None, False),
"s-maxage": (int, True),
}
cc_headers = headers.get("cache-control", "")
retval = {} # type: ignore
for cc_directive in cc_headers.split(","):
if not cc_directive.strip():
continue
parts = cc_directive.split("=", 1)
directive = parts[0].strip()
try:
typ, required = known_directives[directive]
except KeyError:
logger.debug("Ignoring unknown cache-control directive: %s",
directive)
continue
if not typ or not required:
retval[directive] = None
if typ:
try:
retval[directive] = typ(parts[1].strip())
except IndexError:
if required:
logger.debug(
"Missing value for cache-control "
"directive: %s",
directive,
)
except ValueError:
logger.debug(
"Invalid value for cache-control directive "
"%s, must be %s",
directive,
typ.__name__,
)
return retval
async def request(
self, method: str, path: str, headers: typing.Union[Headers, dict],
result_body: io.BytesIO, body: typing.Union[io.BytesIO, bytes] = None) -> Headers:
"""Returns the headers from the server after making the given request
at the given path using the given headers. The body of the response is
written to the result body.
Arguments:
- `method (str)`: The HTTP verb to perform, uppercased, e.g., 'GET'
- `path (str)`: The path within the host, e.g., /api/foo
- `headers (Headers, dict)`: The headers to send with case-insensitive
keys.
- `result_body (io.BaseIO)`: The IO to write the body from the server;
nothing is written if the server does not provide a body.
- `body (bytes, io.BaseIO, None)`: The bytes to send to the server in
the body, specified either as bytes (which are wrapped with BytesIO)
or just an bytes-based io object.
Returns:
- `headers (Headers)`: The returned headers and trailers from the server.
"""
if not self.opened:
await self.open()
if not isinstance(headers, Headers):
headers = Headers(headers)
itr = self.rconn.request(method, self.host, path, headers.multi_items(), body)
raw_headers = await itr.__anext__()
response_headers = Headers(raw_headers)
chunk = await itr.__anext__()
while chunk is not None:
result_body.write(chunk)
chunk = await itr.__anext__()
raw_trailers = await itr.__anext__()
response_trailers = Headers(raw_trailers)
for key, val in response_trailers.items():
response_headers[key] = val
return response_headers
def test_last_modified_is_used_when_cache_control_public(self):
headers = {
"Date": self.now,
"Last-Modified": self.week_ago,
"Cache-Control": "public",
}
modified = self.heuristic.update_headers(Headers(headers), 200)
assert ["expires"] == list(modified.keys())
assert datetime(*parsedate(modified["expires"])
[:6]) > datetime.now() # type: ignore
def __init__(self, token: str):
self.client = AsyncClient(timeout=10)
self.client.headers = Headers({
"Accept":
"application/json",
"Authorization":
f"Bearer {token}",
"User-Agent":
"holo observatory bot/1.0.0"
})
def check_vary_headers(request_headers: Headers,
cached_vary_data: dict) -> bool:
"""Verify our vary headers match."""
# Ensure that the Vary headers for the cached response match our
# request
# TODO: this should not be here, no reason for request headers to be so deep in deserialization.
for header, value in cached_vary_data.items():
if request_headers.get(header, None) != value:
return False
return True
def __init__(
self,
status_code: typing.Optional[int] = None,
headers: typing.Optional[HeaderTypes] = None,
content: typing.Optional[ContentDataTypes] = None,
context: typing.Optional[Kwargs] = None,
) -> None:
self.http_version = 1.1
self.status_code = status_code or 200
self.context = context if context is not None else {}
self._headers = Headers(headers or {})
self._content = content if content is not None else b""
async def request(
self,
method: bytes,
url: typing.Tuple[bytes, bytes, int, bytes],
headers: typing.List[typing.Tuple[bytes, bytes]],
stream: ContentStream,
timeout: typing.Dict[str, typing.Optional[float]] = None,
) -> typing.Tuple[bytes, int, bytes, typing.List[typing.Tuple[
bytes, bytes]], ContentStream]:
headers = Headers()
body = JSONStream({"ok": "ok"})
return b"HTTP/1.1", 200, b"OK", headers, body
async def process_headers(headers: Headers) -> Dict:
"""Filter out unwanted headers and return as a dictionary."""
headers = dict(headers)
header_keys = (
"user-agent",
"referer",
"accept-encoding",
"accept-language",
"x-real-ip",
"x-forwarded-for",
)
return {k: headers.get(k) for k in header_keys}
def get_vary_headers(request_headers: Headers, response: Response):
"""Get vary headers values for persisting in the cache for later checking"""
vary = {}
# Construct our vary headers
if "vary" in response.headers:
varied_headers = response.headers["vary"].split(",")
for header in varied_headers:
header = header.strip()
header_value = request_headers.get(header, None)
vary[header] = header_value
return vary
async def create_session(session: AsyncClient, token: str):
data = {TIMESTAMP: datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
resp = await session.post(constants.BASE_URL +
"/api/portal/CreateSession2",
headers=Headers({
**bearer_header(token),
**{
"Content-Type": "application/json"
}
}),
json=data)
return resp.json()['BrowserSessionId']
def __init__(self) -> None:
"""Define DOI credentials and config.
:param state: can be publish, register, hide or draft.
"""
self.rems_api = environ.get("REMS_API", "")
self.rems_user = environ.get("REMS_USER", "")
self.rems_key = environ.get("REMS_KEY", "")
self.config = CONFIG_INFO["rems"]
self.headers = Headers({
"Content-Type": "application/json",
"Accept": "application/json",
"x-rems-api-key": self.rems_key,
"x-rems-user-id": self.rems_user,
})
def __init__(self,
key_id: str,
rsa_key: RSAPrivateKey,
staging: bool = False,
**kwargs) -> None:
auth = SatispayAuth(key_id, rsa_key)
headers = kwargs.get('headers', Headers())
headers.update({'Accept': 'application/json'})
if staging:
base_url = URL('https://staging.authservices.satispay.com')
else:
base_url = URL('https://authservices.satispay.com')
super().__init__(auth=auth,
headers=headers,
base_url=base_url,
**kwargs)
def _inject_headers(self, headers: dict):
# need to coerce to str
headers = {k: str(v) for k, v in headers.items()}
headers.update(
{"date": get_utc_datetime().strftime("%a, %d %b %y %T %z")})
# inject user information to request headers
user = context_user()
headers.update({
settings.INTERNAL_REQUEST_USER_HEADER.lower():
to_header_value(user)
})
# inject correlation_id
correlation_id = context_correlation_id()
headers.update(
{settings.REQUEST_ID_HEADER_FIELD.lower(): correlation_id})
return Headers(headers)
async def create_draft_doi(self, user: str,
inbox_path: str) -> Union[Dict, None]:
"""Create an auto-generated draft DOI.
We are using just the prefix for the DOI so that it will be autogenerated.
"""
dataset = generate_dataset_id(user, inbox_path, self.ns_url)
suffix = shortuuid.uuid(name=dataset)[:10]
doi_suffix = f"{suffix[:4]}-{suffix[4:]}"
headers = Headers({"Content-Type": "application/json"})
draft_doi_payload = {
"data": {
"type": "dois",
"attributes": {
"doi": f"{self.doi_prefix}/{doi_suffix}"
}
}
}
async with AsyncClient() as client:
response = await client.post(self.doi_api,
auth=(self.doi_user, self.doi_key),
json=draft_doi_payload,
headers=headers)
doi_data = None
if response.status_code == 201:
draft_resp = response.json()
_doi = draft_resp["data"]["attributes"]["doi"]
_suffix = draft_resp["data"]["attributes"]["suffix"]
LOG.debug(f"DOI draft created and response was: {draft_resp}")
LOG.info(f"DOI draft created with doi: {_doi}.")
doi_data = {
"suffix": _suffix,
"fullDOI": _doi,
"dataset": f"{self.ns_url}/{_suffix.lower()}",
}
else:
LOG.error(
f"DOI API create draft request failed with code: {response.status_code}"
)
doi_data = self._check_errors(response, doi_suffix)
return doi_data
def _fetch_stored_request(self, method, path):
cache_key = self._cache_key(method, path)
with self.cache.transact():
cached_at = self.cache.get(cache_key + b'-cached-at')
if cached_at is None:
return (None, None, None)
header = self.cache.get(cache_key + b'-header')
if header is None:
self.cache.delete(cache_key + b'-cached-at')
return (None, None, None)
body = self.cache.get(cache_key + b'-body')
if body is None:
self.cache.delete(cache_key + b'-cached-at')
self.cache.delete(cache_key + b'-header')
return (None, None, None)
return (cached_at, Headers(header), body)
def failure_response():
return Response(
status_code=404,
request=Request(url="https://leomenezessz.github.io/pyppium/",
method="get"),
headers=Headers({
"cache-control": "no-cache",
"content-type": "application/json; charset=utf-8",
"date": "Fri, 04 Sep 2020 06:35:18 GMT",
"server": "nginx",
"status": "404 Not Found",
"vary": "Origin",
"x-powered-by": "Phusion Passenger",
"x-request-id": "632eeff4-a6f4-4016-82aa-75d10805f149",
"x-runtime": "0.093190",
"x-ua-compatible": "IE=Edge,chrome=1",
"content-length": "36",
"connection": "keep-alive",
}),
)
def test_production(self, key_id, rsa_key,
create_payment_production_signature):
route = respx.post(
'https://authservices.satispay.com/g_business/v1/payments')
body_params = {
'callback_url': 'https://test.test?payment_id={uuid}',
'expiration_date': '2019-03-18T16:10:24.000Z',
'external_code': 'test_code',
'metadata': {
'metadata': 'test'
}
}
headers = Headers({'Idempotency-Key': 'test_idempotency_key'})
satispaython.create_payment(key_id, rsa_key, 100, 'EUR', body_params,
headers)
assert route.called
assert route.call_count == 1
request = route.calls.last.request
assert request.method == 'POST'
assert json.loads(request.content.decode()) == {
'flow': 'MATCH_CODE',
'amount_unit': 100,
'currency': 'EUR',
'callback_url': 'https://test.test?payment_id={uuid}',
'expiration_date': '2019-03-18T16:10:24.000Z',
'external_code': 'test_code',
'metadata': {
'metadata': 'test'
}
}
assert request.headers['Idempotency-Key'] == 'test_idempotency_key'
assert request.headers['Accept'] == 'application/json'
assert request.headers['Content-Type'] == 'application/json'
assert request.headers['Host'] == 'authservices.satispay.com'
assert request.headers['Date'] == 'Mon, 18 Mar 2019 15:10:24 +0000'
assert request.headers[
'Digest'] == 'SHA-256=dOjZtX6Has9wFZQDmriLhIfThHD11nuxFZNIjp7FwR0='
assert request.headers['Authorization'] == f'Signature keyId="{key_id}", ' \
f'algorithm="rsa-sha256", ' \
f'headers="(request-target) host date digest", ' \
f'signature="{create_payment_production_signature}"'
def __init__(self, **kwargs):
data = {
'body': {},
'headers': {},
'path': {},
'query': {},
}
data['header'] = data['headers']
skip_validation = kwargs.pop('esi_skip_validation', False)
errors = {}
for param in self.params:
value = kwargs.pop(param.safe_name)
if value == param.default:
continue
try:
param.validate(value)
except ValidationError as e:
e.update_error_dict(errors, param.safe_name)
data[param.part][param.name] = value
if errors and not skip_validation:
raise ValidationError(errors)
cls = self.__class__
body = None
if data['body']:
body = list(data['body'].values())[0]
self.method = cls.method.upper()
self.url = URL(self.path.format(**data['path']),
allow_relative=True,
params=data['query'])
self.headers = Headers(data['header'])
self.stream = encode(body, None, None)
self.timer = ElapsedTimer()
self.prepare()
# Clear out headers we don't need (These will be set by the session)
for key in ["User-Agent"]:
if key in self.headers:
del self.headers[key]
async def app(scope, receive, send):
request = Request(scope, receive)
user_key = request.headers.get('api_key')
if not USERS_CACHE.get(user_key):
USERS_CACHE[user_key] = await auth(user_key)
headers = (tuple(["session_id",
USERS_CACHE[user_key]]), *request.headers.items())
print(headers)
async with httpx.AsyncClient() as client:
url = TARGET_SERVER + request.url.path
r = await client.request(request.method,
url,
headers=Headers(headers),
data=await request.body())
response = UJSONResponse(r.json())
#response = Response(content=r.content, headers=r.headers, status_code=r.status_code, media_type="application/json")
result = await response(scope, receive, send)
return result
async def create_payment(self,
amount_unit: int,
currency: str,
body_params: Optional[dict] = None,
headers: Optional[Headers] = None) -> Response:
target = URL('/g_business/v1/payments')
try:
headers.update({'Content-Type': 'application/json'})
except AttributeError:
headers = Headers({'Content-Type': 'application/json'})
try:
body_params.update({
'flow': 'MATCH_CODE',
'amount_unit': amount_unit,
'currency': currency
})
except AttributeError:
body_params = {
'flow': 'MATCH_CODE',
'amount_unit': amount_unit,
'currency': currency
}
return await self.post(target, json=body_params, headers=headers)
async def request(self,
method,
path,
params=None,
headers=None,
data=None,
json=None) -> ResponseData:
"""Makes a request using the given method to the given path.
Arguments:
- `method (str)`: The HTTP verb (uppercased)
- `path (str)`: The path to make the request to, must just be the
path and must be prefixed with a forward slash (e.g., /api)
- `params (dict, None)`: If specified the keys and values must be
strings and they will be injected into the query arguments.
- `headers (dict, Headers, None)`: If specified these headers will be
sent alongside the request.
- `data (io.BytesIO, bytes, str, None)`: Sent in the body of the
request. If specified as a BytesIO like object the data will be
streamed across the connection, which is very suitable for uploading
files. If specified as a string it is encoding using utf-8. Unlike
requests we do not support `application/x-www-form-urlencoded` style
requests natively.
- `json (any)`: If not `None` then `data` is ignored, and `json`
will be serialized as json to text and then treated as if it were
the value of `data`.
Returns:
- `response (ResponseData)`: The result from the server, fully loaded
in memory.
"""
if params is not None:
base_path, _, current_args = path.partition('?')
query_args = dict(
arg.split('=') for arg in current_args.split('&') if arg)
query_args = {**query_args, **params}
path = base_path + '?' + urlencode(query_args)
content_type_hint = None
if json is not None:
data = jsonlib.dumps(json).encode('utf-8')
content_type_hint = 'application/json; charset=utf-8'
if isinstance(data, str) and content_type_hint is None:
data = data.encode('utf-8')
content_type_hint = 'text/plain; charset=utf-8'
if headers is None:
headers = {}
if content_type_hint is not None and headers.get(
'content-type') is None:
headers['content-type'] = content_type_hint
headers = {**self.default_headers, **headers}
result_body = io.BytesIO()
response_headers = await self.h2conn.request(method, path, headers,
result_body, data)
return ResponseData(self.h2conn.host, path, method, Headers(headers),
response_headers, result_body.getvalue())
def test_expiry_is_no_more_than_twenty_four_hours(self):
headers = {"Date": self.now, "Last-Modified": self.year_ago}
modified = self.heuristic.update_headers(Headers(headers), 200)
assert ["expires"] == list(modified.keys())
assert self.day_ahead == modified["expires"]
def test_last_modified_is_not_used_when_status_is_unknown(self):
headers = {"Date": self.now, "Last-Modified": self.week_ago}
status = 299
assert self.heuristic.update_headers(Headers(headers), status) == {}
def test_last_modified_is_used(self):
headers = {"Date": self.now, "Last-Modified": self.week_ago}
modified = self.heuristic.update_headers(Headers(headers), 200)
assert ["expires"] == list(modified.keys())
assert datetime(*parsedate(modified["expires"])
[:6]) > datetime.now() # type: ignore
def test_expires_is_not_replaced_when_present(self):
headers = {"Expires": self.day_ahead}
assert self.heuristic.update_headers(Headers(headers), 200) == {}
