async def check_response(response: aiohttp.ClientResponse, ) -> None:
"""
Check for specialised K8s errors, and raise with extended information.
"""
if response.status >= 400:
# Read the response's body before it is closed by raise_for_status().
payload: Optional[RawStatus]
try:
payload = await response.json()
except (json.JSONDecodeError, aiohttp.ContentTypeError,
aiohttp.ClientConnectionError):
payload = None
# Better be safe: who knows which sensitive information can be dumped unless kind==Status.
if not isinstance(
payload,
collections.abc.Mapping) or payload.get('kind') != 'Status':
payload = None
cls = (APIUnauthorizedError if response.status == 401 else
APIForbiddenError if response.status == 403 else
APINotFoundError if response.status == 404 else APIError)
# Raise the framework-specific error while keeping the original error in scope.
# This call also closes the response's body, so it cannot be read afterwards.
try:
response.raise_for_status()
except aiohttp.ClientResponseError as e:
raise cls(payload, status=response.status) from e
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None:
"""Check resp for status and if error log additional info."""
try:
resp.raise_for_status()
except aiohttp.ClientResponseError:
log.exception('Error body: %s', await resp.text())
raise
async def _extract_response_data(
resp: aiohttp.ClientResponse,
) -> list | dict:
"""Checks the correctness of the response.
Args:
resp: Response instance.
Returns:
resp['data'] field if expected data.
Raises:
aiohttp.ClientResponseError: if response status >= 400.
aiohttp.ClientPayloadError: if failure result of the request.
#TODO: refactor for process HTTP statuses!
"""
resp.raise_for_status()
json = await resp.json()
if json.get("success"):
_LOG.debug(
"Successfully response",
extra={
"url": resp.url,
},
)
return json.get("data", {})
raise aiohttp.ClientPayloadError(f"Failure server result: {resp.url} {json}")
async def test_iter_content_generator():
"""Test CRLF -> LF newline conversion."""
async def mock_iter_content(n):
for chunk in [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']:
yield chunk
response = ClientResponse(
'get',
ST_URL,
request_info=Mock(),
writer=Mock(),
continue100=None,
timer=TimerNoop(),
traces=[],
loop=Mock(),
session=Mock(),
)
response._headers = {'Content-Type': 'application/json;charset=utf-8'}
with patch.object(response, 'content',
Mock(iter_chunked=mock_iter_content)):
result = [
line async for line in _iter_content_generator(response=response,
decode_unicode=True)
]
assert result == ['1\n2\n', '3', '\n4', '\n5']
result = [
line
async for line in _iter_content_generator(response=response,
decode_unicode=False)
]
assert result == [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None:
"""Check resp for status and if error log additional info."""
# Copied from aiohttp's raise_for_status() -- since it releases the
# response payload, we need to grab the `resp.text` first to help users
# debug.
#
# Useability/performance notes:
# * grabbing the response can be slow for large files, only do it as
# needed
# * we can't know in advance what encoding the files might have unless
# we're certain in advance that the result is an error payload from
# Google (otherwise, it could be a binary blob from GCS, for example)
# * sometimes, errors are expected, so we should try to avoid polluting
# logs in that case
#
# https://github.com/aio-libs/aiohttp/blob/
# 385b03ef21415d062886e1caab74eb5b93fdb887/aiohttp/
# client_reqrep.py#L892-L902
if resp.status >= 400:
assert resp.reason is not None
# Google's error messages are useful, pass 'em through
body = await resp.text(errors='replace')
resp.release()
raise aiohttp.ClientResponseError(resp.request_info,
resp.history,
status=resp.status,
message=f'{resp.reason}: {body}',
headers=resp.headers)
async def from_client_response(cls,
client_response: ClientResponse,
expires: datetime = None):
"""Convert a ClientResponse into a CachedReponse"""
if isinstance(client_response, cls):
return client_response
# Copy most attributes over as is
copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS
response = cls(**{k: getattr(client_response, k) for k in copy_attrs})
# Read response content, and reset StreamReader on original response
if not client_response._released:
await client_response.read()
response._body = client_response._body
client_response.content = CachedStreamReader(client_response._body)
# Set remaining attributes individually
response.expires = expires
response.links = client_response.links
response.real_url = client_response.request_info.real_url
# The encoding may be unset even if the response has been read, and
# get_encoding() does not handle certain edge cases like an empty response body
try:
response.encoding = client_response.get_encoding()
except (RuntimeError, TypeError):
pass
if client_response.history:
response.history = (*[
await cls.from_client_response(r)
for r in client_response.history
], )
return response
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None:
"""Check resp for status and if error log additional info."""
body = await resp.text(errors='replace')
try:
resp.raise_for_status()
except aiohttp.ClientResponseError:
log.exception('got http error response: %s', body)
raise
def _check_status(r: aiohttp.ClientResponse, expected_status):
r.raise_for_status()
if r.status != expected_status:
raise DSWCommunicationError(
reason='Unexpected response status',
message=f'Server responded with unexpected HTTP status {r.status}: '
f'{r.reason} (expecting {expected_status})'
)
async def parse_api_response(resp: ClientResponse):
"""
Checks for response status, parses the json and checks for API error
"""
resp.raise_for_status()
resp = json.loads(await resp.text())
if "error" in resp:
raise APIError(resp["error"])
return resp
async def _raise_for_status(self, resp: aiohttp.ClientResponse) -> None:
if resp.status >= 400:
explanation = await resp.text()
resp.release()
logger.debug("Server responded:\n%s\n%s", explanation, "=" * 40)
raise SPARQLRequestFailed(resp.request_info,
resp.history,
code=resp.status,
message=resp.reason,
explanation=explanation)
def _raise_error_from_response(
self,
resp: aiohttp.ClientResponse,
) -> None:
if resp.status in [401, 403]:
abort(403, "request rejected by backend")
if resp.status == 400:
abort(500, "request rejected by backend")
if not 200 <= resp.status < 300:
resp.raise_for_status()
def make_response(loop,
method,
url,
data=None,
content_type='text/plain',
charset='utf-8'):
if LooseVersion(aiohttp_version) >= LooseVersion('3.3.0'):
response = ClientResponse(method,
URL(url),
writer=mock.Mock(),
continue100=None,
timer=None,
request_info=mock.Mock(),
traces=[],
loop=loop,
session=mock.Mock())
elif LooseVersion(aiohttp_version) >= LooseVersion('3.1.0'):
response = ClientResponse(method,
URL(url),
writer=mock.Mock(),
continue100=None,
timer=None,
request_info=mock.Mock(),
auto_decompress=True,
traces=[],
loop=loop,
session=mock.Mock())
else:
response = ClientResponse(method,
URL(url),
writer=mock.Mock(),
continue100=None,
timer=None,
request_info=mock.Mock(),
auto_decompress=True)
response._post_init(loop, mock.Mock())
def side_effect(*args, **kwargs):
fut = loop.create_future()
fut.set_result(str(data).encode(charset))
return fut
if LooseVersion(aiohttp_version) >= LooseVersion('3.3.0'):
response._headers = {
'Content-Type': '%s; charset=%s' % (content_type, charset)
}
else:
response.headers = {
'Content-Type': '%s; charset=%s' % (content_type, charset)
}
content = response.content = mock.Mock()
if data:
content.read.side_effect = side_effect
return response
def raise_not_200(response: aiohttp.ClientResponse) -> None:
if response.status != 200:
assert response.reason is not None
response.release()
raise aiohttp.ClientResponseError(
response.request_info,
response.history,
status=response.status,
message=response.reason,
headers=response.headers,
)
def aiohttp_raise_for_status(response: aiohttp.ClientResponse):
# workaround aiohttp bug, can remove after fixed in aiohttp
# issue: https://github.com/aio-libs/aiohttp/issues/3906
if response.status >= 400:
response.release()
raise aiohttp.ClientResponseError(
response.request_info,
response.history,
status=response.status,
message=response.reason,
headers=response.headers,
)
def raise_for_status(
resp: aiohttp.ClientResponse) -> aiohttp.ClientResponse:
"""Raise exceptions on failure methods."""
try:
resp.raise_for_status()
except ClientResponseError as err:
if err.status == HTTP_UNAUTHORIZED:
raise AuthException(
f"Unable to authenticate with API: {err}") from err
raise ApiException(f"Error from API: {err}") from err
except ClientError as err:
raise ApiException(f"Error from API: {err}") from err
return resp
async def check_error(resp: ClientResponse) -> Any:
try:
resp_data = await resp.json()
except ContentTypeError:
resp.raise_for_status()
return
except json.JSONDecodeError:
resp.raise_for_status()
raise
if not isinstance(resp_data, dict) or "errors" not in resp_data:
resp.raise_for_status()
return resp_data
try:
error = resp_data["errors"][0]
code = error["code"]
message = error["message"]
except (KeyError, IndexError):
resp.raise_for_status()
raise
if code == 88:
raise RateLimitError(code, message, resp.headers)
elif code == 32:
raise TwitterAuthError(code, message)
raise TwitterError(code, message)
async def handle_error_response(self,
response: aiohttp.ClientResponse) -> None:
if response.status >= 400:
try:
body = await response.json()
except aiohttp.ContentTypeError:
body = await response.text()
response.release()
raise HttpResponseException(response.request_info,
response.history,
status=response.status,
reason=response.reason,
headers=response.headers,
body=body)
def make_http_response(
status=200, reason="OK", method="GET", url="/", headers=None, content=None
):
"""Return a minimal ClientResponse with fields used in tests."""
url = URL(url)
headers = CIMultiDict(headers or {})
request_info = RequestInfo(url=url, method=method, headers=headers)
response = ClientResponse(
method,
url,
writer=None,
continue100=None,
timer=None,
request_info=request_info,
traces=(),
loop=get_event_loop(),
session=None,
)
response.status = status
response.reason = reason
response._headers = headers
if isinstance(content, io.IOBase):
response.content = FakeStreamReader(content)
elif content is not None:
response.content = FakeStreamReader(
io.BytesIO(json_dumps(content).encode("utf8"))
)
response.headers["Content-Type"] = "application/json"
return response
def _patch_get(
self,
client_response: aiohttp.ClientResponse) -> aiohttp.ClientResponse:
""" Wrap aiohttp.ClientResponse text and json coros in run_until_complete. Monkeypatch text and json with wrappers."""
# May iter through methods and wrap all coro's in the future
# however that may not work if a non-coro returns a async context manager for example
text = self._wrap_coro_in_callable(client_response.text)
json = self._wrap_coro_in_callable(client_response.json)
# resign signatures
text = forge.copy(aiohttp.ClientResponse.text, exclude="self")(text)
json = forge.copy(aiohttp.ClientResponse.json, exclude="self")(json)
client_response.text = text
client_response.json = json
return client_response
async def test_authenticate():
st = AsyncSpaceTrackClient('identity', 'wrongpassword')
loop = asyncio.get_event_loop()
response = ClientResponse(
'post', URL('https://www.space-track.org/ajaxauth/login'))
# aiohttp 2.2 uses session
try:
response._post_init(loop)
except TypeError:
response._post_init(loop, st.session)
response.status = 200
response.json = Mock()
async def mock_post(url, data):
response.json.return_value = asyncio.Future()
if data['password'] == 'wrongpassword':
response.json.return_value.set_result({'Login': '******'})
elif data['password'] == 'unknownresponse':
# Space-Track doesn't respond like this, but make sure anything
# other than {'Login': '******'} doesn't raise AuthenticationError
response.json.return_value.set_result({'Login': '******'})
else:
response.json.return_value.set_result('')
return response
with st, patch.object(st.session, 'post', mock_post):
with pytest.raises(AuthenticationError):
await st.authenticate()
assert response.json.call_count == 1
st.password = '******'
await st.authenticate()
# This shouldn't make a HTTP request since we're already authenticated.
await st.authenticate()
assert response.json.call_count == 2
st = AsyncSpaceTrackClient('identity', 'unknownresponse')
with st, patch.object(st.session, 'post', mock_post):
await st.authenticate()
response.close()
async def wrap_async(response: ClientResponse) -> Response:
"""Build a ``requests`` response from a ``aiohttp`` response.
A ``requests.Response`` instance is built to provide synchronous
access to the original response's data. Note that the returned
response does not have proper data for :attr:``elapsed`` or
:attr:``request``.
The response will be consumed if it has not already.
"""
# Ensure the response data is read so that the wrapped response
# does not require any async methods.
await response.read()
wrapped = Response()
wrapped._content = response._body # type: ignore
wrapped._content_consumed = True # type: ignore
wrapped.status_code = response.status
wrapped.headers = CaseInsensitiveDict(response.headers)
wrapped.url = str(response.url) # `aiohttp` uses a `URL` object.
wrapped.encoding = response.get_encoding()
wrapped.history = [await wrap_async(rsp) for rsp in response.history]
wrapped.reason = response.reason or ""
wrapped.cookies = cookiejar_from_dict(response.cookies)
return wrapped
async def raise_for_status(response: ClientResponse):
if response.ok:
return response
try:
result = await response.json()
message = result["message"]
detail = result["detail"]
message = f"{message}: {detail}"
except Exception:
message = await response.text() or response.reason
log.exception(message)
else:
log.error(message)
response.raise_for_status()
async def parse_response(response: ClientResponse, schema: dict) -> Any:
"""
Validate and parse the BMA answer
:param response: Response of aiohttp request
:param schema: The expected response structure
:return: the json data
"""
try:
data = await response.json()
response.close()
if schema is not None:
jsonschema.validate(data, schema)
return data
except (TypeError, json.decoder.JSONDecodeError) as e:
raise jsonschema.ValidationError("Could not parse json : {0}".format(str(e)))
async def _check_response(result: aiohttp.ClientResponse, url, method) -> Dict:
"""
Check url response, and raise exceptions
:param result: http response
:param url: url to show in the log
:param method: 'post', 'get'
:return: json decoded result
"""
if result.status != 200:
raise ServerException(f'{url} {method} failed, status code: {result.status}')
result = await result.json()
if not isinstance(result, dict):
return result
status_code = result.get('code')
if method == 'post':
if status_code is None:
raise ServerException('Empty response')
if status_code == 0: # normal
return result
elif method == 'get':
if status_code is None or status_code == 0:
return result
if status_code in error_code:
raise error_code[status_code]()
else:
raise MiraiException('HTTP API updated, please upgrade python-mirai-core')
async def raise_for_status(response: ClientResponse):
if response.ok:
return response
try:
result = await response.json()
except Exception:
message = response.reason
log.exception(message)
else:
message = result["message"]
detail = result.get("detail")
message = f"{message}" + f": {detail}" if detail else ""
log.error(message)
response.raise_for_status()
async def from_client_response(cls,
client_response: ClientResponse,
expires: datetime = None):
"""Convert a ClientResponse into a CachedReponse"""
# Response may not have been read yet, if fetched by something other than CachedSession
if not client_response._released:
await client_response.read()
# Copy most attributes over as is
copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS
response = cls(**{k: getattr(client_response, k) for k in copy_attrs})
# Set some remaining attributes individually
response._body = client_response._body
response._links = [(k, _to_str_tuples(v))
for k, v in client_response.links.items()]
response.expires = expires
response.real_url = client_response.request_info.real_url
# The encoding may be unset even if the response has been read
try:
response.encoding = client_response.get_encoding()
except RuntimeError:
pass
response.url = str(client_response.url)
if client_response.history:
response.history = (*[
await cls.from_client_response(r)
for r in client_response.history
], )
return response
async def raise_response_error(resp: aiohttp.ClientResponse,
reason: str):
# We raise a TransportServerError if the status code is 400 or higher
# We raise a TransportProtocolError in the other cases
try:
# Raise a ClientResponseError if response status is 400 or higher
resp.raise_for_status()
except ClientResponseError as e:
raise TransportServerError(str(e), e.status) from e
result_text = await resp.text()
raise TransportProtocolError(
f"Server did not return a GraphQL result: "
f"{reason}: "
f"{result_text}")
async def __norm_callback(
response: aiohttp.ClientResponse,
decode: bool = False,
max_size: Optional[int] = None,
intended_content_type: Optional[str] = None) -> Optional[AnyStr]:
content_type = response.headers.get('Content-Type')
if not intended_content_type or not content_type or content_type.startswith(
intended_content_type):
body: Optional[bytes] = None
if max_size is None:
body = await response.read()
elif max_size > 0:
body = await response.content.read(max_size)
if decode and body:
xml_header = body.split(b'\n', 1)[0]
if xml_header.startswith(
b'<?xml'
) and b'?>' in xml_header and b'encoding' in xml_header:
try:
encoding = BeautifulSoup(xml_header,
'lxml-xml').original_encoding
return body.decode(encoding=encoding, errors='replace')
except (LookupError, RuntimeError):
pass
try:
encoding = response.get_encoding()
return body.decode(encoding=encoding, errors='replace')
except (LookupError, RuntimeError):
return body.decode(encoding='utf-8', errors='replace')
return body
return None
def link_extractor(response: ClientResponse, url_filter: URLFilter,
defrag: bool) -> (str, List[str]):
html = response._body.decode('utf-8', errors='ignore')
req_url = response._url
dom = lh.fromstring(html)
response.__setattr__('dom', dom)
response.__setattr__('html', html)
found_urls = []
for href in dom.xpath('//a/@href'):
url = urljoin(str(req_url), href)
if defrag:
url = urldefrag(url)[0]
netloc = urlparse(url).netloc
can_crawl = url_filter.can_crawl(netloc, url)
if can_crawl and valid_url(url):
found_urls.append(url)
return response, found_urls
async def parse_response(response: ClientResponse, schema: dict) -> Any:
"""
Validate and parse the BMA answer
:param response: Response of aiohttp request
:param schema: The expected response structure
:return: the json data
"""
try:
data = await response.json()
response.close()
if schema is not None:
jsonschema.validate(data, schema)
return data
except (TypeError, json.decoder.JSONDecodeError) as e:
raise jsonschema.ValidationError("Could not parse json : {0}".format(
str(e))) from e
def _handle_response(raw_response: aiohttp.ClientResponse) -> List[dict]:
"""From a raw response, return a list of extracted sites in dictionary form.
Relevant dictionary keys are:
"usgs_site_code"
"variableName"
"measurement_unit"
"values"
"series"
Parameters
----------
raw_response : aiohttp.ClientResponse
Request GET response
Returns
-------
List[dict]
A list of handled responses
"""
# TODO: Speed test using orjson instead of native
deserialized_response = raw_response.json()
def extract_metadata(json_time_series):
return {
# Add site code
"usgs_site_code":
json_time_series["sourceInfo"]["siteCode"][0]["value"],
# Add variable name
"variableName":
IVDataService.simplify_variable_name(
json_time_series["variable"]["variableName"]),
# Add units
"measurement_unit":
json_time_series["variable"]["unit"]["unitCode"],
}
flattened_data = []
for response_value_timeSeries in deserialized_response["value"][
"timeSeries"]:
for indicies, site_data in enumerate(
response_value_timeSeries["values"]):
# Create general site metadata dictionary
site_metadata = extract_metadata(response_value_timeSeries)
# Add site time series values and its index number
site_metadata.update({
"values": site_data["value"],
"series": indicies
})
flattened_data.append(site_metadata)
return flattened_data
async def test_authenticate():
st = AsyncSpaceTrackClient('identity', 'wrongpassword')
loop = asyncio.get_event_loop()
response = ClientResponse(
'post', 'https://www.space-track.org/ajaxauth/login')
response._post_init(loop)
response.status = 200
response.json = Mock()
async def mock_post(url, data):
response.json.return_value = asyncio.Future()
if data['password'] == 'wrongpassword':
response.json.return_value.set_result({'Login': '******'})
elif data['password'] == 'unknownresponse':
# Space-Track doesn't respond like this, but make sure anything
# other than {'Login': '******'} doesn't raise AuthenticationError
response.json.return_value.set_result({'Login': '******'})
else:
response.json.return_value.set_result('')
return response
with st, patch.object(st.session, 'post', mock_post):
with pytest.raises(AuthenticationError):
await st.authenticate()
assert response.json.call_count == 1
st.password = '******'
await st.authenticate()
# This shouldn't make a HTTP request since we're already authenticated.
await st.authenticate()
assert response.json.call_count == 2
st = AsyncSpaceTrackClient('identity', 'unknownresponse')
with st, patch.object(st.session, 'post', mock_post):
await st.authenticate()
response.close()
async def test_generic_request():
def mock_authenticate(self):
result = asyncio.Future()
result.set_result(None)
return result
def mock_download_predicate_data(self, class_):
result = asyncio.Future()
data = [
{
'Default': '0000-00-00 00:00:00',
'Extra': '',
'Field': 'PUBLISH_EPOCH',
'Key': '',
'Null': 'NO',
'Type': 'datetime'
},
{
'Default': '',
'Extra': '',
'Field': 'TLE_LINE1',
'Key': '',
'Null': 'NO',
'Type': 'char(71)'
},
{
'Default': '',
'Extra': '',
'Field': 'TLE_LINE2',
'Key': '',
'Null': 'NO',
'Type': 'char(71)'
}
]
result.set_result(data)
return result
st = AsyncSpaceTrackClient('identity', 'password')
loop = asyncio.get_event_loop()
response = ClientResponse(
'get', 'https://www.space-track.org/basicspacedata/query/class'
'/tle_publish/format/tle')
response._post_init(loop)
tle = (
'1 25544U 98067A 08264.51782528 -.00002182 00000-0 -11606-4 0 2927\r\n'
'2 25544 51.6416 247.4627 0006703 130.5360 325.0288 15.72125391563537\r\n')
normalised_tle = tle.replace('\r\n', '\n')
response.status = 200
response.text = Mock()
response.text.return_value = asyncio.Future()
response.text.return_value.set_result(tle)
mock_get = asyncio.Future()
mock_get.set_result(response)
patch_authenticate = patch.object(
AsyncSpaceTrackClient, 'authenticate', mock_authenticate)
patch_download_predicate_data = patch.object(
AsyncSpaceTrackClient, '_download_predicate_data',
mock_download_predicate_data)
patch_get = patch.object(st.session, 'get', return_value=mock_get)
with patch_authenticate, patch_download_predicate_data, patch_get:
assert await st.tle_publish(format='tle') == normalised_tle
response.close()
response = ClientResponse(
'get', 'https://www.space-track.org/basicspacedata/query/class'
'/tle_publish')
response._post_init(loop)
response.status = 200
response.json = Mock()
response.json.return_value = asyncio.Future()
response.json.return_value.set_result({'a': 5})
mock_get = asyncio.Future()
mock_get.set_result(response)
patch_get = patch.object(st.session, 'get', return_value=mock_get)
with patch_authenticate, patch_download_predicate_data, patch_get:
result = await st.tle_publish()
assert result['a'] == 5
response.close()
st.close()
