async def check_response(response: aiohttp.ClientResponse, ) -> None: """ Check for specialised K8s errors, and raise with extended information. """ if response.status >= 400: # Read the response's body before it is closed by raise_for_status(). payload: Optional[RawStatus] try: payload = await response.json() except (json.JSONDecodeError, aiohttp.ContentTypeError, aiohttp.ClientConnectionError): payload = None # Better be safe: who knows which sensitive information can be dumped unless kind==Status. if not isinstance( payload, collections.abc.Mapping) or payload.get('kind') != 'Status': payload = None cls = (APIUnauthorizedError if response.status == 401 else APIForbiddenError if response.status == 403 else APINotFoundError if response.status == 404 else APIError) # Raise the framework-specific error while keeping the original error in scope. # This call also closes the response's body, so it cannot be read afterwards. try: response.raise_for_status() except aiohttp.ClientResponseError as e: raise cls(payload, status=response.status) from e
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None: """Check resp for status and if error log additional info.""" try: resp.raise_for_status() except aiohttp.ClientResponseError: log.exception('Error body: %s', await resp.text()) raise
async def _extract_response_data( resp: aiohttp.ClientResponse, ) -> list | dict: """Checks the correctness of the response. Args: resp: Response instance. Returns: resp['data'] field if expected data. Raises: aiohttp.ClientResponseError: if response status >= 400. aiohttp.ClientPayloadError: if failure result of the request. #TODO: refactor for process HTTP statuses! """ resp.raise_for_status() json = await resp.json() if json.get("success"): _LOG.debug( "Successfully response", extra={ "url": resp.url, }, ) return json.get("data", {}) raise aiohttp.ClientPayloadError(f"Failure server result: {resp.url} {json}")
async def test_iter_content_generator(): """Test CRLF -> LF newline conversion.""" async def mock_iter_content(n): for chunk in [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']: yield chunk response = ClientResponse( 'get', ST_URL, request_info=Mock(), writer=Mock(), continue100=None, timer=TimerNoop(), traces=[], loop=Mock(), session=Mock(), ) response._headers = {'Content-Type': 'application/json;charset=utf-8'} with patch.object(response, 'content', Mock(iter_chunked=mock_iter_content)): result = [ line async for line in _iter_content_generator(response=response, decode_unicode=True) ] assert result == ['1\n2\n', '3', '\n4', '\n5'] result = [ line async for line in _iter_content_generator(response=response, decode_unicode=False) ] assert result == [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None: """Check resp for status and if error log additional info.""" # Copied from aiohttp's raise_for_status() -- since it releases the # response payload, we need to grab the `resp.text` first to help users # debug. # # Useability/performance notes: # * grabbing the response can be slow for large files, only do it as # needed # * we can't know in advance what encoding the files might have unless # we're certain in advance that the result is an error payload from # Google (otherwise, it could be a binary blob from GCS, for example) # * sometimes, errors are expected, so we should try to avoid polluting # logs in that case # # https://github.com/aio-libs/aiohttp/blob/ # 385b03ef21415d062886e1caab74eb5b93fdb887/aiohttp/ # client_reqrep.py#L892-L902 if resp.status >= 400: assert resp.reason is not None # Google's error messages are useful, pass 'em through body = await resp.text(errors='replace') resp.release() raise aiohttp.ClientResponseError(resp.request_info, resp.history, status=resp.status, message=f'{resp.reason}: {body}', headers=resp.headers)
async def from_client_response(cls, client_response: ClientResponse, expires: datetime = None): """Convert a ClientResponse into a CachedReponse""" if isinstance(client_response, cls): return client_response # Copy most attributes over as is copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) # Read response content, and reset StreamReader on original response if not client_response._released: await client_response.read() response._body = client_response._body client_response.content = CachedStreamReader(client_response._body) # Set remaining attributes individually response.expires = expires response.links = client_response.links response.real_url = client_response.request_info.real_url # The encoding may be unset even if the response has been read, and # get_encoding() does not handle certain edge cases like an empty response body try: response.encoding = client_response.get_encoding() except (RuntimeError, TypeError): pass if client_response.history: response.history = (*[ await cls.from_client_response(r) for r in client_response.history ], ) return response
async def _raise_for_status(resp: aiohttp.ClientResponse) -> None: """Check resp for status and if error log additional info.""" body = await resp.text(errors='replace') try: resp.raise_for_status() except aiohttp.ClientResponseError: log.exception('got http error response: %s', body) raise
def _check_status(r: aiohttp.ClientResponse, expected_status): r.raise_for_status() if r.status != expected_status: raise DSWCommunicationError( reason='Unexpected response status', message=f'Server responded with unexpected HTTP status {r.status}: ' f'{r.reason} (expecting {expected_status})' )
async def parse_api_response(resp: ClientResponse): """ Checks for response status, parses the json and checks for API error """ resp.raise_for_status() resp = json.loads(await resp.text()) if "error" in resp: raise APIError(resp["error"]) return resp
async def _raise_for_status(self, resp: aiohttp.ClientResponse) -> None: if resp.status >= 400: explanation = await resp.text() resp.release() logger.debug("Server responded:\n%s\n%s", explanation, "=" * 40) raise SPARQLRequestFailed(resp.request_info, resp.history, code=resp.status, message=resp.reason, explanation=explanation)
def _raise_error_from_response( self, resp: aiohttp.ClientResponse, ) -> None: if resp.status in [401, 403]: abort(403, "request rejected by backend") if resp.status == 400: abort(500, "request rejected by backend") if not 200 <= resp.status < 300: resp.raise_for_status()
def make_response(loop, method, url, data=None, content_type='text/plain', charset='utf-8'): if LooseVersion(aiohttp_version) >= LooseVersion('3.3.0'): response = ClientResponse(method, URL(url), writer=mock.Mock(), continue100=None, timer=None, request_info=mock.Mock(), traces=[], loop=loop, session=mock.Mock()) elif LooseVersion(aiohttp_version) >= LooseVersion('3.1.0'): response = ClientResponse(method, URL(url), writer=mock.Mock(), continue100=None, timer=None, request_info=mock.Mock(), auto_decompress=True, traces=[], loop=loop, session=mock.Mock()) else: response = ClientResponse(method, URL(url), writer=mock.Mock(), continue100=None, timer=None, request_info=mock.Mock(), auto_decompress=True) response._post_init(loop, mock.Mock()) def side_effect(*args, **kwargs): fut = loop.create_future() fut.set_result(str(data).encode(charset)) return fut if LooseVersion(aiohttp_version) >= LooseVersion('3.3.0'): response._headers = { 'Content-Type': '%s; charset=%s' % (content_type, charset) } else: response.headers = { 'Content-Type': '%s; charset=%s' % (content_type, charset) } content = response.content = mock.Mock() if data: content.read.side_effect = side_effect return response
def raise_not_200(response: aiohttp.ClientResponse) -> None: if response.status != 200: assert response.reason is not None response.release() raise aiohttp.ClientResponseError( response.request_info, response.history, status=response.status, message=response.reason, headers=response.headers, )
def aiohttp_raise_for_status(response: aiohttp.ClientResponse): # workaround aiohttp bug, can remove after fixed in aiohttp # issue: https://github.com/aio-libs/aiohttp/issues/3906 if response.status >= 400: response.release() raise aiohttp.ClientResponseError( response.request_info, response.history, status=response.status, message=response.reason, headers=response.headers, )
def raise_for_status( resp: aiohttp.ClientResponse) -> aiohttp.ClientResponse: """Raise exceptions on failure methods.""" try: resp.raise_for_status() except ClientResponseError as err: if err.status == HTTP_UNAUTHORIZED: raise AuthException( f"Unable to authenticate with API: {err}") from err raise ApiException(f"Error from API: {err}") from err except ClientError as err: raise ApiException(f"Error from API: {err}") from err return resp
async def check_error(resp: ClientResponse) -> Any: try: resp_data = await resp.json() except ContentTypeError: resp.raise_for_status() return except json.JSONDecodeError: resp.raise_for_status() raise if not isinstance(resp_data, dict) or "errors" not in resp_data: resp.raise_for_status() return resp_data try: error = resp_data["errors"][0] code = error["code"] message = error["message"] except (KeyError, IndexError): resp.raise_for_status() raise if code == 88: raise RateLimitError(code, message, resp.headers) elif code == 32: raise TwitterAuthError(code, message) raise TwitterError(code, message)
async def handle_error_response(self, response: aiohttp.ClientResponse) -> None: if response.status >= 400: try: body = await response.json() except aiohttp.ContentTypeError: body = await response.text() response.release() raise HttpResponseException(response.request_info, response.history, status=response.status, reason=response.reason, headers=response.headers, body=body)
def make_http_response( status=200, reason="OK", method="GET", url="/", headers=None, content=None ): """Return a minimal ClientResponse with fields used in tests.""" url = URL(url) headers = CIMultiDict(headers or {}) request_info = RequestInfo(url=url, method=method, headers=headers) response = ClientResponse( method, url, writer=None, continue100=None, timer=None, request_info=request_info, traces=(), loop=get_event_loop(), session=None, ) response.status = status response.reason = reason response._headers = headers if isinstance(content, io.IOBase): response.content = FakeStreamReader(content) elif content is not None: response.content = FakeStreamReader( io.BytesIO(json_dumps(content).encode("utf8")) ) response.headers["Content-Type"] = "application/json" return response
def _patch_get( self, client_response: aiohttp.ClientResponse) -> aiohttp.ClientResponse: """ Wrap aiohttp.ClientResponse text and json coros in run_until_complete. Monkeypatch text and json with wrappers.""" # May iter through methods and wrap all coro's in the future # however that may not work if a non-coro returns a async context manager for example text = self._wrap_coro_in_callable(client_response.text) json = self._wrap_coro_in_callable(client_response.json) # resign signatures text = forge.copy(aiohttp.ClientResponse.text, exclude="self")(text) json = forge.copy(aiohttp.ClientResponse.json, exclude="self")(json) client_response.text = text client_response.json = json return client_response
async def test_authenticate(): st = AsyncSpaceTrackClient('identity', 'wrongpassword') loop = asyncio.get_event_loop() response = ClientResponse( 'post', URL('https://www.space-track.org/ajaxauth/login')) # aiohttp 2.2 uses session try: response._post_init(loop) except TypeError: response._post_init(loop, st.session) response.status = 200 response.json = Mock() async def mock_post(url, data): response.json.return_value = asyncio.Future() if data['password'] == 'wrongpassword': response.json.return_value.set_result({'Login': '******'}) elif data['password'] == 'unknownresponse': # Space-Track doesn't respond like this, but make sure anything # other than {'Login': '******'} doesn't raise AuthenticationError response.json.return_value.set_result({'Login': '******'}) else: response.json.return_value.set_result('') return response with st, patch.object(st.session, 'post', mock_post): with pytest.raises(AuthenticationError): await st.authenticate() assert response.json.call_count == 1 st.password = '******' await st.authenticate() # This shouldn't make a HTTP request since we're already authenticated. await st.authenticate() assert response.json.call_count == 2 st = AsyncSpaceTrackClient('identity', 'unknownresponse') with st, patch.object(st.session, 'post', mock_post): await st.authenticate() response.close()
async def wrap_async(response: ClientResponse) -> Response: """Build a ``requests`` response from a ``aiohttp`` response. A ``requests.Response`` instance is built to provide synchronous access to the original response's data. Note that the returned response does not have proper data for :attr:``elapsed`` or :attr:``request``. The response will be consumed if it has not already. """ # Ensure the response data is read so that the wrapped response # does not require any async methods. await response.read() wrapped = Response() wrapped._content = response._body # type: ignore wrapped._content_consumed = True # type: ignore wrapped.status_code = response.status wrapped.headers = CaseInsensitiveDict(response.headers) wrapped.url = str(response.url) # `aiohttp` uses a `URL` object. wrapped.encoding = response.get_encoding() wrapped.history = [await wrap_async(rsp) for rsp in response.history] wrapped.reason = response.reason or "" wrapped.cookies = cookiejar_from_dict(response.cookies) return wrapped
async def raise_for_status(response: ClientResponse): if response.ok: return response try: result = await response.json() message = result["message"] detail = result["detail"] message = f"{message}: {detail}" except Exception: message = await response.text() or response.reason log.exception(message) else: log.error(message) response.raise_for_status()
async def parse_response(response: ClientResponse, schema: dict) -> Any: """ Validate and parse the BMA answer :param response: Response of aiohttp request :param schema: The expected response structure :return: the json data """ try: data = await response.json() response.close() if schema is not None: jsonschema.validate(data, schema) return data except (TypeError, json.decoder.JSONDecodeError) as e: raise jsonschema.ValidationError("Could not parse json : {0}".format(str(e)))
async def _check_response(result: aiohttp.ClientResponse, url, method) -> Dict: """ Check url response, and raise exceptions :param result: http response :param url: url to show in the log :param method: 'post', 'get' :return: json decoded result """ if result.status != 200: raise ServerException(f'{url} {method} failed, status code: {result.status}') result = await result.json() if not isinstance(result, dict): return result status_code = result.get('code') if method == 'post': if status_code is None: raise ServerException('Empty response') if status_code == 0: # normal return result elif method == 'get': if status_code is None or status_code == 0: return result if status_code in error_code: raise error_code[status_code]() else: raise MiraiException('HTTP API updated, please upgrade python-mirai-core')
async def raise_for_status(response: ClientResponse): if response.ok: return response try: result = await response.json() except Exception: message = response.reason log.exception(message) else: message = result["message"] detail = result.get("detail") message = f"{message}" + f": {detail}" if detail else "" log.error(message) response.raise_for_status()
async def from_client_response(cls, client_response: ClientResponse, expires: datetime = None): """Convert a ClientResponse into a CachedReponse""" # Response may not have been read yet, if fetched by something other than CachedSession if not client_response._released: await client_response.read() # Copy most attributes over as is copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) # Set some remaining attributes individually response._body = client_response._body response._links = [(k, _to_str_tuples(v)) for k, v in client_response.links.items()] response.expires = expires response.real_url = client_response.request_info.real_url # The encoding may be unset even if the response has been read try: response.encoding = client_response.get_encoding() except RuntimeError: pass response.url = str(client_response.url) if client_response.history: response.history = (*[ await cls.from_client_response(r) for r in client_response.history ], ) return response
async def raise_response_error(resp: aiohttp.ClientResponse, reason: str): # We raise a TransportServerError if the status code is 400 or higher # We raise a TransportProtocolError in the other cases try: # Raise a ClientResponseError if response status is 400 or higher resp.raise_for_status() except ClientResponseError as e: raise TransportServerError(str(e), e.status) from e result_text = await resp.text() raise TransportProtocolError( f"Server did not return a GraphQL result: " f"{reason}: " f"{result_text}")
async def __norm_callback( response: aiohttp.ClientResponse, decode: bool = False, max_size: Optional[int] = None, intended_content_type: Optional[str] = None) -> Optional[AnyStr]: content_type = response.headers.get('Content-Type') if not intended_content_type or not content_type or content_type.startswith( intended_content_type): body: Optional[bytes] = None if max_size is None: body = await response.read() elif max_size > 0: body = await response.content.read(max_size) if decode and body: xml_header = body.split(b'\n', 1)[0] if xml_header.startswith( b'<?xml' ) and b'?>' in xml_header and b'encoding' in xml_header: try: encoding = BeautifulSoup(xml_header, 'lxml-xml').original_encoding return body.decode(encoding=encoding, errors='replace') except (LookupError, RuntimeError): pass try: encoding = response.get_encoding() return body.decode(encoding=encoding, errors='replace') except (LookupError, RuntimeError): return body.decode(encoding='utf-8', errors='replace') return body return None
def link_extractor(response: ClientResponse, url_filter: URLFilter, defrag: bool) -> (str, List[str]): html = response._body.decode('utf-8', errors='ignore') req_url = response._url dom = lh.fromstring(html) response.__setattr__('dom', dom) response.__setattr__('html', html) found_urls = [] for href in dom.xpath('//a/@href'): url = urljoin(str(req_url), href) if defrag: url = urldefrag(url)[0] netloc = urlparse(url).netloc can_crawl = url_filter.can_crawl(netloc, url) if can_crawl and valid_url(url): found_urls.append(url) return response, found_urls
async def parse_response(response: ClientResponse, schema: dict) -> Any: """ Validate and parse the BMA answer :param response: Response of aiohttp request :param schema: The expected response structure :return: the json data """ try: data = await response.json() response.close() if schema is not None: jsonschema.validate(data, schema) return data except (TypeError, json.decoder.JSONDecodeError) as e: raise jsonschema.ValidationError("Could not parse json : {0}".format( str(e))) from e
def _handle_response(raw_response: aiohttp.ClientResponse) -> List[dict]: """From a raw response, return a list of extracted sites in dictionary form. Relevant dictionary keys are: "usgs_site_code" "variableName" "measurement_unit" "values" "series" Parameters ---------- raw_response : aiohttp.ClientResponse Request GET response Returns ------- List[dict] A list of handled responses """ # TODO: Speed test using orjson instead of native deserialized_response = raw_response.json() def extract_metadata(json_time_series): return { # Add site code "usgs_site_code": json_time_series["sourceInfo"]["siteCode"][0]["value"], # Add variable name "variableName": IVDataService.simplify_variable_name( json_time_series["variable"]["variableName"]), # Add units "measurement_unit": json_time_series["variable"]["unit"]["unitCode"], } flattened_data = [] for response_value_timeSeries in deserialized_response["value"][ "timeSeries"]: for indicies, site_data in enumerate( response_value_timeSeries["values"]): # Create general site metadata dictionary site_metadata = extract_metadata(response_value_timeSeries) # Add site time series values and its index number site_metadata.update({ "values": site_data["value"], "series": indicies }) flattened_data.append(site_metadata) return flattened_data
async def test_authenticate(): st = AsyncSpaceTrackClient('identity', 'wrongpassword') loop = asyncio.get_event_loop() response = ClientResponse( 'post', 'https://www.space-track.org/ajaxauth/login') response._post_init(loop) response.status = 200 response.json = Mock() async def mock_post(url, data): response.json.return_value = asyncio.Future() if data['password'] == 'wrongpassword': response.json.return_value.set_result({'Login': '******'}) elif data['password'] == 'unknownresponse': # Space-Track doesn't respond like this, but make sure anything # other than {'Login': '******'} doesn't raise AuthenticationError response.json.return_value.set_result({'Login': '******'}) else: response.json.return_value.set_result('') return response with st, patch.object(st.session, 'post', mock_post): with pytest.raises(AuthenticationError): await st.authenticate() assert response.json.call_count == 1 st.password = '******' await st.authenticate() # This shouldn't make a HTTP request since we're already authenticated. await st.authenticate() assert response.json.call_count == 2 st = AsyncSpaceTrackClient('identity', 'unknownresponse') with st, patch.object(st.session, 'post', mock_post): await st.authenticate() response.close()
async def test_generic_request(): def mock_authenticate(self): result = asyncio.Future() result.set_result(None) return result def mock_download_predicate_data(self, class_): result = asyncio.Future() data = [ { 'Default': '0000-00-00 00:00:00', 'Extra': '', 'Field': 'PUBLISH_EPOCH', 'Key': '', 'Null': 'NO', 'Type': 'datetime' }, { 'Default': '', 'Extra': '', 'Field': 'TLE_LINE1', 'Key': '', 'Null': 'NO', 'Type': 'char(71)' }, { 'Default': '', 'Extra': '', 'Field': 'TLE_LINE2', 'Key': '', 'Null': 'NO', 'Type': 'char(71)' } ] result.set_result(data) return result st = AsyncSpaceTrackClient('identity', 'password') loop = asyncio.get_event_loop() response = ClientResponse( 'get', 'https://www.space-track.org/basicspacedata/query/class' '/tle_publish/format/tle') response._post_init(loop) tle = ( '1 25544U 98067A 08264.51782528 -.00002182 00000-0 -11606-4 0 2927\r\n' '2 25544 51.6416 247.4627 0006703 130.5360 325.0288 15.72125391563537\r\n') normalised_tle = tle.replace('\r\n', '\n') response.status = 200 response.text = Mock() response.text.return_value = asyncio.Future() response.text.return_value.set_result(tle) mock_get = asyncio.Future() mock_get.set_result(response) patch_authenticate = patch.object( AsyncSpaceTrackClient, 'authenticate', mock_authenticate) patch_download_predicate_data = patch.object( AsyncSpaceTrackClient, '_download_predicate_data', mock_download_predicate_data) patch_get = patch.object(st.session, 'get', return_value=mock_get) with patch_authenticate, patch_download_predicate_data, patch_get: assert await st.tle_publish(format='tle') == normalised_tle response.close() response = ClientResponse( 'get', 'https://www.space-track.org/basicspacedata/query/class' '/tle_publish') response._post_init(loop) response.status = 200 response.json = Mock() response.json.return_value = asyncio.Future() response.json.return_value.set_result({'a': 5}) mock_get = asyncio.Future() mock_get.set_result(response) patch_get = patch.object(st.session, 'get', return_value=mock_get) with patch_authenticate, patch_download_predicate_data, patch_get: result = await st.tle_publish() assert result['a'] == 5 response.close() st.close()