async def _get_request_side_effect(url, *args, **kwargs):
nonlocal retry_count
retry_count += 1
if retry_count <= len(proxies):
raise ProxyError(response=Response(status_code=429))
else:
return Response(status_code=429)
async def test_receive_error_no_retries_configured(self, client_mock):
client_mock.return_value.__aenter__.return_value = client_mock
client_mock.get = CoroutineMock(return_value=Response(status_code=429))
pytrends = TrendReq(retries=0, backoff_factor=0)
pytrends.cookies = {'NID': '12eqf98hnf8032r54'}
with pytest.raises(ResponseError):
await pytrends.top_charts(date=2018)
assert client_mock.get.call_count == 1
async def _get_request_side_effect(url, *args, **kwargs):
nonlocal retry_count
retry_count += 1
#Make fail in a few different ways. On last attempt, return response
if retry_count == pytrend.retries - 1:
raise ConnectionRefusedError()
elif retry_count != pytrend.retries:
return Response(status_code=429)
else:
return trending_searches_200_response
async def test_all_retries_fail(self, client_mock):
client_mock.return_value.__aenter__.return_value = client_mock
client_mock.get = CoroutineMock(return_value=Response(status_code=429))
pytrend = TrendReq(timeout=TIMEOUT, retries=3, backoff_factor=0.1)
with pytest.raises(ResponseError):
await pytrend.build_payload(kw_list=['pizza', 'bagel'])
calls = [
call('https://trends.google.com/?geo=US', timeout=ANY)
for _ in range(pytrend.retries)
]
client_mock.get.assert_has_calls(calls)
async def send(self,
request: Request,
timeout: Timeout = None) -> Response:
stream_id = self._quic.get_next_available_stream_id()
# prepare request
self._http.send_headers(
stream_id=stream_id,
headers=[
(b":method", request.method.encode()),
(b":scheme", request.url.scheme.encode()),
(b":authority", str(request.url.authority).encode()),
(b":path", request.url.full_path.encode()),
] + [(k.encode(), v.encode())
for (k, v) in request.headers.items()
if k not in ("connection", "host")],
)
self._http.send_data(stream_id=stream_id,
data=request.read(),
end_stream=True)
# transmit request
waiter = self._loop.create_future()
self._request_events[stream_id] = deque()
self._request_waiter[stream_id] = waiter
self.transmit()
# process response
events: Deque[H3Event] = await asyncio.shield(waiter)
content = b""
headers = []
status_code = None
for event in events:
if isinstance(event, HeadersReceived):
for header, value in event.headers:
if header == b":status":
status_code = int(value.decode())
elif header[0:1] != b":":
headers.append((header.decode(), value.decode()))
elif isinstance(event, DataReceived):
content += event.data
return Response(
status_code=status_code,
http_version="HTTP/3",
headers=headers,
content=content,
request=request,
)
async def send(self,
request: Request,
timeout: TimeoutTypes = None) -> Response:
scope = {
"type": "http",
"asgi": {
"version": "3.0"
},
"http_version": "1.1",
"method": request.method,
"headers": request.headers.raw,
"scheme": request.url.scheme,
"path": request.url.path,
"query_string": request.url.query.encode("ascii"),
"server": request.url.host,
"client": self.client,
"root_path": self.root_path,
"extensions": ["http.response.template"],
}
status_code = None
headers = None
body_parts = []
response_started = False
response_complete = False
template = None
context = None
request_body_chunks = request.stream.__aiter__()
async def receive() -> dict:
try:
body = await request_body_chunks.__anext__()
except StopAsyncIteration:
return {
"type": "http.request",
"body": b"",
"more_body": False
}
return {"type": "http.request", "body": body, "more_body": True}
async def send(message: dict) -> None:
nonlocal status_code, headers, body_parts
nonlocal response_started, response_complete
nonlocal template, context
if message["type"] == "http.response.start":
assert not response_started
status_code = message["status"]
headers = message.get("headers", [])
response_started = True
elif message["type"] == "http.response.body":
assert not response_complete
body = message.get("body", b"")
more_body = message.get("more_body", False)
if body and request.method != "HEAD":
body_parts.append(body)
if not more_body:
response_complete = True
elif message["type"] == "http.response.template":
template = message["template"]
context = message["context"]
try:
await self.app(scope, receive, send)
except Exception:
if self.raise_app_exceptions or not response_complete:
raise
assert response_complete
assert status_code is not None
assert headers is not None
stream = ByteStream(b"".join(body_parts))
response = Response(
status_code=status_code,
http_version="HTTP/1.1",
headers=headers,
stream=stream,
request=request,
)
response.template = template
response.context = context
return response