async def test_add_trace_request_ctx(aiohttp_client, loop):
actual_request_contexts = []
async def on_request_start(
_: ClientSession,
trace_config_ctx: SimpleNamespace,
__: TraceRequestStartParams,
) -> None:
actual_request_contexts.append(trace_config_ctx)
test_app = App()
trace_config = TraceConfig()
trace_config.on_request_start.append(on_request_start) # type: ignore
retry_client = RetryClient()
retry_client._client = await aiohttp_client(
test_app.get_app(),
trace_configs=[trace_config]
)
async with retry_client.get('/sometimes_error', trace_request_ctx={'foo': 'bar'}):
assert test_app.counter == 3
assert actual_request_contexts == [
SimpleNamespace(
trace_request_ctx={
'foo': 'bar',
'current_attempt': i + 1,
},
)
for i in range(3)
]
def __init__(self):
# pylint: disable=unused-argument
async def on_request_start(
session: ClientSession,
trace_config_ctx: SimpleNamespace,
params: TraceRequestStartParams,
) -> None:
current_attempt = trace_config_ctx.trace_request_ctx[
"current_attempt"]
if current_attempt > 1:
LOG.info("iNat request attempt #%d: %s", current_attempt,
repr(params))
trace_config = TraceConfig()
trace_config.on_request_start.append(on_request_start)
self.session = RetryClient(
raise_for_status=False,
trace_configs=[trace_config],
)
self.request_time = time()
self.places_cache = {}
self.projects_cache = {}
self.users_cache = {}
self.users_login_cache = {}
self.taxa_cache = {}
# api_v1_limiter:
# ---------------
# - Allow a burst of 60 requests (i.e. equal to max_rate) in the initial
# seconds of the 60 second time_period before enforcing a rate limit of
# 60 requests per minute (max_rate).
# - This honours "try to keep it to 60 requests per minute or lower":
# - https://api.inaturalist.org/v1/docs/
# - Since the iNat API doesn't throttle until 100 requests per minute,
# this should ensure we never get throttled.
self.api_v1_limiter = AsyncLimiter(60, 60)
async def test_not_found_error(aiohttp_client, loop):
test_app = App()
app = test_app.get_app()
client = await aiohttp_client(app)
retry_client = RetryClient()
retry_client._client = client
retry_options = RetryOptions(attempts=5, statuses={404})
async with retry_client.get('/not_found_error', retry_options) as response:
assert response.status == 404
assert test_app.counter == 5
await retry_client.close()
await client.close()
async def test_internal_error(aiohttp_client, loop):
test_app = TestApp()
app = test_app.get_app()
client = await aiohttp_client(app)
retry_client = RetryClient()
retry_client._client = client
async with retry_client.get('/internal_error',
retry_attempts=5) as response:
assert response.status == 500
assert test_app.counter == 5
await retry_client.close()
await client.close()
async def _does_existing_artifact_need_to_be_reuploaded(existing_artifact, target_artifact, retry_on_404=False):
should_artifact_be_reuploaded = False
artifact_name = target_artifact["name"]
for field in ("size", "content_type"):
target_value = target_artifact[field]
existing_value = getattr(existing_artifact, field, None)
if existing_value != target_value:
log.info(f'Artifact "{artifact_name}" has its "{field}" differing. Expected: {target_value}. Got: {existing_value}')
should_artifact_be_reuploaded = True
# XXX For an unknown reason, Github sometimes fails to upload assets correctly. In this case:
# the API does tell the artifact exists and has the expected size + content-type, but nothing
# is displayed on the Web UI. Trying to download the URL exposed on the Web UI enables us to
# catch this very issue.
download_url = existing_artifact.browser_download_url
# XXX A given release may be temporarilly 404 when it just got created
retry_for_statuses = {404} if retry_on_404 else {}
async with RetryClient() as client:
# XXX We cannot do simple HEAD requests because Github uses AWS and they forbid them.
# https://github.com/cavaliercoder/grab/issues/43#issuecomment-431076499
async with client.get(download_url, retry_for_statuses=retry_for_statuses) as response:
response_status = response.status
if response_status != 200:
log.warning(
f'Got an unexpected HTTP code when trying to download the existing artifact "{artifact_name}". Expected: 200. Got: {response_status}'
)
should_artifact_be_reuploaded = True
return should_artifact_be_reuploaded
async def _download_from_asyncgen(
items: AsyncGenerator,
params: DownloadParams,
tcp_connections: int = 64,
nb_workers: int = 64,
batch_size: int = 16,
retries: int = 1,
logger: logging.Logger = None,
):
"""Asynchronous downloader that takes an interable and downloads it
Args:
items (Union[Generator, AsyncGenerator]): (async/sync) generator that yiels a standardized dict of urls
params (DownloadParams): Download parameter dict
tcp_connections (int, optional): Maximum number of concurrent TCP connections. Defaults to 128.
nb_workers (int, optional): Maximum number of workers. Defaults to 64.
batch_size (int, optional): Maximum queue batch size. Defaults to 16.
retries (int, optional): Maximum number of attempts. Defaults to 1.
logger (logging.Logger, optional): Logger object. Defaults to None.
Raises:
NotImplementedError: If generator turns out to be invalid.
"""
queue = asyncio.Queue(nb_workers)
progressbar = tqdm(smoothing=0,
unit=" Downloads",
disable=logger.getEffectiveLevel() > logging.INFO)
stats = {"failed": 0, "skipped": 0, "success": 0}
retry_options = ExponentialRetry(attempts=retries)
async with RetryClient(
connector=aiohttp.TCPConnector(limit=tcp_connections),
raise_for_status=True,
retry_options=retry_options,
trust_env=True,
) as session:
loop = asyncio.get_event_loop()
workers = [
loop.create_task(
_download_queue(queue,
session,
stats,
params=params,
progressbar=progressbar,
logger=logger)) for _ in range(nb_workers)
]
# get chunks from async generator and add to async queue
async with aiostream.stream.chunks(items, batch_size).stream() as chnk:
async for batch in chnk:
await queue.put(batch)
await queue.join()
for w in workers:
w.cancel()
return stats
async def create(cls):
self = IdiomScraper()
self.client = RetryClient(
raise_for_status=True, # raise exception if response status >= 400
timeout=ClientTimeout(total=CLIENT_TIMEOUT), # set a timeout value
)
return self
async def async_query_api(self, endpoint, payload=None):
"""Queries a given endpoint on the Eskom loadshedding API with the specified payload
Args:
endpoint (string): The endpoint of the Eskom API
payload (dict, optional): The parameters to apply to the query. Defaults to None.
Returns:
The response object from the request
"""
async with RetryClient() as client:
# The Eskom API occasionally drops incoming connections, implement reies
async with client.get(
url=self.base_url + endpoint,
headers=self.headers,
params=payload,
ssl=self.ssl_context,
retry_attempts=50,
retry_exceptions={
ClientConnectorError,
ServerDisconnectedError,
ConnectionError,
OSError,
},
) as res:
return await res.json()
def get_client_session(self, config: SeekerConfig) -> ClientSession:
async def _on_request_start(
session: ClientSession,
trace_config_ctx: SimpleNamespace,
params: TraceRequestStartParams
) -> None:
current_attempt = \
trace_config_ctx.trace_request_ctx['current_attempt']
if(current_attempt > 1):
logger.warning(
f'::warn ::Retry Attempt #{current_attempt} ' +
f'of {config.max_tries}: {params.url}')
trace_config = TraceConfig()
trace_config.on_request_start.append(_on_request_start)
limit_per_host = max(0, config.connect_limit_per_host)
connector = TCPConnector(
limit_per_host=limit_per_host,
ttl_dns_cache=600 # 10-minute DNS cache
)
retry_options = ExponentialRetry(
attempts=config.max_tries,
max_timeout=config.max_time,
exceptions=[
aiohttp.ClientError,
asyncio.TimeoutError
])
return RetryClient(
raise_for_status=True,
connector=connector,
timeout=ClientTimeout(total=config.timeout),
headers={'User-Agent': config.agent},
retry_options=retry_options,
trace_configs=[trace_config])
async def fetch_url(sem: asyncio.Semaphore, limit: int, offset: int):
async with sem:
log.debug(f'Started request with offset {offset}')
async with RetryClient(auth=self.auth) as s, s.post(
f'http://gta.intel.com/api/results/v2/results?limit={limit}&offset={offset}',
data=json.dumps(payload),
headers={
"Accept": "application/json",
"Content-Type": "application/json",
},
retry_attempts=20,
retry_start_timeout=1,
retry_factor=1.5,
retry_for_statuses=[500],
raise_for_status=True,
retry_exceptions=[
aiohttp.client_exceptions.ClientPayloadError
]) as r:
jsn = await r.json()
res = []
for item in jsn['items']:
if item['testRunUrl'][0] in self.url_list:
res.append(item)
log.debug(f'Finished request with offset {offset}')
return res
async def get_tags_list_async(url: str, query: str, pages_count: int,
timeout: float, retries_count: int) -> List[str]:
"""
Return tags list from `url` with `query`. Works asynchronous.
:param url: URL for `query`
:param query: Query for repos search
:param pages_count: Pages count for tags list creation
:param timeout: timeout between retries in case of status code != 200
:param retries_count: retries count in case of status code != 200
:return: List with tags
"""
tags_list = []
async with RetryClient() as client:
for page_num in range(1, pages_count + 1):
logger.debug(f'Page №{page_num} request')
query_string = url + '?p=' + str(
page_num) + '&q=' + query + '&type=Repositories&s=stars'
request_result = await fetch(client, query_string, timeout,
retries_count)
logger.debug(f'Page №{page_num} processing')
soup = BeautifulSoup(request_result, 'lxml')
for tag in soup.find_all(
'a',
attrs={'class': 'topic-tag topic-tag-link f6 px-2 mx-0'}):
tags_list.append(tag.text.strip())
logger.debug(
f'Tags count after page №{page_num} processing: {len(tags_list)}'
)
return tags_list
def initialize_session(self) -> ClientSession:
# Restrict the size of the connection pool for scraping ethic
conn = aiohttp.TCPConnector(limit_per_host=30)
retry_options = ListRetry(timeouts=[0, 0, 0.6, 1.2],
statuses={500, 502, 503, 504, 514})
retry_client = RetryClient(connector=conn, retry_options=retry_options)
# TODO we should inform the type checker that RetryClient has the same interface
# with that of ClientSession. We can either do it nominally by making
# RetryClient a subclass of ClientSession, or do it structurally by duck typing
# / structural typing / typing.Protocol, etc.
session = cast(ClientSession, retry_client)
origin_get = session.get
def patched_get(self, *args, **kwargs):
# Add random parameter to the URL to break potential cache mechanism of
# the server or the network or the aiohttp library.
salt_key = "锟斤铐"
# TODO can we just use random bytes as salt_value?
salt_value = "".join(random.choices(string.hexdigits, k=10))
params = kwargs.setdefault("params", {})
params[salt_key] = salt_value
return origin_get(*args, **kwargs)
session.get = MethodType(patched_get, session)
return session
async def replace_session(self):
await self.session.close()
proxy_connector = ProxyConnector(**env.TELEGRAPH_PROXY_DICT, loop=self.loop) \
if env.TELEGRAPH_PROXY_DICT else None
self.session = RetryClient(connector=proxy_connector,
timeout=ClientTimeout(total=10),
loop=self.loop,
json_serialize=self._json_serialize)
async def test_hello(aiohttp_client, loop):
test_app = TestApp()
app = test_app.get_app()
client = await aiohttp_client(app)
retry_client = RetryClient()
retry_client._client = client
async with retry_client.get('/ping') as response:
text = await response.text()
assert response.status == 200
assert text == 'Ok!'
assert test_app.counter == 1
await retry_client.close()
await client.close()
async def test_sometimes_error_with_raise_for_status(aiohttp_client, loop):
test_app = TestApp()
app = test_app.get_app()
client = await aiohttp_client(app, raise_for_status=True)
retry_client = RetryClient()
retry_client._client = client
async with retry_client.get('/sometimes_error', retry_attempts=5, retry_exceptions={ClientResponseError}) \
as response:
text = await response.text()
assert response.status == 200
assert text == 'Ok!'
assert test_app.counter == 3
await retry_client.close()
await client.close()
async def test_sometimes_error(aiohttp_client, loop):
test_app = TestApp()
app = test_app.get_app()
client = await aiohttp_client(app)
retry_client = RetryClient()
retry_client._client = client
async with retry_client.get('/sometimes_error',
retry_attempts=5) as response:
text = await response.text()
assert response.status == 200
assert text == 'Ok!'
assert test_app.counter == 3
await retry_client.close()
await client.close()
async def generate_link() -> Optional[str]:
settings = Settings()
assert settings.SPELLTABLE_AUTH_KEY
headers = {
"user-agent": f"spellbot/{__version__}",
"key": settings.SPELLTABLE_AUTH_KEY,
}
data: Optional[dict[str, Any]] = None
raw_data: Optional[bytes] = None
try:
async with RetryClient(
raise_for_status=False,
retry_options=ExponentialRetry(attempts=5),
) as client:
async with client.post(settings.SPELLTABLE_CREATE,
headers=headers) as resp:
# Rather than use `resp.json()`, which respects minetype, let's just
# grab the data and try to decode it ourselves.
# https://github.com/inyutin/aiohttp_retry/issues/55
raw_data = await resp.read()
data = json.loads(raw_data)
if not data or "gameUrl" not in data:
logger.warning(
"warning: gameUrl missing from SpellTable API response (%s): %s",
resp.status,
data,
)
return None
assert data is not None
returned_url = str(data["gameUrl"])
wizards_url = returned_url.replace(
"www.spelltable.com",
"spelltable.wizards.com",
)
return wizards_url
except ClientError as ex:
add_span_error(ex)
logger.warning(
"warning: SpellTable API failure: %s, data: %s, raw: %s",
ex,
data,
raw_data,
exc_info=True,
)
return None
except Exception as ex:
add_span_error(ex)
logger.error(
"error: unexpected exception: %s, data: %s, raw: %s",
ex,
data,
raw_data,
exc_info=True,
)
return None
async def async_get_zone_json(self, zoneJsonUrl):
async with RetryClient() as client:
async with client.get(
url=zoneJsonUrl,
headers=self.headers,
ssl=self.ssl_context,
retry_attemps=10,
) as res:
return await res.json()
async def get_active_tenders(self):
params = {"ticket": self.MERCADO_PUBLICO_TICKET, "estado": "activas"}
try:
async with RetryClient() as client:
async with client.get(url=self.MERCADO_PUBLICO_API_URL,
params=params) as response:
response.raise_for_status()
return await response.json()
except Exception as e:
raise Exception("Request Error: ", e) from None
async def test_override_options(aiohttp_client, loop):
test_app = App()
app = test_app.get_app()
client = await aiohttp_client(app)
retry_options = RetryOptions(attempts=1)
retry_client = RetryClient(retry_options=retry_options)
retry_client._client = client
retry_options = RetryOptions(attempts=5)
async with retry_client.get('/sometimes_error', retry_options) as response:
text = await response.text()
assert response.status == 200
assert text == 'Ok!'
assert test_app.counter == 3
await retry_client.close()
await client.close()
async def create(cls,
epsagon_token,
retry_attempts=DEFAULT_RETRY_ATTEMPTS):
"""
Creates a new EpsagonClient instance
:param epsagon_token: used for authorization
"""
self = cls()
if not epsagon_token:
raise ValueError("Epsagon token must be given")
self.epsagon_token = epsagon_token
retry_options = ExponentialRetry(attempts=retry_attempts,
exceptions=(ClientError, ))
self.client = RetryClient(auth=BasicAuth(login=self.epsagon_token),
headers={
"Content-Type": "application/json",
},
retry_options=retry_options,
raise_for_status=True)
return self
async def download_file(download_url, file_destination):
async with RetryClient() as s3_client:
async with s3_client.get(download_url) as resp:
with open(file_destination, "wb") as fd:
while True:
chunk = await resp.content.read(CHUNK_SIZE)
if not chunk:
break
fd.write(chunk)
log.info(f"'{file_destination}' downloaded")
async def get_session(timeout: Optional[float] = None):
if not timeout:
timeout = 12
proxy_connector = ProxyConnector.from_url(PROXY) if PROXY else None
session = RetryClient(retry_options=RETRY_OPTION,
connector=proxy_connector,
timeout=aiohttp.ClientTimeout(total=timeout),
headers={'User-Agent': env.USER_AGENT})
return session
async def get_tenders_by_date(self, date):
params = {"ticket": self.MERCADO_PUBLICO_TICKET, "fecha": date}
try:
async with RetryClient() as client:
async with client.get(url=self.MERCADO_PUBLICO_API_URL,
params=params,
retry_attempts=self.retry_attempts,
retry_for_statuses=self.
retry_for_statuses) as response:
response.raise_for_status()
return await response.json()
except Exception as e:
raise Exception("Request Error: ", e) from None
async def async_main(token, branch, commit, workflow, artifacts_directory, locales, derived_data_path=None):
headers = {"Authorization": token}
async with RetryClient(headers=headers) as client:
build_slug = await schedule_build(client, branch, commit, workflow, locales, derived_data_path)
log.info("Created new job. Slug: {}".format(build_slug))
try:
await wait_for_job_to_finish(client, build_slug)
log.info("Job {} is successful. Retrieving artifacts...".format(build_slug))
await download_artifacts(client, build_slug, artifacts_directory)
finally:
log.info("Retrieving bitrise log...")
await download_log(client, build_slug, artifacts_directory)
async def async_main(token, *args):
headers = {"Authorization": token}
async with RetryClient(headers=headers) as client:
build_slug = await schedule_build(client, *args)
log.info("Created new job. Slug: {}".format(build_slug))
try:
await wait_for_job_to_finish(client, build_slug)
log.info("Job {} is successful. Retrieving artifacts...".format(
build_slug))
await download_artifacts(client, build_slug)
finally:
log.info("Retrieving bitrise log...")
await download_log(client, build_slug)
async def fetch_elos(self, steam_ids: list[SteamId], *, headers: Optional[dict[str, str]] = None):
if len(steam_ids) == 0:
return None
formatted_steam_ids = "+".join([str(steam_id) for steam_id in steam_ids])
request_url = f"{self.url_base}{self.balance_api}/{formatted_steam_ids}"
retry_options = ExponentialRetry(attempts=3, factor=0.1,
statuses={500, 502, 504},
exceptions={aiohttp.ClientResponseError, aiohttp.ClientPayloadError})
async with RetryClient(raise_for_status=False, retry_options=retry_options,
timeout=ClientTimeout(total=5, connect=3, sock_connect=3, sock_read=5)) as retry_client:
async with retry_client.get(request_url, headers=headers) as result:
if result.status != 200:
return None
return await result.json()
async def _get_status_code(location: str, client: RetryClient,
retry: int) -> int:
_tracker.stats.inc_requests()
status_code = 0
try:
_logger.debug("Requesting status code for %s", location)
async with client.get(location, retry_attempts=retry) as response:
status_code = response.status
except TooManyRedirects:
_logger.debug("Redirection Tango, danced enough with %s", location)
except ClientConnectionError:
_logger.debug("Connection Error occurred while getting %s", location)
return status_code
async def fetch(client: RetryClient, query_string: str, timeout: float,
retries_count: int) -> Text:
"""
Fetch result of query
:param client: Client with retry mechanism
:param query_string: Full query string for repos getting
:param timeout: timeout between retries in case of status code != 200
:param retries_count: retries count in case of status code != 200
:return:
"""
async with client.get(url=query_string,
retry_attempts=retries_count,
retry_start_timeout=timeout,
retry_factor=2,
retry_max_timeout=timeout * (2**retries_count),
retry_for_statuses=[429]) as response:
return await response.text()
async def get_transcript(url):
async with semaphore:
async with RetryClient(raise_for_status=False) as client:
async with client.post(
'https://brain.deepgram.com/v2/listen',
headers={
'Content-type': 'application/json',
'Authorization': 'Basic {}'.format(
base64.b64encode('{}:{}'.format(*creds).encode('utf-8')).decode('utf-8')
)
},
data=json.dumps({
'url': url
}).encode('utf-8')
) as response:
response_content = await response.text()
return json.loads(response_content)
