async def test_add_trace_request_ctx(aiohttp_client, loop): actual_request_contexts = [] async def on_request_start( _: ClientSession, trace_config_ctx: SimpleNamespace, __: TraceRequestStartParams, ) -> None: actual_request_contexts.append(trace_config_ctx) test_app = App() trace_config = TraceConfig() trace_config.on_request_start.append(on_request_start) # type: ignore retry_client = RetryClient() retry_client._client = await aiohttp_client( test_app.get_app(), trace_configs=[trace_config] ) async with retry_client.get('/sometimes_error', trace_request_ctx={'foo': 'bar'}): assert test_app.counter == 3 assert actual_request_contexts == [ SimpleNamespace( trace_request_ctx={ 'foo': 'bar', 'current_attempt': i + 1, }, ) for i in range(3) ]
def __init__(self): # pylint: disable=unused-argument async def on_request_start( session: ClientSession, trace_config_ctx: SimpleNamespace, params: TraceRequestStartParams, ) -> None: current_attempt = trace_config_ctx.trace_request_ctx[ "current_attempt"] if current_attempt > 1: LOG.info("iNat request attempt #%d: %s", current_attempt, repr(params)) trace_config = TraceConfig() trace_config.on_request_start.append(on_request_start) self.session = RetryClient( raise_for_status=False, trace_configs=[trace_config], ) self.request_time = time() self.places_cache = {} self.projects_cache = {} self.users_cache = {} self.users_login_cache = {} self.taxa_cache = {} # api_v1_limiter: # --------------- # - Allow a burst of 60 requests (i.e. equal to max_rate) in the initial # seconds of the 60 second time_period before enforcing a rate limit of # 60 requests per minute (max_rate). # - This honours "try to keep it to 60 requests per minute or lower": # - https://api.inaturalist.org/v1/docs/ # - Since the iNat API doesn't throttle until 100 requests per minute, # this should ensure we never get throttled. self.api_v1_limiter = AsyncLimiter(60, 60)
async def test_not_found_error(aiohttp_client, loop): test_app = App() app = test_app.get_app() client = await aiohttp_client(app) retry_client = RetryClient() retry_client._client = client retry_options = RetryOptions(attempts=5, statuses={404}) async with retry_client.get('/not_found_error', retry_options) as response: assert response.status == 404 assert test_app.counter == 5 await retry_client.close() await client.close()
async def test_internal_error(aiohttp_client, loop): test_app = TestApp() app = test_app.get_app() client = await aiohttp_client(app) retry_client = RetryClient() retry_client._client = client async with retry_client.get('/internal_error', retry_attempts=5) as response: assert response.status == 500 assert test_app.counter == 5 await retry_client.close() await client.close()
async def _does_existing_artifact_need_to_be_reuploaded(existing_artifact, target_artifact, retry_on_404=False): should_artifact_be_reuploaded = False artifact_name = target_artifact["name"] for field in ("size", "content_type"): target_value = target_artifact[field] existing_value = getattr(existing_artifact, field, None) if existing_value != target_value: log.info(f'Artifact "{artifact_name}" has its "{field}" differing. Expected: {target_value}. Got: {existing_value}') should_artifact_be_reuploaded = True # XXX For an unknown reason, Github sometimes fails to upload assets correctly. In this case: # the API does tell the artifact exists and has the expected size + content-type, but nothing # is displayed on the Web UI. Trying to download the URL exposed on the Web UI enables us to # catch this very issue. download_url = existing_artifact.browser_download_url # XXX A given release may be temporarilly 404 when it just got created retry_for_statuses = {404} if retry_on_404 else {} async with RetryClient() as client: # XXX We cannot do simple HEAD requests because Github uses AWS and they forbid them. # https://github.com/cavaliercoder/grab/issues/43#issuecomment-431076499 async with client.get(download_url, retry_for_statuses=retry_for_statuses) as response: response_status = response.status if response_status != 200: log.warning( f'Got an unexpected HTTP code when trying to download the existing artifact "{artifact_name}". Expected: 200. Got: {response_status}' ) should_artifact_be_reuploaded = True return should_artifact_be_reuploaded
async def _download_from_asyncgen( items: AsyncGenerator, params: DownloadParams, tcp_connections: int = 64, nb_workers: int = 64, batch_size: int = 16, retries: int = 1, logger: logging.Logger = None, ): """Asynchronous downloader that takes an interable and downloads it Args: items (Union[Generator, AsyncGenerator]): (async/sync) generator that yiels a standardized dict of urls params (DownloadParams): Download parameter dict tcp_connections (int, optional): Maximum number of concurrent TCP connections. Defaults to 128. nb_workers (int, optional): Maximum number of workers. Defaults to 64. batch_size (int, optional): Maximum queue batch size. Defaults to 16. retries (int, optional): Maximum number of attempts. Defaults to 1. logger (logging.Logger, optional): Logger object. Defaults to None. Raises: NotImplementedError: If generator turns out to be invalid. """ queue = asyncio.Queue(nb_workers) progressbar = tqdm(smoothing=0, unit=" Downloads", disable=logger.getEffectiveLevel() > logging.INFO) stats = {"failed": 0, "skipped": 0, "success": 0} retry_options = ExponentialRetry(attempts=retries) async with RetryClient( connector=aiohttp.TCPConnector(limit=tcp_connections), raise_for_status=True, retry_options=retry_options, trust_env=True, ) as session: loop = asyncio.get_event_loop() workers = [ loop.create_task( _download_queue(queue, session, stats, params=params, progressbar=progressbar, logger=logger)) for _ in range(nb_workers) ] # get chunks from async generator and add to async queue async with aiostream.stream.chunks(items, batch_size).stream() as chnk: async for batch in chnk: await queue.put(batch) await queue.join() for w in workers: w.cancel() return stats
async def create(cls): self = IdiomScraper() self.client = RetryClient( raise_for_status=True, # raise exception if response status >= 400 timeout=ClientTimeout(total=CLIENT_TIMEOUT), # set a timeout value ) return self
async def async_query_api(self, endpoint, payload=None): """Queries a given endpoint on the Eskom loadshedding API with the specified payload Args: endpoint (string): The endpoint of the Eskom API payload (dict, optional): The parameters to apply to the query. Defaults to None. Returns: The response object from the request """ async with RetryClient() as client: # The Eskom API occasionally drops incoming connections, implement reies async with client.get( url=self.base_url + endpoint, headers=self.headers, params=payload, ssl=self.ssl_context, retry_attempts=50, retry_exceptions={ ClientConnectorError, ServerDisconnectedError, ConnectionError, OSError, }, ) as res: return await res.json()
def get_client_session(self, config: SeekerConfig) -> ClientSession: async def _on_request_start( session: ClientSession, trace_config_ctx: SimpleNamespace, params: TraceRequestStartParams ) -> None: current_attempt = \ trace_config_ctx.trace_request_ctx['current_attempt'] if(current_attempt > 1): logger.warning( f'::warn ::Retry Attempt #{current_attempt} ' + f'of {config.max_tries}: {params.url}') trace_config = TraceConfig() trace_config.on_request_start.append(_on_request_start) limit_per_host = max(0, config.connect_limit_per_host) connector = TCPConnector( limit_per_host=limit_per_host, ttl_dns_cache=600 # 10-minute DNS cache ) retry_options = ExponentialRetry( attempts=config.max_tries, max_timeout=config.max_time, exceptions=[ aiohttp.ClientError, asyncio.TimeoutError ]) return RetryClient( raise_for_status=True, connector=connector, timeout=ClientTimeout(total=config.timeout), headers={'User-Agent': config.agent}, retry_options=retry_options, trace_configs=[trace_config])
async def fetch_url(sem: asyncio.Semaphore, limit: int, offset: int): async with sem: log.debug(f'Started request with offset {offset}') async with RetryClient(auth=self.auth) as s, s.post( f'http://gta.intel.com/api/results/v2/results?limit={limit}&offset={offset}', data=json.dumps(payload), headers={ "Accept": "application/json", "Content-Type": "application/json", }, retry_attempts=20, retry_start_timeout=1, retry_factor=1.5, retry_for_statuses=[500], raise_for_status=True, retry_exceptions=[ aiohttp.client_exceptions.ClientPayloadError ]) as r: jsn = await r.json() res = [] for item in jsn['items']: if item['testRunUrl'][0] in self.url_list: res.append(item) log.debug(f'Finished request with offset {offset}') return res
async def get_tags_list_async(url: str, query: str, pages_count: int, timeout: float, retries_count: int) -> List[str]: """ Return tags list from `url` with `query`. Works asynchronous. :param url: URL for `query` :param query: Query for repos search :param pages_count: Pages count for tags list creation :param timeout: timeout between retries in case of status code != 200 :param retries_count: retries count in case of status code != 200 :return: List with tags """ tags_list = [] async with RetryClient() as client: for page_num in range(1, pages_count + 1): logger.debug(f'Page №{page_num} request') query_string = url + '?p=' + str( page_num) + '&q=' + query + '&type=Repositories&s=stars' request_result = await fetch(client, query_string, timeout, retries_count) logger.debug(f'Page №{page_num} processing') soup = BeautifulSoup(request_result, 'lxml') for tag in soup.find_all( 'a', attrs={'class': 'topic-tag topic-tag-link f6 px-2 mx-0'}): tags_list.append(tag.text.strip()) logger.debug( f'Tags count after page №{page_num} processing: {len(tags_list)}' ) return tags_list
def initialize_session(self) -> ClientSession: # Restrict the size of the connection pool for scraping ethic conn = aiohttp.TCPConnector(limit_per_host=30) retry_options = ListRetry(timeouts=[0, 0, 0.6, 1.2], statuses={500, 502, 503, 504, 514}) retry_client = RetryClient(connector=conn, retry_options=retry_options) # TODO we should inform the type checker that RetryClient has the same interface # with that of ClientSession. We can either do it nominally by making # RetryClient a subclass of ClientSession, or do it structurally by duck typing # / structural typing / typing.Protocol, etc. session = cast(ClientSession, retry_client) origin_get = session.get def patched_get(self, *args, **kwargs): # Add random parameter to the URL to break potential cache mechanism of # the server or the network or the aiohttp library. salt_key = "锟斤铐" # TODO can we just use random bytes as salt_value? salt_value = "".join(random.choices(string.hexdigits, k=10)) params = kwargs.setdefault("params", {}) params[salt_key] = salt_value return origin_get(*args, **kwargs) session.get = MethodType(patched_get, session) return session
async def replace_session(self): await self.session.close() proxy_connector = ProxyConnector(**env.TELEGRAPH_PROXY_DICT, loop=self.loop) \ if env.TELEGRAPH_PROXY_DICT else None self.session = RetryClient(connector=proxy_connector, timeout=ClientTimeout(total=10), loop=self.loop, json_serialize=self._json_serialize)
async def test_hello(aiohttp_client, loop): test_app = TestApp() app = test_app.get_app() client = await aiohttp_client(app) retry_client = RetryClient() retry_client._client = client async with retry_client.get('/ping') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() await client.close()
async def test_sometimes_error_with_raise_for_status(aiohttp_client, loop): test_app = TestApp() app = test_app.get_app() client = await aiohttp_client(app, raise_for_status=True) retry_client = RetryClient() retry_client._client = client async with retry_client.get('/sometimes_error', retry_attempts=5, retry_exceptions={ClientResponseError}) \ as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() await client.close()
async def test_sometimes_error(aiohttp_client, loop): test_app = TestApp() app = test_app.get_app() client = await aiohttp_client(app) retry_client = RetryClient() retry_client._client = client async with retry_client.get('/sometimes_error', retry_attempts=5) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() await client.close()
async def generate_link() -> Optional[str]: settings = Settings() assert settings.SPELLTABLE_AUTH_KEY headers = { "user-agent": f"spellbot/{__version__}", "key": settings.SPELLTABLE_AUTH_KEY, } data: Optional[dict[str, Any]] = None raw_data: Optional[bytes] = None try: async with RetryClient( raise_for_status=False, retry_options=ExponentialRetry(attempts=5), ) as client: async with client.post(settings.SPELLTABLE_CREATE, headers=headers) as resp: # Rather than use `resp.json()`, which respects minetype, let's just # grab the data and try to decode it ourselves. # https://github.com/inyutin/aiohttp_retry/issues/55 raw_data = await resp.read() data = json.loads(raw_data) if not data or "gameUrl" not in data: logger.warning( "warning: gameUrl missing from SpellTable API response (%s): %s", resp.status, data, ) return None assert data is not None returned_url = str(data["gameUrl"]) wizards_url = returned_url.replace( "www.spelltable.com", "spelltable.wizards.com", ) return wizards_url except ClientError as ex: add_span_error(ex) logger.warning( "warning: SpellTable API failure: %s, data: %s, raw: %s", ex, data, raw_data, exc_info=True, ) return None except Exception as ex: add_span_error(ex) logger.error( "error: unexpected exception: %s, data: %s, raw: %s", ex, data, raw_data, exc_info=True, ) return None
async def async_get_zone_json(self, zoneJsonUrl): async with RetryClient() as client: async with client.get( url=zoneJsonUrl, headers=self.headers, ssl=self.ssl_context, retry_attemps=10, ) as res: return await res.json()
async def get_active_tenders(self): params = {"ticket": self.MERCADO_PUBLICO_TICKET, "estado": "activas"} try: async with RetryClient() as client: async with client.get(url=self.MERCADO_PUBLICO_API_URL, params=params) as response: response.raise_for_status() return await response.json() except Exception as e: raise Exception("Request Error: ", e) from None
async def test_override_options(aiohttp_client, loop): test_app = App() app = test_app.get_app() client = await aiohttp_client(app) retry_options = RetryOptions(attempts=1) retry_client = RetryClient(retry_options=retry_options) retry_client._client = client retry_options = RetryOptions(attempts=5) async with retry_client.get('/sometimes_error', retry_options) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() await client.close()
async def create(cls, epsagon_token, retry_attempts=DEFAULT_RETRY_ATTEMPTS): """ Creates a new EpsagonClient instance :param epsagon_token: used for authorization """ self = cls() if not epsagon_token: raise ValueError("Epsagon token must be given") self.epsagon_token = epsagon_token retry_options = ExponentialRetry(attempts=retry_attempts, exceptions=(ClientError, )) self.client = RetryClient(auth=BasicAuth(login=self.epsagon_token), headers={ "Content-Type": "application/json", }, retry_options=retry_options, raise_for_status=True) return self
async def download_file(download_url, file_destination): async with RetryClient() as s3_client: async with s3_client.get(download_url) as resp: with open(file_destination, "wb") as fd: while True: chunk = await resp.content.read(CHUNK_SIZE) if not chunk: break fd.write(chunk) log.info(f"'{file_destination}' downloaded")
async def get_session(timeout: Optional[float] = None): if not timeout: timeout = 12 proxy_connector = ProxyConnector.from_url(PROXY) if PROXY else None session = RetryClient(retry_options=RETRY_OPTION, connector=proxy_connector, timeout=aiohttp.ClientTimeout(total=timeout), headers={'User-Agent': env.USER_AGENT}) return session
async def get_tenders_by_date(self, date): params = {"ticket": self.MERCADO_PUBLICO_TICKET, "fecha": date} try: async with RetryClient() as client: async with client.get(url=self.MERCADO_PUBLICO_API_URL, params=params, retry_attempts=self.retry_attempts, retry_for_statuses=self. retry_for_statuses) as response: response.raise_for_status() return await response.json() except Exception as e: raise Exception("Request Error: ", e) from None
async def async_main(token, branch, commit, workflow, artifacts_directory, locales, derived_data_path=None): headers = {"Authorization": token} async with RetryClient(headers=headers) as client: build_slug = await schedule_build(client, branch, commit, workflow, locales, derived_data_path) log.info("Created new job. Slug: {}".format(build_slug)) try: await wait_for_job_to_finish(client, build_slug) log.info("Job {} is successful. Retrieving artifacts...".format(build_slug)) await download_artifacts(client, build_slug, artifacts_directory) finally: log.info("Retrieving bitrise log...") await download_log(client, build_slug, artifacts_directory)
async def async_main(token, *args): headers = {"Authorization": token} async with RetryClient(headers=headers) as client: build_slug = await schedule_build(client, *args) log.info("Created new job. Slug: {}".format(build_slug)) try: await wait_for_job_to_finish(client, build_slug) log.info("Job {} is successful. Retrieving artifacts...".format( build_slug)) await download_artifacts(client, build_slug) finally: log.info("Retrieving bitrise log...") await download_log(client, build_slug)
async def fetch_elos(self, steam_ids: list[SteamId], *, headers: Optional[dict[str, str]] = None): if len(steam_ids) == 0: return None formatted_steam_ids = "+".join([str(steam_id) for steam_id in steam_ids]) request_url = f"{self.url_base}{self.balance_api}/{formatted_steam_ids}" retry_options = ExponentialRetry(attempts=3, factor=0.1, statuses={500, 502, 504}, exceptions={aiohttp.ClientResponseError, aiohttp.ClientPayloadError}) async with RetryClient(raise_for_status=False, retry_options=retry_options, timeout=ClientTimeout(total=5, connect=3, sock_connect=3, sock_read=5)) as retry_client: async with retry_client.get(request_url, headers=headers) as result: if result.status != 200: return None return await result.json()
async def _get_status_code(location: str, client: RetryClient, retry: int) -> int: _tracker.stats.inc_requests() status_code = 0 try: _logger.debug("Requesting status code for %s", location) async with client.get(location, retry_attempts=retry) as response: status_code = response.status except TooManyRedirects: _logger.debug("Redirection Tango, danced enough with %s", location) except ClientConnectionError: _logger.debug("Connection Error occurred while getting %s", location) return status_code
async def fetch(client: RetryClient, query_string: str, timeout: float, retries_count: int) -> Text: """ Fetch result of query :param client: Client with retry mechanism :param query_string: Full query string for repos getting :param timeout: timeout between retries in case of status code != 200 :param retries_count: retries count in case of status code != 200 :return: """ async with client.get(url=query_string, retry_attempts=retries_count, retry_start_timeout=timeout, retry_factor=2, retry_max_timeout=timeout * (2**retries_count), retry_for_statuses=[429]) as response: return await response.text()
async def get_transcript(url): async with semaphore: async with RetryClient(raise_for_status=False) as client: async with client.post( 'https://brain.deepgram.com/v2/listen', headers={ 'Content-type': 'application/json', 'Authorization': 'Basic {}'.format( base64.b64encode('{}:{}'.format(*creds).encode('utf-8')).decode('utf-8') ) }, data=json.dumps({ 'url': url }).encode('utf-8') ) as response: response_content = await response.text() return json.loads(response_content)