def wrap(f): if asyncio and asyncio.iscoroutinefunction(f): r = AsyncRetrying(*dargs, **dkw) else: r = Retrying(*dargs, **dkw) return r.wraps(f)
def wrap(f): if asyncio and asyncio.iscoroutinefunction(f): r = AsyncRetrying(*dargs, **dkw) #elif tornado and tornado.gen.is_coroutine_function(f): # r = TornadoRetrying(*dargs, **dkw) else: r = Retrying(*dargs, **dkw) return r.wraps(f)
async def startrace(self, **kwargs): """ Create a race. Creates a handler for the room and returns a handler object. """ if kwargs.get('goal') and kwargs.get('custom_goal'): # TODO use a specific error class raise Exception('Either a goal or custom_goal can be specified, but not both.') try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(aiohttp.ClientResponseError), wait=wait_exponential(multiplier=1, min=4, max=10)): with attempt: async with self.http.post( url=self.http_uri(f'/o/{self.category_slug}/startrace'), data=kwargs, ssl=self.ssl_context, headers={ 'Authorization': 'Bearer ' + self.access_token, } ) as resp: headers = resp.headers except RetryError as e: raise e.last_attempt._exception from e if 'Location' in headers: race_name = headers['Location'][1:] return await self.join_race_room(race_name) raise Exception('Received an unexpected response when creating a race.')
async def start(self): self.access_token, self.reauthorize_every = await self.authorize() self.loop.create_task(self.reauthorize()) self.loop.create_task(self.refresh_races()) unlisted_rooms = await models.RTGGUnlistedRooms.filter( category=self.category_slug) for unlisted_room in unlisted_rooms: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type( aiohttp.ClientResponseError)): with attempt: async with self.http.get( self.http_uri( f'/{unlisted_room.room_name}/data'), ssl=self.ssl_context, ) as resp: race_data = await resp.json() if race_data['status']['value'] in [ 'finished', 'cancelled' ] or not race_data['unlisted']: await unlisted_room.delete() else: await self.join_race_room(unlisted_room.room_name) except RetryError as e: raise e.last_attempt._exception from e
async def fetch_from_url(url: str, job: str, instance: str, retries: int = 5) -> Optional[str]: """ Fetch content from specified URL URL will be retried up to 'retries' times Args: url: str URL to fetch job: Class name or other description of download type, used when logging instance: Specific software instance being downloaded, used when logging retries: int specifying the number of times to retry URL Returns: The supplied 'Collection' class instance """ count = 0 try: async for attempt in AsyncRetrying(stop=stop_after_delay(retries)): with attempt: count += 1 if count > 1: fastapi_logger.info('Fetching %s "%s" metadata (try %s)', job, instance, count) # Create HTTP session async with aiohttp.ClientSession() as session: # Fetch latest JSON from Ansible Galaxy API async with session.get(url) as response: # Cache latest JSON return await response.text() except RetryError: fastapi_logger.exception('Error fetching %s "%s" URL %s', job, instance, url) return None
async def _connect_ssh(hostname: Hostname, *args, **kwargs) -> AsyncIterator[Machine]: """Connect to the given host, retrying if necessary. A pretty simple wrapper around asyncssh.connect, with a couple of changes: - waits for /var/lib/cloud/instance/boot-finished to exist (AWS Ubuntu has this when the machine is ready) - yields a Machine instead of just connections: a nice wrapper of the connection with a hostname - retries until the machine is ready """ reraise_err = None async for attempt in AsyncRetrying(wait=wait_fixed(2)): with attempt: async with asyncssh.connect(hostname, *args, **kwargs) as conn: # SSH may be ready but really the system isn't until this file exists. await conn.run("test -f /var/lib/cloud/instance/boot-finished", check=True) try: yield Machine(conn, Hostname(hostname), kwargs) except BaseException as err: # pylint: disable=broad-except # Exceptions from "yield" have nothing to do with us. # We reraise them below without retrying. reraise_err = err if reraise_err is not None: raise reraise_err from None
async def get_current_version(self) -> version.Version: async with httpx.AsyncClient(timeout=self.timeout) as client: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), reraise=True, retry=retry_if_exception_type(httpx.ConnectTimeout), wait=wait_random(min=self.timeout, max=self.timeout * 10), ): with attempt: r = await client.get(self.checkurl) except (httpx.HTTPStatusError, httpx.ReadTimeout): return None webpage = r.text # FIXME: this is a hardcode tarball_regex = re.compile(r"\bemacs-[0-9.]+\.tar\.[a-z]*\b") version_regex = re.compile(r"\b[0-9.]+\b") currentversion = None for tarball in tarball_regex.findall(webpage): versionstring = version_regex.findall(tarball)[0] if versionstring.endswith("."): versionstring = versionstring[:-1] emacsversion = version.parse(versionstring) if currentversion is None or currentversion < emacsversion: currentversion = emacsversion return emacsversion
async def generate(self, spoilers="off", tournament=True, allow_quickswap=True): if self.preset_data is None: await self.fetch() try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(ClientResponseError)): with attempt: try: mystery = await mystery_generate(self.preset_data, spoilers=spoilers) if mystery.doors: seed = await AlttprDoorDiscord.create( settings=mystery.settings, spoilers=spoilers != "mystery", ) else: if mystery.customizer: endpoint = "/api/customizer" else: endpoint = "/api/randomizer" mystery.settings['tournament'] = tournament mystery.settings[ 'allow_quickswap'] = allow_quickswap seed = await ALTTPRDiscord.generate( settings=mystery.settings, endpoint=endpoint) except: await models.AuditGeneratedGames.create( randomizer='alttpr', settings=mystery.settings, gentype='mystery failure', genoption=self.preset, customizer=1 if mystery.customizer else 0, doors=mystery.doors) logging.exception( "Failed to generate game, retrying...") raise except RetryError as e: raise e.last_attempt._exception from e await models.AuditGeneratedGames.create( randomizer='alttpr', hash_id=seed.hash, permalink=seed.url, settings=mystery.settings, gentype='mystery', genoption=self.preset, customizer=1 if mystery.customizer else 0, doors=mystery.doors) mystery.seed = seed return mystery
def build(self) -> AsyncRetrying: return AsyncRetrying( wait=self.wait, retry=self.retry_condition, stop=self.stop, before_sleep=self.before_sleep, after=self.after, reraise=self.reraise() )
async def join_race_room(self, race_name, force=False): """ Retrieve current race information for the requested race_name and joins that room, if the room isn't already being handled. If it's already handled, no action is taken. This is useful if the room is unlisted, and as a result refresh_races would be unable to find and join the race room. `race_name` is in the format of categoryslug/adjective-verb-0123 """ def done(task_name, *args): del self.handlers[task_name] self.logger.info(f'Attempting to join {race_name}') if not race_name.split('/')[0] == self.category_slug: # TODO create a real exception class for this raise Exception('Race is not for the bot\'s category category.') try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(aiohttp.ClientResponseError), wait=wait_exponential(multiplier=1, min=4, max=10)): with attempt: async with self.http.get( self.http_uri(f'/{race_name}/data'), ssl=self.ssl_context, ) as resp: data = json.loads(await resp.read()) except RetryError as e: raise e.last_attempt._exception from e name = data['name'] async with self.join_lock: if name in self.handlers: self.logger.info(f'Returning existing handler for {name}') return self.handlers[name] if self.should_handle(data) or force: handler = await self.create_handler(data) self.handlers[name] = TaskHandler() self.handlers[name].task = self.loop.create_task(handler.handle()) self.handlers[name].task.add_done_callback(partial(done, name)) self.handlers[name].handler = handler return handler else: if name in self.state: del self.state[name] self.logger.info( 'Ignoring %(race)s by configuration.' % {'race': data.get('name')} )
async def create_client(url) -> aioredis.Redis: # create redis client client: Optional[aioredis.Redis] = None async for attempt in AsyncRetrying(**retry_upon_init_policy): with attempt: client = await aioredis.create_redis_pool(url, encoding="utf-8") if not client: raise ValueError( "Expected aioredis client instance, got {client}") return client
def wrap(f): if asyncio and asyncio.iscoroutinefunction(f): r = AsyncRetrying(*dargs, **dkw) else: r = Retrying(*dargs, **dkw) @six.wraps(f) def wrapped_f(*args, **kw): return r.call(f, *args, **kw) wrapped_f.retry = r return wrapped_f
async def _create_pg_engine(dsn: DataSourceName, min_size: int, max_size: int) -> Engine: log.info("Creating pg engine for %s", dsn) async for attempt in AsyncRetrying( **PostgresRetryPolicyUponInitialization(log).kwargs): with attempt: engine = await create_pg_engine(dsn, minsize=min_size, maxsize=max_size) await raise_if_not_responsive(engine) return engine # type: ignore # tenacity rules guarantee exit with exc
async def fetch_url(url: str, timeout: int = 5, retry: int = 5): async with httpx.AsyncClient(timeout=timeout) as client: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), reraise=True, retry=retry_if_exception_type(httpx.ConnectTimeout), wait=wait_random(min=timeout, max=timeout * 10), ): with attempt: r = await client.get(url) except (httpx.HTTPStatusError, httpx.ReadTimeout) as e: return str(e) return r.json()
async def make_request(method='get', api_url='', api_key='', data={}, retries=0): """ Make call to external urls using python request :param method: get|post (str) :param api_url: :param api_key: :param data: data to send in the request (dict) :param retries: :return: """ headers = {'content-type': 'application/json'} if api_key and isinstance(data, dict): data['api_key'] = api_key method = method.upper() respx.route(method=method, path=api_url).mock(return_value=httpx.Response( status_code=MOCK_URLS[api_url]['status_code'], json=MOCK_URLS[api_url]['data'])) event_hooks = {} if DEBUG_MODE: event_hooks = {'request': [log_request], 'response': [log_response]} async with httpx.AsyncClient(event_hooks=event_hooks) as client: try: async for attempt in AsyncRetrying( stop=stop_after_attempt(retries), wait=wait_fixed(2)): with attempt: if method == httpretty.GET: response = await client.get(api_url, params=data, headers=headers) elif method == httpretty.POST: response = await client.post( api_url, data=json.dumps(data, default=date_time_json_serialize), headers=headers) if response.status_code in STATUS_FORCE_LIST: response.raise_for_status() return { 'status_code': response.status_code, 'data': response.json() } except RetryError: return { 'status_code': INTERNAL_SERVER_STATUS_CODE, 'data': INTERNAL_SERVER_ERROR_MESSAGE }
async def wrapper(db, *args, **kwargs): try: return await function(db, *args, **kwargs) except (InterfaceError, OperationalError, psycopg2.InterfaceError, psycopg2.OperationalError): try: async for attempt in AsyncRetrying( wait=wait_exponential(multiplier=1, min=2, max=10), before_sleep=before_sleep_log(log, logging.ERROR)): with attempt: db._objects.database._connect() except RetryError: pass else: return await function(db, *args, **kwargs)
async def test_retry_object(vault_prefix, mocker): config = Konfig(vault_backend=AsyncVaultBackend( vault_prefix, retry=AsyncRetrying(retry=retry_if_exception_type(KonfettiError), reraise=True, stop=stop_after_attempt(2)), )) mocker.patch("aiohttp.ClientSession._request", side_effect=KonfettiError) m = mocker.patch.object(config.vault_backend, "_call", wraps=config.vault_backend._call) with pytest.raises(KonfettiError): await config.SECRET assert m.called is True assert m.call_count == 2
async def pg_engine(app: web.Application): cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] pg_cfg = cfg["postgres"] app[f"{__name__}.dsn"] = dsn = DataSourceName( application_name=f"{__name__}_{id(app)}", database=pg_cfg["database"], user=pg_cfg["user"], password=pg_cfg["password"], host=pg_cfg["host"], port=pg_cfg["port"], ) # type: ignore pool_kwargs: Dict[str, Any] = {} if app[APP_CONFIG_KEY]["main"]["testing"]: pool_kwargs["echo"] = True log.info("Creating pg engine for %s", dsn) engine: Optional[Engine] = None async for attempt in AsyncRetrying( **PostgresRetryPolicyUponInitialization(log).kwargs): with attempt: engine = await create_pg_engine(dsn, minsize=pg_cfg["minsize"], maxsize=pg_cfg["maxsize"], **pool_kwargs) assert engine # nosec await raise_if_not_responsive(engine) assert engine # nosec app[APP_DB_ENGINE_KEY] = engine yield # ------------------- if engine is not app.get(APP_DB_ENGINE_KEY): log.critical( "app does not hold right db engine. Somebody has changed it??") engine.close() await engine.wait_closed() log.debug( "engine '%s' after shutdown: closed=%s, size=%d", engine.dsn, engine.closed, engine.size, )
async def generate_game(self): try: async for attempt in AsyncRetrying( stop=stop_after_attempt(3), retry=retry_if_exception_type((aiohttp.ClientResponseError, aiohttp.client_exceptions.ServerDisconnectedError))): with attempt: async with aiohttp.request( method='post', url=f'{self.baseurl}/randomizerWebService', data=self.settings, auth=self.auth, raise_for_status=True) as resp: req = await resp.json(content_type='text/html') return req except RetryError as e: raise e.last_attempt._exception from e
async def get_team(self, team_name): try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(aiohttp.ClientResponseError), wait=wait_exponential(multiplier=1, min=4, max=10)): with attempt: async with self.http.get( self.http_uri(f'/team/{team_name}/data'), ssl=self.ssl_context, ) as resp: data = json.loads(await resp.read()) except RetryError as e: raise e.last_attempt._exception from e return data
async def pg_engine(app: web.Application): engine = None pg_cfg: PostgresSettings = app[APP_CONFIG_KEY].STORAGE_POSTGRES dsn = DataSourceName( application_name=f"{__name__}_{id(app)}", database=pg_cfg.POSTGRES_DB, user=pg_cfg.POSTGRES_USER, password=pg_cfg.POSTGRES_PASSWORD.get_secret_value(), host=pg_cfg.POSTGRES_HOST, port=pg_cfg.POSTGRES_PORT, ) # type: ignore log.info("Creating pg engine for %s", dsn) async for attempt in AsyncRetrying( **PostgresRetryPolicyUponInitialization(log).kwargs ): with attempt: engine = await create_pg_engine( dsn, minsize=pg_cfg.POSTGRES_MINSIZE, maxsize=pg_cfg.POSTGRES_MAXSIZE ) await raise_if_not_responsive(engine) if app[APP_CONFIG_KEY].STORAGE_TESTING: log.info("Initializing tables for %s", dsn) init_pg_tables(dsn, schema=metadata) assert engine # nosec app[APP_DB_ENGINE_KEY] = engine yield # ---------- if engine is not app.get(APP_DB_ENGINE_KEY): log.critical("app does not hold right db engine. Somebody has changed it??") if engine: engine.close() await engine.wait_closed() log.debug( "engine '%s' after shutdown: closed=%s, size=%d", engine.dsn, engine.closed, engine.size, )
async def GetGoogleCookie(self): """ Gets google cookie (used for each and every proxy) Blacklist proxies on error. """ def retry_if_proxies_remaining(ex): should_retry = True if isinstance(ex, ProxyError) and ex.response.status_code == 429: logging.info(( f"Proxy {self.proxies[self.proxy_index]} responded with 429." " Will retry request with another proxy.")) self._rate_limited_proxies.append( self.proxies[self.proxy_index]) del self.proxies[self.proxy_index] elif len(self.proxies) > 0: logging.error(( f"Proxy {self.proxies[self.proxy_index]} caused {str(ex)}." " Blacklisting proxy and will retry request with another." )) self.blacklisted_proxies.append(self.proxies[self.proxy_index]) del self.proxies[self.proxy_index] else: should_retry = False self._iterate_proxy() return should_retry cfg = self._retry_config if len(self.proxies) > 0: cfg = dict(retry=retry_if_exception(retry_if_proxies_remaining), reraise=cfg.get('reraise', True)) try: retryer = AsyncRetrying(**cfg) resp = await retryer.call( self._send_req, 'https://trends.google.com/?geo={geo}'.format( geo=self.hl[-2:]), timeout=self.timeout) finally: self.proxies.extend(self._rate_limited_proxies) self._rate_limited_proxies.clear() cookies = resp.cookies.items() return dict(filter(lambda i: i[0] == 'NID', cookies))
async def _assemble_cached_indexes(app: web.Application): """ Currently the static resources are contain 3 folders: osparc, s4l, tis each of them contain and index.html to be served to as the root of the site for each type of frontend. Caching these 3 items on start. This """ settings: StaticWebserverModuleSettings = get_settings(app) cached_indexes: Dict[str, str] = {} session: ClientSession = get_client_session(app) for frontend_name in FRONTEND_APPS_AVAILABLE: url = URL(settings.static_web_server_url) / frontend_name log.info("Fetching index from %s", url) try: # web-static server might still not be up async for attempt in AsyncRetrying(**RETRY_ON_STARTUP_POLICY): with attempt: response = await session.get(url, raise_for_status=True) body = await response.text() except ClientError as err: log.error("Could not fetch index from static server: %s", err) # ANE: Yes this is supposed to fail the boot process raise RuntimeError( f"Could not fetch index at {str(url)}. Stopping application boot" ) from err # fixes relative paths body = body.replace(f"../resource/{frontend_name}", f"resource/{frontend_name}") body = body.replace("boot.js", f"{frontend_name}/boot.js") log.info("Storing index for %s", url) cached_indexes[frontend_name] = body app[APP_FRONTEND_CACHED_INDEXES_KEY] = cached_indexes
async def generate_random_game(weightset='weighted', weights=None, tournament=True, spoilers="off", festive=False): if weights is None: weights = await get_weights(weightset) try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(ClientResponseError)): with attempt: try: settings, customizer = await generate_random_settings( weights, festive=festive, spoilers=spoilers) settings['tournament'] = tournament seed = await alttpr(settings=settings, customizer=customizer, festive=festive) except ClientResponseError: await audit.insert_generated_game( randomizer='alttpr', hash_id=None, permalink=None, settings=settings, gentype='mystery failure', genoption=weightset, customizer=1 if customizer else 0) raise except RetryError as e: raise e.last_attempt._exception from e await audit.insert_generated_game(randomizer='alttpr', hash_id=seed.hash, permalink=seed.url, settings=settings, gentype='mystery', genoption=weightset, customizer=1 if customizer else 0) return seed
async def authorize(self): """ Get an OAuth2 token from the authentication server. """ try: async for attempt in AsyncRetrying( stop=stop_after_attempt(5), retry=retry_if_exception_type(aiohttp.ClientResponseError)): with attempt: async with self.http.post(self.http_uri('/o/token'), data={ 'client_id': self.client_id, 'client_secret': self.client_secret, 'grant_type': 'client_credentials', }, ssl=self.ssl_context) as resp: data = await resp.json() if not data.get('access_token'): raise Exception('Unable to retrieve access token.') return data.get('access_token'), data.get('expires_in', 36000) except RetryError as e: raise e.last_attempt._exception from e
async def _get_data(self, url, method=GET_METHOD, trim_chars=0, **kwargs): """Send a request to Google and return the JSON response as a Python object :param url: the url to which the request will be sent :param method: the HTTP method ('get' or 'post') :param trim_chars: how many characters should be trimmed off the beginning of the content of the response before this is passed to the JSON parser :param kwargs: any extra key arguments passed to the request builder (usually query parameters or data) :return: """ if self.cookies is None or len(self.proxies) > 0: self.cookies = await self.GetGoogleCookie() retryer = AsyncRetrying(**self._retry_config) response = await retryer.call(self._send_req, url, method=method, timeout=self.timeout, cookies=self.cookies, headers={'accept-language': self.hl}, **kwargs) # check if the response contains json and throw an exception otherwise # Google mostly sends 'application/json' in the Content-Type header, # but occasionally it sends 'application/javascript # and sometimes even 'text/javascript if response.status_code == 200 and 'application/json' in \ response.headers['Content-Type'] or \ 'application/javascript' in response.headers['Content-Type'] or \ 'text/javascript' in response.headers['Content-Type']: # trim initial characters # some responses start with garbage characters, like ")]}'," # these have to be cleaned before being passed to the json parser content = response.text[trim_chars:] # parse json self._iterate_proxy() return json.loads(content) else: # error raise exceptions.ResponseError( 'The request failed: Google returned a ' 'response with code {0}.'.format(response.status_code), response=response)
async def additional_setup(self): with Halo("[infrastructure] starting etcd") as spinner: await self.publisher.ssh.run( f"HOSTNAME={self.publisher.hostname} " "envsubst '$HOSTNAME' " ' < "$HOME/config/etcd.template" ' " | sudo tee /etc/default/etcd " " > /dev/null", check=True, ) await self.publisher.ssh.run("sudo systemctl restart etcd", check=True) # Make sure etcd is healthy async for attempt in AsyncRetrying(wait=wait_fixed(2), stop=stop_after_attempt(20)): with attempt: await self.publisher.ssh.run( ("ETCDCTL_API=3 etcdctl " f"--endpoints {self.publisher.hostname}:2379 " "endpoint health"), check=True, ) spinner.succeed("[infrastructure] etcd healthy")
async def _celery_app(app: web.Application): comp_settings: ComputationSettings = _get_computation_settings(app) celery_app: Optional[Celery] = None async for attempt in AsyncRetrying(**retry_upon_init_policy): with attempt: celery_app = Celery( comp_settings.task_name, broker=comp_settings.broker_url, backend=comp_settings.result_backend, ) if not celery_app: raise ValueError( "Expected celery client app instance, got {celery_app}") app[__APP_CLIENT_CELERY_CLIENT_KEY] = celery_app yield if celery_app is not app[__APP_CLIENT_CELERY_CLIENT_KEY]: log.critical("Invalid celery client in app") celery_app.close()
async def test_retry_using_async_retying(self): thing = NoIOErrorAfterCount(5) retrying = AsyncRetrying() await retrying(_async_function, thing) assert thing.counter == thing.count
def _a_get_retry_object(self) -> AsyncRetrying: """ Instantiates an async retry object :return: instance of AsyncRetrying class """ return AsyncRetrying(**self.retry_args)