async def test_patch_aiohttp_session(*, execution): patch_aiohttp_session() from aiohttp.client import ClientSession trace_configs = [] async with ClientSession(trace_configs=trace_configs): assert trace_configs config = trace_configs[-1] on_request_start = config.on_request_start[-1] session = Mock() ctx = SimpleNamespace() params = Mock() params.headers = {} assert current_test_stack.top is None await on_request_start(session, ctx, params) with current_test_stack.push(execution): await on_request_start(session, ctx, params) assert params.headers assert 'LiveCheck-Test-Id' in params.headers async with ClientSession(trace_configs=None): pass
async def start(heliotrope: Heliotrope, loop: AbstractEventLoop): heliotrope.ctx.mongo = (mongo(heliotrope_app.config.MONGO_DB_URL, io_loop=loop).hitomi.info if mongo else None) heliotrope.ctx.hitomi_requester = HitomiRequester(ClientSession(loop=loop)) heliotrope.ctx.mirroring_manager = Mirroring(ClientSession(loop=loop), heliotrope.ctx.mongo) heliotrope.add_task(heliotrope.ctx.mirroring_manager.mirroring_task(3600))
def _session(self) -> ClientSession: return ClientSession(connector=ProxyConnector.from_url( self.http_proxy), raise_for_status=True, timeout=ClientTimeout(total=60, connect=30, sock_read=10))
async def opensips_cmd(cmd: str) -> dict: data = {"jsonrpc": "2.0", "method": cmd, "id": opensips_cmd_seq.id} async with ClientSession() as session: async with session.post(OSIPS_MI_URL, json=data) as resp: if resp.status == 200: return await resp.json() raise Exception(await resp.text())
async def _make_request(self, context: FContext, payload): sem = asyncio.Semaphore(200) conn = aiohttp.TCPConnector(use_dns_cache=True, loop=self.loop, limit=0) async with sem: async with ClientSession(connector=conn) as session: try: if self._timeout > 0: with async_timeout.timeout(self._timeout / 1000): async with session.post(self._url, data=payload, headers=self._headers) \ as response: return response.status, await response.content.read( ) else: async with session.post( self._url, data=payload, headers=self._headers) as response: return response.status, await response.content.read( ) except asyncio.TimeoutError: raise TTransportException( type=TTransportExceptionType.TIMED_OUT, message='request timed out')
async def get_client(klass) -> ClientSession: sess = getattr(klass, '_session') if sess: return sess client = ClientSession() setattr(klass, '_session', client) return client
async def get_by_id(cls, acc_id) -> Account: async with ClientSession() as session: async with session.get( f"{settings.ACCOUNT_SERVICE_ADDRESS}/accounts/{acc_id}" ) as resp: data = await resp.json() return Account(**data)
async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: """ Get the configuration for Args: url: the URL to query for the config, Defaults to self._data_sources_config_url Returns: DataSourceConfig: the data sources config """ if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) try: async with ClientSession(headers=self._extra_headers) as session: response = await session.get(url) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) else: error_details = await response.json() raise ClientError( f"Fetch data sources failed with status code {response.status}, error: {error_details}" ) except: logger.exception(f"Failed to load data sources config") raise
async def _make_request(self, context: FContext, payload): """ Helper method to make a request over the network. Args: payload: The data to be sent over the network. Return: The status code and body of the response. Throws: TTransportException if the request timed out. """ # construct headers for request request_headers = {} if self._get_request_headers is not None: request_headers = self._get_request_headers(context) # apply the default headers so their values cannot be modified request_headers.update(self._headers) timeout = ClientTimeout(total=context.timeout / 1000) async with ClientSession(timeout=timeout) as session: try: async with session.post(self._url, data=payload, headers=request_headers) \ as response: return response.status, await response.content.read() except ServerTimeoutError: raise TTransportException( type=TTransportExceptionType.TIMED_OUT, message='request timed out')
class ProxyBlobMixin: """Provides methods upload and download. Proxies request to other HTTP service""" http = ClientSession() def __init__(self, base, http_session=None): self.base = base if http_session: self.http = http_session or ClientSession() async def upload(self, stream, content_type=None): headers = {} if content_type is not None: headers['content-type'] = content_type async with self.http.post(f"{self.base}{self.id}", data=stream, headers=headers) as r: return await r.json() async def download(self, blobId): async with self.http.get(f"{self.base}{self.id}/{blobId}") as res: if res.status == 200: return await res.read() elif res.status // 100 == 5: raise errors.serverFail() raise errors.notFound(f'Blob {blobId} not found')
async def _check_fresh(self, dev_db: bool = False, site_db: bool = False, template_db: bool = False, group_db: bool = False): update_funcs = [] if dev_db: update_funcs += [self.update_dev_db] if site_db: update_funcs += [self.update_site_db] if template_db: update_funcs += [self.update_template_db] if group_db: update_funcs += [self.update_group_db] async with ClientSession() as self.central.aio_session: if update_funcs: if await update_funcs[0](): if len(update_funcs) > 1: await asyncio.gather(*[f() for f in update_funcs[1:]]) # update groups first so template update can use the result, and to trigger token_refresh if necessary elif await self.update_group_db(): await asyncio.gather(self.update_dev_db(), self.update_site_db(), self.update_template_db())
def build_session(self, apply_for: Optional[ClientSession] = None, *args, **kwargs) -> ClientSession: trace_config = TraceConfig() trace_config.on_request_start.append(self.on_request_start) trace_config.on_request_end.append(self.on_request_end) session = apply_for or ClientSession( trace_configs=[trace_config], *args, **kwargs) if apply_for is not None: session.trace_configs.append(trace_config) trace_config.freeze() def hook_method(method): def hooked_wrapper(*args, **kwargs): return method( *args, **{ **kwargs, "trace_request_ctx": (args, kwargs) }) return hooked_wrapper session.get = hook_method(session.get) session.post = hook_method(session.post) return session
def __init__( self, base_url: str, *, api_key: Optional[str] = None, admin_insecure: Optional[bool] = False, tenant_jwt: Optional[str] = None, ): if not api_key and not admin_insecure: raise Exception( "api_key property is missing. Use admin_insecure=True if you want" " to use the controller without authentication.") headers = {} if api_key: headers["x-api-key"] = api_key if tenant_jwt: headers["authorization"] = f"Bearer {tenant_jwt}" client_session = ClientSession( headers=headers, raise_for_status=True, ) super().__init__( base_url, client=client_session, extra_service_params={"converter": PydanticConverter()}, )
async def listen_to_runner(runner_url, artifact_manager, dput_host, debsign_keyid: Optional[str] = None, distributions: Optional[List[str]] = None, source_only: bool = False): from aiohttp.client import ClientSession import urllib.parse url = urllib.parse.urljoin(runner_url, "ws/result") async with ClientSession() as session: async for result in pubsub_reader(session, url): if result["code"] != "success": continue if not result['target']: continue if result['target']['name'] != 'debian': continue if not distributions or result['target']['details'][ 'build_distribution'] in distributions: await upload_build_result(result['log_id'], artifact_manager, dput_host, debsign_keyid=debsign_keyid, source_only=source_only)
async def main(): forms: Dict[str, Form] = await get_forms() for task_code in task2_data: parsed_task_code: str = parse_task_code(task_code) if parsed_task_code not in forms: print(f'Error: task code {parsed_task_code} does not exist') else: form: Form = forms[parsed_task_code] req_range: range = get_range(task2_data[task_code]) exist_range: range = get_range([form.min_year, form.max_year]) no_pdf_set: Set[int] = set(req_range) - set(exist_range) yes_pdf_set: Set[int] = set(req_range) & set(exist_range) if len(no_pdf_set): print(f'{task_code}. No PDFs for year(s): {no_pdf_set}') if not len(yes_pdf_set): print('{task_code}. No years to load') else: for year in yes_pdf_set: async with ClientSession() as PdfPage.session: pdf_page: PdfPage = PdfPage(form.url_year(year)) content: bytes = await pdf_page.content() folder: str = os.path.normpath( os.path.join(TASK2_DIR, task_code)) if not os.path.isdir(folder): os.mkdir(folder) filename: str = f'{task_code} - {year}.pdf' file_path: str = os.path.normpath( os.path.join(folder, filename)) async with aiofiles.open(file_path, mode='wb') as f: await f.write(content) print(f'{filename} successfully created')
async def test_borrow_connector_loop(connector: Any, create_session: Any, loop: Any) -> None: session = ClientSession(connector=connector) try: assert session._loop, loop finally: await session.close()
def test_del(self): conn = self.make_open_connector() session = ClientSession(loop=self.loop, connector=conn) with self.assertWarns(ResourceWarning): del session gc.collect()
def test_close(self): conn = self.make_open_connector() session = ClientSession(loop=self.loop, connector=conn) session.close() self.assertIsNone(session.connector) self.assertTrue(conn.closed)
async def test_client_session_timeout_zero() -> None: timeout = client.ClientTimeout(total=10, connect=0, sock_connect=0, sock_read=0) try: async with ClientSession(timeout=timeout) as session: await session.get("http://example.com") except asyncio.TimeoutError: pytest.fail("0 should disable timeout.")
async def on_message(self, target, source, message): if not message.startswith(self.nickname + ": "): return message = message[len(self.nickname + ": "):] m = re.match("reschedule (.*)", message) if m: await self.message(target, "Rescheduling %s" % m.group(1)) return if message == "status": if self._runner_status: status_strs = [ "%s (%s) since %s" % (item["package"], item["suite"], item["start_time"]) for item in self._runner_status["processing"] ] await self.message( target, "Currently processing: " + ", ".join(status_strs) + ".") else: await self.message(target, "Current runner status unknown.") if message == "scan": url = urljoin(self.publisher_url, "scan") async with ClientSession() as session, session.post(url) as resp: if resp.status in (200, 202): await self.message(target, "Merge proposal scan started.") else: await self.message( target, "Merge proposal scan failed: %d." % resp.status)
async def get_urls_filtered() -> List[str]: urls_filtered: List[str] urls_cache: str = os.path.join(CACHE_DIR, 'urls_cache.txt') if os.path.isfile(urls_cache): with open(urls_cache, 'r') as f: urls_filtered = f.readlines() else: async with ClientSession() as Page.session: root_sitemap: RootSitemap = RootSitemap() sitemap_urls: List[str] = await root_sitemap.urls() assert sitemap_urls # be sure that root sitemap loaded correctly sitemaps: List[Sitemap] = [] tasks: List[FetchFuncListStr] = [] for sitemap_url in sitemap_urls: sitemap: Sitemap = Sitemap(sitemap_url) sitemaps.append(sitemap) tasks.append(sitemap.urls()) urls: List[str] = [] # all urls from sitemaps urls_list: Tuple[List[str]] = await asyncio.gather(*tasks) for url_list_item in urls_list: urls += url_list_item urls_filtered: List[str] = list(filter(url_filter, urls)) with open(urls_cache, 'w') as f: f.write('\n'.join(urls_filtered)) return urls_filtered
async def invoke_async(self, runs, url, headers, secrets): results = RunList() tasks = [] async with ClientSession() as session: for run in runs: self.store_run(run) run.spec.secret_sources = secrets or [] tasks.append( asyncio.ensure_future( submit(session, url, run.to_dict(), headers), )) for status, resp, logs in await asyncio.gather(*tasks): if status != 200: logger.error("failed to access {} - {}".format(url, resp)) else: results.append(json.loads(resp)) if logs: parsed = parse_logs(logs) if parsed: print(parsed, '----------') return results
def test_init_cookies_with_list_of_tuples(self): session = ClientSession(cookies=[("c1", "cookie1"), ("c2", "cookie2")], loop=self.loop) self.assertEqual(set(session.cookies), {'c1', 'c2'}) self.assertEqual(session.cookies['c1'].value, 'cookie1') self.assertEqual(session.cookies['c2'].value, 'cookie2') session.close()
async def _invoke_async(self, tasks, url, headers, secrets, generator): results = RunList() runs = [] num_errors = 0 stop = False parallel_runs = generator.options.parallel_runs or 1 semaphore = asyncio.Semaphore(parallel_runs) async with ClientSession() as session: for task in tasks: # TODO: store run using async calls to improve performance self.store_run(task) task.spec.secret_sources = secrets or [] resp = submit(session, url, task, semaphore, headers=headers) runs.append(asyncio.ensure_future(resp, )) for result in asyncio.as_completed(runs): status, resp, logs, task = await result if status != 200: err_message = f"failed to access {url} - {resp}" # TODO: store logs using async calls to improve performance log_std( self._db_conn, task, parse_logs(logs) if logs else None, err_message, silent=True, ) # TODO: update run using async calls to improve performance results.append( self._update_run_state(task=task, err=err_message)) num_errors += 1 else: if logs: log_std(self._db_conn, task, parse_logs(logs)) resp = self._update_run_state(json.loads(resp)) state = get_in(resp, "status.state", "") if state == "error": num_errors += 1 results.append(resp) run_results = get_in(resp, "status.results", {}) stop = generator.eval_stop_condition(run_results) if stop: logger.info( f"reached early stop condition ({generator.options.stop_condition}), stopping iterations!" ) break if num_errors > generator.max_errors: logger.error("max errors reached, stopping iterations!") stop = True break if stop: for task in runs: task.cancel() return results
async def query_db(): await Tortoise.init( db_url=os.environ["DB_URL"], modules={ "models": [ "heliotrope.database.models.hitomi", "heliotrope.database.models.requestcount", ] }, ) await Tortoise.generate_schemas() hitomi = HitomiRequester(ClientSession()) galleryinfo = await hitomi.get_galleryinfo(1536576) galleyinfo_orm_object = await GalleryInfo.create( language_localname=galleryinfo.language_localname, language=galleryinfo.language, date=galleryinfo.date, japanese_title=galleryinfo.japanese_title, title=galleryinfo.title, id=galleryinfo.galleryid, type=galleryinfo.hitomi_type, ) if galleryinfo.files: file_orm_object_list = [] for file_info in galleryinfo.files: file_orm_object = File( index_id=galleryinfo.galleryid, width=file_info.get("width"), hash=file_info.get("hash"), haswebp=file_info.get("haswebp"), name=file_info.get("name"), height=file_info.get("height"), ) await file_orm_object.save() file_orm_object_list.append(file_orm_object) await galleyinfo_orm_object.files.add(*file_orm_object_list) if galleryinfo.tags: tag_orm_object_list = [] for tag_info in galleryinfo.tags: tag_orm_object = Tag( index_id=galleryinfo.galleryid, male=tag_info.get("male"), female=tag_info.get("female"), url=tag_info.get("url"), ) await tag_orm_object.save() tag_orm_object_list.append(tag_orm_object) await galleyinfo_orm_object.tags.add(*tag_orm_object_list) await Index.create(index_id="1536576") info = await hitomi.get_info_using_index(1536576) mongo = AsyncIOMotorClient(os.environ["MONGO_DB_URL"]).hitomi.info await mongo.insert_one(info) await hitomi.session.close()
def test_del(connector, loop, warning): # N.B. don't use session fixture, it stores extra reference internally session = ClientSession(connector=connector, loop=loop) loop.set_exception_handler(lambda loop, ctx: None) with warning(ResourceWarning): del session gc.collect()
def test_connector_loop(self): loop = asyncio.new_event_loop() connector = TCPConnector(loop=loop) with self.assertRaisesRegex(ValueError, "loop argument must agree with connector"): ClientSession(connector=connector, loop=self.loop) connector.close() loop.close()
def test_del(self): conn = self.make_open_connector() session = ClientSession(loop=self.loop, connector=conn) self.loop.set_exception_handler(lambda loop, ctx: None) with self.assertWarns(ResourceWarning): del session gc.collect()
def __init__(self, *args: Any, **kwargs: Any) -> None: self.proxy_url = kwargs.pop("proxy_url") self.permanent_headers = { "Proxy-Authorization-Key": kwargs.pop("authorization"), "Accept": "application/json", } self._session = ClientSession(*args, **kwargs)
async def get_weather(lat, lon): weather_api_key = os.environ['WEATHER_API_KEY'] url = 'https://api.openweathermap.org/data/2.5/weather?lat={}&lon={}&appid={}'.format(lat, lon, weather_api_key) print(url) async with ClientSession() as session: async with session.get(url) as response: result = await response.json() return result