async def test_connector(create_session, loop, mocker): connector = TCPConnector(loop=loop) mocker.spy(connector, 'close') session = create_session(connector=connector) assert session.connector is connector await session.close() assert connector.close.called connector.close()
def test_connector_loop(self): loop = asyncio.new_event_loop() connector = TCPConnector(loop=loop) with self.assertRaisesRegex( ValueError, "loop argument must agree with connector"): ClientSession(connector=connector, loop=self.loop) connector.close() loop.close()
async def make_connector(): return TCPConnector()
def _populate(database_url, legacy_database_url, steemd_http_url, start_block, end_block, accounts_file): CONNECTOR = TCPConnector(loop=loop, limit=100) AIOHTTP_SESSION = aiohttp.ClientSession( loop=loop, connector=CONNECTOR, json_serialize=json.dumps, headers={'Content-Type': 'application/json'}) DB_META = task_load_db_meta(legacy_database_url) try: pool = create_asyncpg_pool(database_url) task_num = 0 # [1/7] confirm db connectivity task_num += 1 task_message = fmt_task_message('Confirm database connectivity', emoji_code_point=u'\U0001F4DE', task_num=task_num) click.echo(task_message) #task_confirm_db_connectivity(database_url) # [2/7] init db if required task_num += 1 task_message = fmt_task_message('Initialising db if required', emoji_code_point=u'\U0001F50C', task_num=task_num) click.echo(task_message) task_init_db_if_required(database_url=database_url) # [3/7] find last irreversible block task_num += 1 if end_block == -1: task_message = fmt_task_message('Finding highest blockchain block', emoji_code_point='\U0001F50E', task_num=task_num) click.echo(task_message) last_chain_block_num = loop.run_until_complete( get_last_irreversible_block_num(steemd_http_url, AIOHTTP_SESSION)) end_block = last_chain_block_num success_msg = fmt_success_message( 'last irreversible block number is %s', last_chain_block_num) click.echo(success_msg) else: task_message = fmt_task_message( f'Using --end_block {end_block} as highest blockchain block to load', emoji_code_point='\U0001F50E', task_num=task_num) click.echo(task_message) # [4/7] build list of blocks missing from db existing_count, missing_count, range_count = loop.run_until_complete( get_existing_and_missing_count(pool, start_block, end_block)) task_message = fmt_task_message( f'Building list of {missing_count} blocks missing from db between {start_block}<<-->>{end_block}', emoji_code_point=u'\U0001F52D', task_num=4) click.echo(task_message) with click.progressbar(length=end_block, **pbar_kwargs) as pbar: pbar.update(existing_count) missing_block_nums = loop.run_until_complete( collect_missing_block_nums(pool, end_block, missing_count, start_block=start_block, pbar=pbar)) # [5.1/7] preload accounts file if accounts_file: task_message = fmt_task_message('Preloading account names', emoji_code_point=u'\U0001F52D', task_num=5) click.echo(task_message) with open(accounts_file) as f: account_names = json.load(f) loop.run_until_complete(preload_account_names(pool, account_names)) del account_names # [5/7] add missing blocks and operations task_message = fmt_task_message( 'Adding missing blocks and operations to db', emoji_code_point=u'\U0001F52D', task_num=5) click.echo(task_message) blocks_progress_bar = tqdm( initial=existing_count, total=range_count, bar_format='{bar}| [{rate_fmt}{postfix}]', ncols=48, dynamic_ncols=False, unit=' blocks', ) ops_progress_bar = tqdm(initial=existing_count * 50, total=range_count * 50, bar_format='{bar}| [{rate_fmt}{postfix}]', ncols=48, dynamic_ncols=False, unit=' ops') loop.run_until_complete( process_blocks(missing_block_nums, steemd_http_url, AIOHTTP_SESSION, pool, DB_META, blocks_pbar=blocks_progress_bar, ops_pbar=ops_progress_bar)) # [6/7] Make second sweep for missing blocks task_message = fmt_task_message( 'Adding missing blocks and operations to db (second sweep)', emoji_code_point=u'\U0001F52D', task_num=6) click.echo(task_message) existing_count, missing_count, range_count = loop.run_until_complete( get_existing_and_missing_count(pool, start_block, end_block)) task_message = fmt_task_message( f'Building list of {missing_count} blocks missing from db between {start_block}<<-->>{end_block}', emoji_code_point=u'\U0001F52D', task_num=4) click.echo(task_message) with click.progressbar(length=end_block, **pbar_kwargs) as pbar: pbar.update(existing_count) missing_block_nums = loop.run_until_complete( collect_missing_block_nums(pool, end_block, missing_count, start_block=start_block, pbar=pbar)) blocks_progress_bar = tqdm( initial=existing_count, total=range_count, dynamic_ncols=False, unit=' blocks', ) ops_progress_bar = tqdm(initial=existing_count * 50, total=range_count * 50, dynamic_ncols=False, unit=' ops') loop.run_until_complete( process_blocks(missing_block_nums, steemd_http_url, AIOHTTP_SESSION, pool, DB_META, blocks_pbar=blocks_progress_bar, ops_pbar=ops_progress_bar)) # [7/7] stream new blocks task_message = fmt_task_message('Streaming blocks', emoji_code_point=u'\U0001F4DD', task_num=7) click.echo(task_message) except KeyboardInterrupt: pass except Exception as e: logger.exception('ERROR') raise e
def test_connector(create_session, loop): connector = TCPConnector(loop=loop) session = create_session(connector=connector) assert session.connector is connector
async def fetch(self, url: str) -> str: proxy, headers = self.args.proxy, self.args.headers async with ClientSession(connector=TCPConnector(ssl=False)) as client: # type: ClientSession async with client.get(url, proxy=proxy, headers=headers) as resp: # type: ClientResponse return await resp.text(encoding='utf-8')
async def create_client_session(app): app['client_session'] = aiohttp.ClientSession(connector=TCPConnector(limit=300))
def client_request( method: str, url: StrOrURL, *, params: Optional[Mapping[str, str]] = None, data: Any = None, json: Any = None, headers: LooseHeaders = None, skip_auto_headers: Optional[Iterable[str]] = None, auth: Optional[BasicAuth] = None, allow_redirects: bool = True, max_redirects: int = 10, compress: Optional[str] = None, chunked: Optional[bool] = None, expect100: bool = False, raise_for_status: Optional[bool] = None, read_until_eof: bool = True, proxy: Optional[StrOrURL] = None, proxy_auth: Optional[BasicAuth] = None, timeout: Union[ClientTimeout, object] = sentinel, cookies: Optional[LooseCookies] = None, version: HttpVersion = http.HttpVersion11, connector: Optional[BaseConnector] = None, loop: Optional[asyncio.AbstractEventLoop] = None ) -> _SessionRequestContextManager: """ as aiohttp.client.request using a client session that does not decompress, i.e auto_decompress=False """ connector_owner = False if connector is None: connector_owner = True connector = TCPConnector(loop=loop, force_close=True) session = ClientSession( loop=loop, cookies=cookies, version=version, timeout=timeout, connector=connector, connector_owner=connector_owner, auto_decompress=False, ) return _SessionRequestContextManager( session._request( method, url, params=params, data=data, json=json, headers=headers, skip_auto_headers=skip_auto_headers, auth=auth, allow_redirects=allow_redirects, max_redirects=max_redirects, compress=compress, chunked=chunked, expect100=expect100, raise_for_status=raise_for_status, read_until_eof=read_until_eof, proxy=proxy, proxy_auth=proxy_auth, ), session, )
def test_connector(self): connector = TCPConnector(loop=self.loop) session = ClientSession(connector=connector, loop=self.loop) self.assertIs(session.connector, connector) session.close()