async def test_connector(create_session, loop, mocker):
connector = TCPConnector(loop=loop)
mocker.spy(connector, 'close')
session = create_session(connector=connector)
assert session.connector is connector
await session.close()
assert connector.close.called
connector.close()
def test_connector_loop(self):
loop = asyncio.new_event_loop()
connector = TCPConnector(loop=loop)
with self.assertRaisesRegex(
ValueError,
"loop argument must agree with connector"):
ClientSession(connector=connector, loop=self.loop)
connector.close()
loop.close()
async def make_connector():
return TCPConnector()
def _populate(database_url, legacy_database_url, steemd_http_url, start_block,
end_block, accounts_file):
CONNECTOR = TCPConnector(loop=loop, limit=100)
AIOHTTP_SESSION = aiohttp.ClientSession(
loop=loop,
connector=CONNECTOR,
json_serialize=json.dumps,
headers={'Content-Type': 'application/json'})
DB_META = task_load_db_meta(legacy_database_url)
try:
pool = create_asyncpg_pool(database_url)
task_num = 0
# [1/7] confirm db connectivity
task_num += 1
task_message = fmt_task_message('Confirm database connectivity',
emoji_code_point=u'\U0001F4DE',
task_num=task_num)
click.echo(task_message)
#task_confirm_db_connectivity(database_url)
# [2/7] init db if required
task_num += 1
task_message = fmt_task_message('Initialising db if required',
emoji_code_point=u'\U0001F50C',
task_num=task_num)
click.echo(task_message)
task_init_db_if_required(database_url=database_url)
# [3/7] find last irreversible block
task_num += 1
if end_block == -1:
task_message = fmt_task_message('Finding highest blockchain block',
emoji_code_point='\U0001F50E',
task_num=task_num)
click.echo(task_message)
last_chain_block_num = loop.run_until_complete(
get_last_irreversible_block_num(steemd_http_url,
AIOHTTP_SESSION))
end_block = last_chain_block_num
success_msg = fmt_success_message(
'last irreversible block number is %s', last_chain_block_num)
click.echo(success_msg)
else:
task_message = fmt_task_message(
f'Using --end_block {end_block} as highest blockchain block to load',
emoji_code_point='\U0001F50E',
task_num=task_num)
click.echo(task_message)
# [4/7] build list of blocks missing from db
existing_count, missing_count, range_count = loop.run_until_complete(
get_existing_and_missing_count(pool, start_block, end_block))
task_message = fmt_task_message(
f'Building list of {missing_count} blocks missing from db between {start_block}<<-->>{end_block}',
emoji_code_point=u'\U0001F52D',
task_num=4)
click.echo(task_message)
with click.progressbar(length=end_block, **pbar_kwargs) as pbar:
pbar.update(existing_count)
missing_block_nums = loop.run_until_complete(
collect_missing_block_nums(pool,
end_block,
missing_count,
start_block=start_block,
pbar=pbar))
# [5.1/7] preload accounts file
if accounts_file:
task_message = fmt_task_message('Preloading account names',
emoji_code_point=u'\U0001F52D',
task_num=5)
click.echo(task_message)
with open(accounts_file) as f:
account_names = json.load(f)
loop.run_until_complete(preload_account_names(pool, account_names))
del account_names
# [5/7] add missing blocks and operations
task_message = fmt_task_message(
'Adding missing blocks and operations to db',
emoji_code_point=u'\U0001F52D',
task_num=5)
click.echo(task_message)
blocks_progress_bar = tqdm(
initial=existing_count,
total=range_count,
bar_format='{bar}| [{rate_fmt}{postfix}]',
ncols=48,
dynamic_ncols=False,
unit=' blocks',
)
ops_progress_bar = tqdm(initial=existing_count * 50,
total=range_count * 50,
bar_format='{bar}| [{rate_fmt}{postfix}]',
ncols=48,
dynamic_ncols=False,
unit=' ops')
loop.run_until_complete(
process_blocks(missing_block_nums,
steemd_http_url,
AIOHTTP_SESSION,
pool,
DB_META,
blocks_pbar=blocks_progress_bar,
ops_pbar=ops_progress_bar))
# [6/7] Make second sweep for missing blocks
task_message = fmt_task_message(
'Adding missing blocks and operations to db (second sweep)',
emoji_code_point=u'\U0001F52D',
task_num=6)
click.echo(task_message)
existing_count, missing_count, range_count = loop.run_until_complete(
get_existing_and_missing_count(pool, start_block, end_block))
task_message = fmt_task_message(
f'Building list of {missing_count} blocks missing from db between {start_block}<<-->>{end_block}',
emoji_code_point=u'\U0001F52D',
task_num=4)
click.echo(task_message)
with click.progressbar(length=end_block, **pbar_kwargs) as pbar:
pbar.update(existing_count)
missing_block_nums = loop.run_until_complete(
collect_missing_block_nums(pool,
end_block,
missing_count,
start_block=start_block,
pbar=pbar))
blocks_progress_bar = tqdm(
initial=existing_count,
total=range_count,
dynamic_ncols=False,
unit=' blocks',
)
ops_progress_bar = tqdm(initial=existing_count * 50,
total=range_count * 50,
dynamic_ncols=False,
unit=' ops')
loop.run_until_complete(
process_blocks(missing_block_nums,
steemd_http_url,
AIOHTTP_SESSION,
pool,
DB_META,
blocks_pbar=blocks_progress_bar,
ops_pbar=ops_progress_bar))
# [7/7] stream new blocks
task_message = fmt_task_message('Streaming blocks',
emoji_code_point=u'\U0001F4DD',
task_num=7)
click.echo(task_message)
except KeyboardInterrupt:
pass
except Exception as e:
logger.exception('ERROR')
raise e
def test_connector(create_session, loop):
connector = TCPConnector(loop=loop)
session = create_session(connector=connector)
assert session.connector is connector
async def fetch(self, url: str) -> str:
proxy, headers = self.args.proxy, self.args.headers
async with ClientSession(connector=TCPConnector(ssl=False)) as client: # type: ClientSession
async with client.get(url, proxy=proxy, headers=headers) as resp: # type: ClientResponse
return await resp.text(encoding='utf-8')
async def create_client_session(app):
app['client_session'] = aiohttp.ClientSession(connector=TCPConnector(limit=300))
def client_request(
method: str,
url: StrOrURL,
*,
params: Optional[Mapping[str, str]] = None,
data: Any = None,
json: Any = None,
headers: LooseHeaders = None,
skip_auto_headers: Optional[Iterable[str]] = None,
auth: Optional[BasicAuth] = None,
allow_redirects: bool = True,
max_redirects: int = 10,
compress: Optional[str] = None,
chunked: Optional[bool] = None,
expect100: bool = False,
raise_for_status: Optional[bool] = None,
read_until_eof: bool = True,
proxy: Optional[StrOrURL] = None,
proxy_auth: Optional[BasicAuth] = None,
timeout: Union[ClientTimeout, object] = sentinel,
cookies: Optional[LooseCookies] = None,
version: HttpVersion = http.HttpVersion11,
connector: Optional[BaseConnector] = None,
loop: Optional[asyncio.AbstractEventLoop] = None
) -> _SessionRequestContextManager:
"""
as aiohttp.client.request using a client session that does not decompress, i.e auto_decompress=False
"""
connector_owner = False
if connector is None:
connector_owner = True
connector = TCPConnector(loop=loop, force_close=True)
session = ClientSession(
loop=loop,
cookies=cookies,
version=version,
timeout=timeout,
connector=connector,
connector_owner=connector_owner,
auto_decompress=False,
)
return _SessionRequestContextManager(
session._request(
method,
url,
params=params,
data=data,
json=json,
headers=headers,
skip_auto_headers=skip_auto_headers,
auth=auth,
allow_redirects=allow_redirects,
max_redirects=max_redirects,
compress=compress,
chunked=chunked,
expect100=expect100,
raise_for_status=raise_for_status,
read_until_eof=read_until_eof,
proxy=proxy,
proxy_auth=proxy_auth,
),
session,
)
def test_connector(self):
connector = TCPConnector(loop=self.loop)
session = ClientSession(connector=connector, loop=self.loop)
self.assertIs(session.connector, connector)
session.close()
