def __init__(self, connector: TCPConnector = None):
"""Initialize client options.
Params:
* **connector**: TCPConnector to be used if provided
"""
self.connector = connector or TCPConnector()
async def performance_aiosonic(url, concurrency, pool_cls=None, timeouts=None):
"""Test aiohttp performance."""
return await timeit_coro(aiosonic.get,
url,
connector=TCPConnector(pool_size=concurrency,
pool_cls=pool_cls),
timeouts=timeouts)
async def test_wait_connections_empty(mocker):
"""Test simple get."""
async with aiosonic.HTTPClient() as client:
assert await client.wait_requests()
connector = TCPConnector(pool_cls=CyclicQueuePool)
async with aiosonic.HTTPClient(connector) as client:
assert await client.wait_requests()
async def test_get_http2(http2_serv):
"""Test simple get to node http2 server."""
url = http2_serv
connector = TCPConnector(timeouts=Timeouts(sock_connect=3, sock_read=4))
res = await aiosonic.get(url, connector=connector, verify=False)
assert res.status_code == 200
assert 'Hello World' == await res.text()
def __init__(self,
connector: TCPConnector = None,
handle_cookies=False,
verify_ssl=True):
"""Initialize client options."""
self.connector = connector or TCPConnector()
self.handle_cookies = handle_cookies
self.cookies_map: Dict[str, cookies.SimpleCookie] = {}
self.verify_ssl = verify_ssl
async def test_read_timeout(app, aiohttp_server, mocker):
"""Test read timeout."""
server = await aiohttp_server(app)
url = 'http://localhost:%d/slow_request' % server.port
connector = TCPConnector(timeouts=Timeouts(sock_read=0.2))
async with aiosonic.HTTPClient(connector) as client:
with pytest.raises(ReadTimeout):
await client.get(url)
await server.close()
async def test_simple_get(app, aiohttp_server):
"""Test simple get."""
server = await aiohttp_server(app)
url = 'http://localhost:%d' % server.port
connector = TCPConnector(timeouts=Timeouts(sock_connect=3, sock_read=4))
async with aiosonic.HTTPClient(connector) as client:
res = await client.get(url)
assert res.status_code == 200
assert await res.content() == b'Hello, world'
assert await res.text() == 'Hello, world'
await server.close()
async def test_connect_timeout(mocker):
"""Test connect timeout."""
url = 'http://localhost:1234'
async def long_connect(*_args, **_kwargs):
await asyncio.sleep(3)
_connect = mocker.patch('aiosonic.connection.Connection._connect')
_connect.return_value = long_connect()
with pytest.raises(ConnectTimeout):
await aiosonic.get(
url, connector=TCPConnector(timeouts=Timeouts(sock_connect=0.2)))
async def test_pool_acquire_timeout(app, aiohttp_server, mocker):
"""Test pool acquirere timeout."""
server = await aiohttp_server(app)
url = 'http://localhost:%d/slow_request' % server.port
connector = TCPConnector(pool_size=1, timeouts=Timeouts(pool_acquire=0.3))
with pytest.raises(ConnectionPoolAcquireTimeout):
await asyncio.gather(
aiosonic.get(url, connector=connector),
aiosonic.get(url, connector=connector),
)
await server.close()
async def test_close_connection(app, aiohttp_server):
"""Test close connection."""
server = await aiohttp_server(app)
url = 'http://localhost:%d/post' % server.port
connector = TCPConnector(pool_size=1, connection_cls=MyConnection)
res = await aiosonic.post(url, data=b'close', connector=connector)
connection = await connector.pool.acquire()
assert res.status_code == 200
assert not connection.keep
assert await res.text() == 'close'
async def test_get_google_http2():
"""Test simple get."""
url = 'https://www.google.com'
connector = TCPConnector(
timeouts=Timeouts(sock_connect=3, sock_read=4))
res = await aiosonic.get(url, connector=connector, headers={
'user-agent': (
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 '
'Safari/537.36')
})
assert res.status_code == 200
assert '<title>Google</title>' in await res.text()
async def test_keep_alive_smart_pool(app, aiohttp_server):
"""Test keepalive smart pool."""
server = await aiohttp_server(app)
url = 'http://localhost:%d' % server.port
urlparsed = urlparse(url)
connector = TCPConnector(pool_size=2, connection_cls=MyConnection)
for _ in range(5):
res = await aiosonic.get(url, connector=connector)
connection = await connector.pool.acquire(urlparsed)
assert res.status_code == 200
assert await res.text() == 'Hello, world'
assert connection.counter == 5
await server.close()
def __init__(self, logger, args, **kwargs):
"""Initialize runner."""
self.logger = logger
self.args = args
self.kwargs = kwargs
self.client = aiosonic.HTTPClient(
connector=TCPConnector(pool_size=args.concurrency),
verify_ssl=not args.insecure)
logger.info('preparing_requests')
self.requests_data = [
self.prepare_request(**kwargs)
for _ in range(args.number_of_requests)
]
logger.info('prepared_requests')
async def test_get_with_cookies(app, aiohttp_server):
"""Test simple get."""
server = await aiohttp_server(app)
url = f'http://localhost:{server.port}/cookies'
connector = TCPConnector(timeouts=Timeouts(sock_connect=3, sock_read=4))
async with aiosonic.HTTPClient(connector, handle_cookies=True) as client:
res = await client.get(url)
assert res.status_code == 200
assert res.cookies
# check if server got cookies
res = await client.get(url)
assert await res.text() == 'Got cookies'
await server.close()
async def test_wait_connections_busy_timeout(mocker):
"""Test simple get."""
async def long_connect(*_args, **_kwargs):
await asyncio.sleep(1)
return True
_connect = mocker.patch('aiosonic.connectors.TCPConnector.wait_free_pool',
new=long_connect)
_connect.return_value = long_connect()
async with aiosonic.HTTPClient() as client:
assert not await client.wait_requests(0)
connector = TCPConnector(pool_cls=CyclicQueuePool)
async with aiosonic.HTTPClient(connector) as client:
assert not await client.wait_requests(0)
async def test_get_python(http2_serv):
"""Test simple get."""
url = http2_serv
connector = TCPConnector(timeouts=Timeouts(sock_connect=3, sock_read=4))
async with aiosonic.HTTPClient(connector) as client:
res = await client.get(
url,
verify=False,
headers={
'user-agent':
('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:70.0)'
' Gecko/20100101 Firefox/70.0')
},
http2=True)
assert 'Hello World' in await res.text()
async def test_request_timeout(app, aiohttp_server, mocker):
"""Test request timeout."""
server = await aiohttp_server(app)
url = 'http://localhost:%d/post_json' % server.port
async def long_request(*_args, **_kwargs):
await asyncio.sleep(3)
_connect = mocker.patch('aiosonic._do_request', new=long_request)
_connect.return_value = long_request()
connector = TCPConnector(timeouts=Timeouts(request_timeout=0.2))
async with aiosonic.HTTPClient(connector) as client:
with pytest.raises(RequestTimeout):
await client.get(url)
await server.close()
async def test_timeouts_overriden(app, aiohttp_server, mocker):
"""Test timeouts overriden."""
server = await aiohttp_server(app)
url = 'http://localhost:%d/slow_request' % server.port
# request takes 1s so this timeout should not be applied
# instead the one provided by request call
connector = TCPConnector(timeouts=Timeouts(sock_read=2))
async with aiosonic.HTTPClient(connector) as client:
response = await client.get(url)
assert response.status_code == 200
with pytest.raises(ReadTimeout):
await client.get(url, timeouts=Timeouts(sock_read=0.3))
await server.close()
async def test_connect_timeout(mocker):
"""Test connect timeout."""
url = 'http://localhost:1234'
async def long_connect(*_args, **_kwargs):
await asyncio.sleep(3)
async def acquire(*_args, **_kwargs):
return mocker.MagicMock(connect=long_connect)
_connect = mocker.patch('aiosonic.pools.SmartPool.acquire', new=acquire)
# _connect.return_value = long_connect()
connector = TCPConnector(timeouts=Timeouts(sock_connect=0.2))
with pytest.raises(ConnectTimeout):
async with aiosonic.HTTPClient(connector) as client:
await client.get(url)
async def test_keep_alive_cyclic_pool(app, aiohttp_server):
"""Test keepalive cyclic pool."""
server = await aiohttp_server(app)
url = 'http://localhost:%d' % server.port
connector = TCPConnector(pool_size=2,
connection_cls=MyConnection,
pool_cls=CyclicQueuePool)
async with aiosonic.HTTPClient(connector) as client:
for _ in range(5):
res = await client.get(url)
async with await connector.pool.acquire() as connection:
assert res.status_code == 200
assert await res.text() == 'Hello, world'
assert connection.counter == 2
await server.close()
async def test_close_old_keeped_conn(app, aiohttp_server):
"""Test close old conn."""
server1 = await aiohttp_server(app)
server2 = await aiohttp_server(app)
url1 = 'http://localhost:%d' % server1.port
url2 = 'http://localhost:%d' % server2.port
connector = TCPConnector(pool_size=1, connection_cls=MyConnection)
await aiosonic.get(url1, connector=connector)
# get used writer
connection = await connector.pool.acquire()
writer = connection.writer
connection = connector.pool.release(connection)
await aiosonic.get(url2, connector=connector)
# python 3.6 doesn't have writer.is_closing
is_closing = getattr(writer, 'is_closing', writer._transport.is_closing)
# check that old writer is closed
assert is_closing()
await server1.close()
await server2.close()
async def request(url: str,
method: str = 'GET',
headers: HeadersType = None,
params: ParamsType = None,
data: DataType = None,
connector: TCPConnector = None,
multipart: bool = False,
verify: bool = True,
ssl: SSLContext = None,
timeouts: Timeouts = None,
follow: bool = False,
http2: bool = False) -> HttpResponse:
"""Do http request.
Params:
* **url**: url of request
* **method**: Http method of request
* **headers**: headers to add in request
* **params**: query params to add in
request if not manually added
* **data**: Data to be sent, this param is ignored for get requests.
* **connector**: TCPConnector to be used if provided
* **multipart**: Tell aiosonic if request is multipart
* **verify**: parameter to indicate whether to verify ssl
* **ssl**: this parameter allows to specify a custom ssl context
* **timeouts**: parameter to indicate timeouts for request
* **follow**: parameter to indicate whether to follow redirects
* **http2**: flag to indicate whether to use http2 (experimental)
"""
if not connector:
key = 'connector_base'
connector = _CACHE[key] = _CACHE.get(key) or TCPConnector()
urlparsed = _get_url_parsed(url)
boundary = None
headers = headers or {}
body: ParsedBodyType = b''
if method != 'GET' and data and not multipart:
body = _setup_body_request(data, headers)
elif multipart:
if not isinstance(data, dict):
raise ValueError('data should be dict')
boundary = 'boundary-%d' % random.randint(10**8, 10**9)
body = await _send_multipart(data, boundary, headers)
max_redirects = 30
while True:
headers_data = partial(_get_header_data,
url=urlparsed,
method=method,
headers=headers,
params=params,
multipart=multipart,
boundary=boundary)
try:
response = await asyncio.wait_for(
_do_request(urlparsed, headers_data, connector, body, verify,
ssl, timeouts, http2),
timeout=(timeouts or connector.timeouts).request_timeout)
if follow and response.status_code in {301, 302}:
max_redirects -= 1
if max_redirects == 0:
raise MaxRedirects()
parsed_full_url = _get_url_parsed(
response.headers[b'location'].decode())
# if full url, will have scheme
if parsed_full_url.scheme:
urlparsed = parsed_full_url
else:
urlparsed = _get_url_parsed(
url.replace(urlparsed.path,
response.headers[b'location'].decode()))
else:
return response
except ConnectTimeout:
raise
except futures._base.TimeoutError:
raise RequestTimeout()
async def performance_aiosonic(url, concurrency, pool_cls=None, timeouts=None):
"""Test aiohttp performance."""
client = aiosonic.HTTPClient(
TCPConnector(pool_size=concurrency, pool_cls=pool_cls))
return await timeit_coro(client.get, url, timeouts=timeouts)