def test_chunked_explicit_size(self, m_http):
req = ClientRequest(
'get', 'http://python.org/', chunked=1024, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('chunked', req.headers['TRANSFER-ENCODING'])
m_http.Request.return_value\
.add_chunking_filter.assert_called_with(1024)
def test_data_stream_exc_chain(self):
fut = asyncio.Future(loop=self.loop)
def gen():
yield from fut
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
inner_exc = ValueError()
@asyncio.coroutine
def exc():
yield from asyncio.sleep(0.01, loop=self.loop)
fut.set_exception(inner_exc)
asyncio.async(exc(), loop=self.loop)
resp = req.send(self.transport, self.protocol)
resp.connection = self.connection
self.loop.run_until_complete(req._writer)
self.assertTrue(self.connection.close.called)
self.assertTrue(self.protocol.set_exception.called)
outer_exc = self.protocol.set_exception.call_args[0][0]
self.assertIsInstance(outer_exc, aiohttp.ClientRequestError)
self.assertIs(inner_exc, outer_exc.__context__)
self.assertIs(inner_exc, outer_exc.__cause__)
def test_chunked_length(self):
req = ClientRequest(
'get', 'http://python.org/',
headers={'CONTENT-LENGTH': '1000'}, chunked=1024, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
self.assertNotIn('CONTENT-LENGTH', req.headers)
def test_data_stream_exc(self):
fut = asyncio.Future(loop=self.loop)
def gen():
yield b'binary data'
yield from fut
return b' result'
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(inspect.isgenerator(req.body))
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
@asyncio.coroutine
def exc():
yield from asyncio.sleep(0.01, loop=self.loop)
fut.set_exception(ValueError)
asyncio.async(exc(), loop=self.loop)
resp = req.send(self.transport, self.protocol)
resp.connection = self.connection
self.loop.run_until_complete(req._writer)
self.assertTrue(self.connection.close.called)
self.assertTrue(self.protocol.set_exception.called)
def test_https_connect(self, ClientRequestMock):
loop_mock = unittest.mock.Mock()
proxy_req = ClientRequest('GET', 'http://proxy.example.com',
loop=loop_mock)
ClientRequestMock.return_value = proxy_req
proxy_resp = ClientResponse('get', 'http://proxy.example.com')
proxy_resp._loop = loop_mock
proxy_req.send = send_mock = unittest.mock.Mock()
send_mock.return_value = proxy_resp
proxy_resp.start = start_mock = unittest.mock.Mock()
self._fake_coroutine(start_mock, unittest.mock.Mock(status=200))
connector = aiohttp.ProxyConnector(
'http://proxy.example.com', loop=loop_mock)
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
self._fake_coroutine(loop_mock.create_connection, (tr, proto))
req = ClientRequest('GET', 'https://www.python.org', loop=self.loop)
self.loop.run_until_complete(connector._create_connection(req))
self.assertEqual(req.path, '/')
self.assertEqual(proxy_req.method, 'CONNECT')
self.assertEqual(proxy_req.path, 'www.python.org:443')
tr.pause_reading.assert_called_once_with()
tr.get_extra_info.assert_called_with('socket', default=None)
self.loop.run_until_complete(proxy_req.close())
proxy_resp.close()
self.loop.run_until_complete(req.close())
def test_https_connect_http_proxy_error(self, ClientRequestMock):
loop_mock = unittest.mock.Mock()
proxy_req = ClientRequest('GET', 'http://proxy.example.com',
loop=loop_mock)
ClientRequestMock.return_value = proxy_req
proxy_resp = ClientResponse('get', 'http://proxy.example.com')
proxy_resp._loop = loop_mock
proxy_req.send = send_mock = unittest.mock.Mock()
send_mock.return_value = proxy_resp
proxy_resp.start = start_mock = unittest.mock.Mock()
self._fake_coroutine(
start_mock, unittest.mock.Mock(status=400, reason='bad request'))
connector = aiohttp.ProxyConnector(
'http://proxy.example.com', loop=loop_mock)
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
tr.get_extra_info.return_value = None
self._fake_coroutine(loop_mock.create_connection, (tr, proto))
req = ClientRequest('GET', 'https://www.python.org', loop=self.loop)
with self.assertRaisesRegex(
aiohttp.HttpProxyError, "400, message='bad request'"):
self.loop.run_until_complete(connector._create_connection(req))
self.loop.run_until_complete(proxy_req.close())
proxy_resp.close()
self.loop.run_until_complete(req.close())
def test_content_encoding(self, m_http):
req = ClientRequest('get', 'http://python.org/',
compress='deflate', loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
self.assertEqual(req.headers['CONTENT-ENCODING'], 'deflate')
m_http.Request.return_value\
.add_compression_filter.assert_called_with('deflate')
def test_custom_response_class(self):
class CustomResponse(ClientResponse):
def read(self, decode=False):
return 'customized!'
req = ClientRequest(
'GET', 'http://python.org/', response_class=CustomResponse,
loop=self.loop)
resp = req.send(self.transport, self.protocol)
self.assertEqual('customized!', resp.read())
def test_post_data(self):
for meth in ClientRequest.POST_METHODS:
req = ClientRequest(
meth, 'http://python.org/',
data={'life': '42'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('/', req.path)
self.assertEqual(b'life=42', req.body)
self.assertEqual('application/x-www-form-urlencoded',
req.headers['CONTENT-TYPE'])
def test_bytes_data(self):
for meth in ClientRequest.POST_METHODS:
req = ClientRequest(
meth, 'http://python.org/',
data=b'binary data', loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('/', req.path)
self.assertEqual(b'binary data', req.body)
self.assertEqual('application/octet-stream',
req.headers['CONTENT-TYPE'])
def test_data_stream_not_bytes(self):
@asyncio.coroutine
def gen():
yield object()
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
req.send(self.transport, self.protocol)
self.loop.run_until_complete(req._writer)
self.assertTrue(self.protocol.set_exception.called)
def test_close(self):
@asyncio.coroutine
def gen():
yield from asyncio.sleep(0.00001, loop=self.loop)
return b'result'
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
req.send(self.transport, self.protocol)
self.loop.run_until_complete(req.close())
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'result'),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_connect(self, ClientRequestMock):
req = ClientRequest('GET', 'http://www.python.org', loop=self.loop)
self.assertEqual(req.path, '/')
loop_mock = unittest.mock.Mock()
connector = aiohttp.ProxyConnector('http://proxy.example.com',
loop=loop_mock)
self.assertIs(loop_mock, connector._loop)
resolve_host = unittest.mock.Mock()
self._fake_coroutine(resolve_host, [unittest.mock.MagicMock()])
connector._resolve_host = resolve_host
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
self._fake_coroutine(loop_mock.create_connection, (tr, proto))
conn = self.loop.run_until_complete(connector.connect(req))
self.assertEqual(req.path, 'http://www.python.org/')
self.assertIs(conn._transport, tr)
self.assertIs(conn._protocol, proto)
# resolve_host.assert_called_once_with('proxy.example.com', 80)
tr.get_extra_info.assert_called_once_with('sslcontext')
ClientRequestMock.assert_called_with(
'GET',
'http://proxy.example.com',
auth=None,
headers={'HOST': 'www.python.org'},
loop=loop_mock)
conn.close()
def test_auth_utf8(self):
proxy_req = ClientRequest('GET',
'http://proxy.example.com',
auth=aiohttp.helpers.BasicAuth(
'юзер', 'пасс', 'utf-8'),
loop=self.loop)
self.assertIn('AUTHORIZATION', proxy_req.headers)
def test_basic_auth_tuple_deprecated(self):
with self.assertWarns(DeprecationWarning):
req = ClientRequest('get',
'http://python.org',
auth=('nkim', '1234'))
self.assertIn('AUTHORIZATION', req.headers)
self.assertEqual('Basic bmtpbToxMjM0', req.headers['AUTHORIZATION'])
def function2540(arg1260, function1139):
var3583 = mock.Mock()
var3583.is_connected.return_value = True
var3002 = ClientRequest('GET', URL('http://localhost1:80'), loop=arg1260)
var742 = aiohttp.BaseConnector(loop=arg1260, limit=0, limit_per_host=0)
var742._conns[function1139] = [(var3583, arg1260.time())]
var742._create_connection = mock.Mock()
var742._create_connection.return_value = helpers.create_future(arg1260)
var742._create_connection.return_value.set_result(var3583)
var3355 = False
var3007 = yield from var742.connect(var3002)
@asyncio.coroutine
def function301():
nonlocal acquired
var2225 = yield from var742.connect(var3002)
var3355 = True
assert (1 == len(var742._acquired))
assert (1 == len(var742._acquired_per_host[function1139]))
var2225.release()
var2804 = helpers.ensure_future(function301(), loop=arg1260)
yield from asyncio.sleep(0.01, loop=arg1260)
assert acquired
var3007.release()
yield from task
var742.close()
def test_headers(self):
req = ClientRequest('get',
'http://python.org/',
headers={'Content-Type': 'text/plain'})
self.assertIn('CONTENT-TYPE', req.headers)
self.assertEqual(req.headers['CONTENT-TYPE'], 'text/plain')
self.assertEqual(req.headers['ACCEPT-ENCODING'], 'gzip, deflate')
def function712(arg516, function1139):
var4045 = mock.Mock()
var4045.is_connected.return_value = True
var384 = ClientRequest('GET', URL('http://localhost1:80'), loop=arg516)
var3117 = aiohttp.BaseConnector(loop=arg516, limit=0, limit_per_host=1)
var3117._conns[function1139] = [(var4045, arg516.time())]
var3117._create_connection = mock.Mock()
var3117._create_connection.return_value = helpers.create_future(arg516)
var3117._create_connection.return_value.set_result(var4045)
var4490 = False
var712 = yield from var3117.connect(var384)
@asyncio.coroutine
def function301():
nonlocal acquired
var993 = yield from var3117.connect(var384)
var4490 = True
var993.release()
var4482 = helpers.ensure_future(function301(), loop=arg516)
yield from asyncio.sleep(0.01, loop=arg516)
assert (not var4490)
var712.release()
yield from asyncio.sleep(0, loop=arg516)
assert acquired
yield from task
var3117.close()
def function2543(arg440, function1139):
var4520 = mock.Mock()
var4520.is_connected.return_value = True
var1008 = ClientRequest('GET', URL('http://localhost1:80'), loop=arg440)
var4698 = aiohttp.BaseConnector(loop=arg440, limit=1000, limit_per_host=1)
var4698._conns[function1139] = [(var4520, arg440.time())]
var4698._create_connection = mock.Mock()
var4698._create_connection.return_value = helpers.create_future(arg440)
var4698._create_connection.return_value.set_result(var4520)
var3298 = False
var4590 = yield from var4698.connect(var1008)
@asyncio.coroutine
def function301():
nonlocal acquired
var916 = yield from var4698.connect(var1008)
var3298 = True
assert (1 == len(var4698._acquired))
assert (1 == len(var4698._acquired_per_host[function1139]))
var916.release()
var3195 = helpers.ensure_future(function301(), loop=arg440)
yield from asyncio.sleep(0.01, loop=arg440)
assert (not var3298)
var4590.release()
yield from asyncio.sleep(0, loop=arg440)
assert acquired
yield from task
var4698.close()
async def test_connect_with_no_limits(loop, key):
proto = mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET', URL('http://localhost1:80'), loop=loop)
conn = aiohttp.BaseConnector(loop=loop, limit=0, limit_per_host=0)
conn._conns[key] = [(proto, loop.time())]
conn._create_connection = mock.Mock()
conn._create_connection.return_value = helpers.create_future(loop)
conn._create_connection.return_value.set_result(proto)
acquired = False
connection1 = await conn.connect(req)
async def f():
nonlocal acquired
connection2 = await conn.connect(req)
acquired = True
assert 1 == len(conn._acquired)
assert 1 == len(conn._acquired_per_host[key])
connection2.release()
task = asyncio.ensure_future(f(), loop=loop)
await asyncio.sleep(0.01, loop=loop)
assert acquired
connection1.release()
await task
conn.close()
def test_get_with_data(self):
for meth in ClientRequest.GET_METHODS:
req = ClientRequest(meth,
'http://python.org/',
data={'life': '42'})
self.assertEqual('/', req.path)
self.assertEqual(b'life=42', req.body)
def test_files_and_bytes_data(self):
with self.assertRaises(ValueError):
with self.assertWarns(DeprecationWarning):
ClientRequest('POST',
'http://python.org/',
data=b'binary data',
files={'file': b'file data'})
def test_connect_with_no_limit_and_limit_per_host(loop, key):
proto = mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET', URL('http://localhost1:80'), loop=loop)
conn = aiohttp.BaseConnector(loop=loop, limit=0, limit_per_host=1)
conn._conns[key] = [(proto, loop.time())]
conn._create_connection = mock.Mock()
conn._create_connection.return_value = helpers.create_future(loop)
conn._create_connection.return_value.set_result(proto)
acquired = False
connection1 = yield from conn.connect(req)
@asyncio.coroutine
def f():
nonlocal acquired
connection2 = yield from conn.connect(req)
acquired = True
connection2.release()
task = helpers.ensure_future(f(), loop=loop)
yield from asyncio.sleep(0.01, loop=loop)
assert not acquired
connection1.release()
yield from asyncio.sleep(0, loop=loop)
assert acquired
yield from task
conn.close()
def test_close_with_acquired_connection(loop):
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET',
URL('http://host:80'),
loop=loop,
response_class=unittest.mock.Mock())
conn = aiohttp.BaseConnector(loop=loop, limit=1)
key = ('host', 80, False)
conn._conns[key] = [(tr, proto, loop.time())]
conn._create_connection = unittest.mock.Mock()
conn._create_connection.return_value = helpers.create_future(loop)
conn._create_connection.return_value.set_result((tr, proto))
connection = yield from conn.connect(req)
assert 1 == len(conn._acquired)
conn.close()
assert 0 == len(conn._acquired)
assert conn.closed
tr.close.assert_called_with()
assert not connection.closed
connection.close()
assert connection.closed
def go():
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET', 'http://host:80',
loop=self.loop,
response_class=unittest.mock.Mock())
conn = aiohttp.BaseConnector(loop=self.loop, limit=1)
key = ('host', 80, False)
conn._conns[key] = [(tr, proto, self.loop.time())]
conn._create_connection = unittest.mock.Mock()
conn._create_connection.return_value = helpers.create_future(
self.loop)
conn._create_connection.return_value.set_result((tr, proto))
connection = yield from conn.connect(req)
self.assertEqual(1, len(conn._acquired))
conn.close()
self.assertEqual(0, len(conn._acquired))
self.assertTrue(conn.closed)
tr.close.assert_called_with()
self.assertFalse(connection.closed)
connection.close()
self.assertTrue(connection.closed)
def test_basic_auth_utf8(self):
req = ClientRequest('get', 'http://python.org',
auth=aiohttp.helpers.BasicAuth('nkim', 'секрет',
'utf-8'))
self.assertIn('AUTHORIZATION', req.headers)
self.assertEqual('Basic bmtpbTrRgdC10LrRgNC10YI=',
req.headers['AUTHORIZATION'])
def test_connect_with_limit_cancelled(loop):
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET',
URL('http://host:80'),
loop=loop,
response_class=unittest.mock.Mock())
conn = aiohttp.BaseConnector(loop=loop, limit=1)
key = ('host', 80, False)
conn._conns[key] = [(tr, proto, loop.time())]
conn._create_connection = unittest.mock.Mock()
conn._create_connection.return_value = helpers.create_future(loop)
conn._create_connection.return_value.set_result((tr, proto))
connection = yield from conn.connect(req)
assert connection._transport == tr
assert 1 == len(conn._acquired[key])
with pytest.raises(asyncio.TimeoutError):
# limit exhausted
yield from asyncio.wait_for(conn.connect(req), 0.01, loop=loop)
connection.close()
def test_connect_with_no_limits(loop, key, transport):
proto = unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET',
URL('http://localhost1:80'),
loop=loop,
response_class=unittest.mock.Mock())
conn = aiohttp.BaseConnector(loop=loop, limit=0, limit_per_host=0)
conn._conns[key] = [(transport, proto, loop.time())]
conn._create_connection = unittest.mock.Mock()
conn._create_connection.return_value = helpers.create_future(loop)
conn._create_connection.return_value.set_result((transport, proto))
acquired = False
connection1 = yield from conn.connect(req)
@asyncio.coroutine
def f():
nonlocal acquired
connection2 = yield from conn.connect(req)
acquired = True
assert 1 == len(conn._acquired)
assert 1 == len(conn._acquired_per_host[key])
connection2.release()
task = helpers.ensure_future(f(), loop=loop)
yield from asyncio.sleep(0.01, loop=loop)
assert acquired
connection1.release()
yield from task
conn.close()
def go():
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET', 'http://host:80',
loop=self.loop,
response_class=unittest.mock.Mock())
conn = aiohttp.BaseConnector(loop=self.loop, limit=1)
key = ('host', 80, False)
conn._conns[key] = [(tr, proto, self.loop.time())]
conn._create_connection = unittest.mock.Mock()
conn._create_connection.return_value = helpers.create_future(
self.loop)
conn._create_connection.return_value.set_result((tr, proto))
connection = yield from conn.connect(req)
self.assertEqual(connection._transport, tr)
self.assertEqual(1, len(conn._acquired[key]))
with self.assertRaises(asyncio.TimeoutError):
# limit exhausted
yield from asyncio.wait_for(conn.connect(req), 0.01,
loop=self.loop)
connection.close()
async def test_connect_with_limit_concurrent(loop):
proto = mock.Mock()
proto.should_close = False
proto.is_connected.return_value = True
req = ClientRequest('GET', URL('http://host:80'), loop=loop)
max_connections = 2
num_connections = 0
conn = aiohttp.BaseConnector(limit=max_connections, loop=loop)
# Use a real coroutine for _create_connection; a mock would mask
# problems that only happen when the method yields.
async def create_connection(req):
nonlocal num_connections
num_connections += 1
await asyncio.sleep(0, loop=loop)
# Make a new transport mock each time because acquired
# transports are stored in a set. Reusing the same object
# messes with the count.
proto = mock.Mock(should_close=False)
proto.is_connected.return_value = True
return proto
conn._create_connection = create_connection
# Simulate something like a crawler. It opens a connection, does
# something with it, closes it, then creates tasks that make more
# connections and waits for them to finish. The crawler is started
# with multiple concurrent requests and stops when it hits a
# predefined maximum number of requests.
max_requests = 10
num_requests = 0
start_requests = max_connections + 1
async def f(start=True):
nonlocal num_requests
if num_requests == max_requests:
return
num_requests += 1
if not start:
connection = await conn.connect(req)
await asyncio.sleep(0, loop=loop)
connection.release()
tasks = [
loop.create_task(f(start=False))
for i in range(start_requests)
]
await asyncio.wait(tasks, loop=loop)
await f()
conn.close()
assert max_connections == num_connections
def test_data_file(self):
req = ClientRequest(
'POST', 'http://python.org/', data=io.BytesIO(b'*' * 2),
loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(isinstance(req.body, io.IOBase))
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
resp = req.send(self.transport, self.protocol)
self.assertIsInstance(req._writer, asyncio.Future)
self.loop.run_until_complete(resp.wait_for_close())
self.assertIsNone(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'*' * 2),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_request_port(self, ClientRequestMock):
proxy_req = ClientRequest('GET', 'http://proxy.example.com',
loop=self.loop)
ClientRequestMock.return_value = proxy_req
loop_mock = unittest.mock.Mock()
connector = aiohttp.ProxyConnector('http://proxy.example.com',
loop=loop_mock)
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
tr.get_extra_info.return_value = None
self._fake_coroutine(loop_mock.create_connection, (tr, proto))
req = ClientRequest('GET', 'http://localhost:1234/path',
loop=self.loop)
self.loop.run_until_complete(connector._create_connection(req))
self.assertEqual(req.path, 'http://localhost:1234/path')
def test_close(self):
@asyncio.coroutine
def gen():
yield from asyncio.sleep(0.00001, loop=self.loop)
return b'result'
req = ClientRequest('POST',
'http://python.org/',
data=gen(),
loop=self.loop)
req.send(self.transport, self.protocol)
self.loop.run_until_complete(req.close())
self.assertEqual(self.transport.write.mock_calls[-3:], [
unittest.mock.call(b'result'),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')
])
def test_host_header(self):
req = ClientRequest('get', 'http://python.org/')
self.assertEqual(req.headers['HOST'], 'python.org')
req = ClientRequest('get', 'http://python.org:80/')
self.assertEqual(req.headers['HOST'], 'python.org:80')
req = ClientRequest('get', 'http://python.org:99/')
self.assertEqual(req.headers['HOST'], 'python.org:99')
req = ClientRequest('get', 'http://python.org/',
headers={'host': 'example.com'})
self.assertEqual(req.headers['HOST'], 'example.com')
req = ClientRequest('get', 'http://python.org/',
headers={'host': 'example.com:99'})
self.assertEqual(req.headers['HOST'], 'example.com:99')
def test_data_file(self):
req = ClientRequest(
'POST', 'http://python.org/', data=io.BytesIO(b'*' * 2),
loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(isinstance(req.body, io.IOBase))
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
resp = req.send(self.transport, self.protocol)
self.assertIsInstance(req._writer, asyncio.Future)
self.loop.run_until_complete(resp.wait_for_close())
self.assertIsNone(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'*' * 2),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_data_continue(self):
req = ClientRequest(
'POST', 'http://python.org/', data=b'data',
expect100=True, loop=self.loop)
def coro():
yield from asyncio.sleep(0.0001, loop=self.loop)
req._continue.set_result(1)
asyncio.async(coro(), loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual(1, len(self.transport.write.mock_calls))
self.loop.run_until_complete(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-1],
unittest.mock.call(b'data'))
def test_data_continue(self):
req = ClientRequest(
'POST', 'http://python.org/', data=b'data',
expect100=True, loop=self.loop)
def coro():
yield from asyncio.sleep(0.0001, loop=self.loop)
req._continue.set_result(1)
asyncio.async(coro(), loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual(1, len(self.transport.write.mock_calls))
self.loop.run_until_complete(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-1],
unittest.mock.call(b'data'))
def go():
proto = unittest.mock.Mock()
proto.is_connected.return_value = True
req = ClientRequest('GET', 'http://host:80',
loop=self.loop,
response_class=unittest.mock.Mock(
_should_close=False))
max_connections = 2
num_connections = 0
conn = aiohttp.BaseConnector(limit=max_connections, loop=self.loop)
# Use a real coroutine for _create_connection; a mock would mask
# problems that only happen when the method yields.
@asyncio.coroutine
def create_connection(req):
nonlocal num_connections
num_connections += 1
yield from asyncio.sleep(0, loop=self.loop)
# Make a new transport mock each time because acquired
# transports are stored in a set. Reusing the same object
# messes with the count.
tr = unittest.mock.Mock()
return tr, proto
conn._create_connection = create_connection
# Simulate something like a crawler. It opens a connection, does
# something with it, closes it, then creates tasks that make more
# connections and waits for them to finish. The crawler is started
# with multiple concurrent requests and stops when it hits a
# predefined maximum number of requests.
max_requests = 10
num_requests = 0
start_requests = max_connections + 1
@asyncio.coroutine
def f(start=True):
nonlocal num_requests
if num_requests == max_requests:
return
num_requests += 1
if not start:
connection = yield from conn.connect(req)
yield from asyncio.sleep(0, loop=self.loop)
connection.release()
tasks = [
asyncio.async(f(start=False), loop=self.loop)
for i in range(start_requests)
]
yield from asyncio.wait(tasks, loop=self.loop)
def test_auth__not_modifying_request(self, ClientRequestMock):
proxy_req = ClientRequest('GET', 'http://*****:*****@proxy.example.com')
ClientRequestMock.return_value = proxy_req
proxy_req_headers = dict(proxy_req.headers)
loop_mock = unittest.mock.Mock()
connector = aiohttp.ProxyConnector(
'http://*****:*****@proxy.example.com', loop=loop_mock)
connector._resolve_host = resolve_mock = unittest.mock.Mock()
self._fake_coroutine(resolve_mock, OSError('nothing personal'))
req = ClientRequest('GET', 'http://www.python.org')
req_headers = dict(req.headers)
with self.assertRaises(aiohttp.ProxyConnectionError):
self.loop.run_until_complete(connector.connect(req))
self.assertEqual(req.headers, req_headers)
self.assertEqual(req.path, '/')
self.assertEqual(proxy_req.headers, proxy_req_headers)
def test_unix_connector_not_found(loop):
connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop)
req = ClientRequest(
'GET', URL('http://www.python.org'),
loop=loop,
)
with pytest.raises(aiohttp.ClientConnectorError):
loop.run_until_complete(connector.connect(req))
def test_chunked(self):
req = ClientRequest(
'get', 'http://python.org/',
headers={'TRANSFER-ENCODING': 'gzip'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('gzip', req.headers['TRANSFER-ENCODING'])
req = ClientRequest(
'get', 'http://python.org/',
headers={'Transfer-encoding': 'chunked'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('chunked', req.headers['TRANSFER-ENCODING'])
def test_expect100(self):
req = ClientRequest('get', 'http://python.org/',
expect100=True, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('100-continue', req.headers['EXPECT'])
self.assertIsNotNone(req._continue)
req = ClientRequest('get', 'http://python.org/',
headers={'expect': '100-continue'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('100-continue', req.headers['EXPECT'])
self.assertIsNotNone(req._continue)
def test_data_stream(self):
def gen():
yield b'binary data'
return b' result'
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(inspect.isgenerator(req.body))
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
resp = req.send(self.transport, self.protocol)
self.assertIsInstance(req._writer, asyncio.Future)
self.loop.run_until_complete(resp.wait_for_close())
self.assertIsNone(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'binary data result'),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_dns_error(loop):
connector = aiohttp.TCPConnector(loop=loop)
connector._resolve_host = make_mocked_coro(
raise_exception=OSError('dont take it serious'))
req = ClientRequest(
'GET', URL('http://www.python.org'),
loop=loop,
)
with pytest.raises(aiohttp.ClientConnectorError):
loop.run_until_complete(connector.connect(req))
def test_data_stream(self):
def gen():
yield b'binary data'
return b' result'
req = ClientRequest(
'POST', 'http://python.org/', data=gen(), loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(inspect.isgenerator(req.body))
self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked')
resp = req.send(self.transport, self.protocol)
self.assertIsInstance(req._writer, asyncio.Future)
self.loop.run_until_complete(resp.wait_for_close())
self.assertIsNone(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'binary data result'),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_https_connect_runtime_error(self, ClientRequestMock):
loop_mock = unittest.mock.Mock()
proxy_req = ClientRequest('GET', 'http://proxy.example.com',
loop=loop_mock)
ClientRequestMock.return_value = proxy_req
proxy_resp = ClientResponse('get', 'http://proxy.example.com')
proxy_resp._loop = loop_mock
proxy_req.send = send_mock = unittest.mock.Mock()
send_mock.return_value = proxy_resp
proxy_resp.start = start_mock = unittest.mock.Mock()
self._fake_coroutine(start_mock, unittest.mock.Mock(status=200))
connector = aiohttp.ProxyConnector(
'http://proxy.example.com', loop=loop_mock)
tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
tr.get_extra_info.return_value = None
self._fake_coroutine(loop_mock.create_connection, (tr, proto))
req = ClientRequest('GET', 'https://www.python.org', loop=self.loop)
with self.assertRaisesRegex(
RuntimeError, "Transport does not expose socket instance"):
self.loop.run_until_complete(connector._create_connection(req))
proxy_req.close()
proxy_resp.close()
req.close()
def test_no_content_length(self):
req = ClientRequest('get', 'http://python.org', loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('0', req.headers.get('CONTENT-LENGTH'))
req = ClientRequest('head', 'http://python.org', loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('0', req.headers.get('CONTENT-LENGTH'))
def test_data_stream_continue(self):
def gen():
yield b'binary data'
return b' result'
req = ClientRequest(
'POST', 'http://python.org/', data=gen(),
expect100=True, loop=self.loop)
self.assertTrue(req.chunked)
self.assertTrue(inspect.isgenerator(req.body))
def coro():
yield from asyncio.sleep(0.0001, loop=self.loop)
req._continue.set_result(1)
asyncio.async(coro(), loop=self.loop)
req.send(self.transport, self.protocol)
self.loop.run_until_complete(req._writer)
self.assertEqual(
self.transport.write.mock_calls[-3:],
[unittest.mock.call(b'binary data result'),
unittest.mock.call(b'\r\n'),
unittest.mock.call(b'0\r\n\r\n')])
def test_expect100(self):
req = ClientRequest('get', 'http://python.org/',
expect100=True, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('100-continue', req.headers['EXPECT'])
self.assertIsNotNone(req._continue)
req = ClientRequest('get', 'http://python.org/',
headers={'expect': '100-continue'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('100-continue', req.headers['EXPECT'])
self.assertIsNotNone(req._continue)
def test_chunked(self):
req = ClientRequest(
'get', 'http://python.org/',
headers={'TRANSFER-ENCODING': 'gzip'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('gzip', req.headers['TRANSFER-ENCODING'])
req = ClientRequest(
'get', 'http://python.org/',
headers={'Transfer-encoding': 'chunked'}, loop=self.loop)
req.send(self.transport, self.protocol)
self.assertEqual('chunked', req.headers['TRANSFER-ENCODING'])
