def __init__(self, rooturl, loop, maxtasks=100):
self.rooturl = rooturl
self.loop = loop
self.todo = set()
self.busy = set()
self.done = {}
self.tasks = set()
self.sem = asyncio.Semaphore(maxtasks)
# connector stores cookies between requests and uses connection pool
self.connector = aiohttp.SocketConnector(share_cookies=True, loop=loop)
def test_tcp_connector(self):
with test_utils.run_server(self.loop, router=Functional) as httpd:
r = self.loop.run_until_complete(
client.request(
'get',
httpd.url('method', 'get'),
connector=aiohttp.SocketConnector(loop=self.loop),
loop=self.loop))
content = self.loop.run_until_complete(r.content.read())
content = content.decode()
self.assertEqual(r.status, 200)
r.close()
def __init__(self, hosts, *, method=None, path=None,
ssl=False, timeout=None,
conn_pool=True, conn_timeout=None, failed_timeout=5.0,
resolve=True, resolve_timeout=360.0, verify_ssl=True,
loop=None):
super().__init__()
if isinstance(hosts, str):
hosts = (hosts,)
if not hosts:
raise ValueError('Hosts are required')
self._hosts = []
for host in hosts:
if isinstance(host, str):
if ':' in host:
host, port = host.split(':')
try:
port = int(port)
except:
raise ValueError('Port has to be integer: %s' % host)
else:
port = 80
else:
host, port = host
self._hosts.append((host, port))
self._method = method
self._path = path
self._timeout = timeout
self._schema = 'https' if ssl else 'http'
self._failed = collections.deque()
self._failed_handle = None
self._failed_timeout = failed_timeout
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
if conn_pool:
self._connector = aiohttp.SocketConnector(
share_cookies=True, conn_timeout=conn_timeout,
resolve=resolve, verify_ssl=verify_ssl, loop=loop)
self._resolve_timeout = resolve_timeout
self._resolve_handle = self._loop.call_later(
self._resolve_timeout, self._cleanup_resolved_host)
else:
self._connector = None
def test_HTTP_200_OK_METHOD_ssl(self):
connector = aiohttp.SocketConnector(verify_ssl=False, loop=self.loop)
with test_utils.run_server(self.loop, use_ssl=True) as httpd:
for meth in ('get', 'post', 'put', 'delete', 'head'):
r = self.loop.run_until_complete(
client.request(meth, httpd.url('method', meth),
loop=self.loop, connector=connector))
content = self.loop.run_until_complete(r.read())
self.assertEqual(r.status, 200)
self.assertEqual(content, b'Test message')
r.close()
def test_session_close(self):
conn = aiohttp.SocketConnector(loop=self.loop)
with test_utils.run_server(self.loop, router=Functional) as httpd:
r = self.loop.run_until_complete(
client.request(
'get', httpd.url('keepalive') + '?close=1',
connector=conn, loop=self.loop))
self.assertEqual(r.status, 200)
content = self.loop.run_until_complete(r.read(True))
self.assertEqual(content['content'], 'requests=1')
r.close()
r = self.loop.run_until_complete(
client.request('get', httpd.url('keepalive'),
connector=conn, loop=self.loop))
self.assertEqual(r.status, 200)
content = self.loop.run_until_complete(r.read(True))
self.assertEqual(content['content'], 'requests=1')
r.close()
def queries():
connector = aiohttp.SocketConnector(share_cookies=True,
loop=self.loop)
# initiate session; set start value to 2
resp = yield from aiohttp.request('GET',
url + "/2",
connector=connector,
loop=self.loop)
data = yield from resp.json()
self.assertEqual(resp.status, 200)
self.assertEqual(data, {'result': 3})
# do increment
resp = yield from aiohttp.request('GET',
url,
connector=connector,
loop=self.loop)
data = yield from resp.json()
self.assertEqual(resp.status, 200)
self.assertEqual(data, {'result': 4})
# try to override start value
resp = yield from aiohttp.request('GET',
url + '/3',
connector=connector,
loop=self.loop)
data = yield from resp.json()
self.assertEqual(resp.status, 200)
self.assertEqual(data, {'result': 5})
# session deleted; try count
resp = yield from aiohttp.request('GET',
url,
connector=connector,
loop=self.loop)
data = yield from resp.json()
self.assertEqual(resp.status, 200)
self.assertEqual(data, {'result': 1})
def request(method, url, *,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
allow_redirects=True,
max_redirects=10,
encoding='utf-8',
version=(1, 1),
timeout=None,
compress=None,
chunked=None,
expect100=False,
connector=None,
loop=None,
read_until_eof=True,
request_class=None):
"""Constructs and sends a request. Returns response object.
:param method: http method
:param url: request url
:param params: (optional) Dictionary or bytes to be sent in the query
string of the new request
:param data: (optional) Dictionary, bytes, or file-like object to
send in the body of the request
:param headers: (optional) Dictionary of HTTP Headers to send with
the request
:param cookies: (optional) Dict object to send with the request
:param files: (optional) Dictionary of 'name': file-like-objects
for multipart encoding upload
:param auth: (optional) Auth tuple to enable Basic HTTP Auth
:param timeout: (optional) Float describing the timeout of the request
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
:param compress: Boolean. Set to True if request has to be compressed
with deflate encoding.
:param chunked: Boolean or Integer. Set to chunk size for chunked
transfer encoding.
:param expect100: Boolean. Expect 100-continue response from server.
:param connector: aiohttp.conntect.SocketConnector instance to support
connection pooling and session cookies.
:param read_until_eof: Read response until eof if response
does not have Content-Length header.
:param request_class: Custom Request class implementation.
:param loop: Optional event loop.
Usage::
>>> import aiohttp
>>> resp = yield from aiohttp.request('GET', 'http://python.org/')
>>> resp
<HttpResponse(python.org/) [200]>
>>> data = yield from resp.read_and_close()
"""
redirects = 0
if loop is None:
loop = asyncio.get_event_loop()
if request_class is None:
request_class = HttpRequest
if connector is None:
connector = aiohttp.SocketConnector(loop=loop)
while True:
req = request_class(
method, url, params=params, headers=headers, data=data,
cookies=cookies, files=files, auth=auth, encoding=encoding,
version=version, compress=compress, chunked=chunked,
loop=loop, expect100=expect100)
try:
conn = yield from connector.connect(req)
resp = req.send(conn.writer, conn.reader)
try:
if timeout:
yield from asyncio.wait_for(
resp.start(conn, read_until_eof), timeout, loop=loop)
else:
yield from resp.start(conn, read_until_eof)
except:
resp.close()
conn.close()
raise
except asyncio.TimeoutError:
raise aiohttp.TimeoutError from None
except aiohttp.BadStatusLine as exc:
raise aiohttp.ClientConnectionError(exc)
except OSError as exc:
raise aiohttp.OsConnectionError(exc)
# redirects
if resp.status in (301, 302) and allow_redirects:
redirects += 1
if max_redirects and redirects >= max_redirects:
resp.close()
break
r_url = resp.get('location') or resp.get('uri')
scheme = urllib.parse.urlsplit(r_url)[0]
if scheme not in ('http', 'https', ''):
raise ValueError('Can redirect only to http or https')
elif not scheme:
r_url = urllib.parse.urljoin(url, r_url)
url = urllib.parse.urldefrag(r_url)[0]
if url:
resp.close()
continue
break
return resp