def __init__(self, connection_pool=None, recorder=None):
if connection_pool is not None:
self._connection_pool = connection_pool
else:
self._connection_pool = ConnectionPool()
self._recorder = recorder
class Client(BaseClient):
'''HTTP/1.1 client.
Args:
connection_pool (ConnectionPool): An instance of
:class:`ConnectionPool`.
recorder (Recorder): An instance of :class:`.recorder.BaseRecorder`.
This HTTP client manages connection pooling to reuse existing
connections if possible.
'''
def __init__(self, connection_pool=None, recorder=None):
if connection_pool is not None:
self._connection_pool = connection_pool
else:
self._connection_pool = ConnectionPool()
self._recorder = recorder
@tornado.gen.coroutine
def fetch(self, request, **kwargs):
'''Fetch a document.
Args:
request (Request): An instance of :class:`Request`.
kwargs: Any keyword arguments to pass to :func:`Connection.fetch`.
Returns:
Response: An instance of :class:`Response`.
Raises:
Exception: See :meth:`.http.connection.Connection.fetch`.
'''
_logger.debug('Client fetch request {0}.'.format(request))
if 'recorder' not in kwargs:
kwargs['recorder'] = self._recorder
elif self._recorder:
kwargs['recorder'] = DemuxRecorder(
(kwargs['recorder'], self._recorder)
)
async_result = toro.AsyncResult()
yield self._connection_pool.put(request, kwargs, async_result)
response = yield async_result.get()
if isinstance(response, Exception):
raise response from response
else:
raise tornado.gen.Return(response)
def close(self):
'''Close the connection pool and recorders.'''
_logger.debug('Client closing.')
self._connection_pool.close()
if self._recorder:
self._recorder.close()
class Client(BaseClient):
'''HTTP/1.1 client.
Args:
connection_pool (ConnectionPool): An instance of
:class:`ConnectionPool`.
recorder (Recorder): An instance of :class:`.recorder.BaseRecorder`.
This HTTP client manages connection pooling to reuse existing
connections if possible.
'''
def __init__(self, connection_pool=None, recorder=None):
if connection_pool is not None:
self._connection_pool = connection_pool
else:
self._connection_pool = ConnectionPool()
self._recorder = recorder
@tornado.gen.coroutine
def fetch(self, request, **kwargs):
'''Fetch a document.
Args:
request (Request): An instance of :class:`Request`.
kwargs: Any keyword arguments to pass to :func:`Connection.fetch`.
Returns:
Response: An instance of :class:`Response`.
Raises:
Exception: See :meth:`.http.connection.Connection.fetch`.
'''
_logger.debug('Client fetch request {0}.'.format(request))
if 'recorder' not in kwargs:
kwargs['recorder'] = self._recorder
elif self._recorder:
kwargs['recorder'] = DemuxRecorder(
(kwargs['recorder'], self._recorder))
async_result = toro.AsyncResult()
yield self._connection_pool.put(request, kwargs, async_result)
response = yield async_result.get()
if isinstance(response, Exception):
raise response from response
else:
raise tornado.gen.Return(response)
def close(self):
'''Close the connection pool and recorders.'''
_logger.debug('Client closing.')
self._connection_pool.close()
if self._recorder:
self._recorder.close()
def test_connection_pool_clean(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
requests = [client.fetch(
Request.new(self.get_url('/'))) for dummy in range(12)]
responses = yield requests
for response in responses:
self.assertEqual(200, response.status_code)
connection_pool.clean()
self.assertEqual(0, len(connection_pool))
def test_connection_pool_min(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
for dummy in range(2):
response = yield client.fetch(
Request.new(self.get_url('/sleep_short')))
self.assertEqual(200, response.status_code)
self.assertEqual(b'12', response.body.content)
self.assertEqual(1, len(connection_pool))
connection_pool_entry = list(connection_pool.values())[0]
self.assertIsInstance(connection_pool_entry, HostConnectionPool)
self.assertEqual(1, len(connection_pool_entry))
def test_connection_pool_over_max(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
requests = [client.fetch(
Request.new(self.get_url('/sleep_short'))) for dummy in range(12)]
responses = yield requests
for response in responses:
self.assertEqual(200, response.status_code)
self.assertEqual(b'12', response.body.content)
self.assertEqual(1, len(connection_pool))
connection_pool_entry = list(connection_pool.values())[0]
self.assertIsInstance(connection_pool_entry, HostConnectionPool)
self.assertEqual(6, len(connection_pool_entry))
def __init__(self, connection_pool=None, recorder=None):
if connection_pool is not None:
self._connection_pool = connection_pool
else:
self._connection_pool = ConnectionPool()
self._recorder = recorder
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, read_timeout=0.2)
host_connection_pool_factory = functools.partial(
HostConnectionPool, connection_factory=connection_factory)
connection_pool = ConnectionPool(host_connection_pool_factory)
client = Client(connection_pool)
for dummy in range(7):
try:
yield client.fetch(
Request.new(self.get_url('/header_early_close')),
recorder=DebugPrintRecorder()
)
except NetworkError:
pass
else:
self.fail()
for dummy in range(7):
response = yield client.fetch(Request.new(self.get_url('/')))
self.assertEqual(200, response.status_code)