def test_client_duration_timeout(self):
client = Client()
with self.assertRaises(DurationTimeout), client.session() as session:
request = Request(self.get_url('/sleep_long'))
yield From(session.fetch(request))
yield From(session.read_content(duration_timeout=0.1))
def test_client_duration_timeout(self):
client = Client()
with self.assertRaises(DurationTimeout), client.session() as session:
request = Request(self.get_url('/sleep_long'))
yield From(session.fetch(request))
yield From(session.read_content(duration_timeout=0.1))
def test_basic_requests(self):
proxy_http_client = Client(recorder=DebugPrintRecorder())
proxy_server = HTTPProxyServer(proxy_http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield From(trollius.start_server(proxy_server, sock=proxy_socket))
connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port))
http_client = Client(connection_pool=connection_pool,
recorder=DebugPrintRecorder())
for dummy in range(3):
with http_client.session() as session:
response = yield From(session.fetch(Request(self.get_url('/'))))
self.assertEqual(200, response.status_code)
file = io.BytesIO()
yield From(session.read_content(file=file))
data = file.getvalue().decode('ascii', 'replace')
self.assertTrue(data.endswith('</html>'))
with http_client.session() as session:
response = yield From(session.fetch(Request(
self.get_url('/always_error'))))
self.assertEqual(500, response.status_code)
self.assertEqual('Dragon In Data Center', response.reason)
file = io.BytesIO()
yield From(session.read_content(file=file))
data = file.getvalue().decode('ascii', 'replace')
self.assertEqual('Error', data)
def test_client_did_not_complete(self):
client = Client()
with warnings.catch_warnings(record=True) as warn_list:
warnings.simplefilter("always")
with client.session() as session:
request = Request(self.get_url('/'))
yield From(session.fetch(request))
self.assertFalse(session.done())
for warn_obj in warn_list:
print(warn_obj)
# Unrelated warnings may occur in PyPy
# https://travis-ci.org/chfoo/wpull/jobs/51420202
self.assertGreaterEqual(len(warn_list), 1)
for warn_obj in warn_list:
if str(warn_obj.message) == 'HTTP session did not complete.':
break
else:
self.fail('Warning did not occur.')
client = Client()
with self.assertRaises(MyException):
with client.session() as session:
request = Request(self.get_url('/'))
yield From(session.fetch(request))
raise MyException('Oops')
def test_client_exception_throw(self):
client = Client()
with client.session() as session:
request = Request('http://wpull-no-exist.invalid')
with self.assertRaises(NetworkError):
yield From(session.fetch(request))
def test_client_exception_throw(self):
client = Client()
with client.session() as session:
request = Request('http://wpull-no-exist.invalid')
with self.assertRaises(NetworkError):
yield From(session.fetch(request))
def test_client_exception_throw(self):
client = Client()
try:
yield client.fetch(Request.new('http://wpull-no-exist.invalid'))
except NetworkError:
pass
else:
self.fail()
def test_connection_pool_clean(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
requests = [client.fetch(
Request.new(self.get_url('/'))) for dummy in range(12)]
responses = yield requests
for response in responses:
self.assertEqual(200, response.status_code)
connection_pool.clean()
self.assertEqual(0, len(connection_pool))
def test_connection_pool_min(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
for dummy in range(2):
response = yield client.fetch(
Request.new(self.get_url('/sleep_short')))
self.assertEqual(200, response.status_code)
self.assertEqual(b'12', response.body.content)
self.assertEqual(1, len(connection_pool))
connection_pool_entry = list(connection_pool.values())[0]
self.assertIsInstance(connection_pool_entry, HostConnectionPool)
self.assertEqual(1, len(connection_pool_entry))
def test_connection_pool_over_max(self):
connection_pool = ConnectionPool()
client = Client(connection_pool)
requests = [client.fetch(
Request.new(self.get_url('/sleep_short'))) for dummy in range(12)]
responses = yield requests
for response in responses:
self.assertEqual(200, response.status_code)
self.assertEqual(b'12', response.body.content)
self.assertEqual(1, len(connection_pool))
connection_pool_entry = list(connection_pool.values())[0]
self.assertIsInstance(connection_pool_entry, HostConnectionPool)
self.assertEqual(6, len(connection_pool_entry))
def test_client_recorder(self):
recorder = MockRecorder()
client = Client(recorder=recorder)
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
yield From(session.read_content())
self.assertEqual(200, response.status_code)
self.assertTrue(recorder.pre_request)
self.assertTrue(recorder.request)
self.assertTrue(recorder.pre_response)
self.assertTrue(recorder.response)
self.assertIn(b'GET', recorder.request_data)
self.assertIn(b'hello', recorder.response_data)
def test_client_recorder(self):
recorder = MockRecorder()
client = Client(recorder=recorder)
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
yield From(session.read_content())
self.assertEqual(200, response.status_code)
self.assertTrue(recorder.pre_request)
self.assertTrue(recorder.request)
self.assertTrue(recorder.pre_response)
self.assertTrue(recorder.response)
self.assertIn(b'GET', recorder.request_data)
self.assertIn(b'hello', recorder.response_data)
def test_basic(self):
client = Client()
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
self.assertEqual(200, response.status_code)
self.assertEqual(request, response.request)
file_obj = io.BytesIO()
yield From(session.read_content(file_obj))
self.assertEqual(b'hello world!', file_obj.getvalue())
self.assertTrue(request.url_info)
self.assertTrue(request.address)
self.assertTrue(response.body)
def test_basic(self):
client = Client()
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
self.assertEqual(200, response.status_code)
self.assertEqual(request, response.request)
file_obj = io.BytesIO()
yield From(session.read_content(file_obj))
self.assertEqual(b'hello world!', file_obj.getvalue())
self.assertTrue(request.url_info)
self.assertTrue(request.address)
self.assertTrue(response.body)
def test_basic(self):
http_client = Client()
client = RichClient(http_client)
session = client.session(Request.new(self.get_url('/')))
self.assertFalse(session.done)
response = yield session.fetch()
self.assertEqual(200, response.status_code)
self.assertTrue(session.done)
def __init__(self,
http_client=None,
request_factory=Request,
redirect_tracker_factory=RedirectTracker,
cookie_jar=None):
super().__init__()
self._http_client = http_client or Client()
self._request_factory = request_factory
self._redirect_tracker_factory = redirect_tracker_factory
self._cookie_jar = cookie_jar
self._loop_type = None
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, read_timeout=0.2)
host_connection_pool_factory = functools.partial(
HostConnectionPool, connection_factory=connection_factory)
connection_pool = ConnectionPool(host_connection_pool_factory)
client = Client(connection_pool)
for dummy in range(7):
try:
yield client.fetch(
Request.new(self.get_url('/header_early_close')),
recorder=DebugPrintRecorder()
)
except NetworkError:
pass
else:
self.fail()
for dummy in range(7):
response = yield client.fetch(Request.new(self.get_url('/')))
self.assertEqual(200, response.status_code)
def test_bad_redirect(self):
http_client = Client()
client = RichClient(http_client)
session = client.session(Request.new(self.get_url('/bad_redirect')))
while not session.done:
try:
yield session.fetch()
except ProtocolError:
return
else:
self.fail()
def test_client_did_not_complete(self):
client = Client()
with warnings.catch_warnings(record=True) as warn_list:
warnings.simplefilter("always")
with client.session() as session:
request = Request(self.get_url('/'))
yield From(session.fetch(request))
self.assertFalse(session.done())
for warn_obj in warn_list:
print(warn_obj)
# Unrelated warnings may occur in PyPy
# https://travis-ci.org/chfoo/wpull/jobs/51420202
self.assertGreaterEqual(len(warn_list), 1)
for warn_obj in warn_list:
if str(warn_obj.message) == 'HTTP session did not complete.':
break
else:
self.fail('Warning did not occur.')
client = Client()
with self.assertRaises(MyException):
with client.session() as session:
request = Request(self.get_url('/'))
yield From(session.fetch(request))
raise MyException('Oops')
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, timeout=2.0)
connection_pool = ConnectionPool(connection_factory=connection_factory)
client = Client(connection_pool=connection_pool)
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/header_early_close'))
try:
yield From(session.fetch(request))
except NetworkError:
pass
else:
self.fail() # pragma: no cover
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
self.assertEqual(200, response.status_code)
yield From(session.read_content())
self.assertTrue(session.done())
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, timeout=2.0)
connection_pool = ConnectionPool(connection_factory=connection_factory)
client = Client(connection_pool=connection_pool)
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/header_early_close'))
try:
yield From(session.fetch(request))
except NetworkError:
pass
else:
self.fail() # pragma: no cover
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/'))
response = yield From(session.fetch(request))
self.assertEqual(200, response.status_code)
yield From(session.read_content())
self.assertTrue(session.done())
def test_redirect(self):
http_client = Client()
client = RichClient(http_client)
session = client.session(Request.new(self.get_url('/redirect')))
status_codes = []
while not session.done:
response = yield session.fetch()
if not status_codes:
self.assertEqual(RichClientResponseType.redirect,
session.response_type)
status_codes.append(response.status_code)
self.assertEqual([301, 200], status_codes)
self.assertTrue(session.done)
self.assertEqual(RichClientResponseType.normal, session.response_type)
def test_no_content(self):
http_client = Client(recorder=DebugPrintRecorder())
proxy = HTTPProxyServer(http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield From(trollius.start_server(proxy, sock=proxy_socket))
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(self.get_url('/no_content'),
proxy_host='localhost',
proxy_port=proxy_port)
response = yield From(
tornado_future_adapter(test_client.fetch(request)))
self.assertEqual(204, response.code)
def test_no_content(self):
http_client = Client(recorder=DebugPrintRecorder())
proxy = HTTPProxyServer(http_client, io_loop=self.io_loop)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
proxy.add_socket(proxy_socket)
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/no_content'),
proxy_host='localhost',
proxy_port=proxy_port
)
response = yield test_client.fetch(request)
self.assertEqual(204, response.code)
def test_basic(self):
cookie_jar = BetterMozillaCookieJar()
policy = DeFactoCookiePolicy(cookie_jar=cookie_jar)
cookie_jar.set_policy(policy)
cookie_jar_wrapper = CookieJarWrapper(cookie_jar)
http_client = Client(recorder=DebugPrintRecorder())
proxy = HTTPProxyServer(http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
def request_callback(request):
print(request)
cookie_jar_wrapper.add_cookie_header(request)
def pre_response_callback(request, response):
print(response)
cookie_jar_wrapper.extract_cookies(response, request)
proxy.request_callback = request_callback
proxy.pre_response_callback = pre_response_callback
yield From(trollius.start_server(proxy, sock=proxy_socket))
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/'),
proxy_host='localhost',
proxy_port=proxy_port,
)
response = yield From(
tornado_future_adapter(test_client.fetch(request)))
self.assertEqual(200, response.code)
self.assertIn(b'Hello!', response.body)
cookies = tuple(cookie_jar)
self.assertEqual('hi', cookies[0].name)
self.assertEqual('hello', cookies[0].value)
def test_post(self):
http_client = Client(recorder=DebugPrintRecorder())
proxy = HTTPProxyServer(http_client, io_loop=self.io_loop)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
proxy.add_socket(proxy_socket)
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/post/'),
proxy_host='localhost',
proxy_port=proxy_port,
body='text=blah',
method='POST'
)
response = yield test_client.fetch(request)
self.assertEqual(200, response.code)
self.assertIn(b'OK', response.body)
def test_client(self):
http_client = Client()
proxy_server = HTTPProxyServer(http_client)
proxy_socket, proxy_host = tornado.testing.bind_unused_port()
proxy_server.add_socket(proxy_socket)
remote_client = PhantomJSClient('localhost:{0}'.format(proxy_host))
with remote_client.remote() as remote:
self.assertIn(remote, remote_client.remotes_busy)
test_remote = remote
for dummy in range(100):
if test_remote in remote_client.remotes_ready:
break
yield wpull. async .sleep(0.1)
self.assertIn(test_remote, remote_client.remotes_ready)
self.assertNotIn(test_remote, remote_client.remotes_busy)
self._data_queue = data_queue
@contextlib.contextmanager
def session(self):
yield ProxyRecorderSession(self._data_queue)
class ProxyRecorderSession(BaseRecorderSession):
'''Proxy Recorder Session.'''
def __init__(self, data_queue):
self._data_queue = data_queue
def response_data(self, data):
'''Callback for the bytes that was received.'''
self._data_queue.put(data)
def response(self, response):
self._data_queue.put(None)
if __name__ == '__main__':
from wpull.http.client import Client
logging.basicConfig(level=logging.DEBUG)
http_client = Client(recorder=ProgressRecorder())
proxy = HTTPProxyServer(http_client)
proxy.listen(8888)
tornado.ioloop.IOLoop.current().start()
