def test_basic_requests(self):
proxy_http_client = Client()
proxy_server = HTTPProxyServer(proxy_http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield from asyncio.start_server(proxy_server, sock=proxy_socket)
connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port))
http_client = Client(connection_pool=connection_pool)
for dummy in range(3):
with http_client.session() as session:
response = yield from session.start(Request(self.get_url('/')))
self.assertEqual(200, response.status_code)
file = io.BytesIO()
yield from session.download(file=file)
data = file.getvalue().decode('ascii', 'replace')
self.assertTrue(data.endswith('</html>'))
with http_client.session() as session:
response = yield from session.start(
Request(self.get_url('/always_error')))
self.assertEqual(500, response.status_code)
self.assertEqual('Dragon In Data Center', response.reason)
file = io.BytesIO()
yield from session.download(file=file)
data = file.getvalue().decode('ascii', 'replace')
self.assertEqual('Error', data)
def test_client_duration_timeout(self):
client = Client()
with self.assertRaises(DurationTimeout), client.session() as session:
request = Request(self.get_url('/sleep_long'))
yield from session.start(request)
yield from session.download(duration_timeout=0.1)
def test_basic_requests(self):
proxy_http_client = Client()
proxy_server = HTTPProxyServer(proxy_http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield from asyncio.start_server(proxy_server, sock=proxy_socket)
connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port))
http_client = Client(connection_pool=connection_pool)
for dummy in range(3):
with http_client.session() as session:
response = yield from session.start(Request(self.get_url('/')))
self.assertEqual(200, response.status_code)
file = io.BytesIO()
yield from session.download(file=file)
data = file.getvalue().decode('ascii', 'replace')
self.assertTrue(data.endswith('</html>'))
with http_client.session() as session:
response = yield from session.start(Request(
self.get_url('/always_error')))
self.assertEqual(500, response.status_code)
self.assertEqual('Dragon In Data Center', response.reason)
file = io.BytesIO()
yield from session.download(file=file)
data = file.getvalue().decode('ascii', 'replace')
self.assertEqual('Error', data)
def test_client_duration_timeout(self):
client = Client()
with self.assertRaises(DurationTimeout), client.session() as session:
request = Request(self.get_url('/sleep_long'))
yield from session.start(request)
yield from session.download(duration_timeout=0.1)
def test_client_did_not_complete(self):
client = Client()
with warnings.catch_warnings(record=True) as warn_list:
warnings.simplefilter("always")
with client.session() as session:
request = Request(self.get_url('/'))
yield from session.start(request)
self.assertFalse(session.done())
for warn_obj in warn_list:
print(warn_obj)
# Unrelated warnings may occur in PyPy
# https://travis-ci.org/ArchiveTeam/wpull/jobs/51420202
self.assertGreaterEqual(len(warn_list), 1)
for warn_obj in warn_list:
if str(warn_obj.message) == 'HTTP session did not complete.':
break
else:
self.fail('Warning did not occur.')
client = Client()
with self.assertRaises(MyException):
with client.session() as session:
request = Request(self.get_url('/'))
yield from session.start(request)
raise MyException('Oops')
def test_client_exception_throw(self):
client = Client()
with client.session() as session:
request = Request('http://wpull-no-exist.invalid')
with self.assertRaises(NetworkError):
yield from session.start(request)
def test_client_exception_throw(self):
client = Client()
with client.session() as session:
request = Request('http://wpull-no-exist.invalid')
with self.assertRaises(NetworkError):
yield from session.start(request)
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, timeout=2.0)
connection_pool = ConnectionPool(connection_factory=connection_factory)
client = Client(connection_pool=connection_pool)
for dummy in range(7):
with self.assertRaises(NetworkError), client.session() as session:
request = Request(self.get_url('/header_early_close'))
yield from session.start(request)
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/'))
response = yield from session.start(request)
self.assertEqual(200, response.status_code)
yield from session.download()
self.assertTrue(session.done())
def test_client_exception_recovery(self):
connection_factory = functools.partial(Connection, timeout=2.0)
connection_pool = ConnectionPool(connection_factory=connection_factory)
client = Client(connection_pool=connection_pool)
for dummy in range(7):
with self.assertRaises(NetworkError), client.session() as session:
request = Request(self.get_url('/header_early_close'))
yield from session.start(request)
for dummy in range(7):
with client.session() as session:
request = Request(self.get_url('/'))
response = yield from session.start(request)
self.assertEqual(200, response.status_code)
yield from session.download()
self.assertTrue(session.done())
def test_basic(self):
client = Client()
with client.session() as session:
request = Request(self.get_url('/'))
response = yield from session.start(request)
self.assertEqual(200, response.status_code)
self.assertEqual(request, response.request)
file_obj = io.BytesIO()
yield from session.download(file_obj)
self.assertEqual(b'hello world!', file_obj.getvalue())
self.assertTrue(request.url_info)
self.assertTrue(request.address)
self.assertTrue(response.body)
def test_basic(self):
client = Client()
with client.session() as session:
request = Request(self.get_url('/'))
response = yield from session.start(request)
self.assertEqual(200, response.status_code)
self.assertEqual(request, response.request)
file_obj = io.BytesIO()
yield from session.download(file_obj)
self.assertEqual(b'hello world!', file_obj.getvalue())
self.assertTrue(request.url_info)
self.assertTrue(request.address)
self.assertTrue(response.body)
def _main_test():
from wpull.protocol.http.client import Client
logging.basicConfig(level=logging.DEBUG)
http_client = Client()
proxy = HTTPProxyServer(http_client)
asyncio.get_event_loop().run_until_complete(asyncio.start_server(proxy, port=8888))
asyncio.get_event_loop().run_forever()
def __init__(self,
http_client: Optional[Client] = None,
request_factory: Callable[..., Request] = Request,
redirect_tracker_factory: Optional[Callable[
..., RedirectTracker]] = RedirectTracker,
cookie_jar: Optional[CookieJarWrapper] = None):
super().__init__()
self._http_client = http_client or Client()
self._request_factory = request_factory
self._redirect_tracker_factory = redirect_tracker_factory
self._cookie_jar = cookie_jar
self._loop_type = None
def test_client_did_not_complete(self):
client = Client()
with warnings.catch_warnings(record=True) as warn_list:
warnings.simplefilter("always")
with client.session() as session:
request = Request(self.get_url('/'))
yield from session.start(request)
self.assertFalse(session.done())
for warn_obj in warn_list:
print(warn_obj)
# Unrelated warnings may occur in PyPy
# https://travis-ci.org/chfoo/wpull/jobs/51420202
self.assertGreaterEqual(len(warn_list), 1)
for warn_obj in warn_list:
if str(warn_obj.message) == 'HTTP session did not complete.':
break
else:
self.fail('Warning did not occur.')
client = Client()
with self.assertRaises(MyException):
with client.session() as session:
request = Request(self.get_url('/'))
yield from session.start(request)
raise MyException('Oops')
def test_basic(self):
cookie_jar = BetterMozillaCookieJar()
policy = DeFactoCookiePolicy(cookie_jar=cookie_jar)
cookie_jar.set_policy(policy)
cookie_jar_wrapper = CookieJarWrapper(cookie_jar)
http_client = Client()
proxy = HTTPProxyServer(http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
client_request = None
def request_callback(client_request_):
nonlocal client_request
client_request = client_request_
print('client request', client_request)
cookie_jar_wrapper.add_cookie_header(client_request)
def server_response_callback(server_response):
print('server response', server_response)
assert client_request
cookie_jar_wrapper.extract_cookies(server_response, client_request)
def new_sesssion_callback(session: HTTPProxySession):
session.event_dispatcher.add_listener(
HTTPProxySession.Event.client_request, request_callback)
session.event_dispatcher.add_listener(
HTTPProxySession.Event.server_end_response,
server_response_callback)
proxy.event_dispatcher.add_listener(
HTTPProxyServer.Event.begin_session, new_sesssion_callback)
yield from asyncio.start_server(proxy, sock=proxy_socket)
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/'),
proxy_host='localhost',
proxy_port=proxy_port,
)
response = yield from tornado_future_adapter(test_client.fetch(request))
self.assertEqual(200, response.code)
self.assertIn(b'Hello!', response.body)
cookies = tuple(cookie_jar)
self.assertEqual('hi', cookies[0].name)
self.assertEqual('hello', cookies[0].value)
def test_no_content(self):
http_client = Client()
proxy = HTTPProxyServer(http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield from asyncio.start_server(proxy, sock=proxy_socket)
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/no_content'),
proxy_host='localhost',
proxy_port=proxy_port
)
response = yield from tornado_future_adapter(test_client.fetch(request))
self.assertEqual(204, response.code)
def test_post(self):
http_client = Client()
proxy = HTTPProxyServer(http_client)
proxy_socket, proxy_port = tornado.testing.bind_unused_port()
yield from asyncio.start_server(proxy, sock=proxy_socket)
_logger.debug('Proxy on port {0}'.format(proxy_port))
test_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
self.get_url('/post/'),
proxy_host='localhost',
proxy_port=proxy_port,
body='text=blah',
method='POST'
)
response = yield from tornado_future_adapter(test_client.fetch(request))
self.assertEqual(200, response.code)
self.assertIn(b'OK', response.body)
