def test_basic_requests(self): proxy_http_client = Client() proxy_server = HTTPProxyServer(proxy_http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield from asyncio.start_server(proxy_server, sock=proxy_socket) connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port)) http_client = Client(connection_pool=connection_pool) for dummy in range(3): with http_client.session() as session: response = yield from session.start(Request(self.get_url('/'))) self.assertEqual(200, response.status_code) file = io.BytesIO() yield from session.download(file=file) data = file.getvalue().decode('ascii', 'replace') self.assertTrue(data.endswith('</html>')) with http_client.session() as session: response = yield from session.start( Request(self.get_url('/always_error'))) self.assertEqual(500, response.status_code) self.assertEqual('Dragon In Data Center', response.reason) file = io.BytesIO() yield from session.download(file=file) data = file.getvalue().decode('ascii', 'replace') self.assertEqual('Error', data)
def test_client_duration_timeout(self): client = Client() with self.assertRaises(DurationTimeout), client.session() as session: request = Request(self.get_url('/sleep_long')) yield from session.start(request) yield from session.download(duration_timeout=0.1)
def test_basic_requests(self): proxy_http_client = Client() proxy_server = HTTPProxyServer(proxy_http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield from asyncio.start_server(proxy_server, sock=proxy_socket) connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port)) http_client = Client(connection_pool=connection_pool) for dummy in range(3): with http_client.session() as session: response = yield from session.start(Request(self.get_url('/'))) self.assertEqual(200, response.status_code) file = io.BytesIO() yield from session.download(file=file) data = file.getvalue().decode('ascii', 'replace') self.assertTrue(data.endswith('</html>')) with http_client.session() as session: response = yield from session.start(Request( self.get_url('/always_error'))) self.assertEqual(500, response.status_code) self.assertEqual('Dragon In Data Center', response.reason) file = io.BytesIO() yield from session.download(file=file) data = file.getvalue().decode('ascii', 'replace') self.assertEqual('Error', data)
def test_client_did_not_complete(self): client = Client() with warnings.catch_warnings(record=True) as warn_list: warnings.simplefilter("always") with client.session() as session: request = Request(self.get_url('/')) yield from session.start(request) self.assertFalse(session.done()) for warn_obj in warn_list: print(warn_obj) # Unrelated warnings may occur in PyPy # https://travis-ci.org/ArchiveTeam/wpull/jobs/51420202 self.assertGreaterEqual(len(warn_list), 1) for warn_obj in warn_list: if str(warn_obj.message) == 'HTTP session did not complete.': break else: self.fail('Warning did not occur.') client = Client() with self.assertRaises(MyException): with client.session() as session: request = Request(self.get_url('/')) yield from session.start(request) raise MyException('Oops')
def test_client_exception_throw(self): client = Client() with client.session() as session: request = Request('http://wpull-no-exist.invalid') with self.assertRaises(NetworkError): yield from session.start(request)
def test_client_exception_recovery(self): connection_factory = functools.partial(Connection, timeout=2.0) connection_pool = ConnectionPool(connection_factory=connection_factory) client = Client(connection_pool=connection_pool) for dummy in range(7): with self.assertRaises(NetworkError), client.session() as session: request = Request(self.get_url('/header_early_close')) yield from session.start(request) for dummy in range(7): with client.session() as session: request = Request(self.get_url('/')) response = yield from session.start(request) self.assertEqual(200, response.status_code) yield from session.download() self.assertTrue(session.done())
def test_basic(self): client = Client() with client.session() as session: request = Request(self.get_url('/')) response = yield from session.start(request) self.assertEqual(200, response.status_code) self.assertEqual(request, response.request) file_obj = io.BytesIO() yield from session.download(file_obj) self.assertEqual(b'hello world!', file_obj.getvalue()) self.assertTrue(request.url_info) self.assertTrue(request.address) self.assertTrue(response.body)
def _main_test(): from wpull.protocol.http.client import Client logging.basicConfig(level=logging.DEBUG) http_client = Client() proxy = HTTPProxyServer(http_client) asyncio.get_event_loop().run_until_complete(asyncio.start_server(proxy, port=8888)) asyncio.get_event_loop().run_forever()
def __init__(self, http_client: Optional[Client] = None, request_factory: Callable[..., Request] = Request, redirect_tracker_factory: Optional[Callable[ ..., RedirectTracker]] = RedirectTracker, cookie_jar: Optional[CookieJarWrapper] = None): super().__init__() self._http_client = http_client or Client() self._request_factory = request_factory self._redirect_tracker_factory = redirect_tracker_factory self._cookie_jar = cookie_jar self._loop_type = None
def test_client_did_not_complete(self): client = Client() with warnings.catch_warnings(record=True) as warn_list: warnings.simplefilter("always") with client.session() as session: request = Request(self.get_url('/')) yield from session.start(request) self.assertFalse(session.done()) for warn_obj in warn_list: print(warn_obj) # Unrelated warnings may occur in PyPy # https://travis-ci.org/chfoo/wpull/jobs/51420202 self.assertGreaterEqual(len(warn_list), 1) for warn_obj in warn_list: if str(warn_obj.message) == 'HTTP session did not complete.': break else: self.fail('Warning did not occur.') client = Client() with self.assertRaises(MyException): with client.session() as session: request = Request(self.get_url('/')) yield from session.start(request) raise MyException('Oops')
def test_basic(self): cookie_jar = BetterMozillaCookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) cookie_jar_wrapper = CookieJarWrapper(cookie_jar) http_client = Client() proxy = HTTPProxyServer(http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() client_request = None def request_callback(client_request_): nonlocal client_request client_request = client_request_ print('client request', client_request) cookie_jar_wrapper.add_cookie_header(client_request) def server_response_callback(server_response): print('server response', server_response) assert client_request cookie_jar_wrapper.extract_cookies(server_response, client_request) def new_sesssion_callback(session: HTTPProxySession): session.event_dispatcher.add_listener( HTTPProxySession.Event.client_request, request_callback) session.event_dispatcher.add_listener( HTTPProxySession.Event.server_end_response, server_response_callback) proxy.event_dispatcher.add_listener( HTTPProxyServer.Event.begin_session, new_sesssion_callback) yield from asyncio.start_server(proxy, sock=proxy_socket) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/'), proxy_host='localhost', proxy_port=proxy_port, ) response = yield from tornado_future_adapter(test_client.fetch(request)) self.assertEqual(200, response.code) self.assertIn(b'Hello!', response.body) cookies = tuple(cookie_jar) self.assertEqual('hi', cookies[0].name) self.assertEqual('hello', cookies[0].value)
def test_no_content(self): http_client = Client() proxy = HTTPProxyServer(http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield from asyncio.start_server(proxy, sock=proxy_socket) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/no_content'), proxy_host='localhost', proxy_port=proxy_port ) response = yield from tornado_future_adapter(test_client.fetch(request)) self.assertEqual(204, response.code)
def test_post(self): http_client = Client() proxy = HTTPProxyServer(http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield from asyncio.start_server(proxy, sock=proxy_socket) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/post/'), proxy_host='localhost', proxy_port=proxy_port, body='text=blah', method='POST' ) response = yield from tornado_future_adapter(test_client.fetch(request)) self.assertEqual(200, response.code) self.assertIn(b'OK', response.body)