def test_client_duration_timeout(self): client = Client() with self.assertRaises(DurationTimeout), client.session() as session: request = Request(self.get_url('/sleep_long')) yield From(session.fetch(request)) yield From(session.read_content(duration_timeout=0.1))
def test_basic_requests(self): proxy_http_client = Client(recorder=DebugPrintRecorder()) proxy_server = HTTPProxyServer(proxy_http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield From(trollius.start_server(proxy_server, sock=proxy_socket)) connection_pool = HTTPProxyConnectionPool(('127.0.0.1', proxy_port)) http_client = Client(connection_pool=connection_pool, recorder=DebugPrintRecorder()) for dummy in range(3): with http_client.session() as session: response = yield From(session.fetch(Request(self.get_url('/')))) self.assertEqual(200, response.status_code) file = io.BytesIO() yield From(session.read_content(file=file)) data = file.getvalue().decode('ascii', 'replace') self.assertTrue(data.endswith('</html>')) with http_client.session() as session: response = yield From(session.fetch(Request( self.get_url('/always_error')))) self.assertEqual(500, response.status_code) self.assertEqual('Dragon In Data Center', response.reason) file = io.BytesIO() yield From(session.read_content(file=file)) data = file.getvalue().decode('ascii', 'replace') self.assertEqual('Error', data)
def test_client_did_not_complete(self): client = Client() with warnings.catch_warnings(record=True) as warn_list: warnings.simplefilter("always") with client.session() as session: request = Request(self.get_url('/')) yield From(session.fetch(request)) self.assertFalse(session.done()) for warn_obj in warn_list: print(warn_obj) # Unrelated warnings may occur in PyPy # https://travis-ci.org/chfoo/wpull/jobs/51420202 self.assertGreaterEqual(len(warn_list), 1) for warn_obj in warn_list: if str(warn_obj.message) == 'HTTP session did not complete.': break else: self.fail('Warning did not occur.') client = Client() with self.assertRaises(MyException): with client.session() as session: request = Request(self.get_url('/')) yield From(session.fetch(request)) raise MyException('Oops')
def test_client_exception_throw(self): client = Client() with client.session() as session: request = Request('http://wpull-no-exist.invalid') with self.assertRaises(NetworkError): yield From(session.fetch(request))
def test_client_exception_throw(self): client = Client() try: yield client.fetch(Request.new('http://wpull-no-exist.invalid')) except NetworkError: pass else: self.fail()
def test_connection_pool_clean(self): connection_pool = ConnectionPool() client = Client(connection_pool) requests = [client.fetch( Request.new(self.get_url('/'))) for dummy in range(12)] responses = yield requests for response in responses: self.assertEqual(200, response.status_code) connection_pool.clean() self.assertEqual(0, len(connection_pool))
def test_connection_pool_min(self): connection_pool = ConnectionPool() client = Client(connection_pool) for dummy in range(2): response = yield client.fetch( Request.new(self.get_url('/sleep_short'))) self.assertEqual(200, response.status_code) self.assertEqual(b'12', response.body.content) self.assertEqual(1, len(connection_pool)) connection_pool_entry = list(connection_pool.values())[0] self.assertIsInstance(connection_pool_entry, HostConnectionPool) self.assertEqual(1, len(connection_pool_entry))
def test_connection_pool_over_max(self): connection_pool = ConnectionPool() client = Client(connection_pool) requests = [client.fetch( Request.new(self.get_url('/sleep_short'))) for dummy in range(12)] responses = yield requests for response in responses: self.assertEqual(200, response.status_code) self.assertEqual(b'12', response.body.content) self.assertEqual(1, len(connection_pool)) connection_pool_entry = list(connection_pool.values())[0] self.assertIsInstance(connection_pool_entry, HostConnectionPool) self.assertEqual(6, len(connection_pool_entry))
def test_client_recorder(self): recorder = MockRecorder() client = Client(recorder=recorder) with client.session() as session: request = Request(self.get_url('/')) response = yield From(session.fetch(request)) yield From(session.read_content()) self.assertEqual(200, response.status_code) self.assertTrue(recorder.pre_request) self.assertTrue(recorder.request) self.assertTrue(recorder.pre_response) self.assertTrue(recorder.response) self.assertIn(b'GET', recorder.request_data) self.assertIn(b'hello', recorder.response_data)
def test_basic(self): client = Client() with client.session() as session: request = Request(self.get_url('/')) response = yield From(session.fetch(request)) self.assertEqual(200, response.status_code) self.assertEqual(request, response.request) file_obj = io.BytesIO() yield From(session.read_content(file_obj)) self.assertEqual(b'hello world!', file_obj.getvalue()) self.assertTrue(request.url_info) self.assertTrue(request.address) self.assertTrue(response.body)
def test_basic(self): http_client = Client() client = RichClient(http_client) session = client.session(Request.new(self.get_url('/'))) self.assertFalse(session.done) response = yield session.fetch() self.assertEqual(200, response.status_code) self.assertTrue(session.done)
def __init__(self, http_client=None, request_factory=Request, redirect_tracker_factory=RedirectTracker, cookie_jar=None): super().__init__() self._http_client = http_client or Client() self._request_factory = request_factory self._redirect_tracker_factory = redirect_tracker_factory self._cookie_jar = cookie_jar self._loop_type = None
def test_client_exception_recovery(self): connection_factory = functools.partial(Connection, read_timeout=0.2) host_connection_pool_factory = functools.partial( HostConnectionPool, connection_factory=connection_factory) connection_pool = ConnectionPool(host_connection_pool_factory) client = Client(connection_pool) for dummy in range(7): try: yield client.fetch( Request.new(self.get_url('/header_early_close')), recorder=DebugPrintRecorder() ) except NetworkError: pass else: self.fail() for dummy in range(7): response = yield client.fetch(Request.new(self.get_url('/'))) self.assertEqual(200, response.status_code)
def test_bad_redirect(self): http_client = Client() client = RichClient(http_client) session = client.session(Request.new(self.get_url('/bad_redirect'))) while not session.done: try: yield session.fetch() except ProtocolError: return else: self.fail()
def test_client_exception_recovery(self): connection_factory = functools.partial(Connection, timeout=2.0) connection_pool = ConnectionPool(connection_factory=connection_factory) client = Client(connection_pool=connection_pool) for dummy in range(7): with client.session() as session: request = Request(self.get_url('/header_early_close')) try: yield From(session.fetch(request)) except NetworkError: pass else: self.fail() # pragma: no cover for dummy in range(7): with client.session() as session: request = Request(self.get_url('/')) response = yield From(session.fetch(request)) self.assertEqual(200, response.status_code) yield From(session.read_content()) self.assertTrue(session.done())
def test_redirect(self): http_client = Client() client = RichClient(http_client) session = client.session(Request.new(self.get_url('/redirect'))) status_codes = [] while not session.done: response = yield session.fetch() if not status_codes: self.assertEqual(RichClientResponseType.redirect, session.response_type) status_codes.append(response.status_code) self.assertEqual([301, 200], status_codes) self.assertTrue(session.done) self.assertEqual(RichClientResponseType.normal, session.response_type)
def test_no_content(self): http_client = Client(recorder=DebugPrintRecorder()) proxy = HTTPProxyServer(http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() yield From(trollius.start_server(proxy, sock=proxy_socket)) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest(self.get_url('/no_content'), proxy_host='localhost', proxy_port=proxy_port) response = yield From( tornado_future_adapter(test_client.fetch(request))) self.assertEqual(204, response.code)
def test_no_content(self): http_client = Client(recorder=DebugPrintRecorder()) proxy = HTTPProxyServer(http_client, io_loop=self.io_loop) proxy_socket, proxy_port = tornado.testing.bind_unused_port() proxy.add_socket(proxy_socket) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/no_content'), proxy_host='localhost', proxy_port=proxy_port ) response = yield test_client.fetch(request) self.assertEqual(204, response.code)
def test_basic(self): cookie_jar = BetterMozillaCookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) cookie_jar_wrapper = CookieJarWrapper(cookie_jar) http_client = Client(recorder=DebugPrintRecorder()) proxy = HTTPProxyServer(http_client) proxy_socket, proxy_port = tornado.testing.bind_unused_port() def request_callback(request): print(request) cookie_jar_wrapper.add_cookie_header(request) def pre_response_callback(request, response): print(response) cookie_jar_wrapper.extract_cookies(response, request) proxy.request_callback = request_callback proxy.pre_response_callback = pre_response_callback yield From(trollius.start_server(proxy, sock=proxy_socket)) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/'), proxy_host='localhost', proxy_port=proxy_port, ) response = yield From( tornado_future_adapter(test_client.fetch(request))) self.assertEqual(200, response.code) self.assertIn(b'Hello!', response.body) cookies = tuple(cookie_jar) self.assertEqual('hi', cookies[0].name) self.assertEqual('hello', cookies[0].value)
def test_post(self): http_client = Client(recorder=DebugPrintRecorder()) proxy = HTTPProxyServer(http_client, io_loop=self.io_loop) proxy_socket, proxy_port = tornado.testing.bind_unused_port() proxy.add_socket(proxy_socket) _logger.debug('Proxy on port {0}'.format(proxy_port)) test_client = tornado.curl_httpclient.CurlAsyncHTTPClient() request = tornado.httpclient.HTTPRequest( self.get_url('/post/'), proxy_host='localhost', proxy_port=proxy_port, body='text=blah', method='POST' ) response = yield test_client.fetch(request) self.assertEqual(200, response.code) self.assertIn(b'OK', response.body)
def test_client(self): http_client = Client() proxy_server = HTTPProxyServer(http_client) proxy_socket, proxy_host = tornado.testing.bind_unused_port() proxy_server.add_socket(proxy_socket) remote_client = PhantomJSClient('localhost:{0}'.format(proxy_host)) with remote_client.remote() as remote: self.assertIn(remote, remote_client.remotes_busy) test_remote = remote for dummy in range(100): if test_remote in remote_client.remotes_ready: break yield wpull. async .sleep(0.1) self.assertIn(test_remote, remote_client.remotes_ready) self.assertNotIn(test_remote, remote_client.remotes_busy)
self._data_queue = data_queue @contextlib.contextmanager def session(self): yield ProxyRecorderSession(self._data_queue) class ProxyRecorderSession(BaseRecorderSession): '''Proxy Recorder Session.''' def __init__(self, data_queue): self._data_queue = data_queue def response_data(self, data): '''Callback for the bytes that was received.''' self._data_queue.put(data) def response(self, response): self._data_queue.put(None) if __name__ == '__main__': from wpull.http.client import Client logging.basicConfig(level=logging.DEBUG) http_client = Client(recorder=ProgressRecorder()) proxy = HTTPProxyServer(http_client) proxy.listen(8888) tornado.ioloop.IOLoop.current().start()