def test_chunked_request_parse(self) -> None: self.parser.parse( build_http_request( httpMethods.POST, b'http://example.org/', headers={ b'Transfer-Encoding': b'chunked', b'Content-Type': b'application/json', }, body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n', ), ) self.assertEqual(self.parser.body, b'{"key":"value"}') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) self.assertEqual( self.parser.build(), build_http_request( httpMethods.POST, b'/', headers={ b'Transfer-Encoding': b'chunked', b'Content-Type': b'application/json', }, body=b'f\r\n{"key":"value"}\r\n0\r\n\r\n', ), )
def test_redirect_to_custom_server_plugin( self, mock_server_conn: mock.Mock) -> None: request = build_http_request(b'GET', b'http://example.org/get', headers={ b'Host': b'example.org', }) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], ] self.protocol_handler.run_once() upstream = urlparse.urlsplit( RedirectToCustomServerPlugin.UPSTREAM_SERVER) mock_server_conn.assert_called_with('localhost', 8899) mock_server_conn.return_value.queue.assert_called_with( build_http_request(b'GET', upstream.path, headers={ b'Host': upstream.netloc, b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }))
async def test_shortlink_plugin_external(self) -> None: request = build_http_request( b'GET', b'http://jaxl.com/', headers={ b'Host': b'jaxl.com', }, no_ua=True, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_called_once_with('jaxl.com', 80) self.mock_server_conn.return_value.queue.assert_called_with( build_http_request( b'GET', b'/', headers={ b'Host': b'jaxl.com', b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, no_ua=True, ), ) self.assertFalse(self.protocol_handler.work.has_buffer())
def test_modify_post_data_plugin(self, mock_server_conn: mock.Mock) -> None: original = b'{"key": "value"}' modified = b'{"key": "modified"}' self._conn.recv.return_value = build_http_request( b'POST', b'http://httpbin.org/post', headers={ b'Host': b'httpbin.org', b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Length': bytes_(len(original)), }, body=original) self.mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], ] self.protocol_handler.run_once() mock_server_conn.assert_called_with('httpbin.org', DEFAULT_HTTP_PORT) mock_server_conn.return_value.queue.assert_called_with( build_http_request(b'POST', b'/post', headers={ b'Host': b'httpbin.org', b'Content-Length': bytes_(len(modified)), b'Content-Type': b'application/json', b'Via': b'1.1 %s' % PROXY_AGENT_HEADER_VALUE, }, body=modified))
async def test_proxy_auth_fails_with_invalid_cred(self) -> None: self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ b'Host': b'upstream.host', httpHeaders.PROXY_AUTHORIZATION: b'Basic hello', }, ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_not_called() self.assertEqual(self.protocol_handler.work.has_buffer(), True) self.assertEqual( self.protocol_handler.work.buffer[0], PROXY_AUTH_FAILED_RESPONSE_PKT, ) self._conn.send.assert_not_called()
def test_static_web_server_serves_404(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: self._conn = mock_fromfd.return_value self._conn.recv.return_value = build_http_request( b'GET', b'/not-found.html') mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_WRITE, data=None), selectors.EVENT_WRITE)], ] flags = Flags(enable_static_server=True) flags.plugins = Flags.load_plugins( b'proxy.http.proxy.HttpProxyPlugin,proxy.http.server.HttpWebServerPlugin' ) self.protocol_handler = HttpProtocolHandler(TcpClientConnection( self._conn, self._addr), flags=flags) self.protocol_handler.initialize() self.protocol_handler.run_once() self.protocol_handler.run_once() self.assertEqual(mock_selector.return_value.select.call_count, 2) self.assertEqual(self._conn.send.call_count, 1) self.assertEqual(self._conn.send.call_args[0][0], HttpWebServerPlugin.DEFAULT_404_RESPONSE)
def test_man_in_the_middle_plugin(self) -> None: request = build_http_request(b'GET', b'/', headers={ b'Host': b'uni.corn', }) self.client_ssl_connection.recv.return_value = request # Client read self.protocol_handler.run_once() self.server.queue.assert_called_once_with(request) # Server write self.protocol_handler.run_once() self.server.flush.assert_called_once() # Server read self.server.recv.return_value = \ build_http_response( httpStatusCodes.OK, reason=b'OK', body=b'Original Response From Upstream') self.protocol_handler.run_once() self.assertEqual( self.protocol_handler.client.buffer[0].tobytes(), build_http_response(httpStatusCodes.OK, reason=b'OK', body=b'Hello from man in the middle'))
async def test_shortlink_plugin(self) -> None: request = build_http_request( b'GET', b'http://t/', headers={ b'Host': b't', }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.work.buffer[0].tobytes(), build_http_response( status_code=httpStatusCodes.SEE_OTHER, reason=b'See Other', headers={ b'Location': b'http://twitter.com/', }, conn_close=True, ), )
async def test_proxy_plugin_on_and_before_upstream_connection(self) -> None: self.plugin.return_value.write_to_descriptors.return_value = False self.plugin.return_value.read_from_descriptors.return_value = False self.plugin.return_value.before_upstream_connection.side_effect = lambda r: r self.plugin.return_value.handle_client_request.side_effect = lambda r: r self.plugin.return_value.resolve_dns.return_value = None, None self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ b'Host': b'upstream.host', }, ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_called_with( 'upstream.host', DEFAULT_HTTP_PORT, ) self.plugin.return_value.before_upstream_connection.assert_called() self.plugin.return_value.handle_client_request.assert_called()
async def test_proxy_plugin_before_upstream_connection_can_teardown(self) -> None: self.plugin.return_value.write_to_descriptors.return_value = False self.plugin.return_value.read_from_descriptors.return_value = False self.plugin.return_value.before_upstream_connection.side_effect = HttpProtocolException() self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ b'Host': b'upstream.host', }, ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_not_called() self.plugin.return_value.before_upstream_connection.assert_called()
async def test_shortlink_plugin_unknown(self) -> None: request = build_http_request( b'GET', b'http://unknown/', headers={ b'Host': b'unknown', }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.work.buffer[0].tobytes(), NOT_FOUND_RESPONSE_PKT, )
async def test_filter_by_url_regex_plugin(self) -> None: request = build_http_request( b'GET', b'http://www.facebook.com/tr/', headers={ b'Host': b'www.facebook.com', }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.assertEqual( self.protocol_handler.work.buffer[0], build_http_response( status_code=httpStatusCodes.NOT_FOUND, reason=b'Blocked', conn_close=True, ), )
async def test_static_web_server_serves_404(self) -> None: self._conn.recv.return_value = build_http_request( b'GET', b'/not-found.html', ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_WRITE, data=None, ), selectors.EVENT_WRITE, )], ] await self.protocol_handler._run_once() await self.protocol_handler._run_once() self.assertEqual(self.mock_selector.return_value.select.call_count, 2) self.assertEqual(self._conn.send.call_count, 1) self.assertEqual( self._conn.send.call_args[0][0], NOT_FOUND_RESPONSE_PKT, )
async def test_redirect_to_custom_server_plugin_skips_https(self) -> None: request = build_http_request( b'CONNECT', b'jaxl.com:443', headers={ b'Host': b'jaxl.com:443', }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_called_with('jaxl.com', 443) self.assertEqual( self.protocol_handler.work.buffer[0].tobytes(), PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, )
async def test_filter_by_upstream_host_plugin(self) -> None: request = build_http_request( b'GET', b'http://facebook.com/', headers={ b'Host': b'facebook.com', }, ) self._conn.recv.return_value = request self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_not_called() self.assertEqual( self.protocol_handler.work.buffer[0], build_http_response( status_code=httpStatusCodes.I_AM_A_TEAPOT, reason=b'I\'m a tea pot', conn_close=True, ), )
def test_proposed_rest_api_plugin(self, mock_server_conn: mock.Mock) -> None: path = b'/v1/users/' self._conn.recv.return_value = build_http_request( b'GET', b'http://%s%s' % (ProposedRestApiPlugin.API_SERVER, path), headers={ b'Host': ProposedRestApiPlugin.API_SERVER, }) self.mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], ] self.protocol_handler.run_once() mock_server_conn.assert_not_called() self.assertEqual( self.protocol_handler.client.buffer[0].tobytes(), build_http_response( httpStatusCodes.OK, reason=b'OK', headers={b'Content-Type': b'application/json'}, body=bytes_( json.dumps(ProposedRestApiPlugin.REST_API_SPEC[path]))))
def test_static_web_server_serves(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: # Setup a static directory static_server_dir = os.path.join(tempfile.gettempdir(), 'static') index_file_path = os.path.join(static_server_dir, 'index.html') html_file_content = b'''<html><head></head><body><h1>Proxy.py Testing</h1></body></html>''' os.makedirs(static_server_dir, exist_ok=True) with open(index_file_path, 'wb') as f: f.write(html_file_content) self._conn = mock_fromfd.return_value self._conn.recv.return_value = build_http_request( b'GET', b'/index.html') mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], [(selectors.SelectorKey(fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_WRITE, data=None), selectors.EVENT_WRITE)], ] flags = Flags(enable_static_server=True, static_server_dir=static_server_dir) flags.plugins = Flags.load_plugins( b'proxy.http.proxy.HttpProxyPlugin,proxy.http.server.HttpWebServerPlugin' ) self.protocol_handler = HttpProtocolHandler(TcpClientConnection( self._conn, self._addr), flags=flags) self.protocol_handler.initialize() self.protocol_handler.run_once() self.protocol_handler.run_once() self.assertEqual(mock_selector.return_value.select.call_count, 2) self.assertEqual(self._conn.send.call_count, 1) encoded_html_file_content = gzip.compress(html_file_content) self.assertEqual( self._conn.send.call_args[0][0], build_http_response(200, reason=b'OK', headers={ b'Content-Type': b'text/html', b'Cache-Control': b'max-age=86400', b'Content-Encoding': b'gzip', b'Connection': b'close', b'Content-Length': bytes_(len(encoded_html_file_content)), }, body=encoded_html_file_content))
def test_is_not_http_1_1_keep_alive_for_http_1_0(self) -> None: self.parser.parse( build_http_request( httpMethods.GET, b'/', protocol_version=b'HTTP/1.0', )) self.assertFalse(self.parser.is_http_1_1_keep_alive())
def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None: self.parser.parse( build_http_request(httpMethods.GET, b'/', headers={ b'Connection': b'close', })) self.assertFalse(self.parser.is_http_1_1_keep_alive())
def test_is_not_http_1_1_keep_alive_with_close_header(self) -> None: self.parser.parse( build_http_request( httpMethods.GET, b'/', conn_close=True, ), ) self.assertFalse(self.parser.is_http_1_1_keep_alive)
def test_is_http_1_1_keep_alive_with_non_close_connection_header( self) -> None: self.parser.parse( build_http_request( httpMethods.GET, b'/', headers={ b'Connection': b'keep-alive', }, ), ) self.assertTrue(self.parser.is_http_1_1_keep_alive)
def test_build_request(self) -> None: self.assertEqual( build_http_request(b'GET', b'http://localhost:12345', b'HTTP/1.1'), CRLF.join([b'GET http://localhost:12345 HTTP/1.1', CRLF])) self.assertEqual( build_http_request(b'GET', b'http://localhost:12345', b'HTTP/1.1', headers={b'key': b'value'}), CRLF.join( [b'GET http://localhost:12345 HTTP/1.1', b'key: value', CRLF])) self.assertEqual( build_http_request(b'GET', b'http://localhost:12345', b'HTTP/1.1', headers={b'key': b'value'}, body=b'Hello from py'), CRLF.join([ b'GET http://localhost:12345 HTTP/1.1', b'key: value', CRLF ]) + b'Hello from py')
def test_modify_post_data_plugin(self) -> None: original = b'{"key": "value"}' modified = b'{"key": "modified"}' self.client_ssl_connection.recv.return_value = build_http_request( b'POST', b'/', headers={ b'Host': b'uni.corn', b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Length': bytes_(len(original)), }, body=original) self.protocol_handler.run_once() self.server.queue.assert_called_with( build_http_request(b'POST', b'/', headers={ b'Host': b'uni.corn', b'Content-Length': bytes_(len(modified)), b'Content-Type': b'application/json', }, body=modified))
async def test_static_web_server_serves(self) -> None: self._conn.recv.return_value = build_http_request( b'GET', b'/index.html', ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_WRITE, data=None, ), selectors.EVENT_WRITE, )], ] await self.protocol_handler._run_once() await self.protocol_handler._run_once() self.assertEqual(self.mock_selector.return_value.select.call_count, 2) self.assertEqual(self._conn.send.call_count, 1) encoded_html_file_content = gzip.compress(self.html_file_content) # parse response and verify response = HttpParser(httpParserTypes.RESPONSE_PARSER) response.parse(self._conn.send.call_args[0][0]) self.assertEqual(response.code, b'200') self.assertEqual(response.header(b'content-type'), b'text/html') self.assertEqual(response.header(b'cache-control'), b'max-age=86400') self.assertEqual(response.header(b'content-encoding'), b'gzip') self.assertEqual(response.header(b'connection'), b'close') self.assertEqual( response.header(b'content-length'), bytes_(len(encoded_html_file_content)), ) assert response.body self.assertEqual( gzip.decompress(response.body), self.html_file_content, )
async def test_man_in_the_middle_plugin(self) -> None: await self.protocol_handler._run_once() self.assertEqual(self.mock_sign_csr.call_count, 1) self.assertEqual(self.mock_gen_csr.call_count, 1) self.assertEqual(self.mock_gen_public_key.call_count, 1) self.mock_server_conn.assert_called_once_with('uni.corn', 443) self.server.connect.assert_called() self.assertEqual( self.protocol_handler.work.connection, self.client_ssl_connection, ) self.assertEqual(self.server.connection, self.server_ssl_connection) self._conn.send.assert_called_with( PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) self.assertFalse(self.protocol_handler.work.has_buffer()) # request = build_http_request( b'GET', b'/', headers={ b'Host': b'uni.corn', }, no_ua=True, ) self.client_ssl_connection.recv.return_value = request # Client read await self.protocol_handler._run_once() self.server.queue.assert_called_once_with(request) # Server write await self.protocol_handler._run_once() self.server.flush.assert_called_once() # Server read self.server.recv.return_value = okResponse( content=b'Original Response From Upstream', ) await self.protocol_handler._run_once() response = HttpParser(httpParserTypes.RESPONSE_PARSER) response.parse(self.protocol_handler.work.buffer[0]) assert response.body self.assertEqual( gzip.decompress(response.body), b'Hello from man in the middle', )
async def test_modify_post_data_plugin(self) -> None: await self.protocol_handler._run_once() self.assertEqual(self.mock_sign_csr.call_count, 1) self.assertEqual(self.mock_gen_csr.call_count, 1) self.assertEqual(self.mock_gen_public_key.call_count, 1) self.mock_server_conn.assert_called_once_with('uni.corn', 443) self.server.connect.assert_called() self.assertEqual( self.protocol_handler.work.connection, self.client_ssl_connection, ) self.assertEqual(self.server.connection, self.server_ssl_connection) self._conn.send.assert_called_with( PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT, ) self.assertFalse(self.protocol_handler.work.has_buffer()) # original = b'{"key": "value"}' modified = b'{"key": "modified"}' self.client_ssl_connection.recv.return_value = build_http_request( b'POST', b'/', headers={ b'Host': b'uni.corn', b'Content-Length': bytes_(len(original)), b'Content-Type': b'application/x-www-form-urlencoded', }, body=original, no_ua=True, ) await self.protocol_handler._run_once() self.server.queue.assert_called_once() # pkt = build_http_request( # b'POST', b'/', # headers={ # b'Host': b'uni.corn', # b'Content-Length': bytes_(len(modified)), # b'Content-Type': b'application/json', # }, # body=modified, # ) response = HttpParser.response( self.server.queue.call_args_list[0][0][0].tobytes(), ) self.assertEqual(response.body, modified)
def test_with_proxy(self) -> None: """Makes a HTTP request to in-build web server via proxy server.""" assert self.PROXY with socket_connection(('localhost', self.PROXY.flags.port)) as conn: conn.send( build_http_request( httpMethods.GET, b'http://localhost:%d/' % self.PROXY.flags.port, headers={ b'Host': b'localhost:%d' % self.PROXY.flags.port, }, ), ) response = conn.recv(DEFAULT_CLIENT_RECVBUF_SIZE) self.assertEqual( response, NOT_FOUND_RESPONSE_PKT.tobytes(), )
def test_with_proxy(self) -> None: """Makes a HTTP request to in-build web server via proxy server.""" with socket_connection(('localhost', self.PROXY_PORT)) as conn: conn.send( build_http_request(httpMethods.GET, b'http://localhost:%d/' % self.PROXY_PORT, headers={ b'Host': b'localhost:%d' % self.PROXY_PORT, })) response = conn.recv(DEFAULT_CLIENT_RECVBUF_SIZE) self.assertEqual( response, build_http_response(httpStatusCodes.NOT_FOUND, reason=b'NOT FOUND', headers={ b'Server': PROXY_AGENT_HEADER_VALUE, b'Connection': b'close' }))
def test_proxy_plugin_before_upstream_connection_can_teardown( self, mock_server_conn: mock.Mock) -> None: self.plugin.return_value.before_upstream_connection.side_effect = HttpProtocolException() self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', headers={ b'Host': b'upstream.host' }) self.mock_selector.return_value.select.side_effect = [ [(selectors.SelectorKey( fileobj=self._conn, fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], ] self.protocol_handler.run_once() self.plugin.return_value.before_upstream_connection.assert_called() mock_server_conn.assert_not_called()
async def test_proposed_rest_api_plugin(self) -> None: path = b'/v1/users/' self._conn.recv.return_value = build_http_request( b'GET', b'http://%s%s' % ( ProposedRestApiPlugin.API_SERVER, path, ), headers={ b'Host': ProposedRestApiPlugin.API_SERVER, }, ) self.mock_selector.return_value.select.side_effect = [ [( selectors.SelectorKey( fileobj=self._conn.fileno(), fd=self._conn.fileno(), events=selectors.EVENT_READ, data=None, ), selectors.EVENT_READ, )], ] await self.protocol_handler._run_once() self.mock_server_conn.assert_not_called() response = HttpParser(httpParserTypes.RESPONSE_PARSER) response.parse(self.protocol_handler.work.buffer[0]) assert response.body self.assertEqual( response.header(b'content-type'), b'application/json', ) self.assertEqual( response.header(b'content-encoding'), b'gzip', ) self.assertEqual( gzip.decompress(response.body), bytes_(json.dumps(ProposedRestApiPlugin.REST_API_SPEC[path], ), ), )