def test_http_get(self): # Send request line self.proxy.client.conn.queue((b'GET http://localhost:%d HTTP/1.1' % self.http_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HttpParser.states.COMPLETE) # Send headers and blank line, thus completing HTTP request self.proxy.client.conn.queue(CRLF.join([ b'User-Agent: proxy.py/%s' % version, b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Proxy-Connection: Keep-Alive', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.assertEqual(self.proxy.server.addr, (b'localhost', self.http_server_port)) # Flush data queued for server self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) # Receive full response from server data = self.proxy.server.recv() while data: self.proxy._process_response(data) logging.info(self.proxy.response.state) if self.proxy.response.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() # Verify 200 success response code self.assertEqual(self.proxy.response.state, HttpParser.states.COMPLETE) self.assertEqual(int(self.proxy.response.code), 200)
def test_post_partial_parse(self): self.parser.parse( CRLF.join([ b'POST http://localhost HTTP/1.1', b'Host: localhost', b'Content-Length: 7', b'Content-Type: application/x-www-form-urlencoded' ])) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.HEADERS_COMPLETE) self.parser.parse(b'a=b') self.assertEqual(self.parser.state, HttpParser.states.RCVING_BODY) self.assertEqual(self.parser.body, b'a=b') self.assertEqual(self.parser.buffer, b'') self.parser.parse(b'&c=d') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'')
def test_authenticated_proxy_http_get(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) self.proxy.client.conn.queue((b'GET http://localhost:%d HTTP/1.1' % self.http_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.proxy.client.conn.queue(CRLF.join([ b'User-Agent: proxy.py/%s' % version, b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.assertEqual(self.proxy.server.addr, (b'localhost', self.http_server_port)) self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) data = self.proxy.server.recv() while data: self.proxy._process_response(data) if self.proxy.response.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() self.assertEqual(self.proxy.response.state, HttpParser.states.COMPLETE) self.assertEqual(int(self.proxy.response.code), 200)
def test_proxy_connection_failed(self): with self.assertRaises(ProxyConnectionFailed): self.proxy._process_request(CRLF.join([ b'GET http://unknown.domain HTTP/1.1', b'Host: unknown.domain', CRLF ]))
def test_proxy_connection_failed(self): with self.assertRaises(ProxyConnectionFailed): self.proxy._process_request( CRLF.join([ b'GET http://unknown.domain HTTP/1.1', b'Host: unknown.domain', CRLF ]))
def test_http_get(self): # Send request line self.proxy.client.conn.queue((b'GET http://localhost:%d HTTP/1.1' % self.http_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HttpParser.states.COMPLETE) # Send headers and blank line, thus completing HTTP request self.proxy.client.conn.queue( CRLF.join([ b'User-Agent: proxy.py/%s' % version, b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Proxy-Connection: Keep-Alive', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.assertEqual(self.proxy.server.addr, (b'localhost', self.http_server_port)) # Flush data queued for server self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) # Receive full response from server data = self.proxy.server.recv() while data: self.proxy._process_response(data) logging.info(self.proxy.response.state) if self.proxy.response.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() # Verify 200 success response code self.assertEqual(self.proxy.response.state, HttpParser.states.COMPLETE) self.assertEqual(int(self.proxy.response.code), 200)
def test_post_partial_parse(self): self.parser.parse(CRLF.join([ b'POST http://localhost HTTP/1.1', b'Host: localhost', b'Content-Length: 7', b'Content-Type: application/x-www-form-urlencoded' ])) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.HEADERS_COMPLETE) self.parser.parse(b'a=b') self.assertEqual(self.parser.state, HttpParser.states.RCVING_BODY) self.assertEqual(self.parser.body, b'a=b') self.assertEqual(self.parser.buffer, b'') self.parser.parse(b'&c=d') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'')
def test_authenticated_proxy_http_get(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) self.proxy.client.conn.queue((b'GET http://localhost:%d HTTP/1.1' % self.http_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.proxy.client.conn.queue( CRLF.join([ b'User-Agent: proxy.py/%s' % version, b'Host: localhost:%d' % self.http_server_port, b'Accept: */*', b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HttpParser.states.COMPLETE) self.assertEqual(self.proxy.server.addr, (b'localhost', self.http_server_port)) self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) data = self.proxy.server.recv() while data: self.proxy._process_response(data) if self.proxy.response.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() self.assertEqual(self.proxy.response.state, HttpParser.states.COMPLETE) self.assertEqual(int(self.proxy.response.code), 200)
def test_get_partial_parse2(self): self.parser.parse( CRLF.join([b'GET http://localhost:8080 HTTP/1.1', b'Host: '])) self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.buffer, b'Host: ') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(b'localhost:8080' + CRLF) self.assertDictContainsSubset({b'host': (b'Host', b'localhost:8080')}, self.parser.headers) self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(b'Content-Type: text/plain' + CRLF) self.assertEqual(self.parser.buffer, b'') self.assertDictContainsSubset( {b'content-type': (b'Content-Type', b'text/plain')}, self.parser.headers) self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)
def test_authenticated_proxy_http_tunnel(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) self.proxy.client.conn.queue( CRLF.join([ b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port, b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % version, b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertFalse(self.proxy.server is None) self.assertEqual(self.proxy.client.buffer, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) parser = HttpParser(HttpParser.types.RESPONSE_PARSER) parser.parse(self.proxy.client.buffer) self.assertEqual(parser.state, HttpParser.states.HEADERS_COMPLETE) self.assertEqual(int(parser.code), 200) self.proxy.client.flush() self.assertEqual(self.proxy.client.buffer_size(), 0) self.proxy.client.conn.queue( CRLF.join([ b'GET / HTTP/1.1', b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % version, CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) parser = HttpParser(HttpParser.types.RESPONSE_PARSER) data = self.proxy.server.recv() while data: parser.parse(data) if parser.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() self.assertEqual(parser.state, HttpParser.states.COMPLETE) self.assertEqual(int(parser.code), 200)
def test_proxy_authentication_failed(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) with self.assertRaises(ProxyAuthenticationFailed): self.proxy._process_request(CRLF.join([ b'GET http://abhinavsingh.com HTTP/1.1', b'Host: abhinavsingh.com', CRLF ]))
def test_authenticated_proxy_http_tunnel(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) self.proxy.client.conn.queue(CRLF.join([ b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port, b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % version, b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.assertFalse(self.proxy.server is None) self.assertEqual(self.proxy.client.buffer, PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) parser = HttpParser(HttpParser.types.RESPONSE_PARSER) parser.parse(self.proxy.client.buffer) self.assertEqual(parser.state, HttpParser.states.HEADERS_COMPLETE) self.assertEqual(int(parser.code), 200) self.proxy.client.flush() self.assertEqual(self.proxy.client.buffer_size(), 0) self.proxy.client.conn.queue(CRLF.join([ b'GET / HTTP/1.1', b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % version, CRLF ])) self.proxy._process_request(self.proxy.client.recv()) self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) parser = HttpParser(HttpParser.types.RESPONSE_PARSER) data = self.proxy.server.recv() while data: parser.parse(data) if parser.state == HttpParser.states.COMPLETE: break data = self.proxy.server.recv() self.assertEqual(parser.state, HttpParser.states.COMPLETE) self.assertEqual(int(parser.code), 200)
def test_proxy_authentication_failed(self): self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(b'user:pass')) with self.assertRaises(ProxyAuthenticationFailed): self.proxy._process_request( CRLF.join([ b'GET http://abhinavsingh.com HTTP/1.1', b'Host: abhinavsingh.com', CRLF ]))
def test_get_full_parse(self): raw = CRLF.join([ b'GET %s HTTP/1.1', b'Host: %s', CRLF ]) self.parser.parse(raw % (b'https://example.com/path/dir/?a=b&c=d#p=q', b'example.com')) self.assertEqual(self.parser.build_url(), b'/path/dir/?a=b&c=d#p=q') self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'example.com') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertDictContainsSubset({b'host': (b'Host', b'example.com')}, self.parser.headers) self.assertEqual(raw % (b'/path/dir/?a=b&c=d#p=q', b'example.com'), self.parser.build(del_headers=[b'host'], add_headers=[(b'Host', b'example.com')]))
def test_request_parse_without_content_length(self): """Case when incoming request doesn't contain a content-length header. From http://w3-org.9356.n7.nabble.com/POST-with-empty-body-td103965.html 'A POST with no content-length and no body is equivalent to a POST with Content-Length: 0 and nothing following, as could perfectly happen when you upload an empty file for instance.' See https://github.com/abhinavsingh/proxy.py/issues/20 for details. """ self.parser.parse( CRLF.join([ b'POST http://localhost HTTP/1.1', b'Host: localhost', b'Content-Type: application/x-www-form-urlencoded', CRLF ])) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)
def test_get_full_parse(self): raw = CRLF.join([b'GET %s HTTP/1.1', b'Host: %s', CRLF]) self.parser.parse( raw % (b'https://example.com/path/dir/?a=b&c=d#p=q', b'example.com')) self.assertEqual(self.parser.build_url(), b'/path/dir/?a=b&c=d#p=q') self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'example.com') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertDictContainsSubset({b'host': (b'Host', b'example.com')}, self.parser.headers) self.assertEqual( raw % (b'/path/dir/?a=b&c=d#p=q', b'example.com'), self.parser.build(del_headers=[b'host'], add_headers=[(b'Host', b'example.com')]))
def test_request_parse_without_content_length(self): """Case when incoming request doesn't contain a content-length header. From http://w3-org.9356.n7.nabble.com/POST-with-empty-body-td103965.html 'A POST with no content-length and no body is equivalent to a POST with Content-Length: 0 and nothing following, as could perfectly happen when you upload an empty file for instance.' See https://github.com/abhinavsingh/proxy.py/issues/20 for details. """ self.parser.parse(CRLF.join([ b'POST http://localhost HTTP/1.1', b'Host: localhost', b'Content-Type: application/x-www-form-urlencoded', CRLF ])) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)
def test_post_full_parse(self): raw = CRLF.join([ b'POST %s HTTP/1.1', b'Host: localhost', b'Content-Length: 7', b'Content-Type: application/x-www-form-urlencoded' + CRLF, b'a=b&c=d' ]) self.parser.parse(raw % b'http://localhost') self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertDictContainsSubset({b'content-type': (b'Content-Type', b'application/x-www-form-urlencoded')}, self.parser.headers) self.assertDictContainsSubset({b'content-length': (b'Content-Length', b'7')}, self.parser.headers) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertEqual(len(self.parser.build()), len(raw % b'/'))
def test_response_parse_without_content_length(self): """Case when server response doesn't contain a content-length header for non-chunk response types. HttpParser by itself has no way to know if more data should be expected. In example below, parser reaches state HttpParser.states.HEADERS_COMPLETE and it is responsibility of callee to change state to HttpParser.states.COMPLETE when server stream closes. See https://github.com/abhinavsingh/proxy.py/issues/20 for details. """ self.parser.type = HttpParser.types.RESPONSE_PARSER self.parser.parse(b'HTTP/1.0 200 OK' + CRLF) self.assertEqual(self.parser.code, b'200') self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(CRLF.join([ b'Server: BaseHTTP/0.3 Python/2.7.10', b'Date: Thu, 13 Dec 2018 16:24:09 GMT', CRLF ])) self.assertEqual(self.parser.state, HttpParser.states.HEADERS_COMPLETE)
def test_response_parse_without_content_length(self): """Case when server response doesn't contain a content-length header for non-chunk response types. HttpParser by itself has no way to know if more data should be expected. In example below, parser reaches state HttpParser.states.HEADERS_COMPLETE and it is responsibility of callee to change state to HttpParser.states.COMPLETE when server stream closes. See https://github.com/abhinavsingh/proxy.py/issues/20 for details. """ self.parser.type = HttpParser.types.RESPONSE_PARSER self.parser.parse(b'HTTP/1.0 200 OK' + CRLF) self.assertEqual(self.parser.code, b'200') self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse( CRLF.join([ b'Server: BaseHTTP/0.3 Python/2.7.10', b'Date: Thu, 13 Dec 2018 16:24:09 GMT', CRLF ])) self.assertEqual(self.parser.state, HttpParser.states.HEADERS_COMPLETE)
def test_post_full_parse(self): raw = CRLF.join([ b'POST %s HTTP/1.1', b'Host: localhost', b'Content-Length: 7', b'Content-Type: application/x-www-form-urlencoded' + CRLF, b'a=b&c=d' ]) self.parser.parse(raw % b'http://localhost') self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertDictContainsSubset( { b'content-type': (b'Content-Type', b'application/x-www-form-urlencoded') }, self.parser.headers) self.assertDictContainsSubset( {b'content-length': (b'Content-Length', b'7')}, self.parser.headers) self.assertEqual(self.parser.body, b'a=b&c=d') self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, HttpParser.states.COMPLETE) self.assertEqual(len(self.parser.build()), len(raw % b'/'))
def test_get_partial_parse1(self): self.parser.parse(CRLF.join([b'GET http://localhost:8080 HTTP/1.1'])) self.assertEqual(self.parser.method, None) self.assertEqual(self.parser.url, None) self.assertEqual(self.parser.version, None) self.assertEqual(self.parser.state, HttpParser.states.INITIALIZED) self.parser.parse(CRLF) self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(b'Host: localhost:8080') self.assertDictEqual(self.parser.headers, dict()) self.assertEqual(self.parser.buffer, b'Host: localhost:8080') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(CRLF * 2) self.assertDictContainsSubset({b'host': (b'Host', b'localhost:8080')}, self.parser.headers) self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)
def test_get_partial_parse2(self): self.parser.parse(CRLF.join([ b'GET http://localhost:8080 HTTP/1.1', b'Host: ' ])) self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.buffer, b'Host: ') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(b'localhost:8080' + CRLF) self.assertDictContainsSubset({b'host': (b'Host', b'localhost:8080')}, self.parser.headers) self.assertEqual(self.parser.buffer, b'') self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(b'Content-Type: text/plain' + CRLF) self.assertEqual(self.parser.buffer, b'') self.assertDictContainsSubset({b'content-type': (b'Content-Type', b'text/plain')}, self.parser.headers) self.assertEqual(self.parser.state, HttpParser.states.RCVING_HEADERS) self.parser.parse(CRLF) self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)
def test_get_partial_parse1(self): self.parser.parse(CRLF.join([ b'GET http://localhost:8080 HTTP/1.1' ])) self.assertEqual(self.parser.method, None) self.assertEqual(self.parser.url, None) self.assertEqual(self.parser.version, None) self.assertEqual(self.parser.state, HttpParser.states.INITIALIZED) self.parser.parse(CRLF) self.assertEqual(self.parser.method, b'GET') self.assertEqual(self.parser.url.hostname, b'localhost') self.assertEqual(self.parser.url.port, 8080) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(b'Host: localhost:8080') self.assertDictEqual(self.parser.headers, dict()) self.assertEqual(self.parser.buffer, b'Host: localhost:8080') self.assertEqual(self.parser.state, HttpParser.states.LINE_RCVD) self.parser.parse(CRLF * 2) self.assertDictContainsSubset({b'host': (b'Host', b'localhost:8080')}, self.parser.headers) self.assertEqual(self.parser.state, HttpParser.states.COMPLETE)