def handle_101_switching_protocols(self, flow): """ Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request. """ # Check for WebSockets handshake is_websockets = (flow and websockets.check_handshake(flow.request.headers) and websockets.check_handshake(flow.response.headers)) if is_websockets and not self.config.options.websockets: self.log( "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.", "info") if is_websockets and self.config.options.websockets: layer = pwebsockets.WebSocketsLayer(self, flow) else: layer = self.ctx.next_layer(self) layer()
def handle_101_switching_protocols(self, flow): """ Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request. """ # Check for WebSockets handshake is_websockets = ( flow and websockets.check_handshake(flow.request.headers) and websockets.check_handshake(flow.response.headers) ) if is_websockets and not self.config.options.websockets: self.log( "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.", "info" ) if is_websockets and self.config.options.websockets: layer = pwebsockets.WebSocketsLayer(self, flow) else: layer = self.ctx.next_layer(self) layer()
def _setup_connection(self): client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() request = http.Request( "authority", "CONNECT", "", "localhost", self.server.server.address.port, "", "HTTP/1.1", content=b'') client.wfile.write(http.http1.assemble_request(request)) client.wfile.flush() response = http.http1.read_response(client.rfile, request) if self.ssl: client.convert_to_ssl() assert client.ssl_established request = http.Request( "relative", "GET", "http", "localhost", self.server.server.address.port, "/ws", "HTTP/1.1", headers=http.Headers( connection="upgrade", upgrade="websocket", sec_websocket_version="13", sec_websocket_key="1234", ), content=b'') client.wfile.write(http.http1.assemble_request(request)) client.wfile.flush() response = http.http1.read_response(client.rfile, request) assert websockets.check_handshake(response.headers) return client
def _setup_connection(self): client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() request = http.Request("authority", "CONNECT", "", "localhost", self.server.server.address.port, "", "HTTP/1.1", content=b'') client.wfile.write(http.http1.assemble_request(request)) client.wfile.flush() response = http.http1.read_response(client.rfile, request) if self.ssl: client.convert_to_ssl() assert client.ssl_established request = http.Request("relative", "GET", "http", "localhost", self.server.server.address.port, "/ws", "HTTP/1.1", headers=http.Headers( connection="upgrade", upgrade="websocket", sec_websocket_version="13", sec_websocket_key="1234", ), content=b'') client.wfile.write(http.http1.assemble_request(request)) client.wfile.flush() response = http.http1.read_response(client.rfile, request) assert websockets.check_handshake(response.headers) return client
def handle(self): try: request = http.http1.read_request(self.rfile) assert websockets.check_handshake(request.headers) response = http.Response( "HTTP/1.1", 101, reason=http.status_codes.RESPONSES.get(101), headers=http.Headers( connection='upgrade', upgrade='websocket', sec_websocket_accept=b'', ), content=b'', ) self.wfile.write(http.http1.assemble_response(response)) self.wfile.flush() self.server.handle_websockets(self.rfile, self.wfile) except: traceback.print_exc()
def __call__(self): if self.mode == "transparent": self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: try: request = self.get_request_from_client() # Make sure that the incoming request matches our expectations self.validate_request(request) except netlib.exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return except netlib.exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) self.log( "request", "warn", "HTTP protocol error in client request: %s" % e ) return self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication # Proxy Authentication conceptually does not work in transparent mode. # We catch this misconfiguration on startup. Here, we sort out requests # after a successful CONNECT request (which do not need to be validated anymore) if not (self.http_authenticated or self.authenticate(request)): return flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request try: # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) flow.error = models.Error(str(e)) self.channel.ask("error", flow) return # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth self.process_request_hook(flow) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): # we only support RFC6455 with WebSockets version 13 # allow inline scripts to manupulate the client handshake self.channel.ask("websockets_handshake", flow) if not flow.response: self.establish_server_connection( flow.request.host, flow.request.port, flow.request.scheme ) self.get_response_from_server(flow) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. flow = self.channel.ask("responseheaders", flow) self.log("response", "debug", [repr(flow.response)]) flow = self.channel.ask("response", flow) self.send_response_to_client(flow) if self.check_close_connection(flow): return # Handle 101 Switching Protocols # It may be useful to pass additional args (such as the upgrade header) # to next_layer in the future if flow.response.status_code == 101: layer = self.ctx.next_layer(self, flow) layer() return # Upstream Proxy Mode: Handle CONNECT if flow.request.first_line_format == "authority" and flow.response.status_code == 200: self.handle_upstream_mode_connect(flow.request.copy()) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not flow.response: flow.error = models.Error(str(e)) self.channel.ask("error", flow) return else: six.reraise(exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: if flow: flow.live = False
def handle_http_request(self, logger): """ Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect log: A dictionary, or None """ with logger.ctx() as lg: try: req = self.protocol.read_request(self.rfile) except HttpReadDisconnect: return None, None except HttpException as s: s = str(s) lg(s) return None, dict(type="error", msg=s) if req.method == 'CONNECT': return self.protocol.handle_http_connect( [req.host, req.port, req.http_version], lg) method = req.method path = req.path http_version = req.http_version headers = req.headers clientcert = None if self.clientcert: clientcert = dict( cn=self.clientcert.cn, subject=self.clientcert.subject, serial=self.clientcert.serial, notbefore=self.clientcert.notbefore.isoformat(), notafter=self.clientcert.notafter.isoformat(), keyinfo=self.clientcert.keyinfo, ) retlog = dict( type="crafted", protocol="http", request=dict( path=path, method=method, headers=headers.fields, http_version=http_version, sni=self.sni, remote_address=self.address(), clientcert=clientcert, ), cipher=None, ) if self.ssl_established: retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() valid_websocket_handshake = websockets.check_handshake(headers) self.settings.websocket_key = websockets.get_client_key(headers) # If this is a websocket initiation, we respond with a proper # server response, unless over-ridden. if valid_websocket_handshake: anchor_gen = language.parse_pathod("ws") else: anchor_gen = None for regex, spec in self.server.anchors: if regex.match(path): anchor_gen = language.parse_pathod(spec, self.use_http2) break else: if m(path.startswith(self.server.craftanchor)): spec = urllib.parse.unquote( path)[len(self.server.craftanchor):] if spec: try: anchor_gen = language.parse_pathod( spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) anchor_gen = iter([ self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % (v.msg + v.marked())) ]) else: if self.use_http2: anchor_gen = iter([ self.make_http_error_response( "Spec Error", "HTTP/2 only supports request/response with the craft anchor point: %s" % self.server.craftanchor) ]) if not anchor_gen: anchor_gen = iter([ self.make_http_error_response( "Not found", "No valid craft request found") ]) spec = next(anchor_gen) if self.use_http2 and isinstance(spec, language.http2.Response): spec.stream_id = req.stream_id lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted(spec, lg) if nexthandler and valid_websocket_handshake: self.protocol = protocols.websockets.WebsocketsProtocol(self) return self.protocol.handle_websocket, retlog else: return nexthandler, retlog
def handle_http_request(self, logger): """ Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect log: A dictionary, or None """ with logger.ctx() as lg: try: req = self.protocol.read_request(self.rfile) except HttpReadDisconnect: return None, None except HttpException as s: s = str(s) lg(s) return None, dict(type="error", msg=s) if req.method == 'CONNECT': return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg) method = req.method path = req.path http_version = req.http_version headers = req.headers clientcert = None if self.clientcert: clientcert = dict( cn=self.clientcert.cn, subject=self.clientcert.subject, serial=self.clientcert.serial, notbefore=self.clientcert.notbefore.isoformat(), notafter=self.clientcert.notafter.isoformat(), keyinfo=self.clientcert.keyinfo, ) retlog = dict( type="crafted", protocol="http", request=dict( path=path, method=method, headers=headers.fields, http_version=http_version, sni=self.sni, remote_address=self.address(), clientcert=clientcert, ), cipher=None, ) if self.ssl_established: retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() valid_websockets_handshake = websockets.check_handshake(headers) self.settings.websocket_key = websockets.get_client_key(headers) # If this is a websocket initiation, we respond with a proper # server response, unless over-ridden. if valid_websockets_handshake: anchor_gen = language.parse_pathod("ws") else: anchor_gen = None for regex, spec in self.server.anchors: if regex.match(path): anchor_gen = language.parse_pathod(spec, self.use_http2) break else: if m(path.startswith(self.server.craftanchor)): spec = urllib.parse.unquote(path)[len(self.server.craftanchor):] if spec: try: anchor_gen = language.parse_pathod(spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) anchor_gen = iter([self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % ( v.msg + v.marked() ) )]) else: if self.use_http2: anchor_gen = iter([self.make_http_error_response( "Spec Error", "HTTP/2 only supports request/response with the craft anchor point: %s" % self.server.craftanchor )]) if not anchor_gen: anchor_gen = iter([self.make_http_error_response( "Not found", "No valid craft request found" )]) spec = next(anchor_gen) if self.use_http2 and isinstance(spec, language.http2.Response): spec.stream_id = req.stream_id lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( spec, lg ) if nexthandler and valid_websockets_handshake: self.protocol = protocols.websockets.WebsocketsProtocol(self) return self.protocol.handle_websocket, retlog else: return nexthandler, retlog
def __call__(self): if self.mode == "transparent": self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: try: request = self.get_request_from_client() # Make sure that the incoming request matches our expectations self.validate_request(request) except netlib.exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return except netlib.exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) six.reraise( exceptions.ProtocolException, exceptions.ProtocolException( "HTTP protocol error in client request: {}".format(e) ), sys.exc_info()[2] ) self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication # Proxy Authentication conceptually does not work in transparent mode. # We catch this misconfiguration on startup. Here, we sort out requests # after a successful CONNECT request (which do not need to be validated anymore) if not (self.http_authenticated or self.authenticate(request)): return flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request try: # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) flow.error = models.Error(str(e)) self.channel.ask("error", flow) return # update host header in reverse proxy mode if self.config.options.mode == "reverse": if six.PY2: flow.request.headers["Host"] = self.config.upstream_server.address.host.encode() else: flow.request.headers["Host"] = self.config.upstream_server.address.host # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth self.process_request_hook(flow) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): # We only support RFC6455 with WebSockets version 13 # allow inline scripts to manipulate the client handshake self.channel.ask("websockets_handshake", flow) if not flow.response: self.establish_server_connection( flow.request.host, flow.request.port, flow.request.scheme ) self.get_response_from_server(flow) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. flow = self.channel.ask("responseheaders", flow) self.log("response", "debug", [repr(flow.response)]) flow = self.channel.ask("response", flow) self.send_response_to_client(flow) if self.check_close_connection(flow): return # Handle 101 Switching Protocols if flow.response.status_code == 101: return self.handle_101_switching_protocols(flow) # Upstream Proxy Mode: Handle CONNECT if flow.request.first_line_format == "authority" and flow.response.status_code == 200: self.handle_upstream_mode_connect(flow.request.copy()) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not flow.response: flow.error = models.Error(str(e)) self.channel.ask("error", flow) return else: six.reraise(exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: if flow: flow.live = False
def test_check_handshake(self, input, expected): h = http.Headers(input) assert websockets.check_handshake(h) == expected
def _next_layer(self, top_layer, flow): if flow is not None: # We already have a flow, try to derive the next information from it # Check for WebSockets handshake is_websockets = ( flow and websockets.check_handshake(flow.request.headers) and websockets.check_handshake(flow.response.headers) ) if isinstance(top_layer, protocol.HttpLayer) and is_websockets: return protocol.WebSocketsLayer(top_layer, flow) try: d = top_layer.client_conn.rfile.peek(3) except netlib.exceptions.TcpException as e: six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(str(e)), sys.exc_info()[2]) client_tls = protocol.is_tls_record_magic(d) # 1. check for --ignore if self.config.check_ignore: ignore = self.config.check_ignore(top_layer.server_conn.address) if not ignore and client_tls: try: client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: ignore = self.config.check_ignore((client_hello.sni, 443)) if ignore: return protocol.RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. # An inline script may upgrade from http to https, # in which case we need some form of TLS layer. if isinstance(top_layer, modes.ReverseProxy): return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls) if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer): return protocol.TlsLayer(top_layer, client_tls, client_tls) # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. if isinstance(top_layer, protocol.TlsLayer): if isinstance(top_layer.ctx, modes.HttpProxy): return protocol.Http1Layer(top_layer, "regular") if isinstance(top_layer.ctx, modes.HttpUpstreamProxy): return protocol.Http1Layer(top_layer, "upstream") # 4. Check for other TLS cases (e.g. after CONNECT). if client_tls: return protocol.TlsLayer(top_layer, True, True) # 4. Check for --tcp if self.config.check_tcp(top_layer.server_conn.address): return protocol.RawTCPLayer(top_layer) # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, protocol.TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() if alpn == b'h2': return protocol.Http2Layer(top_layer, 'transparent') if alpn == b'http/1.1': return protocol.Http1Layer(top_layer, 'transparent') # 6. Check for raw tcp mode is_ascii = ( len(d) == 3 and # expect A-Za-z all(65 <= x <= 90 or 97 <= x <= 122 for x in six.iterbytes(d)) ) if self.config.options.rawtcp and not is_ascii: return protocol.RawTCPLayer(top_layer) # 7. Assume HTTP1 by default return protocol.Http1Layer(top_layer, 'transparent')